C2VEAFormatConverter: Encode without copy and format conversion if applicable
In current C2VEAFormatConverter design, it always does format conversion and
copy for each frame even if the source and target format are identical.
This CL we added the scheme to avoid unnecessary copy while detecting the
source format is already the same as target.
Bug: 73059339
Bug: 118544836
Test: CtsMediaTestCases android.media.cts.MediaRecorderTest on Soraka
Test: nexus7 camera App on Soraka
Change-Id: I3ab1387cbc7508777d44f0f4319f5e95cd2da17d
(cherry picked from commit 2a1d153e160b6ccb22b7249899161f9cb5cc9097)
diff --git a/C2VEAComponent.cpp b/C2VEAComponent.cpp
index a5981d1..633a5a9 100644
--- a/C2VEAComponent.cpp
+++ b/C2VEAComponent.cpp
@@ -734,7 +734,8 @@
reportError(status);
return;
}
- // Send format-converted input buffer to VEA for encode.
+ // Send format-converted input buffer to VEA for encode. |convertedBlock| will be the
+ // same as |inputBlock| if zero-copy is applied.
sendInputBufferToAccelerator(convertedBlock, index, timestamp, force_keyframe);
} else {
// Send input buffer to VEA for encode.
diff --git a/C2VEAFormatConverter.cpp b/C2VEAFormatConverter.cpp
index 3d39e2a..2b67d8b 100644
--- a/C2VEAFormatConverter.cpp
+++ b/C2VEAFormatConverter.cpp
@@ -22,6 +22,13 @@
namespace android {
namespace {
+// The constant expression of mapping the pixel format conversion pair (src, dst) to a unique
+// integer.
+constexpr int convertMap(media::VideoPixelFormat src, media::VideoPixelFormat dst) {
+ return static_cast<int>(src) * (static_cast<int>(
+ media::VideoPixelFormat::PIXEL_FORMAT_MAX) + 1) + static_cast<int>(dst);
+}
+
// The helper function to copy a plane pixel by pixel. It assumes bytesPerPixel is 1.
void copyPlaneByPixel(const uint8_t* src, int srcStride, int srcColInc, uint8_t* dst, int dstStride,
int dstColInc, int width, int height) {
@@ -116,7 +123,6 @@
}
BlockEntry* entry = mAvailableQueue.front();
- mAvailableQueue.pop();
std::shared_ptr<C2GraphicBlock> outputBlock = entry->mBlock;
const C2GraphicView& inputView = inputBlock.map().get();
@@ -134,6 +140,7 @@
const int dstStrideUV = outputLayout.planes[C2PlanarLayout::PLANE_U].rowInc; // only for NV12
media::VideoPixelFormat inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_UNKNOWN;
+ *status = C2_OK;
if (inputLayout.type == C2PlanarLayout::TYPE_YUV) {
const uint8_t* srcY = inputView.data()[C2PlanarLayout::PLANE_Y];
const uint8_t* srcU = inputView.data()[C2PlanarLayout::PLANE_U];
@@ -143,46 +150,57 @@
const int srcStrideV = inputLayout.planes[C2PlanarLayout::PLANE_V].rowInc;
if (inputLayout.rootPlanes == 3) {
inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_YV12;
- if (mOutFormat == media::VideoPixelFormat::PIXEL_FORMAT_I420) {
- libyuv::I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY,
- dstStrideY, dstU, dstStrideU, dstV, dstStrideV,
- mVisibleSize.width(), mVisibleSize.height());
- } else { // media::VideoPixelFormat::PIXEL_FORMAT_NV12
- libyuv::I420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY,
- dstStrideY, dstUV, dstStrideUV, mVisibleSize.width(),
- mVisibleSize.height());
- }
} else if (inputLayout.rootPlanes == 2) {
- if (srcV > srcU) {
- inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_NV12;
- if (mOutFormat == media::VideoPixelFormat::PIXEL_FORMAT_I420) {
- libyuv::NV12ToI420(srcY, srcStrideY, srcU, srcStrideU, dstY, dstStrideY, dstU,
- dstStrideU, dstV, dstStrideV, mVisibleSize.width(),
- mVisibleSize.height());
- } else { // media::VideoPixelFormat::PIXEL_FORMAT_NV12
- // TODO(johnylin): remove this copy in the future for zero-copy, we would use a
- // specified c2_status_t to indicate caller there is no need to
- // convert. Moreover, we need to manage returnBlock() wisely.
- libyuv::CopyPlane(srcY, srcStrideY, dstY, dstStrideY, mVisibleSize.width(),
- mVisibleSize.height());
- libyuv::CopyPlane(srcU, srcStrideU, dstUV, dstStrideUV, mVisibleSize.width(),
- mVisibleSize.height() / 2);
- }
- } else {
- inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_NV21;
- if (mOutFormat == media::VideoPixelFormat::PIXEL_FORMAT_I420) {
- libyuv::NV21ToI420(srcY, srcStrideY, srcV, srcStrideV, dstY, dstStrideY, dstU,
- dstStrideU, dstV, dstStrideV, mVisibleSize.width(),
- mVisibleSize.height());
- } else { // media::VideoPixelFormat::PIXEL_FORMAT_NV12
- libyuv::CopyPlane(srcY, srcStrideY, dstY, dstStrideY, mVisibleSize.width(),
- mVisibleSize.height());
- copyPlaneByPixel(srcU, srcStrideU, 2, dstUV, dstStrideUV, 2,
- mVisibleSize.width() / 2, mVisibleSize.height() / 2);
- copyPlaneByPixel(srcV, srcStrideV, 2, dstUV + 1, dstStrideUV, 2,
- mVisibleSize.width() / 2, mVisibleSize.height() / 2);
- }
- }
+ inputFormat = (srcV > srcU) ? media::VideoPixelFormat::PIXEL_FORMAT_NV12
+ : media::VideoPixelFormat::PIXEL_FORMAT_NV21;
+ }
+
+ if (inputFormat == mOutFormat) {
+ ALOGV("Zero-Copy is applied");
+ mGraphicBlocks.emplace_back(new BlockEntry(frameIndex));
+ return inputBlock;
+ }
+
+ switch (convertMap(inputFormat, mOutFormat)) {
+ case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_YV12,
+ media::VideoPixelFormat::PIXEL_FORMAT_I420):
+ libyuv::I420Copy(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY, dstStrideY,
+ dstU, dstStrideU, dstV, dstStrideV, mVisibleSize.width(),
+ mVisibleSize.height());
+ break;
+ case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_YV12,
+ media::VideoPixelFormat::PIXEL_FORMAT_NV12):
+ libyuv::I420ToNV12(srcY, srcStrideY, srcU, srcStrideU, srcV, srcStrideV, dstY,
+ dstStrideY, dstUV, dstStrideUV, mVisibleSize.width(),
+ mVisibleSize.height());
+ break;
+ case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_NV12,
+ media::VideoPixelFormat::PIXEL_FORMAT_I420):
+ libyuv::NV12ToI420(srcY, srcStrideY, srcU, srcStrideU, dstY, dstStrideY, dstU,
+ dstStrideU, dstV, dstStrideV, mVisibleSize.width(),
+ mVisibleSize.height());
+ break;
+ case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_NV21,
+ media::VideoPixelFormat::PIXEL_FORMAT_I420):
+ libyuv::NV21ToI420(srcY, srcStrideY, srcV, srcStrideV, dstY, dstStrideY, dstU,
+ dstStrideU, dstV, dstStrideV, mVisibleSize.width(),
+ mVisibleSize.height());
+ break;
+ case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_NV21,
+ media::VideoPixelFormat::PIXEL_FORMAT_NV12):
+ libyuv::CopyPlane(srcY, srcStrideY, dstY, dstStrideY, mVisibleSize.width(),
+ mVisibleSize.height());
+ copyPlaneByPixel(srcU, srcStrideU, 2, dstUV, dstStrideUV, 2,
+ mVisibleSize.width() / 2, mVisibleSize.height() / 2);
+ copyPlaneByPixel(srcV, srcStrideV, 2, dstUV + 1, dstStrideUV, 2,
+ mVisibleSize.width() / 2, mVisibleSize.height() / 2);
+ break;
+ default:
+ ALOGE("Unsupported pixel format conversion from %s to %s",
+ media::VideoPixelFormatToString(inputFormat).c_str(),
+ media::VideoPixelFormatToString(mOutFormat).c_str());
+ *status = C2_CORRUPTED;
+ return inputBlock; // This is actually redundant and should not be used.
}
} else if (inputLayout.type == C2PlanarLayout::TYPE_RGB) {
// There is only RGBA_8888 specified in C2AllocationGralloc::map(), no BGRA_8888. Maybe
@@ -190,10 +208,16 @@
inputFormat = media::VideoPixelFormat::PIXEL_FORMAT_ABGR;
const uint8_t* srcRGB = inputView.data()[C2PlanarLayout::PLANE_R];
const int srcStrideRGB = inputLayout.planes[C2PlanarLayout::PLANE_R].rowInc;
- if (mOutFormat == media::VideoPixelFormat::PIXEL_FORMAT_I420) {
+
+ switch (convertMap(inputFormat, mOutFormat)) {
+ case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_ABGR,
+ media::VideoPixelFormat::PIXEL_FORMAT_I420):
libyuv::ABGRToI420(srcRGB, srcStrideRGB, dstY, dstStrideY, dstU, dstStrideU, dstV,
dstStrideV, mVisibleSize.width(), mVisibleSize.height());
- } else { // media::VideoPixelFormat::PIXEL_FORMAT_NV12
+ break;
+ case convertMap(media::VideoPixelFormat::PIXEL_FORMAT_ABGR,
+ media::VideoPixelFormat::PIXEL_FORMAT_NV12):
+ {
// There is no libyuv function to convert ABGR to NV12. Therefore, we first convert to
// I420 on dst-Y plane and temporary U/V plane. Then we copy U and V pixels from
// temporary planes to dst-UV interleavedly.
@@ -204,19 +228,25 @@
libyuv::MergeUVPlane(mTempPlaneU.get(), tempStride, mTempPlaneV.get(), tempStride,
dstUV, dstStrideUV, mVisibleSize.width() / 2,
mVisibleSize.height() / 2);
+ break;
}
- }
-
- if (inputFormat == media::VideoPixelFormat::PIXEL_FORMAT_UNKNOWN) {
- ALOGE("Failed to parse input pixel format");
+ default:
+ ALOGE("Unsupported pixel format conversion from %s to %s",
+ media::VideoPixelFormatToString(inputFormat).c_str(),
+ media::VideoPixelFormatToString(mOutFormat).c_str());
+ *status = C2_CORRUPTED;
+ return inputBlock; // This is actually redundant and should not be used.
+ }
+ } else {
+ ALOGE("Unsupported input layout type");
*status = C2_CORRUPTED;
return inputBlock; // This is actually redundant and should not be used.
}
ALOGV("convertBlock(frame_index=%" PRIu64 ", format=%s)", frameIndex,
media::VideoPixelFormatToString(inputFormat).c_str());
- entry->mConvertedFrameIndex = frameIndex;
- *status = C2_OK;
+ entry->mAssociatedFrameIndex = frameIndex;
+ mAvailableQueue.pop();
return outputBlock->share(C2Rect(mVisibleSize.width(), mVisibleSize.height()), C2Fence());
}
@@ -226,14 +256,21 @@
auto iter = std::find_if(
mGraphicBlocks.begin(), mGraphicBlocks.end(),
[frameIndex](const std::unique_ptr<BlockEntry>& be) {
- return be->mConvertedFrameIndex == frameIndex; });
+ return be->mAssociatedFrameIndex == frameIndex; });
if (iter == mGraphicBlocks.end()) {
- ALOGE("Failed to find graphic block by converted frame index: %" PRIu64 "", frameIndex);
+ ALOGE("Failed to find graphic block by converted/zero-copied frame index: %" PRIu64 "",
+ frameIndex);
return C2_BAD_INDEX;
}
- (*iter)->mConvertedFrameIndex = kNoFrameConverted;
- mAvailableQueue.push(iter->get());
+ if ((*iter)->mBlock) {
+ // Returned block is format converted.
+ (*iter)->mAssociatedFrameIndex = kNoFrameAssociated;
+ mAvailableQueue.push(iter->get());
+ } else {
+ // Returned block is zero-copied.
+ mGraphicBlocks.erase(iter);
+ }
return C2_OK;
}
diff --git a/include/C2VEAFormatConverter.h b/include/C2VEAFormatConverter.h
index a1eeff2..47b1a91 100644
--- a/include/C2VEAFormatConverter.h
+++ b/include/C2VEAFormatConverter.h
@@ -29,10 +29,11 @@
uint32_t inputCount,
const media::Size& codedSize);
- // Convert the input block into the alternative block with required pixel format and return it.
+ // Convert the input block into the alternative block with required pixel format and return it,
+ // or return the original block if zero-copy is applied.
C2ConstGraphicBlock convertBlock(uint64_t frameIndex, const C2ConstGraphicBlock& inputBlock,
c2_status_t* status /* non-null */);
- // Return the block ownership when VEA no longer needs it.
+ // Return the block ownership when VEA no longer needs it, or erase the zero-copy BlockEntry.
c2_status_t returnBlock(uint64_t frameIndex);
// Check if there is available block for conversion.
bool isReady() const { return !mAvailableQueue.empty(); }
@@ -42,16 +43,24 @@
// kMinInputBufferArraySize from CCodecBufferChannel.
static constexpr uint32_t kMinInputBufferCount = 8;
// The constant used by BlockEntry to indicate no frame is associated with the BlockEntry.
- static constexpr uint64_t kNoFrameConverted = ~static_cast<uint64_t>(0);
+ static constexpr uint64_t kNoFrameAssociated = ~static_cast<uint64_t>(0);
- // Each block entry contains the shared pointer of allocated graphic block for conversion, and
- // |mConvertedFrameIndex| for recording the frame index of the input frame which is currently
- // converted from.
+ // There are 2 types of BlockEntry:
+ // 1. If |mBlock| is an allocated graphic block (not nullptr). This BlockEntry is for
+ // conversion, and |mAssociatedFrameIndex| records the frame index of the input frame which
+ // is currently converted from. This is created on initialize() and released while
+ // C2VEAFormatConverter is destroyed.
+ // 2. If |mBlock| is nullptr. This BlockEntry is only used to record zero-copied frame index to
+ // |mAssociatedFrameIndex|. This is created on zero-copy is applied during convertBlock(),
+ // and released on returnBlock() in associated with returned frame index.
struct BlockEntry {
+ // Constructor of convertible entry.
BlockEntry(std::shared_ptr<C2GraphicBlock> block) : mBlock(std::move(block)) {}
+ // Constructir of zero-copy entry.
+ BlockEntry(uint64_t frameIndex) : mAssociatedFrameIndex(frameIndex) {}
std::shared_ptr<C2GraphicBlock> mBlock;
- uint64_t mConvertedFrameIndex = kNoFrameConverted;
+ uint64_t mAssociatedFrameIndex = kNoFrameAssociated;
};
C2VEAFormatConverter() = default;
