Memory Domain Runtime: Explicit memory copying.
- Implement ANNMemory_copy.
- Add validation tests.
Bug: 141353602
Bug: 141363565
Test: NNT_static
Change-Id: Id0ec2431cf9cfff650cd8885de4d15abee306a9c
Merged-In: Id0ec2431cf9cfff650cd8885de4d15abee306a9c
(cherry picked from commit 9af11e783aadacf78330a5e6e4e8255f41789c13)
diff --git a/runtime/ExecutionBuilder.cpp b/runtime/ExecutionBuilder.cpp
index 6c18e1d..174faa0 100644
--- a/runtime/ExecutionBuilder.cpp
+++ b/runtime/ExecutionBuilder.cpp
@@ -603,7 +603,7 @@
for (const auto& output : mOutputs) {
if (output.state != ModelArgumentInfo::MEMORY) continue;
const Memory* memory = mMemories[output.locationAndLength.poolIndex];
- NN_RET_CHECK(memory->getValidator().updateDimensions(output.dimensions));
+ NN_RET_CHECK(memory->getValidator().updateMetadata({.dimensions = output.dimensions}));
}
return true;
}
diff --git a/runtime/Memory.cpp b/runtime/Memory.cpp
index 561df07..f8f90f7 100644
--- a/runtime/Memory.cpp
+++ b/runtime/Memory.cpp
@@ -26,6 +26,7 @@
#include <vector>
#include "CompilationBuilder.h"
+#include "CpuExecutor.h"
#include "ExecutionBurstController.h"
#include "Manager.h"
#include "MemoryUtils.h"
@@ -52,6 +53,11 @@
return true;
}
+ Metadata getMetadata() const override { return {.logicalSize = kSize}; }
+ bool updateMetadata(const Metadata& metadata) override {
+ return metadata.logicalSize == 0 || metadata.logicalSize == kSize;
+ }
+
private:
const uint32_t kSize;
};
@@ -73,6 +79,9 @@
<< ") for Non-BLOB format AHardwareBuffer.";
return true;
}
+
+ Metadata getMetadata() const override { return {}; }
+ bool updateMetadata(const Metadata&) override { return true; }
};
// The validator for a memory created from ANNMemory_createFromDesc.
@@ -80,10 +89,10 @@
// with both offset and length set to zero.
class DeviceMemoryValidator : public MemoryValidatorBase {
public:
- DeviceMemoryValidator(std::set<CompilationRole> roles, hal::OperandType type,
+ DeviceMemoryValidator(std::set<CompilationRole> roles, Operand operand,
std::vector<uint32_t> dimensions)
: kCompilationRoles(std::move(roles)),
- mDataType(type),
+ kOperand(std::move(operand)),
kInitialDimensions(std::move(dimensions)),
mUpdatedDimensions(kInitialDimensions) {}
@@ -95,7 +104,7 @@
NN_RET_CHECK(offset == 0 && length == 0)
<< "non-zero offset and/or length for driver-allocated memory.";
if (type) {
- const bool isTensor = TypeManager::get()->isTensorType(mDataType);
+ const bool isTensor = TypeManager::get()->isTensorType(kOperand.type);
NN_RET_CHECK(isTensor || type->dimensionCount == 0)
<< "invalid dimensions for scalar memory.";
std::vector<uint32_t> dimensions(type->dimensions,
@@ -119,19 +128,40 @@
return true;
}
- bool updateDimensions(const std::vector<uint32_t>& dimensions) override {
- NN_RET_CHECK(TypeManager::get()->isTensorType(mDataType) || dimensions.empty());
- auto combined = combineDimensions(dimensions, kInitialDimensions);
+ Metadata getMetadata() const override {
+ CHECK(mInitialized);
+ return {.logicalSize = TypeManager::get()->getSizeOfData(kOperand.type, mUpdatedDimensions),
+ .dimensions = mUpdatedDimensions,
+ .operand = kOperand};
+ }
+
+ bool updateMetadata(const Metadata& metadata) override {
+ NN_RET_CHECK(!metadata.operand.has_value() ||
+ (metadata.operand->type == kOperand.type &&
+ metadata.operand->scale == kOperand.scale &&
+ metadata.operand->zeroPoint == kOperand.zeroPoint &&
+ metadata.operand->extraParams == kOperand.extraParams));
+
+ NN_RET_CHECK(metadata.dimensions.empty() ||
+ TypeManager::get()->isTensorType(kOperand.type));
+ auto combined = combineDimensions(metadata.dimensions, kInitialDimensions);
NN_RET_CHECK(combined.has_value());
+ NN_RET_CHECK(metadata.logicalSize == 0 ||
+ metadata.logicalSize ==
+ TypeManager::get()->getSizeOfData(kOperand.type, combined.value()));
mUpdatedDimensions = std::move(combined.value());
return true;
}
void setInitialized(bool initialized) override { mInitialized = initialized; }
+ bool isInitialized() const override { return mInitialized; }
private:
const std::set<CompilationRole> kCompilationRoles;
- OperandType mDataType;
+
+ // Keep track of the data type, scale, zero point, and extra parameters of the target operand.
+ // Other fields will be ignored, including dimensions, lifetime, location, etc.
+ const Operand kOperand;
// The dimensions of the memory when the memory object is created.
// May have unknown dimensions or rank.
@@ -182,6 +212,90 @@
mUsedBy.emplace(burst.get(), burst);
}
+static int copyHidlMemories(const hidl_memory& src, const hidl_memory& dst) {
+ if (src.size() != dst.size()) {
+ LOG(ERROR) << "ANeuralNetworksMemory_copy -- incompatible memory size";
+ return ANEURALNETWORKS_BAD_DATA;
+ }
+ auto srcPool = RunTimePoolInfo::createFromHidlMemory(src);
+ auto dstPool = RunTimePoolInfo::createFromHidlMemory(dst);
+ if (!srcPool.has_value() || !dstPool.has_value()) {
+ LOG(ERROR) << "ANeuralNetworksMemory_copy -- unable to map memory";
+ return ANEURALNETWORKS_UNMAPPABLE;
+ }
+ CHECK(srcPool->getBuffer() != nullptr);
+ CHECK(dstPool->getBuffer() != nullptr);
+ std::copy(srcPool->getBuffer(), srcPool->getBuffer() + src.size(), dstPool->getBuffer());
+ dstPool->flush();
+ return ANEURALNETWORKS_NO_ERROR;
+}
+
+static int copyIBufferToHidlMemory(const sp<IBuffer>& src, const hidl_memory& dst) {
+ const auto ret = src->copyTo(dst);
+ if (!ret.isOk()) {
+ LOG(ERROR) << "ANeuralNetworksMemory_copy failure: " << ret.description();
+ return ANEURALNETWORKS_OP_FAILED;
+ }
+ return convertErrorStatusToResultCode(static_cast<ErrorStatus>(ret));
+}
+
+static int copyHidlMemoryToIBuffer(const hidl_memory& src, const sp<IBuffer>& dst,
+ const std::vector<uint32_t>& dimensions) {
+ const auto ret = dst->copyFrom(src, dimensions);
+ if (!ret.isOk()) {
+ LOG(ERROR) << "ANeuralNetworksMemory_copy failure: " << ret.description();
+ return ANEURALNETWORKS_OP_FAILED;
+ }
+ return convertErrorStatusToResultCode(static_cast<ErrorStatus>(ret));
+}
+
+static int copyIBuffers(const sp<IBuffer>& src, const sp<IBuffer>& dst,
+ const MemoryValidatorBase::Metadata& srcMetadata) {
+ // TODO(xusongw): Use BLOB mode AHardwareBuffer.
+ hidl_memory hidlMemory = allocateSharedMemory(srcMetadata.logicalSize);
+ if (!hidlMemory.valid()) return ANEURALNETWORKS_OUT_OF_MEMORY;
+ NN_RETURN_IF_ERROR(copyIBufferToHidlMemory(src, hidlMemory));
+ NN_RETURN_IF_ERROR(copyHidlMemoryToIBuffer(hidlMemory, dst, srcMetadata.dimensions));
+ return ANEURALNETWORKS_NO_ERROR;
+}
+
+static int copyInternal(const Memory& src, const Memory& dst) {
+ if (&src == &dst) return ANEURALNETWORKS_NO_ERROR;
+
+ if (!src.getValidator().isInitialized()) {
+ LOG(ERROR) << "ANeuralNetworksMemory_copy -- uninitialized source memory";
+ return ANEURALNETWORKS_BAD_DATA;
+ }
+
+ const auto srcMetadata = src.getValidator().getMetadata();
+ if (!dst.getValidator().updateMetadata(srcMetadata)) {
+ LOG(ERROR) << "ANeuralNetworksMemory_copy -- incompatible memories";
+ return ANEURALNETWORKS_BAD_DATA;
+ }
+
+ bool srcHasHidlMemory = src.getHidlMemory().valid();
+ bool dstHasHidlMemory = dst.getHidlMemory().valid();
+ bool srcHasIBuffer = src.getIBuffer() != nullptr;
+ bool dstHasIBuffer = dst.getIBuffer() != nullptr;
+ if (srcHasIBuffer && dstHasIBuffer) {
+ return copyIBuffers(src.getIBuffer(), dst.getIBuffer(), srcMetadata);
+ } else if (srcHasHidlMemory && dstHasHidlMemory) {
+ return copyHidlMemories(src.getHidlMemory(), dst.getHidlMemory());
+ } else if (srcHasHidlMemory && dstHasIBuffer) {
+ return copyHidlMemoryToIBuffer(src.getHidlMemory(), dst.getIBuffer(),
+ srcMetadata.dimensions);
+ } else if (srcHasIBuffer && dstHasHidlMemory) {
+ return copyIBufferToHidlMemory(src.getIBuffer(), dst.getHidlMemory());
+ }
+ return ANEURALNETWORKS_OP_FAILED;
+}
+
+int Memory::copy(const Memory& src, const Memory& dst) {
+ int n = copyInternal(src, dst);
+ dst.getValidator().setInitialized(n == ANEURALNETWORKS_NO_ERROR);
+ return n;
+}
+
bool MemoryBuilder::badState(const char* name) const {
if (mFinished) {
LOG(ERROR) << "ANeuralNetworksMemoryDesc_" << name << " can't modify after finished";
@@ -375,7 +489,7 @@
if (n == ANEURALNETWORKS_NO_ERROR) {
CHECK(memory != nullptr);
auto validator =
- std::make_unique<DeviceMemoryValidator>(mRoles, mOperand->type, mDesc.dimensions);
+ std::make_unique<DeviceMemoryValidator>(mRoles, mOperand.value(), mDesc.dimensions);
memory->setValidator(std::move(validator));
}
return {n, std::move(memory)};
diff --git a/runtime/Memory.h b/runtime/Memory.h
index 5394338..5044b87 100644
--- a/runtime/Memory.h
+++ b/runtime/Memory.h
@@ -133,9 +133,27 @@
// Validate the memory dimensional information at the beginning of a computation.
virtual bool validateInputDimensions(const std::vector<uint32_t>&) const { return true; }
- virtual bool updateDimensions(const std::vector<uint32_t>&) { return true; }
+ // The validation metadata for this memory.
+ struct Metadata {
+ // The byte size of the memory when it is transformed to a closely packed layout.
+ // Set to 0 if unknown (e.g. non-BLOB mode AHWB or device memory with dynamic shape).
+ uint32_t logicalSize;
+
+ // The dimensions of the memory. Set to empty if undefined.
+ std::vector<uint32_t> dimensions;
+
+ // The data type, scale, zero point, and extra parameters of the target operand.
+ // Other fields will be ignored, including dimensions, lifetime, location, etc.
+ // Set to std::nullopt if undefined.
+ std::optional<hal::Operand> operand;
+ };
+ virtual Metadata getMetadata() const;
+
+ // Try update the memory metadata with the provided metadata. Return false if incompatible.
+ virtual bool updateMetadata(const Metadata& metadata);
virtual void setInitialized(bool) {}
+ virtual bool isInitialized() const { return true; }
};
// Represents a memory region.
@@ -151,6 +169,7 @@
hal::Request::MemoryPool getMemoryPool() const;
const hal::hidl_memory& getHidlMemory() const { return kHidlMemory; }
const sp<hal::IBuffer>& getIBuffer() const { return kBuffer; }
+ virtual uint32_t getSize() const { return getHidlMemory().size(); }
MemoryValidatorBase& getValidator() const {
CHECK(mValidator != nullptr);
@@ -169,6 +188,8 @@
// the bursts' memory cache.
void usedBy(const std::shared_ptr<ExecutionBurstController>& burst) const;
+ static int copy(const Memory& src, const Memory& dst);
+
protected:
Memory(hal::hidl_memory memory);
Memory(hal::hidl_memory memory, std::unique_ptr<MemoryValidatorBase> validator);
diff --git a/runtime/NeuralNetworks.cpp b/runtime/NeuralNetworks.cpp
index f9d60d0..6b4114e 100644
--- a/runtime/NeuralNetworks.cpp
+++ b/runtime/NeuralNetworks.cpp
@@ -909,10 +909,15 @@
return ANEURALNETWORKS_NO_ERROR;
}
-int ANeuralNetworksMemory_copy(const ANeuralNetworksMemory* /*src*/,
- const ANeuralNetworksMemory* /*dst*/) {
- // TODO(xusongw): Implement.
- return ANEURALNETWORKS_OP_FAILED;
+int ANeuralNetworksMemory_copy(const ANeuralNetworksMemory* src, const ANeuralNetworksMemory* dst) {
+ NNTRACE_RT(NNTRACE_PHASE_EXECUTION, "ANeuralNetworksMemory_copy");
+ if (!src || !dst) {
+ LOG(ERROR) << "ANeuralNetworksMemory_copy passed a nullptr";
+ return ANEURALNETWORKS_UNEXPECTED_NULL;
+ }
+ const Memory* s = reinterpret_cast<const Memory*>(src);
+ const Memory* d = reinterpret_cast<const Memory*>(dst);
+ return Memory::copy(*s, *d);
}
int ANeuralNetworksMemory_createFromFd(size_t size, int prot, int fd, size_t offset,
diff --git a/runtime/test/TestValidation.cpp b/runtime/test/TestValidation.cpp
index 45c80b5..1401441 100644
--- a/runtime/test/TestValidation.cpp
+++ b/runtime/test/TestValidation.cpp
@@ -225,10 +225,25 @@
}
virtual void TearDown() {
ANeuralNetworksMemoryDesc_free(mDesc);
+ for (auto* memory : mMemories) ANeuralNetworksMemory_free(memory);
+ for (int fd : mFds) close(fd);
ValidationTestCompilation::TearDown();
}
+ ANeuralNetworksMemory* createAshmem(uint32_t size) {
+ int fd = ASharedMemory_create("nnMemory", size);
+ EXPECT_GT(fd, 0);
+ mFds.push_back(fd);
+ ANeuralNetworksMemory* ashmem = nullptr;
+ EXPECT_EQ(ANeuralNetworksMemory_createFromFd(size, PROT_READ | PROT_WRITE, fd, 0, &ashmem),
+ ANEURALNETWORKS_NO_ERROR);
+ mMemories.push_back(ashmem);
+ return ashmem;
+ }
+
ANeuralNetworksMemoryDesc* mDesc = nullptr;
+ std::vector<ANeuralNetworksMemory*> mMemories;
+ std::vector<int> mFds;
};
class ValidationTestExecutionDeviceMemory : public ValidationTest {
@@ -2418,6 +2433,64 @@
ANeuralNetworksMemory_free(memory);
}
+TEST_F(ValidationTestMemoryDesc, MemoryCopying) {
+ ASSERT_EQ(ANeuralNetworksCompilation_finish(mCompilation), ANEURALNETWORKS_NO_ERROR);
+
+ uint32_t goodSize = sizeof(float) * 2, badSize1 = sizeof(float), badSize2 = sizeof(float) * 4;
+ ANeuralNetworksMemory* goodAshmem = createAshmem(goodSize);
+ ANeuralNetworksMemory* badAshmem1 = createAshmem(badSize1);
+ ANeuralNetworksMemory* badAshmem2 = createAshmem(badSize2);
+
+ ANeuralNetworksMemory *deviceMemory1 = nullptr, *deviceMemory2 = nullptr;
+ EXPECT_EQ(ANeuralNetworksMemoryDesc_create(&mDesc), ANEURALNETWORKS_NO_ERROR);
+ EXPECT_EQ(ANeuralNetworksMemoryDesc_addInputRole(mDesc, mCompilation, 0, 1.0f),
+ ANEURALNETWORKS_NO_ERROR);
+ EXPECT_EQ(ANeuralNetworksMemoryDesc_finish(mDesc), ANEURALNETWORKS_NO_ERROR);
+ EXPECT_EQ(ANeuralNetworksMemory_createFromDesc(mDesc, &deviceMemory1),
+ ANEURALNETWORKS_NO_ERROR);
+ EXPECT_EQ(ANeuralNetworksMemory_createFromDesc(mDesc, &deviceMemory2),
+ ANEURALNETWORKS_NO_ERROR);
+
+ EXPECT_EQ(ANeuralNetworksMemory_copy(nullptr, deviceMemory1), ANEURALNETWORKS_UNEXPECTED_NULL);
+ EXPECT_EQ(ANeuralNetworksMemory_copy(deviceMemory1, nullptr), ANEURALNETWORKS_UNEXPECTED_NULL);
+
+ // Ashmem -> Ashmem
+ // Bad memory size.
+ EXPECT_EQ(ANeuralNetworksMemory_copy(goodAshmem, badAshmem1), ANEURALNETWORKS_BAD_DATA);
+ EXPECT_EQ(ANeuralNetworksMemory_copy(goodAshmem, badAshmem2), ANEURALNETWORKS_BAD_DATA);
+ EXPECT_EQ(ANeuralNetworksMemory_copy(badAshmem1, goodAshmem), ANEURALNETWORKS_BAD_DATA);
+ EXPECT_EQ(ANeuralNetworksMemory_copy(badAshmem2, goodAshmem), ANEURALNETWORKS_BAD_DATA);
+
+ // Ashmem -> Device Memory
+ // Bad memory size.
+ EXPECT_EQ(ANeuralNetworksMemory_copy(badAshmem1, deviceMemory1), ANEURALNETWORKS_BAD_DATA);
+ EXPECT_EQ(ANeuralNetworksMemory_copy(badAshmem2, deviceMemory1), ANEURALNETWORKS_BAD_DATA);
+
+ // Device Memory -> Ashmem
+ // Uninitialized source device memory.
+ EXPECT_EQ(ANeuralNetworksMemory_copy(deviceMemory1, goodAshmem), ANEURALNETWORKS_BAD_DATA);
+ // Bad memory size.
+ EXPECT_EQ(ANeuralNetworksMemory_copy(goodAshmem, deviceMemory1), ANEURALNETWORKS_NO_ERROR);
+ EXPECT_EQ(ANeuralNetworksMemory_copy(deviceMemory1, badAshmem1), ANEURALNETWORKS_BAD_DATA);
+ // Uninitialized source device memory (after a failed copy).
+ EXPECT_EQ(ANeuralNetworksMemory_copy(badAshmem1, deviceMemory1), ANEURALNETWORKS_BAD_DATA);
+ EXPECT_EQ(ANeuralNetworksMemory_copy(deviceMemory1, goodAshmem), ANEURALNETWORKS_BAD_DATA);
+ // Bad memory size.
+ EXPECT_EQ(ANeuralNetworksMemory_copy(goodAshmem, deviceMemory1), ANEURALNETWORKS_NO_ERROR);
+ EXPECT_EQ(ANeuralNetworksMemory_copy(deviceMemory1, badAshmem2), ANEURALNETWORKS_BAD_DATA);
+
+ // Device Memory -> Device Memory
+ // Uninitialized source device memory.
+ EXPECT_EQ(ANeuralNetworksMemory_copy(deviceMemory2, deviceMemory1), ANEURALNETWORKS_BAD_DATA);
+
+ // TODO: Additionally validate the following:
+ // - Device memories with incompatible dimensions
+ // - Deinitialized device memory
+
+ ANeuralNetworksMemory_free(deviceMemory1);
+ ANeuralNetworksMemory_free(deviceMemory2);
+}
+
#ifndef NNTEST_ONLY_PUBLIC_API
TEST(ValidationTestDevice, GetExtensionSupport) {
bool result;
