| /* | |
| * Copyright (C) 2018 The Android Open Source Project | |
| * | |
| * Licensed under the Apache License, Version 2.0 (the "License"); | |
| * you may not use this file except in compliance with the License. | |
| * You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| #define LOG_TAG "Operations" | |
| #include <tensorflow/lite/kernels/internal/common.h> | |
| #include <algorithm> | |
| #include <cfloat> | |
| #include <cmath> | |
| #include <vector> | |
| #include "CpuOperationUtils.h" | |
| #include "Operations.h" | |
| #include "Tracing.h" | |
| namespace android { | |
| namespace nn { | |
| #define ANDROID_NN_GROUPED_CONV_PARAMETERS \ | |
| uint32_t numBatches = getSizeOfDimension(inputShape, 0); \ | |
| uint32_t inputHeight = getSizeOfDimension(inputShape, 1); \ | |
| uint32_t inputWidth = getSizeOfDimension(inputShape, 2); \ | |
| uint32_t inputDepth = getSizeOfDimension(inputShape, 3); \ | |
| uint32_t filterHeight = getSizeOfDimension(filterShape, 1); \ | |
| uint32_t filterWidth = getSizeOfDimension(filterShape, 2); \ | |
| uint32_t filterDepth = getSizeOfDimension(filterShape, 3); \ | |
| uint32_t outputHeight = getSizeOfDimension(outputShape, 1); \ | |
| uint32_t outputWidth = getSizeOfDimension(outputShape, 2); \ | |
| uint32_t outputDepth = getSizeOfDimension(outputShape, 3); \ | |
| uint32_t outputGroupDepth = outputDepth / numGroups; | |
| bool groupedConvFloat32(const float* inputData, const Shape& inputShape, const float* filterData, | |
| const Shape& filterShape, const float* biasData, const Shape& biasShape, | |
| int32_t padding_left, int32_t padding_right, int32_t padding_top, | |
| int32_t padding_bottom, int32_t stride_width, int32_t stride_height, | |
| int32_t numGroups, int32_t activation, float* outputData, | |
| const Shape& outputShape) { | |
| NNTRACE_TRANS("groupConvFloat32"); | |
| ANDROID_NN_GROUPED_CONV_PARAMETERS | |
| float output_activation_min = 0.0f, output_activation_max = 0.0f; | |
| CalculateActivationRangeFloat(activation, &output_activation_min, &output_activation_max); | |
| const float* inputBase = inputData; | |
| float* outPtr = outputData; | |
| for (uint32_t b = 0; b < numBatches; b++) { | |
| for (uint32_t h = 0; h < outputHeight; h++) { | |
| for (uint32_t w = 0; w < outputWidth; w++) { | |
| const float* filterBase = filterData; | |
| for (uint32_t g = 0; g < numGroups; g++) { | |
| for (uint32_t d = 0; d < outputGroupDepth; d++) { | |
| int32_t wInputOrigin = | |
| static_cast<int32_t>(w) * stride_width - padding_left; | |
| int32_t hInputOrigin = | |
| static_cast<int32_t>(h) * stride_height - padding_top; | |
| float sum = 0.0f; | |
| for (uint32_t i = 0; i < filterHeight; i++) { | |
| for (uint32_t j = 0; j < filterWidth; j++) { | |
| for (uint32_t k = 0; k < filterDepth; k++) { | |
| int32_t hInput = hInputOrigin + static_cast<int32_t>(i); | |
| int32_t wInput = wInputOrigin + static_cast<int32_t>(j); | |
| uint32_t dInput = filterDepth * g + k; | |
| if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) && | |
| wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) { | |
| uint32_t filterIndex = | |
| i * filterWidth * filterDepth + j * filterDepth + k; | |
| uint32_t inputIndex = hInput * inputWidth * inputDepth + | |
| wInput * inputDepth + dInput; | |
| sum += filterBase[filterIndex] * inputBase[inputIndex]; | |
| } | |
| } | |
| } | |
| } | |
| sum += biasData[g * outputGroupDepth + d]; | |
| sum = std::max(std::min(sum, output_activation_max), output_activation_min); | |
| outPtr[d] = sum; | |
| filterBase += filterHeight * filterWidth * filterDepth; | |
| } | |
| outPtr += outputGroupDepth; | |
| } | |
| } | |
| } | |
| inputBase += inputHeight * inputWidth * inputDepth; | |
| } | |
| return true; | |
| } | |
| template <typename T> | |
| bool groupedConvQuant8(const T* inputData, const Shape& inputShape, const T* filterData, | |
| const Shape& filterShape, const int32_t* biasData, const Shape& biasShape, | |
| int32_t padding_left, int32_t padding_right, int32_t padding_top, | |
| int32_t padding_bottom, int32_t stride_width, int32_t stride_height, | |
| int32_t numGroups, int32_t activation, T* outputData, | |
| const Shape& outputShape) { | |
| NNTRACE_TRANS("groupConvQuant8"); | |
| ANDROID_NN_GROUPED_CONV_PARAMETERS | |
| int32_t inputOffset = -inputShape.offset; | |
| int32_t filterOffset = -filterShape.offset; | |
| int32_t outputOffset = outputShape.offset; | |
| double realMultiplier = 0.0; | |
| int32_t outputMultiplier = 0; | |
| int32_t outputShift = 0; | |
| NN_RET_CHECK(GetQuantizedConvolutionMultipler(inputShape, filterShape, biasShape, outputShape, | |
| &realMultiplier)); | |
| int exponent; | |
| NN_RET_CHECK(QuantizeMultiplier(realMultiplier, &outputMultiplier, &exponent)); | |
| outputShift = -exponent; | |
| int32_t output_activation_min = 0, output_activation_max = 0; | |
| CalculateActivationRange<T>(activation, outputShape, &output_activation_min, | |
| &output_activation_max); | |
| const T* inputBase = inputData; | |
| T* outPtr = outputData; | |
| for (uint32_t b = 0; b < numBatches; b++) { | |
| for (uint32_t h = 0; h < outputHeight; h++) { | |
| for (uint32_t w = 0; w < outputWidth; w++) { | |
| const T* filterBase = filterData; | |
| for (uint32_t g = 0; g < numGroups; g++) { | |
| for (uint32_t d = 0; d < outputGroupDepth; d++) { | |
| int32_t wInputOrigin = | |
| static_cast<int32_t>(w) * stride_width - padding_left; | |
| int32_t hInputOrigin = | |
| static_cast<int32_t>(h) * stride_height - padding_top; | |
| int32_t sum = 0.0f; | |
| for (uint32_t i = 0; i < filterHeight; i++) { | |
| for (uint32_t j = 0; j < filterWidth; j++) { | |
| for (uint32_t k = 0; k < filterDepth; k++) { | |
| int32_t hInput = hInputOrigin + static_cast<int32_t>(i); | |
| int32_t wInput = wInputOrigin + static_cast<int32_t>(j); | |
| uint32_t dInput = filterDepth * g + k; | |
| if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) && | |
| wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) { | |
| uint32_t filterIndex = | |
| i * filterWidth * filterDepth + j * filterDepth + k; | |
| uint32_t inputIndex = hInput * inputWidth * inputDepth + | |
| wInput * inputDepth + dInput; | |
| sum += (static_cast<int32_t>(filterBase[filterIndex]) + | |
| filterOffset) * | |
| (static_cast<int32_t>(inputBase[inputIndex]) + | |
| inputOffset); | |
| } | |
| } | |
| } | |
| } | |
| sum += biasData[g * outputGroupDepth + d]; | |
| sum = tflite::MultiplyByQuantizedMultiplier(sum, outputMultiplier, | |
| -outputShift); | |
| sum += outputOffset; | |
| sum = std::max(std::min(sum, output_activation_max), output_activation_min); | |
| outPtr[d] = static_cast<T>(sum); | |
| filterBase += filterHeight * filterWidth * filterDepth; | |
| } | |
| outPtr += outputGroupDepth; | |
| } | |
| } | |
| } | |
| inputBase += inputHeight * inputWidth * inputDepth; | |
| } | |
| return true; | |
| } | |
| template bool groupedConvQuant8<int8_t>(const int8_t* inputData, const Shape& inputShape, | |
| const int8_t* filterData, const Shape& filterShape, | |
| const int32_t* biasData, const Shape& biasShape, | |
| int32_t padding_left, int32_t padding_right, | |
| int32_t padding_top, int32_t padding_bottom, | |
| int32_t stride_width, int32_t stride_height, | |
| int32_t numGroups, int32_t activation, int8_t* outputData, | |
| const Shape& outputShape); | |
| template bool groupedConvQuant8<uint8_t>(const uint8_t* inputData, const Shape& inputShape, | |
| const uint8_t* filterData, const Shape& filterShape, | |
| const int32_t* biasData, const Shape& biasShape, | |
| int32_t padding_left, int32_t padding_right, | |
| int32_t padding_top, int32_t padding_bottom, | |
| int32_t stride_width, int32_t stride_height, | |
| int32_t numGroups, int32_t activation, uint8_t* outputData, | |
| const Shape& outputShape); | |
| template <typename T> | |
| bool groupedConvQuant8PerChannel(const T* inputData, const Shape& inputShape, | |
| const int8_t* filterData, const Shape& filterShape, | |
| const float* filterScales, const int32_t* biasData, | |
| const Shape& biasShape, int32_t padding_left, | |
| int32_t padding_right, int32_t padding_top, int32_t padding_bottom, | |
| int32_t stride_width, int32_t stride_height, int32_t numGroups, | |
| int32_t activation, T* outputData, const Shape& outputShape) { | |
| NNTRACE_TRANS("groupConvQuant8"); | |
| ANDROID_NN_GROUPED_CONV_PARAMETERS | |
| int32_t inputOffset = -inputShape.offset; | |
| int32_t outputOffset = outputShape.offset; | |
| auto realMultiplier = std::vector<double>(outputDepth, .0f); | |
| auto outputMultiplier = std::vector<int32_t>(outputDepth, 0); | |
| auto outputShift = std::vector<int32_t>(outputDepth, 0); | |
| for (int i = 0; i < outputDepth; ++i) { | |
| Shape filterChannelShape = filterShape; | |
| filterChannelShape.scale = filterScales[i]; | |
| Shape biasChannelShape = biasShape; | |
| biasChannelShape.scale = filterScales[i] * inputShape.scale; | |
| NN_RET_CHECK(GetQuantizedConvolutionMultipler( | |
| inputShape, filterChannelShape, biasChannelShape, outputShape, &realMultiplier[i])); | |
| int exponent; | |
| NN_RET_CHECK(QuantizeMultiplier(realMultiplier[i], &outputMultiplier[i], &exponent)); | |
| outputShift[i] = -exponent; | |
| } | |
| int32_t output_activation_min = 0, output_activation_max = 0; | |
| CalculateActivationRange<T>(activation, outputShape, &output_activation_min, | |
| &output_activation_max); | |
| const T* inputBase = inputData; | |
| T* outPtr = outputData; | |
| for (uint32_t b = 0; b < numBatches; b++) { | |
| for (uint32_t h = 0; h < outputHeight; h++) { | |
| for (uint32_t w = 0; w < outputWidth; w++) { | |
| const int8_t* filterBase = filterData; | |
| for (uint32_t g = 0; g < numGroups; g++) { | |
| for (uint32_t d = 0; d < outputGroupDepth; d++) { | |
| int32_t wInputOrigin = | |
| static_cast<int32_t>(w) * stride_width - padding_left; | |
| int32_t hInputOrigin = | |
| static_cast<int32_t>(h) * stride_height - padding_top; | |
| int32_t sum = 0.0f; | |
| for (uint32_t i = 0; i < filterHeight; i++) { | |
| for (uint32_t j = 0; j < filterWidth; j++) { | |
| for (uint32_t k = 0; k < filterDepth; k++) { | |
| int32_t hInput = hInputOrigin + static_cast<int32_t>(i); | |
| int32_t wInput = wInputOrigin + static_cast<int32_t>(j); | |
| uint32_t dInput = filterDepth * g + k; | |
| if (hInput >= 0 && hInput < static_cast<int32_t>(inputHeight) && | |
| wInput >= 0 && wInput < static_cast<int32_t>(inputWidth)) { | |
| uint32_t filterIndex = | |
| i * filterWidth * filterDepth + j * filterDepth + k; | |
| uint32_t inputIndex = hInput * inputWidth * inputDepth + | |
| wInput * inputDepth + dInput; | |
| sum += (static_cast<int32_t>(filterBase[filterIndex])) * | |
| (static_cast<int32_t>(inputBase[inputIndex]) + | |
| inputOffset); | |
| } | |
| } | |
| } | |
| } | |
| int channelIndex = g * outputGroupDepth + d; | |
| sum += biasData[channelIndex]; | |
| sum = tflite::MultiplyByQuantizedMultiplier( | |
| sum, outputMultiplier[channelIndex], -outputShift[channelIndex]); | |
| sum += outputOffset; | |
| sum = std::max(std::min(sum, output_activation_max), output_activation_min); | |
| outPtr[d] = static_cast<T>(sum); | |
| filterBase += filterHeight * filterWidth * filterDepth; | |
| } | |
| outPtr += outputGroupDepth; | |
| } | |
| } | |
| } | |
| inputBase += inputHeight * inputWidth * inputDepth; | |
| } | |
| return true; | |
| } | |
| bool groupedConvFloat16(const _Float16* inputData, const Shape& inputShape, | |
| const _Float16* filterData, const Shape& filterShape, | |
| const _Float16* biasData, const Shape& biasShape, int32_t padding_left, | |
| int32_t padding_right, int32_t padding_top, int32_t padding_bottom, | |
| int32_t stride_width, int32_t stride_height, int32_t numGroups, | |
| int32_t activation, _Float16* outputData, const Shape& outputShape) { | |
| NNTRACE_TRANS("groupConvFloat16"); | |
| std::vector<float> inputData_float32(getNumberOfElements(inputShape)); | |
| std::vector<float> filterData_float32(getNumberOfElements(filterShape)); | |
| std::vector<float> biasData_float32(getNumberOfElements(biasShape)); | |
| std::vector<float> outputData_float32(getNumberOfElements(outputShape)); | |
| convertFloat16ToFloat32(inputData, &inputData_float32); | |
| convertFloat16ToFloat32(filterData, &filterData_float32); | |
| convertFloat16ToFloat32(biasData, &biasData_float32); | |
| groupedConvFloat32(inputData_float32.data(), inputShape, filterData_float32.data(), filterShape, | |
| biasData_float32.data(), biasShape, padding_left, padding_right, padding_top, | |
| padding_bottom, stride_width, stride_height, numGroups, activation, | |
| outputData_float32.data(), outputShape); | |
| convertFloat32ToFloat16(outputData_float32, outputData); | |
| return true; | |
| } | |
| template bool groupedConvQuant8PerChannel<uint8_t>( | |
| const uint8_t* inputData, const Shape& inputShape, const int8_t* filterData, | |
| const Shape& filterShape, const float* filterScales, const int32_t* biasData, | |
| const Shape& biasShape, int32_t padding_left, int32_t padding_right, int32_t padding_top, | |
| int32_t padding_bottom, int32_t stride_width, int32_t stride_height, int32_t numGroups, | |
| int32_t activation, uint8_t* outputData, const Shape& outputShape); | |
| template bool groupedConvQuant8PerChannel<int8_t>( | |
| const int8_t* inputData, const Shape& inputShape, const int8_t* filterData, | |
| const Shape& filterShape, const float* filterScales, const int32_t* biasData, | |
| const Shape& biasShape, int32_t padding_left, int32_t padding_right, int32_t padding_top, | |
| int32_t padding_bottom, int32_t stride_width, int32_t stride_height, int32_t numGroups, | |
| int32_t activation, int8_t* outputData, const Shape& outputShape); | |
| #undef ANDROID_NN_GROUPED_CONV_PARAMETERS | |
| } // namespace nn | |
| } // namespace android |