toolkit/RenderScriptToolkit.h

/*
 * Copyright (C) 2021 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ANDROID_RENDERSCRIPT_TOOLKIT_TOOLKIT_H
#define ANDROID_RENDERSCRIPT_TOOLKIT_TOOLKIT_H

#include <cstdint>
#include <memory>

namespace android {
namespace renderscript {

class TaskProcessor;

/**
 * Define a range of data to process.
 *
 * This class is used to restrict a Toolkit operation to a rectangular subset of the input
 * tensor.
 *
 * @property startX The index of the first value to be included on the X axis.
 * @property endX The index after the last value to be included on the X axis.
 * @property startY The index of the first value to be included on the Y axis.
 * @property endY The index after the last value to be included on the Y axis.
 */
struct Restriction {
    size_t startX;
    size_t endX;
    size_t startY;
    size_t endY;
};

/**
 * A collection of high-performance graphic utility functions like blur and blend.
 *
 * This toolkit provides ten image manipulation functions: blend, blur, color matrix, convolve,
 * histogram, histogramDot, lut, lut3d, resize, and YUV to RGB. These functions execute
 * multithreaded on the CPU.
 *
 * These functions work over raw byte arrays. You'll need to specify the width and height of
 * the data to be processed, as well as the number of bytes per pixel. For most use cases,
 * this will be 4.
 *
 * You should instantiate the Toolkit once and reuse it throughout your application.
 * On instantiation, the Toolkit creates a thread pool that's used for processing all the functions.
 * You can limit the number of pool threads used by the Toolkit via the constructor. The pool
 * threads are destroyed once the Toolkit is destroyed, after any pending work is done.
 *
 * This library is thread safe. You can call methods from different pool threads. The functions will
 * execute sequentially.
 *
 * A Java/Kotlin Toolkit is available. It calls this library through JNI.
 *
 * This toolkit can be used as a replacement for most RenderScript Intrinsic functions. Compared
 * to RenderScript, it's simpler to use and more than twice as fast on the CPU. However RenderScript
 * Intrinsics allow more flexibility for the type of allocation supported. In particular, this
 * toolkit does not support allocations of floats.
 */
class RenderScriptToolkit {
    /** Each Toolkit method call is converted to a Task. The processor owns the thread pool. It
     * tiles the tasks and schedule them over the pool threads.
     */
    std::unique_ptr<TaskProcessor> processor;

   public:
    /**
     * Creates the pool threads that are used for processing the method calls.
     */
    RenderScriptToolkit(int numberOfThreads = 0);
    /**
     * Destroys the thread pool. This stops any in-progress work; the Toolkit methods called from
     * other pool threads will return without having completed the work. Because of the undefined
     * state of the output buffers, an application should avoid destroying the Toolkit if other pool
     * threads are executing Toolkit methods.
     */
    ~RenderScriptToolkit();

    /**
     * Determines how a source buffer is blended into a destination buffer.
     *
     * See {@link RenderScriptToolkit::blend}.
     *
     * blend only works on 4 byte RGBA data. In the descriptions below, ".a" represents
     * the alpha channel.
     */
    enum class BlendingMode {
        /**
         * dest = 0
         *
         * The destination is cleared, i.e. each pixel is set to (0, 0, 0, 0)
         */
        CLEAR = 0,
        /**
         * dest = src
         *
         * Sets each pixel of the destination to the corresponding one in the source.
         */
        SRC = 1,
        /**
         * dest = dest
         *
         * Leaves the destination untouched. This is a no-op.
         */
        DST = 2,
        /**
         * dest = src + dest * (1.0 - src.a)
         */
        SRC_OVER = 3,
        /**
         * dest = dest + src * (1.0 - dest.a)
         */
        DST_OVER = 4,
        /**
         * dest = src * dest.a
         */
        SRC_IN = 5,
        /**
         * dest = dest * src.a
         */
        DST_IN = 6,
        /**
         * dest = src * (1.0 - dest.a)
         */
        SRC_OUT = 7,
        /**
         * dest = dest * (1.0 - src.a)
         */
        DST_OUT = 8,
        /**
         * dest.rgb = src.rgb * dest.a + (1.0 - src.a) * dest.rgb, dest.a = dest.a
         */
        SRC_ATOP = 9,
        /**
         * dest = dest.rgb * src.a + (1.0 - dest.a) * src.rgb, dest.a = src.a
         */
        DST_ATOP = 10,
        /**
         * dest = {src.r ^ dest.r, src.g ^ dest.g, src.b ^ dest.b, src.a ^ dest.a}
         *
         * Note: this is NOT the Porter/Duff XOR mode; this is a bitwise xor.
         */
        XOR = 11,
        /**
         * dest = src * dest
         */
        MULTIPLY = 12,
        /**
         * dest = min(src + dest, 1.0)
         */
        ADD = 13,
        /**
         * dest = max(dest - src, 0.0)
         */
        SUBTRACT = 14
    };

    /**
     * Blend a source buffer with the destination buffer.
     *
     * Blends a source buffer and a destination buffer, placing the result in the destination
     * buffer. The blending is done pairwise between two corresponding RGBA values found in
     * each buffer. The mode parameter specifies one of fifteen blending operations.
     * See {@link BlendingMode}.
     *
     * An optional range parameter can be set to restrict the operation to a rectangular subset
     * of each buffer. If provided, the range must be wholly contained with the dimensions
     * described by sizeX and sizeY.
     *
     * The source and destination buffers must have the same dimensions. Both buffers should be
     * large enough for sizeX * sizeY * 4 bytes. The buffers have a row-major layout.
     *
     * @param mode The specific blending operation to do.
     * @param source The RGBA input buffer.
     * @param dest The destination buffer. Used for input and output.
     * @param sizeX The width of both buffers, as a number of RGBA values.
     * @param sizeY The height of both buffers, as a number of RGBA values.
     * @param restriction When not null, restricts the operation to a 2D range of pixels.
     */
    void blend(BlendingMode mode, const uint8_t* _Nonnull source, uint8_t* _Nonnull dst,
               size_t sizeX, size_t sizeY, const Restriction* _Nullable restriction = nullptr);

    /**
     * Blur an image.
     *
     * Performs a Gaussian blur of the input image and stores the result in the out buffer.
     *
     * The radius determines which pixels are used to compute each blurred pixels. This Toolkit
     * accepts values between 1 and 25. Larger values create a more blurred effect but also
     * take longer to compute. When the radius extends past the edge, the edge pixel will
     * be used as replacement for the pixel that's out off boundary.
     *
     * Each input pixel can either be represented by four bytes (RGBA format) or one byte
     * for the less common blurring of alpha channel only image.
     *
     * An optional range parameter can be set to restrict the operation to a rectangular subset
     * of each buffer. If provided, the range must be wholly contained with the dimensions
     * described by sizeX and sizeY.
     *
     * The input and output buffers must have the same dimensions. Both buffers should be
     * large enough for sizeX * sizeY * vectorSize bytes. The buffers have a row-major layout.
     *
     * @param in The buffer of the image to be blurred.
     * @param out The buffer that receives the blurred image.
     * @param sizeX The width of both buffers, as a number of 1 or 4 byte cells.
     * @param sizeY The height of both buffers, as a number of 1 or 4 byte cells.
     * @param vectorSize Either 1 or 4, the number of bytes in each cell, i.e. A vs. RGBA.
     * @param radius The radius of the pixels used to blur.
     * @param restriction When not null, restricts the operation to a 2D range of pixels.
     */
    void blur(const uint8_t* _Nonnull in, uint8_t* _Nonnull out, size_t sizeX, size_t sizeY,
              size_t vectorSize, int radius, const Restriction* _Nullable restriction = nullptr);

    /**
     * Identity matrix that can be passed to the {@link RenderScriptToolkit::colorMatrix} method.
     *
     * Using this matrix will result in no change to the pixel through multiplication although
     * the pixel value can still be modified by the add vector, or transformed to a different
     * format.
     */
    static constexpr float kIdentityMatrix[] =  {
            1.0f, 0.0f, 0.0f, 0.0f,
            0.0f, 1.0f, 0.0f, 0.0f,
            0.0f, 0.0f, 1.0f, 0.0f,
            0.0f, 0.0f, 0.0f, 1.0f
    };

    /**
     * Matrix to turn color pixels to a grey scale.
     *
     * Use this matrix with the {@link RenderScriptToolkit::colorMatrix} method to convert an
     * image from color to greyscale.
     */
    static constexpr float kGreyScaleColorMatrix[] = {
            0.299f, 0.299f, 0.299f, 0.0f,
            0.587f, 0.587f, 0.587f, 0.0f,
            0.114f, 0.114f, 0.114f, 0.0f,
            0.0f,   0.0f,   0.0f,   1.0f
    };

    /**
     * Matrix to convert RGB to YUV.
     *
     * Use this matrix with the {@link RenderScriptToolkit::colorMatrix} method to convert the
     * first three bytes of each pixel from RGB to YUV. This leaves the last byte (the alpha
     * channel) untouched.
     *
     * This is a simplistic conversion. Most YUV buffers have more complicated format, not supported
     * by this method.
     */
    static constexpr float kRgbToYuvMatrix[] = {
            0.299f, -0.14713f,  0.615f,   0.0f,
            0.587f, -0.28886f, -0.51499f, 0.0f,
            0.114f,  0.436f,   -0.10001f, 0.0f,
            0.0f,    0.0f,      0.0f,     1.0f
    };

    /**
     * Matrix to convert YUV to RGB.
     *
     * Use this matrix with the {@link RenderScriptToolkit::colorMatrix} method to convert the
     * first three bytes of each pixel from YUV to RGB. This leaves the last byte (the alpha
     * channel) untouched.
     *
     * This is a simplistic conversion. Most YUV buffers have more complicated format, not supported
     * by this method. Use {@link RenderScriptToolkit::yuvToRgb} to convert these buffers.
     */
    static constexpr float kYuvToRgbMatrix[] = {
            1.0f,      1.0f,     1.0f,     0.0f,
            0.0f,     -0.39465f, 2.03211f, 0.0f,
            1.13983f, -0.5806f,  0.0f,     0.0f,
            0.0f,      0.0f,     0.0f,     1.0f
    };

    /**
     * Transform an image using a color matrix.
     *
     * Converts a 2D array of vectors of unsigned bytes, multiplying each vectors by a 4x4 matrix
     * and adding an optional vector.
     *
     * Each input vector is composed of 1-4 unsigned bytes. If less than 4 bytes, it's extended to
     * 4, padding with zeroes. The unsigned bytes are converted from 0-255 to 0.0-1.0 floats
     * before the multiplication is done.
     *
     * The resulting value is normalized from 0.0-1.0 to a 0-255 value and stored in the output.
     * If the output vector size is less than four, the unused channels are discarded.
     *
     * If addVector is null, a vector of zeroes is added, i.e. a noop.
     *
     * Check kIdentityMatrix, kGreyScaleColorMatrix, kRgbToYuvMatrix, and kYuvToRgbMatrix for sample
     * matrices. The YUV conversion may not work for all color spaces.
     *
     * @param in The buffer of the image to be converted.
     * @param out The buffer that receives the converted image.
     * @param inputVectorSize The number of bytes in each input cell, a value from 1 to 4.
     * @param outputVectorSize The number of bytes in each output cell, a value from 1 to 4.
     * @param sizeX The width of both buffers, as a number of 1 to 4 byte cells.
     * @param sizeY The height of both buffers, as a number of 1 to 4 byte cells.
     * @param matrix The 4x4 matrix to multiply, in row major format.
     * @param addVector A vector of four floats that's added to the result of the multiplication.
     * @param restriction When not null, restricts the operation to a 2D range of pixels.
     */
    void colorMatrix(const void* _Nonnull in, void* _Nonnull out, size_t inputVectorSize,
                     size_t outputVectorSize, size_t sizeX, size_t sizeY,
                     const float* _Nonnull matrix, const float* _Nullable addVector = nullptr,
                     const Restriction* _Nullable restriction = nullptr);

    /**
     * Convolve a ByteArray.
     *
     * Applies a 3x3 or 5x5 convolution to the input array using the provided coefficients.
     *
     * For 3x3 convolutions, 9 coefficients must be provided. For 5x5, 25 coefficients are needed.
     * The coefficients should be provided in row-major format.
     *
     * When the square extends past the edge, the edge values will be used as replacement for the
     * values that's are off boundary.
     *
     * Each input cell can either be represented by one to four bytes. Each byte is multiplied
     * and accumulated independently of the other bytes of the cell.
     *
     * An optional range parameter can be set to restrict the operation to a rectangular subset
     * of each buffer. If provided, the range must be wholly contained with the dimensions
     * described by sizeX and sizeY.
     *
     * The input and output buffers must have the same dimensions. Both buffers should be
     * large enough for sizeX * sizeY * vectorSize bytes. The buffers have a row-major layout.
     *
     * @param in The buffer of the image to be blurred.
     * @param out The buffer that receives the blurred image.
     * @param vectorSize The number of bytes in each cell, a value from 1 to 4.
     * @param sizeX The width of both buffers, as a number of 1 or 4 byte cells.
     * @param sizeY The height of both buffers, as a number of 1 or 4 byte cells.
     * @param coefficients 9 or 25 multipliers.
     * @param restriction When not null, restricts the operation to a 2D range of pixels.
     */
    void convolve3x3(const void* _Nonnull in, void* _Nonnull out, size_t vectorSize, size_t sizeX,
                     size_t sizeY, const float* _Nonnull coefficients,
                     const Restriction* _Nullable restriction = nullptr);

    void convolve5x5(const void* _Nonnull in, void* _Nonnull out, size_t vectorSize, size_t sizeX,
                     size_t sizeY, const float* _Nonnull coefficients,
                     const Restriction* _Nullable restriction = nullptr);

    /**
     * Compute the histogram of an image.
     *
     * Tallies how many times each of the 256 possible values of a byte is found in the input.
     *
     * An input cell can be represented by one to four bytes. The tally is done independently
     * for each of the bytes of the cell. Correspondingly, the out array will have
     * 256 * vectorSize entries. The counts for value 0 are consecutive, followed by those for
     * value 1, etc.
     *
     * An optional range parameter can be set to restrict the operation to a rectangular subset
     * of each buffer. If provided, the range must be wholly contained with the dimensions
     * described by sizeX and sizeY.
     *
     * The source buffers should be large enough for sizeX * sizeY * vectorSize bytes. The buffers
     * have a row-major layout. The out buffer should be large enough for 256 * vectorSize ints.
     *
     * @param in The buffer of the image to be analyzed.
     * @param out The resulting vector of counts.
     * @param sizeX The width of the input buffers, as a number of 1 or 4 byte cells.
     * @param sizeY The height of the input buffers, as a number of 1 or 4 byte cells.
     * @param vectorSize The number of bytes in each cell, a value from 1 to 4.
     * @param restriction When not null, restricts the operation to a 2D range of pixels.
     */
    void histogram(const uint8_t* _Nonnull in, int32_t* _Nonnull out, size_t sizeX, size_t sizeY,
                   size_t vectorSize, const Restriction* _Nullable restriction = nullptr);

    /**
     * Compute the histogram of the dot product of an image.
     *
     * This method supports cells of 1 to 4 bytes in length. For each cell of the array,
     * the dot product of its bytes with the provided coefficients is computed. The resulting
     * floating point value is converted to an unsigned byte and tallied in the histogram.
     *
     * If coefficients is null, the coefficients used for RGBA luminosity calculation will be used,
     * i.e. the values [0.299f, 0.587f, 0.114f, 0.f].
     *
     * Each coefficients must be >= 0 and their sum must be 1.0 or less. There must be the same
     * number of coefficients as vectorSize.
     *
     * An optional range parameter can be set to restrict the operation to a rectangular subset
     * of each buffer. If provided, the range must be wholly contained with the dimensions
     * described by sizeX and sizeY.
     *
     * The source buffers should be large enough for sizeX * sizeY * vectorSize bytes. The buffers
     * have a row-major layout. The out array should be large enough for 256 ints.
     *
     * @param in The buffer of the image to be analyzed.
     * @param out The resulting vector of counts.
     * @param sizeX The width of the input buffers, as a number of 1 or 4 byte cells.
     * @param sizeY The height of the input buffers, as a number of 1 or 4 byte cells.
     * @param vectorSize The number of bytes in each cell, a value from 1 to 4.
     * @param coefficients The values used for the dot product. Can be nullptr.
     * @param restriction When not null, restricts the operation to a 2D range of pixels.
     */
    void histogramDot(const uint8_t* _Nonnull in, int32_t* _Nonnull out, size_t sizeX, size_t sizeY,
                      size_t vectorSize, const float* _Nullable coefficients,
                      const Restriction* _Nullable restriction = nullptr);

    /**
     * Transform an image using a look up table
     *
     * Transforms an image by using a per-channel lookup table. Each channel of the input has an
     * independent lookup table. The tables are 256 entries in size and can cover the full value
     * range of a byte.
     *
     * The input array should be in RGBA format, where four consecutive bytes form an cell.
     *
     * An optional range parameter can be set to restrict the operation to a rectangular subset
     * of each buffer. If provided, the range must be wholly contained with the dimensions
     * described by sizeX and sizeY.
     *
     * The input and output buffers must have the same dimensions. Both buffers should be
     * large enough for sizeX * sizeY * vectorSize bytes. The buffers have a row-major layout.
     *
     * @param in The buffer of the image to be transformed.
     * @param out The buffer that receives the transformed image.
     * @param sizeX The width of both buffers, as a number of 4 byte cells.
     * @param sizeY The height of both buffers, as a number of 4 byte cells.
     * @param red An array of 256 values that's used to convert the R channel.
     * @param green An array of 256 values that's used to convert the G channel.
     * @param blue An array of 256 values that's used to convert the B channel.
     * @param alpha An array of 256 values that's used to convert the A channel.
     * @param restriction When not null, restricts the operation to a 2D range of pixels.
     */
    void lut(const uint8_t* _Nonnull in, uint8_t* _Nonnull out, size_t sizeX, size_t sizeY,
             const uint8_t* _Nonnull red, const uint8_t* _Nonnull green,
             const uint8_t* _Nonnull blue, const uint8_t* _Nonnull alpha,
             const Restriction* _Nullable restriction = nullptr);

    /**
     * Transform an image using a 3D look up table
     *
     * Transforms an image, converting RGB to RGBA by using a 3D lookup table. The incoming R, G,
     * and B values are normalized to the dimensions of the provided 3D buffer. The eight nearest
     * values in that 3D buffer are sampled and linearly interpolated. The resulting RGBA entry
     * is stored in the output.
     *
     * The input array should be in RGBA format, where four consecutive bytes form an cell.
     * The fourth byte of each input cell is ignored.
     *
     * An optional range parameter can be set to restrict the operation to a rectangular subset
     * of each buffer. If provided, the range must be wholly contained with the dimensions
     * described by sizeX and sizeY.
     *
     * The input and output buffers must have the same dimensions. Both buffers should be
     * large enough for sizeX * sizeY * vectorSize bytes. The buffers have a row-major layout.
     *
     * @param in The buffer of the image to be transformed.
     * @param out The buffer that receives the transformed image.
     * @param sizeX The width of both buffers, as a number of 4 byte cells.
     * @param sizeY The height of both buffers, as a number of 4 byte cells.
     * @param cube The translation cube, in row major-format.
     * @param cubeSizeX The number of RGBA entries in the cube in the X direction.
     * @param cubeSizeY The number of RGBA entries in the cube in the Y direction.
     * @param cubeSizeZ The number of RGBA entries in the cube in the Z direction.
     * @param restriction When not null, restricts the operation to a 2D range of pixels.
     */
    void lut3d(const uint8_t* _Nonnull in, uint8_t* _Nonnull out, size_t sizeX, size_t sizeY,
               const uint8_t* _Nonnull cube, size_t cubeSizeX, size_t cubeSizeY, size_t cubeSizeZ,
               const Restriction* _Nullable restriction = nullptr);

    /**
     * Resize an image.
     *
     * Resizes an image using bicubic interpolation.
     *
     * This method supports cells of 1 to 4 bytes in length. Each byte of the cell is
     * interpolated independently from the others.
     *
     * An optional range parameter can be set to restrict the operation to a rectangular subset
     * of the output buffer. The corresponding scaled range of the input will be used.  If provided,
     * the range must be wholly contained with the dimensions described by outputSizeX and
     * outputSizeY.
     *
     * The input and output buffers have a row-major layout. Both buffers should be
     * large enough for sizeX * sizeY * vectorSize bytes.
     *
     * @param in The buffer of the image to be resized.
     * @param out The buffer that receives the resized image.
     * @param inputSizeX The width of the input buffer, as a number of 1-4 byte cells.
     * @param inputSizeY The height of the input buffer, as a number of 1-4 byte cells.
     * @param vectorSize The number of bytes in each cell of both buffers. A value from 1 to 4.
     * @param outputSizeX The width of the output buffer, as a number of 1-4 byte cells.
     * @param outputSizeY The height of the output buffer, as a number of 1-4 byte cells.
     * @param restriction When not null, restricts the operation to a 2D range of pixels.
     */
    void resize(const uint8_t* _Nonnull in, uint8_t* _Nonnull out, size_t inputSizeX,
                size_t inputSizeY, size_t vectorSize, size_t outputSizeX, size_t outputSizeY,
                const Restriction* _Nullable restriction = nullptr);

    /**
     * The YUV formats supported by yuvToRgb.
     */
    enum class YuvFormat {
        NV21 = 0x11,
        YV12 = 0x32315659,
    };

    /**
     * Convert an image from YUV to RGB.
     *
     * Converts an Android YUV buffer to RGB. The input allocation should be
     * supplied in a supported YUV format as a YUV cell Allocation.
     * The output is RGBA; the alpha channel will be set to 255.
     *
     * Note that for YV12 and a sizeX that's not a multiple of 32, the
     * RenderScript Intrinsic may not have converted the image correctly.
     * This Toolkit method should.
     *
     * @param in The buffer of the image to be converted.
     * @param out The buffer that receives the converted image.
     * @param sizeX The width in pixels of the image. Must be even.
     * @param sizeY The height in pixels of the image.
     * @param format Either YV12 or NV21.
     */
    void yuvToRgb(const uint8_t* _Nonnull in, uint8_t* _Nonnull out, size_t sizeX, size_t sizeY,
                  YuvFormat format);
};

}  // namespace renderscript
}  // namespace android

#endif  // ANDROID_RENDERSCRIPT_TOOLKIT_TOOLKIT_H