src/qs8-gavgpool/multipass-neon.c.in - platform/external/XNNPACK - Git at Google

 // Copyright 2020 Google LLC
 //
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree.

 $assert DATATYPE in ["QS8", "QU8"]
 $assert CHANNEL_TILE % 8 == 0
 $assert CHANNEL_TILE >= 8
 $assert ROW_TILE >= 3
 $assert ROW_SUBTILE >= 3
 $assert ROW_SUBTILE <= ROW_TILE
 $assert REQUANTIZATION in ["FP32", "RNDNU"]
 $ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 #include <assert.h>

 #include <arm_neon.h>

 #include <xnnpack/gavgpool.h>
 $if ARMV8:
   #include <xnnpack/intrinsics-polyfill.h>
 #include <xnnpack/math.h>


 $PARAMS_STRUCT = REQUANTIZATION.lower() + "_" + ("neonv8" if ARMV8 else "neon")
 $XINT8_T = {"QS8": "int8_t", "QU8": "uint8_t"}[DATATYPE]
 $XINT8X8_T = {"QS8": "int8x8_t", "QU8": "uint8x8_t"}[DATATYPE]
 $XINT8X16_T = {"QS8": "int8x16_t", "QU8": "uint8x16_t"}[DATATYPE]
 $XINT16X8_T = {"QS8": "int16x8_t", "QU8": "uint16x8_t"}[DATATYPE]
 $VLD1_X8 = {"QS8": "vld1_s8", "QU8": "vld1_u8"}[DATATYPE]
 $VLD1_DUP_X8 = {"QS8": "vld1_dup_s8", "QU8": "vld1_dup_u8"}[DATATYPE]
 $VLD1Q_DUP_X8 = {"QS8": "vld1q_dup_s8", "QU8": "vld1q_dup_u8"}[DATATYPE]
 $VST1_X8 = {"QS8": "vst1_s8", "QU8": "vst1_u8"}[DATATYPE]
 $VST1Q_X8 = {"QS8": "vst1q_s8", "QU8": "vst1q_u8"}[DATATYPE]
 $VST1_LANE_X8 = {"QS8": "vst1_lane_s8", "QU8": "vst1_lane_u8"}[DATATYPE]
 $VADDL_X8 = {"QS8": "vaddl_s8", "QU8": "vaddl_u8"}[DATATYPE]
 $VADDW_X8 = {"QS8": "vaddw_s8", "QU8": "vaddw_u8"}[DATATYPE]
 $VMIN_X8 = {"QS8": "vmin_s8", "QU8": "vmin_u8"}[DATATYPE]
 $VMINQ_X8 = {"QS8": "vminq_s8", "QU8": "vminq_u8"}[DATATYPE]
 $VMAX_X8 = {"QS8": "vmax_s8", "QU8": "vmax_u8"}[DATATYPE]
 $VMAXQ_X8 = {"QS8": "vmaxq_s8", "QU8": "vmaxq_u8"}[DATATYPE]
 $VEXT_X8 = {"QS8": "vext_s8", "QU8": "vext_u8"}[DATATYPE]
 $VQMOVXN_S16 = {"QS8": "vqmovn_s16", "QU8": "vqmovun_s16"}[DATATYPE]
 $VQMOVXN_HIGH_S16 = {"QS8": "vqmovn_high_s16", "QU8": "vqmovun_high_s16"}[DATATYPE]
 $VGET_LOW_X8 = {"QS8": "vget_low_s8", "QU8": "vget_low_u8"}[DATATYPE]
 $VCOMBINE_X8 = {"QS8": "vcombine_s8", "QU8": "vcombine_u8"}[DATATYPE]
 $VREINTERPRET_U32_X8 = {"QS8": "vreinterpret_u32_s8", "QU8": "vreinterpret_u32_u8"}[DATATYPE]
 $VREINTERPRET_U16_X8 = {"QS8": "vreinterpret_u16_s8", "QU8": "vreinterpret_u16_u8"}[DATATYPE]
 $ISA = "neonv8" if ARMV8 else "neon"
 void xnn_${DATATYPE.lower()}_gavgpool_minmax_${REQUANTIZATION.lower()}_ukernel_${ROW_TILE}p${ROW_SUBTILE}x__${ISA}_c${CHANNEL_TILE}(
     size_t rows,
     size_t channels,
     const ${XINT8_T}* input,
     size_t input_stride,
     const ${XINT8_T}* zero,
     int32_t* buffer,
     ${XINT8_T}* output,
     const union xnn_${DATATYPE.lower()}_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
 {
   assert(rows > ${ROW_TILE});
   assert(channels != 0);

   const ${XINT8_T}* i0 = input;
   $for M in range(1, ROW_TILE):
     const ${XINT8_T}* i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M-1} + input_stride);
   $if CHANNEL_TILE <= 16:
     const size_t input_increment = ${ROW_TILE} * input_stride - round_up_po2(channels, ${CHANNEL_TILE}) * sizeof(${XINT8_T});
   $else:
     const size_t input_increment = ${ROW_TILE} * input_stride - round_up_po2(channels, 8) * sizeof(${XINT8_T});

   const int32x4_t vinit_bias = vld1q_dup_s32(&params->${PARAMS_STRUCT}.init_bias);
   int32_t* b = buffer;
   size_t c = channels;
   for (; ${"c >= %d" % CHANNEL_TILE if CHANNEL_TILE > 16 else "c != 0"}; ${("c -= %d" if CHANNEL_TILE > 16 else "c = doz(c, %d)") % CHANNEL_TILE}) {
     $for M in range(2):
       $for C in range(0, CHANNEL_TILE, 8):
         const ${XINT8X8_T} vi${M}x${ABC[C:C+8]} = ${VLD1_X8}(i${M}); i${M} += 8;

     $for C in range(0, CHANNEL_TILE, 8):
       const ${XINT8X8_T} vi2x${ABC[C:C+8]} = ${VLD1_X8}(i2); i2 += 8;
       ${XINT16X8_T} vsum${ABC[C:C+8]} = ${VADDL_X8}(vi0x${ABC[C:C+8]}, vi1x${ABC[C:C+8]});

     $for M in range(2, ROW_TILE):
       $for C in range(0, CHANNEL_TILE, 8):
         $if M + 1 != ROW_TILE:
           const ${XINT8X8_T} vi${M+1}x${ABC[C:C+8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
         vsum${ABC[C:C+8]} = ${VADDW_X8}(vsum${ABC[C:C+8]}, vi${M}x${ABC[C:C+8]});

     $for C in range(0, CHANNEL_TILE, 8):
       $if DATATYPE == "QS8":
         const int32x4_t vacc${ABC[C:C+4]} = vaddw_s16(vinit_bias, vget_low_s16(vsum${ABC[C:C+8]}));
         const int32x4_t vacc${ABC[C+4:C+8]} = vaddw_s16(vinit_bias, vget_high_s16(vsum${ABC[C:C+8]}));
       $else:
         const int32x4_t vacc${ABC[C:C+4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vinit_bias), vget_low_u16(vsum${ABC[C:C+8]})));
         const int32x4_t vacc${ABC[C+4:C+8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vinit_bias), vget_high_u16(vsum${ABC[C:C+8]})));

     $for C in range(0, CHANNEL_TILE, 4):
       vst1q_s32(b, vacc${ABC[C:C+4]}); b += 4;
   }
   $if CHANNEL_TILE > 16:
     if XNN_UNLIKELY(c != 0) {
       do {
         $for M in range(3):
           const ${XINT8X8_T} vi${M}x${ABC[0:8]} = ${VLD1_X8}(i${M}); i${M} += 8;
         ${XINT16X8_T} vsum${ABC[0:8]} = ${VADDL_X8}(vi0x${ABC[0:8]}, vi1x${ABC[0:8]});

         $for M in range(2, ROW_TILE):
           $if M + 1 != ROW_TILE:
             const ${XINT8X8_T} vi${M+1}x${ABC[0:8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
           vsum${ABC[0:8]} = ${VADDW_X8}(vsum${ABC[0:8]}, vi${M}x${ABC[0:8]});

         $if DATATYPE == "QS8":
           const int32x4_t vacc${ABC[0:4]} = vaddw_s16(vinit_bias, vget_low_s16(vsum${ABC[0:8]}));
           const int32x4_t vacc${ABC[4:8]} = vaddw_s16(vinit_bias, vget_high_s16(vsum${ABC[0:8]}));
         $else:
           const int32x4_t vacc${ABC[0:4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vinit_bias), vget_low_u16(vsum${ABC[0:8]})));
           const int32x4_t vacc${ABC[4:8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vinit_bias), vget_high_u16(vsum${ABC[0:8]})));

         vst1q_s32(b, vacc${ABC[0:4]}); b += 4;
         vst1q_s32(b, vacc${ABC[4:8]}); b += 4;

         c = doz(c, 8);
       } while (c != 0);
     }

   for (rows -= ${ROW_TILE}; rows > ${ROW_SUBTILE}; rows -= ${ROW_SUBTILE}) {
     $for M in range(ROW_SUBTILE):
       i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M + ROW_TILE - ROW_SUBTILE} + input_increment);

     int32_t* b = buffer;
     size_t c = channels;
     for (; ${"c >= %d" % CHANNEL_TILE if CHANNEL_TILE > 16 else "c != 0"}; ${("c -= %d" if CHANNEL_TILE > 16 else "c = doz(c, %d)") % CHANNEL_TILE}) {
       $for M in range(2):
         $for C in range(0, CHANNEL_TILE, 8):
           const ${XINT8X8_T} vi${M}x${ABC[C:C+8]} = ${VLD1_X8}(i${M}); i${M} += 8;

       $for C in range(0, CHANNEL_TILE, 8):
         const ${XINT8X8_T} vi2x${ABC[C:C+8]} = ${VLD1_X8}(i2); i2 += 8;
         ${XINT16X8_T} vsum${ABC[C:C+8]} = ${VADDL_X8}(vi0x${ABC[C:C+8]}, vi1x${ABC[C:C+8]});

       $for M in range(2, ROW_TILE):
         $for C in range(0, CHANNEL_TILE, 8):
           $if M + 1 != ROW_TILE:
             const ${XINT8X8_T} vi${M+1}x${ABC[C:C+8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
           $else:
             $if C == 0:
               int32x4_t vacc${ABC[C:C+4]} = vld1q_s32(b);
             $else:
               int32x4_t vacc${ABC[C:C+4]} = vld1q_s32(b + ${C});
             int32x4_t vacc${ABC[C+4:C+8]} = vld1q_s32(b + ${C+4});
           vsum${ABC[C:C+8]} = ${VADDW_X8}(vsum${ABC[C:C+8]}, vi${M}x${ABC[C:C+8]});

       $for C in range(0, CHANNEL_TILE, 8):
         $if DATATYPE == "QS8":
           vacc${ABC[C:C+4]} = vaddw_s16(vacc${ABC[C:C+4]}, vget_low_s16(vsum${ABC[C:C+8]}));
           vacc${ABC[C+4:C+8]} = vaddw_s16(vacc${ABC[C+4:C+8]}, vget_high_s16(vsum${ABC[C:C+8]}));
         $else:
           vacc${ABC[C:C+4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[C:C+4]}), vget_low_u16(vsum${ABC[C:C+8]})));
           vacc${ABC[C+4:C+8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[C+4:C+8]}), vget_high_u16(vsum${ABC[C:C+8]})));

       $for C in range(0, CHANNEL_TILE, 4):
         vst1q_s32(b, vacc${ABC[C:C+4]}); b += 4;
     }
     $if CHANNEL_TILE > 16:
       if XNN_UNLIKELY(c != 0) {
         do {
           $for M in range(3):
             const ${XINT8X8_T} vi${M}x${ABC[0:8]} = ${VLD1_X8}(i${M}); i${M} += 8;
           ${XINT16X8_T} vsum${ABC[0:8]} = ${VADDL_X8}(vi0x${ABC[0:8]}, vi1x${ABC[0:8]});

           $for M in range(2, ROW_TILE):
             $if M + 1 != ROW_TILE:
               const ${XINT8X8_T} vi${M+1}x${ABC[0:8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
             $else:
               int32x4_t vacc${ABC[0:4]} = vld1q_s32(b);
               int32x4_t vacc${ABC[4:8]} = vld1q_s32(b + 4);
             vsum${ABC[0:8]} = ${VADDW_X8}(vsum${ABC[0:8]}, vi${M}x${ABC[0:8]});

           $if DATATYPE == "QS8":
             vacc${ABC[0:4]} = vaddw_s16(vacc${ABC[0:4]}, vget_low_s16(vsum${ABC[0:8]}));
             vacc${ABC[4:8]} = vaddw_s16(vacc${ABC[4:8]}, vget_high_s16(vsum${ABC[0:8]}));
           $else:
             vacc${ABC[0:4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[0:4]}), vget_low_u16(vsum${ABC[0:8]})));
             vacc${ABC[4:8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[4:8]}), vget_high_u16(vsum${ABC[0:8]})));

           vst1q_s32(b, vacc${ABC[0:4]}); b += 4;
           vst1q_s32(b, vacc${ABC[4:8]}); b += 4;

           c = doz(c, 8);
         } while (c != 0);
       }
   }

   i0 = (const ${XINT8_T}*) ((uintptr_t) i${ROW_TILE - ROW_SUBTILE} + input_increment);
   $for M in range(1, ROW_SUBTILE):
     i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M + ROW_TILE - ROW_SUBTILE} + input_increment);
     $if M % 2 == 1:
       if XNN_UNPREDICTABLE(rows < ${M+1}) {
         i${M} = zero;
       }
     $else:
       if XNN_UNPREDICTABLE(rows <= ${M}) {
         i${M} = zero;
       }

   $if REQUANTIZATION == "FP32":
     const float32x4_t vscale = vld1q_dup_f32(&params->${PARAMS_STRUCT}.scale);
     $if ARMV8:
       const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->fp32_neonv8.output_zero_point);
     $else:
       const float32x4_t vmagic_bias = vld1q_dup_f32(&params->fp32_neon.magic_bias);
       const int32x4_t vmagic_bias_less_output_zero_point = vld1q_dup_s32(&params->fp32_neon.magic_bias_less_output_zero_point);
   $elif REQUANTIZATION == "RNDNU":
     const int32x4_t vleft_pre_shift = vld1q_dup_s32(&params->rndnu_neon.left_pre_shift);
     const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
     const int32x4_t vleft_post_shift = vld1q_dup_s32(&params->rndnu_neon.left_post_shift);
     const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
   $if CHANNEL_TILE > 8:
     const ${XINT8X16_T} voutput_min = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
     const ${XINT8X16_T} voutput_max = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);
   $else:
     const ${XINT8X8_T} voutput_min = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
     const ${XINT8X8_T} voutput_max = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);
   for (; channels >= ${CHANNEL_TILE}; channels -= ${CHANNEL_TILE}) {
     $for M in range(2):
       $for C in range(0, CHANNEL_TILE, 8):
         const ${XINT8X8_T} vi${M}x${ABC[C:C+8]} = ${VLD1_X8}(i${M}); i${M} += 8;

     $for C in range(0, CHANNEL_TILE, 8):
       const ${XINT8X8_T} vi2x${ABC[C:C+8]} = ${VLD1_X8}(i2); i2 += 8;
       ${XINT16X8_T} vsum${ABC[C:C+8]} = ${VADDL_X8}(vi0x${ABC[C:C+8]}, vi1x${ABC[C:C+8]});

     $for M in range(2, ROW_TILE):
       $for C in range(0, CHANNEL_TILE, 8):
         $if M + 1 != ROW_TILE:
           const ${XINT8X8_T} vi${M+1}x${ABC[C:C+8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
         $else:
           int32x4_t vacc${ABC[C:C+4]} = vld1q_s32(buffer); buffer += 4;
           int32x4_t vacc${ABC[C+4:C+8]} = vld1q_s32(buffer); buffer += 4;
         vsum${ABC[C:C+8]} = ${VADDW_X8}(vsum${ABC[C:C+8]}, vi${M}x${ABC[C:C+8]});

     $for C in range(0, CHANNEL_TILE, 8):
       $if DATATYPE == "QS8":
         vacc${ABC[C:C+4]} = vaddw_s16(vacc${ABC[C:C+4]}, vget_low_s16(vsum${ABC[C:C+8]}));
         vacc${ABC[C+4:C+8]} = vaddw_s16(vacc${ABC[C+4:C+8]}, vget_high_s16(vsum${ABC[C:C+8]}));
       $else:
         vacc${ABC[C:C+4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[C:C+4]}), vget_low_u16(vsum${ABC[C:C+8]})));
         vacc${ABC[C+4:C+8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[C+4:C+8]}), vget_high_u16(vsum${ABC[C:C+8]})));

     $if REQUANTIZATION == "FP32":
       $for C in range(0, CHANNEL_TILE, 4):
         float32x4_t vfpacc${ABC[C:C+4]} = vcvtq_f32_s32(vacc${ABC[C:C+4]});

       $for C in range(0, CHANNEL_TILE, 4):
         vfpacc${ABC[C:C+4]} = vmulq_f32(vfpacc${ABC[C:C+4]}, vscale);

       $if ARMV8:
         $for C in range(0, CHANNEL_TILE, 4):
           vacc${ABC[C:C+4]} = vcvtnq_s32_f32(vfpacc${ABC[C:C+4]});
       $else:
         $for C in range(0, CHANNEL_TILE, 4):
           vacc${ABC[C:C+4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[C:C+4]}, vmagic_bias));

         $for C in range(0, CHANNEL_TILE, 4):
           vacc${ABC[C:C+4]} = vqsubq_s32(vacc${ABC[C:C+4]}, vmagic_bias_less_output_zero_point);
     $elif REQUANTIZATION == "RNDNU":
       $for C in range(0, CHANNEL_TILE, 4):
         vacc${ABC[C:C+4]} = vqshlq_s32(vacc${ABC[C:C+4]}, vleft_pre_shift);

       $for C in range(0, CHANNEL_TILE, 4):
         vacc${ABC[C:C+4]} = vqdmulhq_s32(vacc${ABC[C:C+4]}, vmultiplier);

       $for C in range(0, CHANNEL_TILE, 4):
         vacc${ABC[C:C+4]} = vrshlq_s32(vacc${ABC[C:C+4]}, vleft_post_shift);

     #if XNN_ARCH_ARM64
       $for C in range(0, CHANNEL_TILE, 8):
         int16x8_t vacc${ABC[C:C+8]} = vqmovn_high_s32(vqmovn_s32(vacc${ABC[C:C+4]}), vacc${ABC[C+4:C+8]});
     #else  // !XNN_ARCH_ARM64
       $for C in range(0, CHANNEL_TILE, 8):
         int16x8_t vacc${ABC[C:C+8]} = vcombine_s16(vqmovn_s32(vacc${ABC[C:C+4]}), vqmovn_s32(vacc${ABC[C+4:C+8]}));
     #endif  // !XNN_ARCH_ARM64

     $if REQUANTIZATION != "FP32" or ARMV8:
       $for C in range(0, CHANNEL_TILE, 8):
         vacc${ABC[C:C+8]} = vqaddq_s16(vacc${ABC[C:C+8]}, voutput_zero_point);

     #if XNN_ARCH_ARM64
       $for C in range(0, CHANNEL_TILE, 16):
         $if C + 8 < CHANNEL_TILE:
           ${XINT8X16_T} vout${ABC[C:C+16]} = ${VQMOVXN_HIGH_S16}(${VQMOVXN_S16}(vacc${ABC[C:C+8]}), vacc${ABC[C+8:C+16]});
         $else:
           ${XINT8X8_T} vout${ABC[C:C+8]} = ${VQMOVXN_S16}(vacc${ABC[C:C+8]});
     #else  // !XNN_ARCH_ARM64
       $for C in range(0, CHANNEL_TILE, 16):
         $if C + 8 < CHANNEL_TILE:
           ${XINT8X16_T} vout${ABC[C:C+16]} = ${VCOMBINE_X8}(${VQMOVXN_S16}(vacc${ABC[C:C+8]}), ${VQMOVXN_S16}(vacc${ABC[C+8:C+16]}));
         $else:
           ${XINT8X8_T} vout${ABC[C:C+8]} = ${VQMOVXN_S16}(vacc${ABC[C:C+8]});
     #endif  // !XNN_ARCH_ARM64

     $for C in range(0, CHANNEL_TILE, 16):
       $if C + 8 < CHANNEL_TILE:
         vout${ABC[C:C+16]} = ${VMAXQ_X8}(vout${ABC[C:C+16]}, voutput_min);
       $elif CHANNEL_TILE > 8:
         vout${ABC[C:C+8]} = ${VMAX_X8}(vout${ABC[C:C+8]}, ${VGET_LOW_X8}(voutput_min));
       $else:
         vout${ABC[C:C+8]} = ${VMAX_X8}(vout${ABC[C:C+8]}, voutput_min);

     $for C in range(0, CHANNEL_TILE, 16):
       $if C + 8 < CHANNEL_TILE:
         vout${ABC[C:C+16]} = ${VMINQ_X8}(vout${ABC[C:C+16]}, voutput_max);
       $elif CHANNEL_TILE > 8:
         vout${ABC[C:C+8]} = ${VMIN_X8}(vout${ABC[C:C+8]}, ${VGET_LOW_X8}(voutput_max));
       $else:
         vout${ABC[C:C+8]} = ${VMIN_X8}(vout${ABC[C:C+8]}, voutput_max);

     $for C in range(0, CHANNEL_TILE, 16):
       $if C + 8 < CHANNEL_TILE:
         ${VST1Q_X8}(output, vout${ABC[C:C+16]}); output += 16;
       $else:
         ${VST1_X8}(output, vout${ABC[C:C+8]}); output += 8;
   }
   if XNN_UNLIKELY(channels != 0) {
     ${"do " if CHANNEL_TILE > 8 else ""}{
       $for M in range(3):
         $if CHANNEL_TILE > 8:
           const ${XINT8X8_T} vi${M}x${ABC[0:8]} = ${VLD1_X8}(i${M}); i${M} += 8;
         $else:
           const ${XINT8X8_T} vi${M}x${ABC[0:8]} = ${VLD1_X8}(i${M});
       ${XINT16X8_T} vsum${ABC[0:8]} = ${VADDL_X8}(vi0x${ABC[0:8]}, vi1x${ABC[0:8]});

       $for M in range(2, ROW_TILE):
         $if M + 1 != ROW_TILE:
           $if CHANNEL_TILE > 8:
             const ${XINT8X8_T} vi${M+1}x${ABC[0:8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
           $else:
             const ${XINT8X8_T} vi${M+1}x${ABC[0:8]} = ${VLD1_X8}(i${M+1});
         $else:
           int32x4_t vacc${ABC[0:4]} = vld1q_s32(buffer); buffer += 4;
           int32x4_t vacc${ABC[4:8]} = vld1q_s32(buffer); buffer += 4;
         vsum${ABC[0:8]} = ${VADDW_X8}(vsum${ABC[0:8]}, vi${M}x${ABC[0:8]});

       $if DATATYPE == "QS8":
         vacc${ABC[0:4]} = vaddw_s16(vacc${ABC[0:4]}, vget_low_s16(vsum${ABC[0:8]}));
         vacc${ABC[4:8]} = vaddw_s16(vacc${ABC[4:8]}, vget_high_s16(vsum${ABC[0:8]}));
       $else:
         vacc${ABC[0:4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[0:4]}), vget_low_u16(vsum${ABC[0:8]})));
         vacc${ABC[4:8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[4:8]}), vget_high_u16(vsum${ABC[0:8]})));

       $if REQUANTIZATION == "FP32":
         float32x4_t vfpacc${ABC[0:4]} = vcvtq_f32_s32(vacc${ABC[0:4]});
         float32x4_t vfpacc${ABC[4:8]} = vcvtq_f32_s32(vacc${ABC[4:8]});

         vfpacc${ABC[0:4]} = vmulq_f32(vfpacc${ABC[0:4]}, vscale);
         vfpacc${ABC[4:8]} = vmulq_f32(vfpacc${ABC[4:8]}, vscale);

         $if ARMV8:
           vacc${ABC[0:4]} = vcvtnq_s32_f32(vfpacc${ABC[0:4]});
           vacc${ABC[4:8]} = vcvtnq_s32_f32(vfpacc${ABC[4:8]});
         $else:
           vacc${ABC[0:4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[0:4]}, vmagic_bias));
           vacc${ABC[4:8]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[4:8]}, vmagic_bias));

           vacc${ABC[0:4]} = vqsubq_s32(vacc${ABC[0:4]}, vmagic_bias_less_output_zero_point);
           vacc${ABC[4:8]} = vqsubq_s32(vacc${ABC[4:8]}, vmagic_bias_less_output_zero_point);
       $elif REQUANTIZATION == "RNDNU":
         vacc${ABC[0:4]} = vqshlq_s32(vacc${ABC[0:4]}, vleft_pre_shift);
         vacc${ABC[4:8]} = vqshlq_s32(vacc${ABC[4:8]}, vleft_pre_shift);

         vacc${ABC[0:4]} = vqdmulhq_s32(vacc${ABC[0:4]}, vmultiplier);
         vacc${ABC[4:8]} = vqdmulhq_s32(vacc${ABC[4:8]}, vmultiplier);

         vacc${ABC[0:4]} = vrshlq_s32(vacc${ABC[0:4]}, vleft_post_shift);
         vacc${ABC[4:8]} = vrshlq_s32(vacc${ABC[4:8]}, vleft_post_shift);

       #if XNN_ARCH_ARM64
         int16x8_t vacc${ABC[0:8]} = vqmovn_high_s32(vqmovn_s32(vacc${ABC[0:4]}), vacc${ABC[4:8]});
       #else
         int16x8_t vacc${ABC[0:8]} = vcombine_s16(vqmovn_s32(vacc${ABC[0:4]}), vqmovn_s32(vacc${ABC[4:8]}));
       #endif
       $if REQUANTIZATION != "FP32" or ARMV8:
         vacc${ABC[0:8]} = vqaddq_s16(vacc${ABC[0:8]}, voutput_zero_point);

       ${XINT8X8_T} vout${ABC[0:8]} = ${VQMOVXN_S16}(vacc${ABC[0:8]});
       $if CHANNEL_TILE > 8:
         vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_min));
         vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_max));

         if XNN_LIKELY(channels >= 8) {
           ${VST1_X8}(output, vout${ABC[0:8]}); output += 8;
           channels -= 8;
         } else {
           if (channels & 4) {
             vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4;
             vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
           }
           if (channels & 2) {
             vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2;
             vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
           }
           if (channels & 1) {
             ${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0); output += 1;
           }
           channels = 0;
         }
       $else:
         vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, voutput_min);
         vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, voutput_max);

         if (channels & 4) {
           vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4;
           vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
         }
         if (channels & 2) {
           vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2;
           vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
         }
         if (channels & 1) {
           ${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0);
         }
     }${" while (channels != 0);" if CHANNEL_TILE > 8 else ""}
   }
 }
	// Copyright 2020 Google LLC
	//
	// This source code is licensed under the BSD-style license found in the
	// LICENSE file in the root directory of this source tree.

	$assert DATATYPE in ["QS8", "QU8"]
	$assert CHANNEL_TILE % 8 == 0
	$assert CHANNEL_TILE >= 8
	$assert ROW_TILE >= 3
	$assert ROW_SUBTILE >= 3
	$assert ROW_SUBTILE <= ROW_TILE
	$assert REQUANTIZATION in ["FP32", "RNDNU"]
	$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
	#include <assert.h>

	#include <arm_neon.h>

	#include <xnnpack/gavgpool.h>
	$if ARMV8:
	#include <xnnpack/intrinsics-polyfill.h>
	#include <xnnpack/math.h>


	$PARAMS_STRUCT = REQUANTIZATION.lower() + "_" + ("neonv8" if ARMV8 else "neon")
	$XINT8_T = {"QS8": "int8_t", "QU8": "uint8_t"}[DATATYPE]
	$XINT8X8_T = {"QS8": "int8x8_t", "QU8": "uint8x8_t"}[DATATYPE]
	$XINT8X16_T = {"QS8": "int8x16_t", "QU8": "uint8x16_t"}[DATATYPE]
	$XINT16X8_T = {"QS8": "int16x8_t", "QU8": "uint16x8_t"}[DATATYPE]
	$VLD1_X8 = {"QS8": "vld1_s8", "QU8": "vld1_u8"}[DATATYPE]
	$VLD1_DUP_X8 = {"QS8": "vld1_dup_s8", "QU8": "vld1_dup_u8"}[DATATYPE]
	$VLD1Q_DUP_X8 = {"QS8": "vld1q_dup_s8", "QU8": "vld1q_dup_u8"}[DATATYPE]
	$VST1_X8 = {"QS8": "vst1_s8", "QU8": "vst1_u8"}[DATATYPE]
	$VST1Q_X8 = {"QS8": "vst1q_s8", "QU8": "vst1q_u8"}[DATATYPE]
	$VST1_LANE_X8 = {"QS8": "vst1_lane_s8", "QU8": "vst1_lane_u8"}[DATATYPE]
	$VADDL_X8 = {"QS8": "vaddl_s8", "QU8": "vaddl_u8"}[DATATYPE]
	$VADDW_X8 = {"QS8": "vaddw_s8", "QU8": "vaddw_u8"}[DATATYPE]
	$VMIN_X8 = {"QS8": "vmin_s8", "QU8": "vmin_u8"}[DATATYPE]
	$VMINQ_X8 = {"QS8": "vminq_s8", "QU8": "vminq_u8"}[DATATYPE]
	$VMAX_X8 = {"QS8": "vmax_s8", "QU8": "vmax_u8"}[DATATYPE]
	$VMAXQ_X8 = {"QS8": "vmaxq_s8", "QU8": "vmaxq_u8"}[DATATYPE]
	$VEXT_X8 = {"QS8": "vext_s8", "QU8": "vext_u8"}[DATATYPE]
	$VQMOVXN_S16 = {"QS8": "vqmovn_s16", "QU8": "vqmovun_s16"}[DATATYPE]
	$VQMOVXN_HIGH_S16 = {"QS8": "vqmovn_high_s16", "QU8": "vqmovun_high_s16"}[DATATYPE]
	$VGET_LOW_X8 = {"QS8": "vget_low_s8", "QU8": "vget_low_u8"}[DATATYPE]
	$VCOMBINE_X8 = {"QS8": "vcombine_s8", "QU8": "vcombine_u8"}[DATATYPE]
	$VREINTERPRET_U32_X8 = {"QS8": "vreinterpret_u32_s8", "QU8": "vreinterpret_u32_u8"}[DATATYPE]
	$VREINTERPRET_U16_X8 = {"QS8": "vreinterpret_u16_s8", "QU8": "vreinterpret_u16_u8"}[DATATYPE]
	$ISA = "neonv8" if ARMV8 else "neon"
	void xnn_${DATATYPE.lower()}_gavgpool_minmax_${REQUANTIZATION.lower()}_ukernel_${ROW_TILE}p${ROW_SUBTILE}x__${ISA}_c${CHANNEL_TILE}(
	size_t rows,
	size_t channels,
	const ${XINT8_T}* input,
	size_t input_stride,
	const ${XINT8_T}* zero,
	int32_t* buffer,
	${XINT8_T}* output,
	const union xnn_${DATATYPE.lower()}_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
	{
	assert(rows > ${ROW_TILE});
	assert(channels != 0);

	const ${XINT8_T}* i0 = input;
	$for M in range(1, ROW_TILE):
	const ${XINT8_T}* i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M-1} + input_stride);
	$if CHANNEL_TILE <= 16:
	const size_t input_increment = ${ROW_TILE} * input_stride - round_up_po2(channels, ${CHANNEL_TILE}) * sizeof(${XINT8_T});
	$else:
	const size_t input_increment = ${ROW_TILE} * input_stride - round_up_po2(channels, 8) * sizeof(${XINT8_T});

	const int32x4_t vinit_bias = vld1q_dup_s32(&params->${PARAMS_STRUCT}.init_bias);
	int32_t* b = buffer;
	size_t c = channels;
	for (; ${"c >= %d" % CHANNEL_TILE if CHANNEL_TILE > 16 else "c != 0"}; ${("c -= %d" if CHANNEL_TILE > 16 else "c = doz(c, %d)") % CHANNEL_TILE}) {
	$for M in range(2):
	$for C in range(0, CHANNEL_TILE, 8):
	const ${XINT8X8_T} vi${M}x${ABC[C:C+8]} = ${VLD1_X8}(i${M}); i${M} += 8;

	$for C in range(0, CHANNEL_TILE, 8):
	const ${XINT8X8_T} vi2x${ABC[C:C+8]} = ${VLD1_X8}(i2); i2 += 8;
	${XINT16X8_T} vsum${ABC[C:C+8]} = ${VADDL_X8}(vi0x${ABC[C:C+8]}, vi1x${ABC[C:C+8]});

	$for M in range(2, ROW_TILE):
	$for C in range(0, CHANNEL_TILE, 8):
	$if M + 1 != ROW_TILE:
	const ${XINT8X8_T} vi${M+1}x${ABC[C:C+8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
	vsum${ABC[C:C+8]} = ${VADDW_X8}(vsum${ABC[C:C+8]}, vi${M}x${ABC[C:C+8]});

	$for C in range(0, CHANNEL_TILE, 8):
	$if DATATYPE == "QS8":
	const int32x4_t vacc${ABC[C:C+4]} = vaddw_s16(vinit_bias, vget_low_s16(vsum${ABC[C:C+8]}));
	const int32x4_t vacc${ABC[C+4:C+8]} = vaddw_s16(vinit_bias, vget_high_s16(vsum${ABC[C:C+8]}));
	$else:
	const int32x4_t vacc${ABC[C:C+4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vinit_bias), vget_low_u16(vsum${ABC[C:C+8]})));
	const int32x4_t vacc${ABC[C+4:C+8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vinit_bias), vget_high_u16(vsum${ABC[C:C+8]})));

	$for C in range(0, CHANNEL_TILE, 4):
	vst1q_s32(b, vacc${ABC[C:C+4]}); b += 4;
	}
	$if CHANNEL_TILE > 16:
	if XNN_UNLIKELY(c != 0) {
	do {
	$for M in range(3):
	const ${XINT8X8_T} vi${M}x${ABC[0:8]} = ${VLD1_X8}(i${M}); i${M} += 8;
	${XINT16X8_T} vsum${ABC[0:8]} = ${VADDL_X8}(vi0x${ABC[0:8]}, vi1x${ABC[0:8]});

	$for M in range(2, ROW_TILE):
	$if M + 1 != ROW_TILE:
	const ${XINT8X8_T} vi${M+1}x${ABC[0:8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
	vsum${ABC[0:8]} = ${VADDW_X8}(vsum${ABC[0:8]}, vi${M}x${ABC[0:8]});

	$if DATATYPE == "QS8":
	const int32x4_t vacc${ABC[0:4]} = vaddw_s16(vinit_bias, vget_low_s16(vsum${ABC[0:8]}));
	const int32x4_t vacc${ABC[4:8]} = vaddw_s16(vinit_bias, vget_high_s16(vsum${ABC[0:8]}));
	$else:
	const int32x4_t vacc${ABC[0:4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vinit_bias), vget_low_u16(vsum${ABC[0:8]})));
	const int32x4_t vacc${ABC[4:8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vinit_bias), vget_high_u16(vsum${ABC[0:8]})));

	vst1q_s32(b, vacc${ABC[0:4]}); b += 4;
	vst1q_s32(b, vacc${ABC[4:8]}); b += 4;

	c = doz(c, 8);
	} while (c != 0);
	}

	for (rows -= ${ROW_TILE}; rows > ${ROW_SUBTILE}; rows -= ${ROW_SUBTILE}) {
	$for M in range(ROW_SUBTILE):
	i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M + ROW_TILE - ROW_SUBTILE} + input_increment);

	int32_t* b = buffer;
	size_t c = channels;
	for (; ${"c >= %d" % CHANNEL_TILE if CHANNEL_TILE > 16 else "c != 0"}; ${("c -= %d" if CHANNEL_TILE > 16 else "c = doz(c, %d)") % CHANNEL_TILE}) {
	$for M in range(2):
	$for C in range(0, CHANNEL_TILE, 8):
	const ${XINT8X8_T} vi${M}x${ABC[C:C+8]} = ${VLD1_X8}(i${M}); i${M} += 8;

	$for C in range(0, CHANNEL_TILE, 8):
	const ${XINT8X8_T} vi2x${ABC[C:C+8]} = ${VLD1_X8}(i2); i2 += 8;
	${XINT16X8_T} vsum${ABC[C:C+8]} = ${VADDL_X8}(vi0x${ABC[C:C+8]}, vi1x${ABC[C:C+8]});

	$for M in range(2, ROW_TILE):
	$for C in range(0, CHANNEL_TILE, 8):
	$if M + 1 != ROW_TILE:
	const ${XINT8X8_T} vi${M+1}x${ABC[C:C+8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
	$else:
	$if C == 0:
	int32x4_t vacc${ABC[C:C+4]} = vld1q_s32(b);
	$else:
	int32x4_t vacc${ABC[C:C+4]} = vld1q_s32(b + ${C});
	int32x4_t vacc${ABC[C+4:C+8]} = vld1q_s32(b + ${C+4});
	vsum${ABC[C:C+8]} = ${VADDW_X8}(vsum${ABC[C:C+8]}, vi${M}x${ABC[C:C+8]});

	$for C in range(0, CHANNEL_TILE, 8):
	$if DATATYPE == "QS8":
	vacc${ABC[C:C+4]} = vaddw_s16(vacc${ABC[C:C+4]}, vget_low_s16(vsum${ABC[C:C+8]}));
	vacc${ABC[C+4:C+8]} = vaddw_s16(vacc${ABC[C+4:C+8]}, vget_high_s16(vsum${ABC[C:C+8]}));
	$else:
	vacc${ABC[C:C+4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[C:C+4]}), vget_low_u16(vsum${ABC[C:C+8]})));
	vacc${ABC[C+4:C+8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[C+4:C+8]}), vget_high_u16(vsum${ABC[C:C+8]})));

	$for C in range(0, CHANNEL_TILE, 4):
	vst1q_s32(b, vacc${ABC[C:C+4]}); b += 4;
	}
	$if CHANNEL_TILE > 16:
	if XNN_UNLIKELY(c != 0) {
	do {
	$for M in range(3):
	const ${XINT8X8_T} vi${M}x${ABC[0:8]} = ${VLD1_X8}(i${M}); i${M} += 8;
	${XINT16X8_T} vsum${ABC[0:8]} = ${VADDL_X8}(vi0x${ABC[0:8]}, vi1x${ABC[0:8]});

	$for M in range(2, ROW_TILE):
	$if M + 1 != ROW_TILE:
	const ${XINT8X8_T} vi${M+1}x${ABC[0:8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
	$else:
	int32x4_t vacc${ABC[0:4]} = vld1q_s32(b);
	int32x4_t vacc${ABC[4:8]} = vld1q_s32(b + 4);
	vsum${ABC[0:8]} = ${VADDW_X8}(vsum${ABC[0:8]}, vi${M}x${ABC[0:8]});

	$if DATATYPE == "QS8":
	vacc${ABC[0:4]} = vaddw_s16(vacc${ABC[0:4]}, vget_low_s16(vsum${ABC[0:8]}));
	vacc${ABC[4:8]} = vaddw_s16(vacc${ABC[4:8]}, vget_high_s16(vsum${ABC[0:8]}));
	$else:
	vacc${ABC[0:4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[0:4]}), vget_low_u16(vsum${ABC[0:8]})));
	vacc${ABC[4:8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[4:8]}), vget_high_u16(vsum${ABC[0:8]})));

	vst1q_s32(b, vacc${ABC[0:4]}); b += 4;
	vst1q_s32(b, vacc${ABC[4:8]}); b += 4;

	c = doz(c, 8);
	} while (c != 0);
	}
	}

	i0 = (const ${XINT8_T}*) ((uintptr_t) i${ROW_TILE - ROW_SUBTILE} + input_increment);
	$for M in range(1, ROW_SUBTILE):
	i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M + ROW_TILE - ROW_SUBTILE} + input_increment);
	$if M % 2 == 1:
	if XNN_UNPREDICTABLE(rows < ${M+1}) {
	i${M} = zero;
	}
	$else:
	if XNN_UNPREDICTABLE(rows <= ${M}) {
	i${M} = zero;
	}

	$if REQUANTIZATION == "FP32":
	const float32x4_t vscale = vld1q_dup_f32(&params->${PARAMS_STRUCT}.scale);
	$if ARMV8:
	const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->fp32_neonv8.output_zero_point);
	$else:
	const float32x4_t vmagic_bias = vld1q_dup_f32(&params->fp32_neon.magic_bias);
	const int32x4_t vmagic_bias_less_output_zero_point = vld1q_dup_s32(&params->fp32_neon.magic_bias_less_output_zero_point);
	$elif REQUANTIZATION == "RNDNU":
	const int32x4_t vleft_pre_shift = vld1q_dup_s32(&params->rndnu_neon.left_pre_shift);
	const int32x4_t vmultiplier = vld1q_dup_s32(&params->rndnu_neon.multiplier);
	const int32x4_t vleft_post_shift = vld1q_dup_s32(&params->rndnu_neon.left_post_shift);
	const int16x8_t voutput_zero_point = vld1q_dup_s16(&params->rndnu_neon.output_zero_point);
	$if CHANNEL_TILE > 8:
	const ${XINT8X16_T} voutput_min = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
	const ${XINT8X16_T} voutput_max = ${VLD1Q_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);
	$else:
	const ${XINT8X8_T} voutput_min = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_min);
	const ${XINT8X8_T} voutput_max = ${VLD1_DUP_X8}(&params->${PARAMS_STRUCT}.output_max);
	for (; channels >= ${CHANNEL_TILE}; channels -= ${CHANNEL_TILE}) {
	$for M in range(2):
	$for C in range(0, CHANNEL_TILE, 8):
	const ${XINT8X8_T} vi${M}x${ABC[C:C+8]} = ${VLD1_X8}(i${M}); i${M} += 8;

	$for C in range(0, CHANNEL_TILE, 8):
	const ${XINT8X8_T} vi2x${ABC[C:C+8]} = ${VLD1_X8}(i2); i2 += 8;
	${XINT16X8_T} vsum${ABC[C:C+8]} = ${VADDL_X8}(vi0x${ABC[C:C+8]}, vi1x${ABC[C:C+8]});

	$for M in range(2, ROW_TILE):
	$for C in range(0, CHANNEL_TILE, 8):
	$if M + 1 != ROW_TILE:
	const ${XINT8X8_T} vi${M+1}x${ABC[C:C+8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
	$else:
	int32x4_t vacc${ABC[C:C+4]} = vld1q_s32(buffer); buffer += 4;
	int32x4_t vacc${ABC[C+4:C+8]} = vld1q_s32(buffer); buffer += 4;
	vsum${ABC[C:C+8]} = ${VADDW_X8}(vsum${ABC[C:C+8]}, vi${M}x${ABC[C:C+8]});

	$for C in range(0, CHANNEL_TILE, 8):
	$if DATATYPE == "QS8":
	vacc${ABC[C:C+4]} = vaddw_s16(vacc${ABC[C:C+4]}, vget_low_s16(vsum${ABC[C:C+8]}));
	vacc${ABC[C+4:C+8]} = vaddw_s16(vacc${ABC[C+4:C+8]}, vget_high_s16(vsum${ABC[C:C+8]}));
	$else:
	vacc${ABC[C:C+4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[C:C+4]}), vget_low_u16(vsum${ABC[C:C+8]})));
	vacc${ABC[C+4:C+8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[C+4:C+8]}), vget_high_u16(vsum${ABC[C:C+8]})));

	$if REQUANTIZATION == "FP32":
	$for C in range(0, CHANNEL_TILE, 4):
	float32x4_t vfpacc${ABC[C:C+4]} = vcvtq_f32_s32(vacc${ABC[C:C+4]});

	$for C in range(0, CHANNEL_TILE, 4):
	vfpacc${ABC[C:C+4]} = vmulq_f32(vfpacc${ABC[C:C+4]}, vscale);

	$if ARMV8:
	$for C in range(0, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = vcvtnq_s32_f32(vfpacc${ABC[C:C+4]});
	$else:
	$for C in range(0, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[C:C+4]}, vmagic_bias));

	$for C in range(0, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = vqsubq_s32(vacc${ABC[C:C+4]}, vmagic_bias_less_output_zero_point);
	$elif REQUANTIZATION == "RNDNU":
	$for C in range(0, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = vqshlq_s32(vacc${ABC[C:C+4]}, vleft_pre_shift);

	$for C in range(0, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = vqdmulhq_s32(vacc${ABC[C:C+4]}, vmultiplier);

	$for C in range(0, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = vrshlq_s32(vacc${ABC[C:C+4]}, vleft_post_shift);

	#if XNN_ARCH_ARM64
	$for C in range(0, CHANNEL_TILE, 8):
	int16x8_t vacc${ABC[C:C+8]} = vqmovn_high_s32(vqmovn_s32(vacc${ABC[C:C+4]}), vacc${ABC[C+4:C+8]});
	#else // !XNN_ARCH_ARM64
	$for C in range(0, CHANNEL_TILE, 8):
	int16x8_t vacc${ABC[C:C+8]} = vcombine_s16(vqmovn_s32(vacc${ABC[C:C+4]}), vqmovn_s32(vacc${ABC[C+4:C+8]}));
	#endif // !XNN_ARCH_ARM64

	$if REQUANTIZATION != "FP32" or ARMV8:
	$for C in range(0, CHANNEL_TILE, 8):
	vacc${ABC[C:C+8]} = vqaddq_s16(vacc${ABC[C:C+8]}, voutput_zero_point);

	#if XNN_ARCH_ARM64
	$for C in range(0, CHANNEL_TILE, 16):
	$if C + 8 < CHANNEL_TILE:
	${XINT8X16_T} vout${ABC[C:C+16]} = ${VQMOVXN_HIGH_S16}(${VQMOVXN_S16}(vacc${ABC[C:C+8]}), vacc${ABC[C+8:C+16]});
	$else:
	${XINT8X8_T} vout${ABC[C:C+8]} = ${VQMOVXN_S16}(vacc${ABC[C:C+8]});
	#else // !XNN_ARCH_ARM64
	$for C in range(0, CHANNEL_TILE, 16):
	$if C + 8 < CHANNEL_TILE:
	${XINT8X16_T} vout${ABC[C:C+16]} = ${VCOMBINE_X8}(${VQMOVXN_S16}(vacc${ABC[C:C+8]}), ${VQMOVXN_S16}(vacc${ABC[C+8:C+16]}));
	$else:
	${XINT8X8_T} vout${ABC[C:C+8]} = ${VQMOVXN_S16}(vacc${ABC[C:C+8]});
	#endif // !XNN_ARCH_ARM64

	$for C in range(0, CHANNEL_TILE, 16):
	$if C + 8 < CHANNEL_TILE:
	vout${ABC[C:C+16]} = ${VMAXQ_X8}(vout${ABC[C:C+16]}, voutput_min);
	$elif CHANNEL_TILE > 8:
	vout${ABC[C:C+8]} = ${VMAX_X8}(vout${ABC[C:C+8]}, ${VGET_LOW_X8}(voutput_min));
	$else:
	vout${ABC[C:C+8]} = ${VMAX_X8}(vout${ABC[C:C+8]}, voutput_min);

	$for C in range(0, CHANNEL_TILE, 16):
	$if C + 8 < CHANNEL_TILE:
	vout${ABC[C:C+16]} = ${VMINQ_X8}(vout${ABC[C:C+16]}, voutput_max);
	$elif CHANNEL_TILE > 8:
	vout${ABC[C:C+8]} = ${VMIN_X8}(vout${ABC[C:C+8]}, ${VGET_LOW_X8}(voutput_max));
	$else:
	vout${ABC[C:C+8]} = ${VMIN_X8}(vout${ABC[C:C+8]}, voutput_max);

	$for C in range(0, CHANNEL_TILE, 16):
	$if C + 8 < CHANNEL_TILE:
	${VST1Q_X8}(output, vout${ABC[C:C+16]}); output += 16;
	$else:
	${VST1_X8}(output, vout${ABC[C:C+8]}); output += 8;
	}
	if XNN_UNLIKELY(channels != 0) {
	${"do " if CHANNEL_TILE > 8 else ""}{
	$for M in range(3):
	$if CHANNEL_TILE > 8:
	const ${XINT8X8_T} vi${M}x${ABC[0:8]} = ${VLD1_X8}(i${M}); i${M} += 8;
	$else:
	const ${XINT8X8_T} vi${M}x${ABC[0:8]} = ${VLD1_X8}(i${M});
	${XINT16X8_T} vsum${ABC[0:8]} = ${VADDL_X8}(vi0x${ABC[0:8]}, vi1x${ABC[0:8]});

	$for M in range(2, ROW_TILE):
	$if M + 1 != ROW_TILE:
	$if CHANNEL_TILE > 8:
	const ${XINT8X8_T} vi${M+1}x${ABC[0:8]} = ${VLD1_X8}(i${M+1}); i${M+1} += 8;
	$else:
	const ${XINT8X8_T} vi${M+1}x${ABC[0:8]} = ${VLD1_X8}(i${M+1});
	$else:
	int32x4_t vacc${ABC[0:4]} = vld1q_s32(buffer); buffer += 4;
	int32x4_t vacc${ABC[4:8]} = vld1q_s32(buffer); buffer += 4;
	vsum${ABC[0:8]} = ${VADDW_X8}(vsum${ABC[0:8]}, vi${M}x${ABC[0:8]});

	$if DATATYPE == "QS8":
	vacc${ABC[0:4]} = vaddw_s16(vacc${ABC[0:4]}, vget_low_s16(vsum${ABC[0:8]}));
	vacc${ABC[4:8]} = vaddw_s16(vacc${ABC[4:8]}, vget_high_s16(vsum${ABC[0:8]}));
	$else:
	vacc${ABC[0:4]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[0:4]}), vget_low_u16(vsum${ABC[0:8]})));
	vacc${ABC[4:8]} = vreinterpretq_s32_u32(vaddw_u16(vreinterpretq_u32_s32(vacc${ABC[4:8]}), vget_high_u16(vsum${ABC[0:8]})));

	$if REQUANTIZATION == "FP32":
	float32x4_t vfpacc${ABC[0:4]} = vcvtq_f32_s32(vacc${ABC[0:4]});
	float32x4_t vfpacc${ABC[4:8]} = vcvtq_f32_s32(vacc${ABC[4:8]});

	vfpacc${ABC[0:4]} = vmulq_f32(vfpacc${ABC[0:4]}, vscale);
	vfpacc${ABC[4:8]} = vmulq_f32(vfpacc${ABC[4:8]}, vscale);

	$if ARMV8:
	vacc${ABC[0:4]} = vcvtnq_s32_f32(vfpacc${ABC[0:4]});
	vacc${ABC[4:8]} = vcvtnq_s32_f32(vfpacc${ABC[4:8]});
	$else:
	vacc${ABC[0:4]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[0:4]}, vmagic_bias));
	vacc${ABC[4:8]} = vreinterpretq_s32_f32(vaddq_f32(vfpacc${ABC[4:8]}, vmagic_bias));

	vacc${ABC[0:4]} = vqsubq_s32(vacc${ABC[0:4]}, vmagic_bias_less_output_zero_point);
	vacc${ABC[4:8]} = vqsubq_s32(vacc${ABC[4:8]}, vmagic_bias_less_output_zero_point);
	$elif REQUANTIZATION == "RNDNU":
	vacc${ABC[0:4]} = vqshlq_s32(vacc${ABC[0:4]}, vleft_pre_shift);
	vacc${ABC[4:8]} = vqshlq_s32(vacc${ABC[4:8]}, vleft_pre_shift);

	vacc${ABC[0:4]} = vqdmulhq_s32(vacc${ABC[0:4]}, vmultiplier);
	vacc${ABC[4:8]} = vqdmulhq_s32(vacc${ABC[4:8]}, vmultiplier);

	vacc${ABC[0:4]} = vrshlq_s32(vacc${ABC[0:4]}, vleft_post_shift);
	vacc${ABC[4:8]} = vrshlq_s32(vacc${ABC[4:8]}, vleft_post_shift);

	#if XNN_ARCH_ARM64
	int16x8_t vacc${ABC[0:8]} = vqmovn_high_s32(vqmovn_s32(vacc${ABC[0:4]}), vacc${ABC[4:8]});
	#else
	int16x8_t vacc${ABC[0:8]} = vcombine_s16(vqmovn_s32(vacc${ABC[0:4]}), vqmovn_s32(vacc${ABC[4:8]}));
	#endif
	$if REQUANTIZATION != "FP32" or ARMV8:
	vacc${ABC[0:8]} = vqaddq_s16(vacc${ABC[0:8]}, voutput_zero_point);

	${XINT8X8_T} vout${ABC[0:8]} = ${VQMOVXN_S16}(vacc${ABC[0:8]});
	$if CHANNEL_TILE > 8:
	vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_min));
	vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, ${VGET_LOW_X8}(voutput_max));

	if XNN_LIKELY(channels >= 8) {
	${VST1_X8}(output, vout${ABC[0:8]}); output += 8;
	channels -= 8;
	} else {
	if (channels & 4) {
	vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4;
	vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
	}
	if (channels & 2) {
	vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2;
	vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
	}
	if (channels & 1) {
	${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0); output += 1;
	}
	channels = 0;
	}
	$else:
	vout${ABC[0:8]} = ${VMAX_X8}(vout${ABC[0:8]}, voutput_min);
	vout${ABC[0:8]} = ${VMIN_X8}(vout${ABC[0:8]}, voutput_max);

	if (channels & 4) {
	vst1_lane_u32((void*) output, ${VREINTERPRET_U32_X8}(vout${ABC[0:8]}), 0); output += 4;
	vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 4);
	}
	if (channels & 2) {
	vst1_lane_u16((void*) output, ${VREINTERPRET_U16_X8}(vout${ABC[0:8]}), 0); output += 2;
	vout${ABC[0:8]} = ${VEXT_X8}(vout${ABC[0:8]}, vout${ABC[0:8]}, 2);
	}
	if (channels & 1) {
	${VST1_LANE_X8}(output, vout${ABC[0:8]}, 0);
	}
	}${" while (channels != 0);" if CHANNEL_TILE > 8 else ""}
	}
	}