src/qs8-gavgpool/multipass-sse4.c.in - platform/external/XNNPACK - Git at Google

 // Copyright 2020 Google LLC
 //
 // This source code is licensed under the BSD-style license found in the
 // LICENSE file in the root directory of this source tree.

 $assert DATATYPE in ["QS8", "QU8"]
 $assert CHANNEL_TILE % 8 == 0
 $assert CHANNEL_TILE >= 8
 $assert ROW_TILE >= 3
 $assert ROW_SUBTILE >= 3
 $assert ROW_SUBTILE <= ROW_TILE
 $assert REQUANTIZATION == "FP32"
 $ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
 #include <assert.h>

 #include <smmintrin.h>

 #include <xnnpack/gavgpool.h>
 #include <xnnpack/math.h>
 #include <xnnpack/unaligned.h>


 $XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t"
 $_MM_CVTEPX8_EPI16 = {"QS8": "_mm_cvtepi8_epi16", "QU8": "_mm_cvtepu8_epi16"}[DATATYPE]
 $_MM_CVTEPX16_EPI32 = {"QS8": "_mm_cvtepi16_epi32", "QU8": "_mm_cvtepu16_epi32"}[DATATYPE]
 $_MM_PACKXS_EPI16 = {"QS8": "_mm_packs_epi16", "QU8": "_mm_packus_epi16"}[DATATYPE]
 $_MM_MAX_EPX8 = {"QS8": "_mm_max_epi8", "QU8": "_mm_max_epu8"}[DATATYPE]
 void xnn_${DATATYPE.lower()}_gavgpool_minmax_fp32_ukernel_${ROW_TILE}p${ROW_SUBTILE}x__sse41_c${CHANNEL_TILE}(
     size_t rows,
     size_t channels,
     const ${XINT8_T}* input,
     size_t input_stride,
     const ${XINT8_T}* zero,
     int32_t* buffer,
     ${XINT8_T}* output,
     const union xnn_${DATATYPE.lower()}_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
 {
   assert(rows > ${ROW_TILE});
   assert(channels != 0);

   const ${XINT8_T}* i0 = input;
   $for M in range(1, ROW_TILE):
     const ${XINT8_T}* i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M-1} + input_stride);
   $if CHANNEL_TILE <= 16:
     const size_t input_increment = ${ROW_TILE} * input_stride - round_up_po2(channels, ${CHANNEL_TILE}) * sizeof(${XINT8_T});
   $else:
     const size_t input_increment = ${ROW_TILE} * input_stride - round_up_po2(channels, 8) * sizeof(${XINT8_T});

   const __m128i vinit_bias = _mm_load_si128((const __m128i*) params->fp32_sse4.init_bias);
   int32_t* b = buffer;
   size_t c = channels;
   for (; ${"c >= %d" % CHANNEL_TILE if CHANNEL_TILE > 16 else "c != 0"}; ${("c -= %d" if CHANNEL_TILE > 16 else "c = doz(c, %d)") % CHANNEL_TILE}) {
     $for M in range(2):
       const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
       $for C in range(8, CHANNEL_TILE, 8):
         const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
       i${M} += ${CHANNEL_TILE};

     __m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
     const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
     $for C in range(8, CHANNEL_TILE, 8):
       __m128i vacc${ABC[C:C+8]} = _mm_add_epi16(vxi0x${ABC[C:C+8]}, vxi1x${ABC[C:C+8]});
       const __m128i vxi2x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i2 + ${C})));
     i2 += ${CHANNEL_TILE};

     $for M in range(3, ROW_TILE):
       vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
       const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
       $for C in range(8, CHANNEL_TILE, 8):
         vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${M-1}x${ABC[C:C+8]});
         const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
       i${M} += ${CHANNEL_TILE};

     $for C in range(0, CHANNEL_TILE, 8):
       vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${ROW_TILE-1}x${ABC[C:C+8]});

     $if DATATYPE == "QU8":
       const __m128i vzero = _mm_setzero_si128();
     $for C in range(0, CHANNEL_TILE, 8):
       __m128i vacc${ABC[C:C+4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[C:C+8]});
       $if DATATYPE == "QS8":
         __m128i vacc${ABC[C+4:C+8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vacc${ABC[C:C+8]}), 16);
       $else:
         __m128i vacc${ABC[C+4:C+8]} = _mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vzero);

     $for C in range(0, CHANNEL_TILE, 4):
       vacc${ABC[C:C+4]} = _mm_add_epi32(vacc${ABC[C:C+4]}, vinit_bias);

     _mm_store_si128((__m128i*) b, vacc${ABC[0:4]});
     $for C in range(4, CHANNEL_TILE, 4):
       _mm_store_si128((__m128i*) (b + ${C}), vacc${ABC[C:C+4]});
     b += ${CHANNEL_TILE};
   }
   $if CHANNEL_TILE > 16:
     if XNN_UNLIKELY(c != 0) {
       do {
         $for M in range(2):
           const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
           i${M} += 8;

         __m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
         const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
         i2 += 8;

         $for M in range(3, ROW_TILE):
           vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
           const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
           i${M} += 8;

         vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${ROW_TILE-1}x${ABC[0:8]});

         __m128i vacc${ABC[0:4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[0:8]});
         $if DATATYPE == "QS8":
           __m128i vacc${ABC[4:8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[0:8]}, vacc${ABC[0:8]}), 16);
         $else:
           __m128i vacc${ABC[4:8]} = _mm_unpackhi_epi16(vacc${ABC[0:8]}, _mm_setzero_si128());

         vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, vinit_bias);
         vacc${ABC[4:8]} = _mm_add_epi32(vacc${ABC[4:8]}, vinit_bias);

         _mm_store_si128((__m128i*) b, vacc${ABC[0:4]});
         _mm_store_si128((__m128i*) (b + 4), vacc${ABC[4:8]});
         b += 8;

         c = doz(c, 8);
       } while (c != 0);
     }

   for (rows -= ${ROW_TILE}; rows > ${ROW_SUBTILE}; rows -= ${ROW_SUBTILE}) {
     $for M in range(ROW_SUBTILE):
       i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M + ROW_TILE - ROW_SUBTILE} + input_increment);

     int32_t* b = buffer;
     size_t c = channels;
     for (; ${"c >= %d" % CHANNEL_TILE if CHANNEL_TILE > 16 else "c != 0"}; ${("c -= %d" if CHANNEL_TILE > 16 else "c = doz(c, %d)") % CHANNEL_TILE}) {
       $for M in range(2):
         const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
         $for C in range(8, CHANNEL_TILE, 8):
           const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
         i${M} += ${CHANNEL_TILE};

       __m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
       const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
       $for C in range(8, CHANNEL_TILE, 8):
         __m128i vacc${ABC[C:C+8]} = _mm_add_epi16(vxi0x${ABC[C:C+8]}, vxi1x${ABC[C:C+8]});
         const __m128i vxi2x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i2 + ${C})));
       i2 += ${CHANNEL_TILE};

       $for M in range(3, ROW_SUBTILE):
         vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
         const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
         $for C in range(8, CHANNEL_TILE, 8):
           vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${M-1}x${ABC[C:C+8]});
           const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
         i${M} += ${CHANNEL_TILE};

       $for C in range(0, CHANNEL_TILE, 8):
         vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${ROW_SUBTILE-1}x${ABC[C:C+8]});

       $if DATATYPE == "QU8":
          const __m128i vzero = _mm_setzero_si128();
       $for C in range(0, CHANNEL_TILE, 8):
         __m128i vacc${ABC[C:C+4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[C:C+8]});
         $if DATATYPE == "QS8":
           __m128i vacc${ABC[C+4:C+8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vacc${ABC[C:C+8]}), 16);
         $else:
           __m128i vacc${ABC[C+4:C+8]} = _mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vzero);

       vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, _mm_load_si128((const __m128i*) b));
       $for C in range(4, CHANNEL_TILE, 4):
         vacc${ABC[C:C+4]} = _mm_add_epi32(vacc${ABC[C:C+4]}, _mm_load_si128((const __m128i*) (b + ${C})));

       _mm_store_si128((__m128i*) b, vacc${ABC[0:4]});
       $for C in range(4, CHANNEL_TILE, 4):
         _mm_store_si128((__m128i*) (b + ${C}), vacc${ABC[C:C+4]});
       b += ${CHANNEL_TILE};
     }
     $if CHANNEL_TILE > 16:
       if XNN_UNLIKELY(c != 0) {
         do {
           $for M in range(2):
             const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
             i${M} += 8;

           __m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
           const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
           i2 += 8;

           $for M in range(3, ROW_SUBTILE):
             vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
             const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
             i${M} += 8;

           vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${ROW_SUBTILE-1}x${ABC[0:8]});

           __m128i vacc${ABC[0:4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[0:8]});
           $if DATATYPE == "QS8":
             __m128i vacc${ABC[4:8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[0:8]}, vacc${ABC[0:8]}), 16);
           $else:
             __m128i vacc${ABC[4:8]} = _mm_unpackhi_epi16(vacc${ABC[0:8]}, _mm_setzero_si128());

           vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, _mm_load_si128((const __m128i*) b));
           vacc${ABC[4:8]} = _mm_add_epi32(vacc${ABC[4:8]}, _mm_load_si128((const __m128i*) (b + 4)));

           _mm_store_si128((__m128i*) b, vacc${ABC[0:4]});
           _mm_store_si128((__m128i*) (b + 4), vacc${ABC[4:8]});
           b += 8;

           c = doz(c, 8);
         } while (c != 0);
       }
   }

   i0 = (const ${XINT8_T}*) ((uintptr_t) i${ROW_TILE - ROW_SUBTILE} + input_increment);
   $for M in range(1, ROW_SUBTILE):
     i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M + ROW_TILE - ROW_SUBTILE} + input_increment);
     $if M % 2 == 1:
       if XNN_UNPREDICTABLE(rows < ${M+1}) {
         i${M} = zero;
       }
     $else:
       if XNN_UNPREDICTABLE(rows <= ${M}) {
         i${M} = zero;
       }

   const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
   const __m128 voutput_max_less_zero_point = _mm_load_ps(params->fp32_sse4.output_max_less_zero_point);
   const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
   const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse4.output_min);
   for (; channels >= ${CHANNEL_TILE}; channels -= ${CHANNEL_TILE}) {
     $for M in range(2):
       const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
       $for C in range(8, CHANNEL_TILE, 8):
         const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
       i${M} += ${CHANNEL_TILE};

     __m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
     const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
     $for C in range(8, CHANNEL_TILE, 8):
       __m128i vacc${ABC[C:C+8]} = _mm_add_epi16(vxi0x${ABC[C:C+8]}, vxi1x${ABC[C:C+8]});
       const __m128i vxi2x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i2 + ${C})));
     i2 += ${CHANNEL_TILE};

     $for M in range(3, ROW_SUBTILE):
       vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
       const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
       $for C in range(8, CHANNEL_TILE, 8):
         vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${M-1}x${ABC[C:C+8]});
         const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
       i${M} += ${CHANNEL_TILE};

     $for C in range(0, CHANNEL_TILE, 8):
       vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${ROW_SUBTILE-1}x${ABC[C:C+8]});

     $if DATATYPE == "QU8":
       const __m128i vzero = _mm_setzero_si128();
     $for C in range(0, CHANNEL_TILE, 8):
       __m128i vacc${ABC[C:C+4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[C:C+8]});
       $if DATATYPE == "QS8":
         __m128i vacc${ABC[C+4:C+8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vacc${ABC[C:C+8]}), 16);
       $else:
         __m128i vacc${ABC[C+4:C+8]} = _mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vzero);

     vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, _mm_load_si128((const __m128i*) buffer));
     $for C in range(4, CHANNEL_TILE, 4):
       vacc${ABC[C:C+4]} = _mm_add_epi32(vacc${ABC[C:C+4]}, _mm_load_si128((const __m128i*) (buffer + ${C})));
     buffer += ${CHANNEL_TILE};

     $for C in range(0, CHANNEL_TILE, 4):
       __m128 vfpacc${ABC[C:C+4]} = _mm_cvtepi32_ps(vacc${ABC[C:C+4]});

     $for C in range(0, CHANNEL_TILE, 4):
       vfpacc${ABC[C:C+4]} = _mm_mul_ps(vfpacc${ABC[C:C+4]}, vscale);

     $for C in range(0, CHANNEL_TILE, 4):
       vfpacc${ABC[C:C+4]} = _mm_min_ps(vfpacc${ABC[C:C+4]}, voutput_max_less_zero_point);

     $for C in range(0, CHANNEL_TILE, 4):
       vacc${ABC[C:C+4]} = _mm_cvtps_epi32(vfpacc${ABC[C:C+4]});

     $for C in range(0, CHANNEL_TILE, 8):
       __m128i vout${ABC[C:C+8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[C:C+4]}, vacc${ABC[C+4:C+8]}), voutput_zero_point);

     $for C in range(0, CHANNEL_TILE, 16):
       $if C + 8 < CHANNEL_TILE:
         __m128i vout${ABC[C:C+16]} = ${_MM_PACKXS_EPI16}(vout${ABC[C:C+8]}, vout${ABC[C+8:C+16]});
       $else:
         __m128i vout${ABC[C:C+8]}${ABC[C:C+8]} = ${_MM_PACKXS_EPI16}(vout${ABC[C:C+8]}, vout${ABC[C:C+8]});

     $for C in range(0, CHANNEL_TILE, 16):
       $if C + 8 < CHANNEL_TILE:
         vout${ABC[C:C+16]} = ${_MM_MAX_EPX8}(vout${ABC[C:C+16]}, voutput_min);
       $else:
         vout${ABC[C:C+8]}${ABC[C:C+8]} = ${_MM_MAX_EPX8}(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_min);

     $if CHANNEL_TILE > 8:
       _mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
     $else:
       _mm_storel_epi64((__m128i*) output, vout${ABC[0:8]}${ABC[0:8]});
     $for C in range(16, CHANNEL_TILE, 16):
       $if C + 8 < CHANNEL_TILE:
         _mm_storeu_si128((__m128i*) (output + ${C}), vout${ABC[C:C+16]});
       $else:
         _mm_storel_epi64((__m128i*) (output + ${C}), vout${ABC[C:C+8]}${ABC[C:C+8]});
     output += ${CHANNEL_TILE};
   }
   if XNN_UNLIKELY(channels != 0) {
     ${"do " if CHANNEL_TILE > 8 else ""}{
       $for M in range(2):
         const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
         i${M} += 8;

       __m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
       const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
       i2 += 8;

       $for M in range(3, ROW_SUBTILE):
         vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
         const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
         i${M} += 8;

       vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${ROW_SUBTILE-1}x${ABC[0:8]});

       __m128i vacc${ABC[0:4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[0:8]});
       $if DATATYPE == "QS8":
         __m128i vacc${ABC[4:8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[0:8]}, vacc${ABC[0:8]}), 16);
       $else:
         __m128i vacc${ABC[4:8]} = _mm_unpackhi_epi16(vacc${ABC[0:8]}, _mm_setzero_si128());

       vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, _mm_load_si128((const __m128i*) buffer));
       vacc${ABC[4:8]} = _mm_add_epi32(vacc${ABC[4:8]}, _mm_load_si128((const __m128i*) (buffer + 4)));
       buffer += 8;

       __m128 vfpacc${ABC[0:4]} = _mm_cvtepi32_ps(vacc${ABC[0:4]});
       __m128 vfpacc${ABC[4:8]} = _mm_cvtepi32_ps(vacc${ABC[4:8]});

       vfpacc${ABC[0:4]} = _mm_mul_ps(vfpacc${ABC[0:4]}, vscale);
       vfpacc${ABC[4:8]} = _mm_mul_ps(vfpacc${ABC[4:8]}, vscale);

       vfpacc${ABC[0:4]} = _mm_min_ps(vfpacc${ABC[0:4]}, voutput_max_less_zero_point);
       vfpacc${ABC[4:8]} = _mm_min_ps(vfpacc${ABC[4:8]}, voutput_max_less_zero_point);

       vacc${ABC[0:4]} = _mm_cvtps_epi32(vfpacc${ABC[0:4]});
       vacc${ABC[4:8]} = _mm_cvtps_epi32(vfpacc${ABC[4:8]});

       __m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[0:4]}, vacc${ABC[4:8]}), voutput_zero_point);

       __m128i vout${ABC[0:8]}${ABC[0:8]} = ${_MM_PACKXS_EPI16}(vout${ABC[0:8]}, vout${ABC[0:8]});
       vout${ABC[0:8]}${ABC[0:8]} = ${_MM_MAX_EPX8}(vout${ABC[0:8]}${ABC[0:8]}, voutput_min);

       $if CHANNEL_TILE > 8:
         if XNN_LIKELY(channels >= 8) {
           _mm_storel_epi64((__m128i*) output, vout${ABC[0:8]}${ABC[0:8]});
           output += 8;
           channels -= 8;
         } else {
           if (channels & 4) {
             unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}));
             vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi64(vout${ABC[0:8]}${ABC[0:8]}, 32);
             output += 4;
           }
           if (channels & 2) {
             unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout${ABC[0:8]}${ABC[0:8]}, 0));
             vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi32(vout${ABC[0:8]}${ABC[0:8]}, 16);
             output += 2;
           }
           if (channels & 1) {
             *output = (${XINT8_T}) _mm_extract_epi8(vout${ABC[0:8]}${ABC[0:8]}, 0);
             output += 1;
           }
           channels = 0;
         }
       $else:
         if (channels & 4) {
           unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}));
           vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi64(vout${ABC[0:8]}${ABC[0:8]}, 32);
           output += 4;
         }
         if (channels & 2) {
           unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout${ABC[0:8]}${ABC[0:8]}, 0));
           vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi32(vout${ABC[0:8]}${ABC[0:8]}, 16);
           output += 2;
         }
         if (channels & 1) {
           *output = (${XINT8_T}) _mm_extract_epi8(vout${ABC[0:8]}${ABC[0:8]}, 0);
         }
     }${" while (channels != 0);" if CHANNEL_TILE > 8 else ""}
   }
 }
	// Copyright 2020 Google LLC
	//
	// This source code is licensed under the BSD-style license found in the
	// LICENSE file in the root directory of this source tree.

	$assert DATATYPE in ["QS8", "QU8"]
	$assert CHANNEL_TILE % 8 == 0
	$assert CHANNEL_TILE >= 8
	$assert ROW_TILE >= 3
	$assert ROW_SUBTILE >= 3
	$assert ROW_SUBTILE <= ROW_TILE
	$assert REQUANTIZATION == "FP32"
	$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
	#include <assert.h>

	#include <smmintrin.h>

	#include <xnnpack/gavgpool.h>
	#include <xnnpack/math.h>
	#include <xnnpack/unaligned.h>


	$XINT8_T = "uint8_t" if DATATYPE == "QU8" else "int8_t"
	$_MM_CVTEPX8_EPI16 = {"QS8": "_mm_cvtepi8_epi16", "QU8": "_mm_cvtepu8_epi16"}[DATATYPE]
	$_MM_CVTEPX16_EPI32 = {"QS8": "_mm_cvtepi16_epi32", "QU8": "_mm_cvtepu16_epi32"}[DATATYPE]
	$_MM_PACKXS_EPI16 = {"QS8": "_mm_packs_epi16", "QU8": "_mm_packus_epi16"}[DATATYPE]
	$_MM_MAX_EPX8 = {"QS8": "_mm_max_epi8", "QU8": "_mm_max_epu8"}[DATATYPE]
	void xnn_${DATATYPE.lower()}_gavgpool_minmax_fp32_ukernel_${ROW_TILE}p${ROW_SUBTILE}x__sse41_c${CHANNEL_TILE}(
	size_t rows,
	size_t channels,
	const ${XINT8_T}* input,
	size_t input_stride,
	const ${XINT8_T}* zero,
	int32_t* buffer,
	${XINT8_T}* output,
	const union xnn_${DATATYPE.lower()}_avgpool_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
	{
	assert(rows > ${ROW_TILE});
	assert(channels != 0);

	const ${XINT8_T}* i0 = input;
	$for M in range(1, ROW_TILE):
	const ${XINT8_T}* i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M-1} + input_stride);
	$if CHANNEL_TILE <= 16:
	const size_t input_increment = ${ROW_TILE} * input_stride - round_up_po2(channels, ${CHANNEL_TILE}) * sizeof(${XINT8_T});
	$else:
	const size_t input_increment = ${ROW_TILE} * input_stride - round_up_po2(channels, 8) * sizeof(${XINT8_T});

	const __m128i vinit_bias = _mm_load_si128((const __m128i*) params->fp32_sse4.init_bias);
	int32_t* b = buffer;
	size_t c = channels;
	for (; ${"c >= %d" % CHANNEL_TILE if CHANNEL_TILE > 16 else "c != 0"}; ${("c -= %d" if CHANNEL_TILE > 16 else "c = doz(c, %d)") % CHANNEL_TILE}) {
	$for M in range(2):
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	$for C in range(8, CHANNEL_TILE, 8):
	const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
	i${M} += ${CHANNEL_TILE};

	__m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
	const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
	$for C in range(8, CHANNEL_TILE, 8):
	__m128i vacc${ABC[C:C+8]} = _mm_add_epi16(vxi0x${ABC[C:C+8]}, vxi1x${ABC[C:C+8]});
	const __m128i vxi2x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i2 + ${C})));
	i2 += ${CHANNEL_TILE};

	$for M in range(3, ROW_TILE):
	vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	$for C in range(8, CHANNEL_TILE, 8):
	vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${M-1}x${ABC[C:C+8]});
	const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
	i${M} += ${CHANNEL_TILE};

	$for C in range(0, CHANNEL_TILE, 8):
	vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${ROW_TILE-1}x${ABC[C:C+8]});

	$if DATATYPE == "QU8":
	const __m128i vzero = _mm_setzero_si128();
	$for C in range(0, CHANNEL_TILE, 8):
	__m128i vacc${ABC[C:C+4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[C:C+8]});
	$if DATATYPE == "QS8":
	__m128i vacc${ABC[C+4:C+8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vacc${ABC[C:C+8]}), 16);
	$else:
	__m128i vacc${ABC[C+4:C+8]} = _mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vzero);

	$for C in range(0, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = _mm_add_epi32(vacc${ABC[C:C+4]}, vinit_bias);

	_mm_store_si128((__m128i*) b, vacc${ABC[0:4]});
	$for C in range(4, CHANNEL_TILE, 4):
	_mm_store_si128((__m128i*) (b + ${C}), vacc${ABC[C:C+4]});
	b += ${CHANNEL_TILE};
	}
	$if CHANNEL_TILE > 16:
	if XNN_UNLIKELY(c != 0) {
	do {
	$for M in range(2):
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	i${M} += 8;

	__m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
	const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
	i2 += 8;

	$for M in range(3, ROW_TILE):
	vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	i${M} += 8;

	vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${ROW_TILE-1}x${ABC[0:8]});

	__m128i vacc${ABC[0:4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[0:8]});
	$if DATATYPE == "QS8":
	__m128i vacc${ABC[4:8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[0:8]}, vacc${ABC[0:8]}), 16);
	$else:
	__m128i vacc${ABC[4:8]} = _mm_unpackhi_epi16(vacc${ABC[0:8]}, _mm_setzero_si128());

	vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, vinit_bias);
	vacc${ABC[4:8]} = _mm_add_epi32(vacc${ABC[4:8]}, vinit_bias);

	_mm_store_si128((__m128i*) b, vacc${ABC[0:4]});
	_mm_store_si128((__m128i*) (b + 4), vacc${ABC[4:8]});
	b += 8;

	c = doz(c, 8);
	} while (c != 0);
	}

	for (rows -= ${ROW_TILE}; rows > ${ROW_SUBTILE}; rows -= ${ROW_SUBTILE}) {
	$for M in range(ROW_SUBTILE):
	i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M + ROW_TILE - ROW_SUBTILE} + input_increment);

	int32_t* b = buffer;
	size_t c = channels;
	for (; ${"c >= %d" % CHANNEL_TILE if CHANNEL_TILE > 16 else "c != 0"}; ${("c -= %d" if CHANNEL_TILE > 16 else "c = doz(c, %d)") % CHANNEL_TILE}) {
	$for M in range(2):
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	$for C in range(8, CHANNEL_TILE, 8):
	const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
	i${M} += ${CHANNEL_TILE};

	__m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
	const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
	$for C in range(8, CHANNEL_TILE, 8):
	__m128i vacc${ABC[C:C+8]} = _mm_add_epi16(vxi0x${ABC[C:C+8]}, vxi1x${ABC[C:C+8]});
	const __m128i vxi2x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i2 + ${C})));
	i2 += ${CHANNEL_TILE};

	$for M in range(3, ROW_SUBTILE):
	vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	$for C in range(8, CHANNEL_TILE, 8):
	vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${M-1}x${ABC[C:C+8]});
	const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
	i${M} += ${CHANNEL_TILE};

	$for C in range(0, CHANNEL_TILE, 8):
	vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${ROW_SUBTILE-1}x${ABC[C:C+8]});

	$if DATATYPE == "QU8":
	const __m128i vzero = _mm_setzero_si128();
	$for C in range(0, CHANNEL_TILE, 8):
	__m128i vacc${ABC[C:C+4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[C:C+8]});
	$if DATATYPE == "QS8":
	__m128i vacc${ABC[C+4:C+8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vacc${ABC[C:C+8]}), 16);
	$else:
	__m128i vacc${ABC[C+4:C+8]} = _mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vzero);

	vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, _mm_load_si128((const __m128i*) b));
	$for C in range(4, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = _mm_add_epi32(vacc${ABC[C:C+4]}, _mm_load_si128((const __m128i*) (b + ${C})));

	_mm_store_si128((__m128i*) b, vacc${ABC[0:4]});
	$for C in range(4, CHANNEL_TILE, 4):
	_mm_store_si128((__m128i*) (b + ${C}), vacc${ABC[C:C+4]});
	b += ${CHANNEL_TILE};
	}
	$if CHANNEL_TILE > 16:
	if XNN_UNLIKELY(c != 0) {
	do {
	$for M in range(2):
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	i${M} += 8;

	__m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
	const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
	i2 += 8;

	$for M in range(3, ROW_SUBTILE):
	vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	i${M} += 8;

	vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${ROW_SUBTILE-1}x${ABC[0:8]});

	__m128i vacc${ABC[0:4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[0:8]});
	$if DATATYPE == "QS8":
	__m128i vacc${ABC[4:8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[0:8]}, vacc${ABC[0:8]}), 16);
	$else:
	__m128i vacc${ABC[4:8]} = _mm_unpackhi_epi16(vacc${ABC[0:8]}, _mm_setzero_si128());

	vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, _mm_load_si128((const __m128i*) b));
	vacc${ABC[4:8]} = _mm_add_epi32(vacc${ABC[4:8]}, _mm_load_si128((const __m128i*) (b + 4)));

	_mm_store_si128((__m128i*) b, vacc${ABC[0:4]});
	_mm_store_si128((__m128i*) (b + 4), vacc${ABC[4:8]});
	b += 8;

	c = doz(c, 8);
	} while (c != 0);
	}
	}

	i0 = (const ${XINT8_T}*) ((uintptr_t) i${ROW_TILE - ROW_SUBTILE} + input_increment);
	$for M in range(1, ROW_SUBTILE):
	i${M} = (const ${XINT8_T}*) ((uintptr_t) i${M + ROW_TILE - ROW_SUBTILE} + input_increment);
	$if M % 2 == 1:
	if XNN_UNPREDICTABLE(rows < ${M+1}) {
	i${M} = zero;
	}
	$else:
	if XNN_UNPREDICTABLE(rows <= ${M}) {
	i${M} = zero;
	}

	const __m128 vscale = _mm_load_ps(params->fp32_sse4.scale);
	const __m128 voutput_max_less_zero_point = _mm_load_ps(params->fp32_sse4.output_max_less_zero_point);
	const __m128i voutput_zero_point = _mm_load_si128((const __m128i*) params->fp32_sse4.output_zero_point);
	const __m128i voutput_min = _mm_load_si128((const __m128i*) params->fp32_sse4.output_min);
	for (; channels >= ${CHANNEL_TILE}; channels -= ${CHANNEL_TILE}) {
	$for M in range(2):
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	$for C in range(8, CHANNEL_TILE, 8):
	const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
	i${M} += ${CHANNEL_TILE};

	__m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
	const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
	$for C in range(8, CHANNEL_TILE, 8):
	__m128i vacc${ABC[C:C+8]} = _mm_add_epi16(vxi0x${ABC[C:C+8]}, vxi1x${ABC[C:C+8]});
	const __m128i vxi2x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i2 + ${C})));
	i2 += ${CHANNEL_TILE};

	$for M in range(3, ROW_SUBTILE):
	vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	$for C in range(8, CHANNEL_TILE, 8):
	vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${M-1}x${ABC[C:C+8]});
	const __m128i vxi${M}x${ABC[C:C+8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) (i${M} + ${C})));
	i${M} += ${CHANNEL_TILE};

	$for C in range(0, CHANNEL_TILE, 8):
	vacc${ABC[C:C+8]} = _mm_add_epi16(vacc${ABC[C:C+8]}, vxi${ROW_SUBTILE-1}x${ABC[C:C+8]});

	$if DATATYPE == "QU8":
	const __m128i vzero = _mm_setzero_si128();
	$for C in range(0, CHANNEL_TILE, 8):
	__m128i vacc${ABC[C:C+4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[C:C+8]});
	$if DATATYPE == "QS8":
	__m128i vacc${ABC[C+4:C+8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vacc${ABC[C:C+8]}), 16);
	$else:
	__m128i vacc${ABC[C+4:C+8]} = _mm_unpackhi_epi16(vacc${ABC[C:C+8]}, vzero);

	vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, _mm_load_si128((const __m128i*) buffer));
	$for C in range(4, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = _mm_add_epi32(vacc${ABC[C:C+4]}, _mm_load_si128((const __m128i*) (buffer + ${C})));
	buffer += ${CHANNEL_TILE};

	$for C in range(0, CHANNEL_TILE, 4):
	__m128 vfpacc${ABC[C:C+4]} = _mm_cvtepi32_ps(vacc${ABC[C:C+4]});

	$for C in range(0, CHANNEL_TILE, 4):
	vfpacc${ABC[C:C+4]} = _mm_mul_ps(vfpacc${ABC[C:C+4]}, vscale);

	$for C in range(0, CHANNEL_TILE, 4):
	vfpacc${ABC[C:C+4]} = _mm_min_ps(vfpacc${ABC[C:C+4]}, voutput_max_less_zero_point);

	$for C in range(0, CHANNEL_TILE, 4):
	vacc${ABC[C:C+4]} = _mm_cvtps_epi32(vfpacc${ABC[C:C+4]});

	$for C in range(0, CHANNEL_TILE, 8):
	__m128i vout${ABC[C:C+8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[C:C+4]}, vacc${ABC[C+4:C+8]}), voutput_zero_point);

	$for C in range(0, CHANNEL_TILE, 16):
	$if C + 8 < CHANNEL_TILE:
	__m128i vout${ABC[C:C+16]} = ${_MM_PACKXS_EPI16}(vout${ABC[C:C+8]}, vout${ABC[C+8:C+16]});
	$else:
	__m128i vout${ABC[C:C+8]}${ABC[C:C+8]} = ${_MM_PACKXS_EPI16}(vout${ABC[C:C+8]}, vout${ABC[C:C+8]});

	$for C in range(0, CHANNEL_TILE, 16):
	$if C + 8 < CHANNEL_TILE:
	vout${ABC[C:C+16]} = ${_MM_MAX_EPX8}(vout${ABC[C:C+16]}, voutput_min);
	$else:
	vout${ABC[C:C+8]}${ABC[C:C+8]} = ${_MM_MAX_EPX8}(vout${ABC[C:C+8]}${ABC[C:C+8]}, voutput_min);

	$if CHANNEL_TILE > 8:
	_mm_storeu_si128((__m128i*) output, vout${ABC[0:16]});
	$else:
	_mm_storel_epi64((__m128i*) output, vout${ABC[0:8]}${ABC[0:8]});
	$for C in range(16, CHANNEL_TILE, 16):
	$if C + 8 < CHANNEL_TILE:
	_mm_storeu_si128((__m128i*) (output + ${C}), vout${ABC[C:C+16]});
	$else:
	_mm_storel_epi64((__m128i*) (output + ${C}), vout${ABC[C:C+8]}${ABC[C:C+8]});
	output += ${CHANNEL_TILE};
	}
	if XNN_UNLIKELY(channels != 0) {
	${"do " if CHANNEL_TILE > 8 else ""}{
	$for M in range(2):
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	i${M} += 8;

	__m128i vacc${ABC[0:8]} = _mm_add_epi16(vxi0x${ABC[0:8]}, vxi1x${ABC[0:8]});
	const __m128i vxi2x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i2));
	i2 += 8;

	$for M in range(3, ROW_SUBTILE):
	vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${M-1}x${ABC[0:8]});
	const __m128i vxi${M}x${ABC[0:8]} = ${_MM_CVTEPX8_EPI16}(_mm_loadl_epi64((const __m128i*) i${M}));
	i${M} += 8;

	vacc${ABC[0:8]} = _mm_add_epi16(vacc${ABC[0:8]}, vxi${ROW_SUBTILE-1}x${ABC[0:8]});

	__m128i vacc${ABC[0:4]} = ${_MM_CVTEPX16_EPI32}(vacc${ABC[0:8]});
	$if DATATYPE == "QS8":
	__m128i vacc${ABC[4:8]} = _mm_srai_epi32(_mm_unpackhi_epi16(vacc${ABC[0:8]}, vacc${ABC[0:8]}), 16);
	$else:
	__m128i vacc${ABC[4:8]} = _mm_unpackhi_epi16(vacc${ABC[0:8]}, _mm_setzero_si128());

	vacc${ABC[0:4]} = _mm_add_epi32(vacc${ABC[0:4]}, _mm_load_si128((const __m128i*) buffer));
	vacc${ABC[4:8]} = _mm_add_epi32(vacc${ABC[4:8]}, _mm_load_si128((const __m128i*) (buffer + 4)));
	buffer += 8;

	__m128 vfpacc${ABC[0:4]} = _mm_cvtepi32_ps(vacc${ABC[0:4]});
	__m128 vfpacc${ABC[4:8]} = _mm_cvtepi32_ps(vacc${ABC[4:8]});

	vfpacc${ABC[0:4]} = _mm_mul_ps(vfpacc${ABC[0:4]}, vscale);
	vfpacc${ABC[4:8]} = _mm_mul_ps(vfpacc${ABC[4:8]}, vscale);

	vfpacc${ABC[0:4]} = _mm_min_ps(vfpacc${ABC[0:4]}, voutput_max_less_zero_point);
	vfpacc${ABC[4:8]} = _mm_min_ps(vfpacc${ABC[4:8]}, voutput_max_less_zero_point);

	vacc${ABC[0:4]} = _mm_cvtps_epi32(vfpacc${ABC[0:4]});
	vacc${ABC[4:8]} = _mm_cvtps_epi32(vfpacc${ABC[4:8]});

	__m128i vout${ABC[0:8]} = _mm_adds_epi16(_mm_packs_epi32(vacc${ABC[0:4]}, vacc${ABC[4:8]}), voutput_zero_point);

	__m128i vout${ABC[0:8]}${ABC[0:8]} = ${_MM_PACKXS_EPI16}(vout${ABC[0:8]}, vout${ABC[0:8]});
	vout${ABC[0:8]}${ABC[0:8]} = ${_MM_MAX_EPX8}(vout${ABC[0:8]}${ABC[0:8]}, voutput_min);

	$if CHANNEL_TILE > 8:
	if XNN_LIKELY(channels >= 8) {
	_mm_storel_epi64((__m128i*) output, vout${ABC[0:8]}${ABC[0:8]});
	output += 8;
	channels -= 8;
	} else {
	if (channels & 4) {
	unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}));
	vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi64(vout${ABC[0:8]}${ABC[0:8]}, 32);
	output += 4;
	}
	if (channels & 2) {
	unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout${ABC[0:8]}${ABC[0:8]}, 0));
	vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi32(vout${ABC[0:8]}${ABC[0:8]}, 16);
	output += 2;
	}
	if (channels & 1) {
	*output = (${XINT8_T}) _mm_extract_epi8(vout${ABC[0:8]}${ABC[0:8]}, 0);
	output += 1;
	}
	channels = 0;
	}
	$else:
	if (channels & 4) {
	unaligned_store_u32(output, (uint32_t) _mm_cvtsi128_si32(vout${ABC[0:8]}${ABC[0:8]}));
	vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi64(vout${ABC[0:8]}${ABC[0:8]}, 32);
	output += 4;
	}
	if (channels & 2) {
	unaligned_store_u16(output, (uint16_t) _mm_extract_epi16(vout${ABC[0:8]}${ABC[0:8]}, 0));
	vout${ABC[0:8]}${ABC[0:8]} = _mm_srli_epi32(vout${ABC[0:8]}${ABC[0:8]}, 16);
	output += 2;
	}
	if (channels & 1) {
	*output = (${XINT8_T}) _mm_extract_epi8(vout${ABC[0:8]}${ABC[0:8]}, 0);
	}
	}${" while (channels != 0);" if CHANNEL_TILE > 8 else ""}
	}
	}