// Copyright 2020 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

$assert CHANNEL_TILE % 4 == 0
$assert HEIGHT_TILE == 2
$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
$VMULADDQ_LANE_F32 = "vfmaq_lane_f32" if FMA else "vmlaq_lane_f32"

#include <assert.h>

#include <arm_neon.h>

#include <xnnpack/conv.h>
#include <xnnpack/math.h>


void xnn_f32_conv_hwc_ukernel_3x3s2p1c3x${CHANNEL_TILE}__${"neonfma" if FMA else "neon"}_${HEIGHT_TILE}x1(
    size_t input_height,
    size_t input_width,
    size_t output_y_start,
    size_t output_y_end,
    const float* input,
    const float* zero,
    const float* weights,
    float* output,
    size_t input_padding_top,
    size_t output_channels,
    size_t output_height_stride,
    size_t output_width_stride,
    const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
{
  assert(input_width != 0);
  assert(output_y_end > output_y_start);
  assert(input_padding_top <= 1);
  assert(output_channels != 0);

  const size_t input_height_stride = input_width * 3 /* channels */ * sizeof(float);
  const size_t input_width_decrement = input_width * 3 /* channels */ * sizeof(float);
  const size_t output_width = (input_width + 1) / 2;
  const size_t output_channel_decrement = output_width * output_width_stride - ${CHANNEL_TILE} * sizeof(float);
  const size_t output_height_increment = output_height_stride * 2 - round_up_po2(output_channels, ${CHANNEL_TILE}) * sizeof(float);

  // Adjustment for padding processed below
  const float* i0 = (const float*) ((uintptr_t) input +
    input_height_stride * (output_y_start * 2 /* vertical stride */ - input_padding_top));
  $for Y in range(HEIGHT_TILE + 3 - 1):
    const float* i${Y+1} = (const float*) ((uintptr_t) i${Y} + input_height_stride);
  float* o0 = (float*) ((uintptr_t) output + output_height_stride * output_y_start);
  $for Y in range(HEIGHT_TILE - 1):
    float* o${Y+1} = (float*) ((uintptr_t) o${Y} + output_height_stride);

  if XNN_UNPREDICTABLE(output_y_start < input_padding_top) {
    i0 = zero;
  }

  $if FMA:
    const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
    const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);

  for (size_t output_y = output_y_start; output_y < output_y_end; output_y += 2) {
    const size_t input_y2 = output_y * 2 + 2 - input_padding_top;
    const size_t input_y4 = input_y2 + 2;
    if XNN_UNPREDICTABLE(input_y2 > input_height) {
      i1 = zero;
    }
    if XNN_UNPREDICTABLE(input_y2 >= input_height) {
      i2 = zero;
    }
    if XNN_UNPREDICTABLE(input_y4 > input_height) {
      i3 = zero;
    }
    if XNN_UNPREDICTABLE(input_y4 >= input_height) {
      i4 = zero;
    }
    if XNN_UNPREDICTABLE(output_y + 2 > output_y_end) {
      o1 = o0;
    }

    const float* w = weights;
    size_t c = output_channels;
    do {
      // viMx0 = ( iM0c2, iM0c1, iM0c0, --- )
      $for Y in range(HEIGHT_TILE + 3):
        float32x4_t vi${Y}x0 = vmovq_n_f32(0.0f);

      size_t iw = input_width;
      for (; iw >= 2; iw -= 2) {
        float32x4_t vo0c${ABC[0:4]} = vld1q_f32(w);
        $for C in range(4, CHANNEL_TILE, 4):
          float32x4_t vo0c${ABC[C:C+4]} = vld1q_f32(w + ${C});
        $for Y in range(1, HEIGHT_TILE):
          $for C in range(0, CHANNEL_TILE, 4):
            float32x4_t vo${Y}c${ABC[C:C+4]} = vo0c${ABC[C:C+4]};

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk00c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk10c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 2});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk20c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 3});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk00c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 4});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk10c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 5});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk20c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 6});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk00c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 7});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk10c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 8});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk20c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 9});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 1);

        // viMx1 = ( iM2c0, iM1c2, iM1c1, iM1c0 )
        $for Y in range(HEIGHT_TILE + 3):
          const float32x4_t vi${Y}x1 = vld1q_f32(i${Y}); i${Y} += 4;

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk01c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 10});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk11c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 11});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk21c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 12});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk01c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 13});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk11c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 14});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk21c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 15});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk01c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 16});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk11c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 17});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk21c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 18});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk02c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 19});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk02c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x1), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk12c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 20});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk12c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x1), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk22c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 21});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk22c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x1), 1);

        // viMx2 = ( iM2c2, iM2c1 )
        $for Y in range(HEIGHT_TILE + 3):
          const float32x2_t vi${Y}x2 = vld1_f32(i${Y}); i${Y} += 2;

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk02c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 22});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk02c1x${ABC[C:C+4]}, vi${Y*2}x2, 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk12c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 23});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk12c1x${ABC[C:C+4]}, vi${Y*2+1}x2, 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk22c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 24});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk22c1x${ABC[C:C+4]}, vi${Y*2+2}x2, 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk02c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 25});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk02c2x${ABC[C:C+4]}, vi${Y*2}x2, 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk12c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 26});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk12c2x${ABC[C:C+4]}, vi${Y*2+1}x2, 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk22c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 27});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk22c2x${ABC[C:C+4]}, vi${Y*2+2}x2, 1);

        $for Y in range(HEIGHT_TILE + 3):
          vi${Y}x0 = vcombine_f32(vget_high_f32(vi${Y}x1), vi${Y}x2);

        $if not FMA:
          const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
          const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = vmaxq_f32(vo${Y}c${ABC[C:C+4]}, vmin);

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = vminq_f32(vo${Y}c${ABC[C:C+4]}, vmax);

        if XNN_LIKELY(c >= ${CHANNEL_TILE}) {
          $for Y in reversed(range(HEIGHT_TILE)):
            vst1q_f32(o${Y}, vo${Y}c${ABC[0:4]});
            $for C in range(4, CHANNEL_TILE, 4):
              vst1q_f32(o${Y} + 4, vo${Y}c${ABC[C:C+4]});
            o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
        } else {
          $for Y in range(HEIGHT_TILE):
            float* o${Y}_tmp = o${Y};
          $for LOG2_CHANNEL_TILE in reversed(range(CHANNEL_TILE.bit_length())):
            $if CHANNEL_TILE != 1 << LOG2_CHANNEL_TILE:
              $if LOG2_CHANNEL_TILE == 1:
                $for Y in range(HEIGHT_TILE):
                  float32x2_t vo${Y}c${ABC[0:2]} = vget_low_f32(vo${Y}c${ABC[0:4]});
              if (c & ${1 << LOG2_CHANNEL_TILE}) {
                $if LOG2_CHANNEL_TILE >= 2:
                  $for C in range(0, 1 << (LOG2_CHANNEL_TILE - 1), 4):
                    $for Y in reversed(range(HEIGHT_TILE)):
                      vst1q_f32(o${Y}_tmp, vo${Y}c${ABC[C:C+4]}); o${Y}_tmp += 4;
                      vo${Y}c${ABC[C:C+4]} = vo${Y}c${ABC[C+(1<<LOG2_CHANNEL_TILE):C+(1<<LOG2_CHANNEL_TILE)+4]};
                $elif LOG2_CHANNEL_TILE == 1:
                  $for Y in reversed(range(HEIGHT_TILE)):
                    vst1_f32(o${Y}_tmp, vo${Y}c${ABC[0:2]}); o${Y}_tmp += 2;
                    vo${Y}c${ABC[0:2]} = vget_high_f32(vo${Y}c${ABC[0:4]});
                $elif LOG2_CHANNEL_TILE == 0:
                  $for Y in reversed(range(HEIGHT_TILE)):
                    vst1_lane_f32(o${Y}_tmp, vo${Y}c${ABC[0:2]}, 0);
              }

          $for Y in range(HEIGHT_TILE):
            o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
        }
      }
      assert(iw < 2);
      if XNN_UNLIKELY(iw & 1) {
        float32x4_t vo0c${ABC[0:4]} = vld1q_f32(w);
        $for C in range(4, CHANNEL_TILE, 4):
          float32x4_t vo0c${ABC[C:C+4]} = vld1q_f32(w + ${C});
        $for Y in range(1, HEIGHT_TILE):
          $for C in range(0, CHANNEL_TILE, 4):
            float32x4_t vo${Y}c${ABC[C:C+4]} = vo0c${ABC[C:C+4]};

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk00c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk10c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 2});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk20c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 3});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk00c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 4});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk10c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 5});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk20c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 6});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk00c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 7});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk10c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 8});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk20c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 9});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 1);

        // viMx1 = ( iM2c0, iM1c2, iM1c1, iM1c0 )
        $for Y in range(HEIGHT_TILE + 3):
          const float32x4_t vi${Y}x1 = vld1q_f32(i${Y}); i${Y} += 3;

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk01c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 10});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk11c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 11});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk21c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 12});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk01c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 13});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk11c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 14});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk21c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 15});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 1);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk01c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 16});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk11c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 17});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x1), 0);

        $for C in range(0, CHANNEL_TILE, 4):
          const float32x4_t vk21c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 18});

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x1), 0);

        $if not FMA:
          const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
          const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = vmaxq_f32(vo${Y}c${ABC[C:C+4]}, vmin);

        $for C in range(0, CHANNEL_TILE, 4):
          $for Y in range(HEIGHT_TILE):
            vo${Y}c${ABC[C:C+4]} = vminq_f32(vo${Y}c${ABC[C:C+4]}, vmax);

        if XNN_LIKELY(c >= ${CHANNEL_TILE}) {
          $for Y in reversed(range(HEIGHT_TILE)):
            vst1q_f32(o${Y}, vo${Y}c${ABC[0:4]});
            $for C in range(4, CHANNEL_TILE, 4):
              vst1q_f32(o${Y} + 4, vo${Y}c${ABC[C:C+4]});
            o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
        } else {
          $for Y in range(HEIGHT_TILE):
            float* o${Y}_tmp = o${Y};
          $for LOG2_CHANNEL_TILE in reversed(range(CHANNEL_TILE.bit_length())):
            $if CHANNEL_TILE != 1 << LOG2_CHANNEL_TILE:
              $if LOG2_CHANNEL_TILE == 1:
                $for Y in range(HEIGHT_TILE):
                  float32x2_t vo${Y}c${ABC[0:2]} = vget_low_f32(vo${Y}c${ABC[0:4]});
              if (c & ${1 << LOG2_CHANNEL_TILE}) {
                $if LOG2_CHANNEL_TILE >= 2:
                  $for C in range(0, 1 << (LOG2_CHANNEL_TILE - 1), 4):
                    $for Y in reversed(range(HEIGHT_TILE)):
                      vst1q_f32(o${Y}_tmp, vo${Y}c${ABC[C:C+4]}); o${Y}_tmp += 4;
                      vo${Y}c${ABC[C:C+4]} = vo${Y}c${ABC[C+(1<<LOG2_CHANNEL_TILE):C+(1<<LOG2_CHANNEL_TILE)+4]};
                $elif LOG2_CHANNEL_TILE == 1:
                  $for Y in reversed(range(HEIGHT_TILE)):
                    vst1_f32(o${Y}_tmp, vo${Y}c${ABC[0:2]}); o${Y}_tmp += 2;
                    vo${Y}c${ABC[0:2]} = vget_high_f32(vo${Y}c${ABC[0:4]});
                $elif LOG2_CHANNEL_TILE == 0:
                  $for Y in reversed(range(HEIGHT_TILE)):
                    vst1_lane_f32(o${Y}_tmp, vo${Y}c${ABC[0:2]}, 0);
              }
          $for Y in range(HEIGHT_TILE):
            o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
        }
      }
      // Move output pointers back to the position of the first pixel in a row,
      // and forward to the next block of output channels
      o0 = (float*) ((uintptr_t) o0 - output_channel_decrement);
      o1 = (float*) ((uintptr_t) o1 - output_channel_decrement);
      // Revert input pointers to the position of the first pixel in a row
      i0 = (const float*) ((uintptr_t) i0 - input_width_decrement);
      i1 = (const float*) ((uintptr_t) i1 - input_width_decrement);
      i2 = (const float*) ((uintptr_t) i2 - input_width_decrement);
      i3 = (const float*) ((uintptr_t) i3 - input_width_decrement);
      i4 = (const float*) ((uintptr_t) i4 - input_width_decrement);
      // Move to the block of weights for the next ${CHANNEL_TILE} output channels
      w += ${CHANNEL_TILE * 28};
      c = doz(c, ${CHANNEL_TILE});
    } while (c != 0);
    // Move output pointers back to the position of the first channel, and forward to the next block of rows
    o0 = (float*) ((uintptr_t) o0 + output_height_increment);
    o1 = (float*) ((uintptr_t) o1 + output_height_increment);
    // Move input pointers forward to the next four rows
    i0 = i4;
    i1 = (const float*) ((uintptr_t) i0 + input_height_stride);
    i2 = (const float*) ((uintptr_t) i1 + input_height_stride);
    i3 = (const float*) ((uintptr_t) i2 + input_height_stride);
    i4 = (const float*) ((uintptr_t) i3 + input_height_stride);
  }
}