Google Git
Sign in
android / platform / external / XNNPACK / refs/heads/main / . / src / f32-conv-hwc / 3x3s2p1c3-neon-x2.c.in
blob: ba40cd6d11bc8e283542b3a15341f4d3957b8175 [file] [log] [blame] [edit]
// Copyright 2020 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.
$assert CHANNEL_TILE % 4 == 0
$assert HEIGHT_TILE == 2
$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
$VMULADDQ_LANE_F32 = "vfmaq_lane_f32" if FMA else "vmlaq_lane_f32"
#include <assert.h>
#include <arm_neon.h>
#include <xnnpack/conv.h>
#include <xnnpack/math.h>
void xnn_f32_conv_hwc_ukernel_3x3s2p1c3x${CHANNEL_TILE}__${"neonfma" if FMA else "neon"}_${HEIGHT_TILE}x2(
size_t input_height,
size_t input_width,
size_t output_y_start,
size_t output_y_end,
const float* input,
const float* zero,
const float* weights,
float* output,
size_t input_padding_top,
size_t output_channels,
size_t output_height_stride,
size_t output_width_stride,
const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
{
assert(input_width != 0);
assert(output_y_end > output_y_start);
assert(input_padding_top <= 1);
assert(output_channels != 0);
const size_t input_height_stride = input_width * 3 /* channels */ * sizeof(float);
const size_t input_width_decrement = input_width * 3 /* channels */ * sizeof(float);
const size_t output_width = (input_width + 1) / 2;
const size_t output_channel_decrement = output_width * output_width_stride - ${CHANNEL_TILE} * sizeof(float);
const size_t output_height_increment = output_height_stride * 2 - round_up_po2(output_channels, ${CHANNEL_TILE}) * sizeof(float);
// Adjustment for padding processed below
const float* i0 = (const float*) ((uintptr_t) input +
input_height_stride * (output_y_start * 2 /* vertical stride */ - input_padding_top));
$for Y in range(HEIGHT_TILE + 3 - 1):
const float* i${Y+1} = (const float*) ((uintptr_t) i${Y} + input_height_stride);
float* o0 = (float*) ((uintptr_t) output + output_height_stride * output_y_start);
$for Y in range(HEIGHT_TILE - 1):
float* o${Y+1} = (float*) ((uintptr_t) o${Y} + output_height_stride);
if XNN_UNPREDICTABLE(output_y_start < input_padding_top) {
i0 = zero;
}
$if FMA:
const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);
for (size_t output_y = output_y_start; output_y < output_y_end; output_y += 2) {
const size_t input_y2 = output_y * 2 + 2 - input_padding_top;
const size_t input_y4 = input_y2 + 2;
if XNN_UNPREDICTABLE(input_y2 > input_height) {
i1 = zero;
}
if XNN_UNPREDICTABLE(input_y2 >= input_height) {
i2 = zero;
}
if XNN_UNPREDICTABLE(input_y4 > input_height) {
i3 = zero;
}
if XNN_UNPREDICTABLE(input_y4 >= input_height) {
i4 = zero;
}
if XNN_UNPREDICTABLE(output_y + 2 > output_y_end) {
o1 = o0;
}
const float* w = weights;
size_t c = output_channels;
do {
// viMx0 = ( iM0c2, iM0c1, iM0c0, --- )
$for Y in range(HEIGHT_TILE + 3):
float32x4_t vi${Y}x0 = vmovq_n_f32(0.0f);
size_t iw = input_width;
for (; iw >= 4; iw -= 4) {
float32x4_t vo0x0c${ABC[0:4]} = vld1q_f32(w);
$for C in range(4, CHANNEL_TILE, 4):
float32x4_t vo0x0c${ABC[C:C+4]} = vld1q_f32(w + ${C});
$for Y in range(1, HEIGHT_TILE):
$for C in range(0, CHANNEL_TILE, 4):
float32x4_t vo${Y}x0c${ABC[C:C+4]} = vo0x0c${ABC[C:C+4]};
$for Y in range(HEIGHT_TILE):
$for C in range(0, CHANNEL_TILE, 4):
float32x4_t vo${Y}x1c${ABC[C:C+4]} = vo0x0c${ABC[C:C+4]};
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk00c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE});
// viMx1 = ( iM2c0, iM1c2, iM1c1, iM1c0 )
$for Y in range(HEIGHT_TILE + 3):
const float32x4_t vi${Y}x1 = vld1q_f32(i${Y}); i${Y} += 4;
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk00c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk00c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk10c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 2});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk10c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk10c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk20c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 3});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk20c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk20c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk00c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 4});
// viMx2 = ( iM3c1, iM3c0, iM2c2, iM2c1 )
$for Y in range(HEIGHT_TILE + 3):
const float32x4_t vi${Y}x2 = vld1q_f32(i${Y}); i${Y} += 4;
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk00c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk00c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x2), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk10c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 5});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk10c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk10c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x2), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk20c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 6});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk20c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk20c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x2), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk00c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 7});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk00c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk00c2x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x2), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk10c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 8});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk10c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk10c2x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x2), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk20c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 9});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk20c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk20c2x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x2), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk01c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 10});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk01c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk01c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x2), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk11c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 11});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk11c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk11c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x2), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk21c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 12});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk21c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk21c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x2), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk01c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 13});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk01c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk01c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x2), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk11c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 14});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk11c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk11c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x2), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk21c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 15});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk21c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk21c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x2), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk01c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 16});
// viMx3 = ( iM4c2, iM4c1, iM4c0, iM3c2 )
$for Y in range(HEIGHT_TILE + 3):
const float32x4_t vi${Y}x3 = vld1q_f32(i${Y}); i${Y} += 4;
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk01c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk01c2x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x3), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk11c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 17});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk11c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk11c2x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x3), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk21c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 18});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk21c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk21c2x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x3), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk02c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 19});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk02c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk02c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x3), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk12c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 20});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk12c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk12c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x3), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk22c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 21});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk22c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk22c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x3), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk02c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 22});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk02c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x2), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk02c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x3), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk12c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 23});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk12c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x2), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk12c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x3), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk22c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 24});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk22c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x2), 0);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk22c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x3), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk02c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 25});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk02c2x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x2), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk02c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x3), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk12c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 26});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk12c2x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x2), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk12c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x3), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk22c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 27});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x0c${ABC[C:C+4]}, vk22c2x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x2), 1);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}x1c${ABC[C:C+4]}, vk22c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x3), 1);
$for Y in range(HEIGHT_TILE + 3):
vi${Y}x0 = vi${Y}x3;
$if not FMA:
const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = vmaxq_f32(vo${Y}x0c${ABC[C:C+4]}, vmin);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = vmaxq_f32(vo${Y}x1c${ABC[C:C+4]}, vmin);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x0c${ABC[C:C+4]} = vminq_f32(vo${Y}x0c${ABC[C:C+4]}, vmax);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}x1c${ABC[C:C+4]} = vminq_f32(vo${Y}x1c${ABC[C:C+4]}, vmax);
if XNN_LIKELY(c >= ${CHANNEL_TILE}) {
$for Y in reversed(range(HEIGHT_TILE)):
vst1q_f32(o${Y}, vo${Y}x0c${ABC[0:4]});
$for C in range(4, CHANNEL_TILE, 4):
vst1q_f32(o${Y} + 4, vo${Y}x0c${ABC[C:C+4]});
o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
$for Y in reversed(range(HEIGHT_TILE)):
vst1q_f32(o${Y}, vo${Y}x1c${ABC[0:4]});
$for C in range(4, CHANNEL_TILE, 4):
vst1q_f32(o${Y} + 4, vo${Y}x1c${ABC[C:C+4]});
o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
} else {
$for Y in range(HEIGHT_TILE):
float* o${Y}_tmp = o${Y};
$for LOG2_CHANNEL_TILE in reversed(range(CHANNEL_TILE.bit_length())):
$if CHANNEL_TILE != 1 << LOG2_CHANNEL_TILE:
$if LOG2_CHANNEL_TILE == 1:
$for Y in range(HEIGHT_TILE):
float32x2_t vo${Y}x0c${ABC[0:2]} = vget_low_f32(vo${Y}x0c${ABC[0:4]});
$for Y in range(HEIGHT_TILE):
float32x2_t vo${Y}x1c${ABC[0:2]} = vget_low_f32(vo${Y}x1c${ABC[0:4]});
if (c & ${1 << LOG2_CHANNEL_TILE}) {
$if LOG2_CHANNEL_TILE >= 2:
$for C in range(0, 1 << (LOG2_CHANNEL_TILE - 1), 4):
$for Y in reversed(range(HEIGHT_TILE)):
vst1q_f32((float*) ((uintptr_t) o${Y}_tmp + output_width_stride), vo${Y}x1c${ABC[C:C+4]});
vo${Y}x1c${ABC[C:C+4]} = vo${Y}x1c${ABC[C+(1<<LOG2_CHANNEL_TILE):C+(1<<LOG2_CHANNEL_TILE)+4]};
$for Y in reversed(range(HEIGHT_TILE)):
vst1q_f32(o${Y}_tmp, vo${Y}x0c${ABC[C:C+4]}); o${Y}_tmp += 4;
vo${Y}x0c${ABC[C:C+4]} = vo${Y}x0c${ABC[C+(1<<LOG2_CHANNEL_TILE):C+(1<<LOG2_CHANNEL_TILE)+4]};
$elif LOG2_CHANNEL_TILE == 1:
$for Y in reversed(range(HEIGHT_TILE)):
vst1_f32((float*) ((uintptr_t) o${Y}_tmp + output_width_stride), vo${Y}x1c${ABC[0:2]});
vo${Y}x1c${ABC[0:2]} = vget_high_f32(vo${Y}x1c${ABC[0:4]});
$for Y in reversed(range(HEIGHT_TILE)):
vst1_f32(o${Y}_tmp, vo${Y}x0c${ABC[0:2]}); o${Y}_tmp += 2;
vo${Y}x0c${ABC[0:2]} = vget_high_f32(vo${Y}x0c${ABC[0:4]});
$elif LOG2_CHANNEL_TILE == 0:
$for Y in reversed(range(HEIGHT_TILE)):
vst1_lane_f32(o${Y}_tmp, vo${Y}x0c${ABC[0:2]}, 0);
$for Y in reversed(range(HEIGHT_TILE)):
vst1_lane_f32((float*) ((uintptr_t) o${Y}_tmp + output_width_stride), vo${Y}x1c${ABC[0:2]}, 0);
}
$for Y in range(HEIGHT_TILE):
o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride * 2);
}
}
assert(iw < 4);
if XNN_UNLIKELY(iw & 2) {
float32x4_t vo0c${ABC[0:4]} = vld1q_f32(w);
$for C in range(4, CHANNEL_TILE, 4):
float32x4_t vo0c${ABC[C:C+4]} = vld1q_f32(w + ${C});
$for Y in range(1, HEIGHT_TILE):
$for C in range(0, CHANNEL_TILE, 4):
float32x4_t vo${Y}c${ABC[C:C+4]} = vo0c${ABC[C:C+4]};
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk00c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk10c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 2});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk20c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 3});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk00c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 4});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk10c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 5});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk20c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 6});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk00c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 7});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk10c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 8});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk20c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 9});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 1);
// viMx1 = ( iM2c0, iM1c2, iM1c1, iM1c0 )
$for Y in range(HEIGHT_TILE + 3):
const float32x4_t vi${Y}x1 = vld1q_f32(i${Y}); i${Y} += 4;
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk01c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 10});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk11c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 11});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk21c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 12});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk01c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 13});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk11c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 14});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk21c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 15});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk01c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 16});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk11c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 17});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk21c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 18});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk02c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 19});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk02c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk12c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 20});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk12c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk22c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 21});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk22c0x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x1), 1);
// viMx2 = ( iM2c2, iM2c1 )
$for Y in range(HEIGHT_TILE + 3):
const float32x2_t vi${Y}x2 = vld1_f32(i${Y}); i${Y} += 2;
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk02c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 22});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk02c1x${ABC[C:C+4]}, vi${Y*2}x2, 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk12c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 23});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk12c1x${ABC[C:C+4]}, vi${Y*2+1}x2, 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk22c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 24});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk22c1x${ABC[C:C+4]}, vi${Y*2+2}x2, 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk02c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 25});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk02c2x${ABC[C:C+4]}, vi${Y*2}x2, 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk12c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 26});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk12c2x${ABC[C:C+4]}, vi${Y*2+1}x2, 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk22c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 27});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk22c2x${ABC[C:C+4]}, vi${Y*2+2}x2, 1);
$for Y in range(HEIGHT_TILE + 3):
vi${Y}x0 = vcombine_f32(vget_high_f32(vi${Y}x1), vi${Y}x2);
$if not FMA:
const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = vmaxq_f32(vo${Y}c${ABC[C:C+4]}, vmin);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = vminq_f32(vo${Y}c${ABC[C:C+4]}, vmax);
if XNN_LIKELY(c >= ${CHANNEL_TILE}) {
$for Y in reversed(range(HEIGHT_TILE)):
vst1q_f32(o${Y}, vo${Y}c${ABC[0:4]});
$for C in range(4, CHANNEL_TILE, 4):
vst1q_f32(o${Y} + 4, vo${Y}c${ABC[C:C+4]});
o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
} else {
$for Y in range(HEIGHT_TILE):
float* o${Y}_tmp = o${Y};
$for LOG2_CHANNEL_TILE in reversed(range(CHANNEL_TILE.bit_length())):
$if CHANNEL_TILE != 1 << LOG2_CHANNEL_TILE:
$if LOG2_CHANNEL_TILE == 1:
$for Y in range(HEIGHT_TILE):
float32x2_t vo${Y}c${ABC[0:2]} = vget_low_f32(vo${Y}c${ABC[0:4]});
if (c & ${1 << LOG2_CHANNEL_TILE}) {
$if LOG2_CHANNEL_TILE >= 2:
$for C in range(0, 1 << (LOG2_CHANNEL_TILE - 1), 4):
$for Y in reversed(range(HEIGHT_TILE)):
vst1q_f32(o${Y}_tmp, vo${Y}c${ABC[C:C+4]}); o${Y}_tmp += 4;
vo${Y}c${ABC[C:C+4]} = vo${Y}c${ABC[C+(1<<LOG2_CHANNEL_TILE):C+(1<<LOG2_CHANNEL_TILE)+4]};
$elif LOG2_CHANNEL_TILE == 1:
$for Y in reversed(range(HEIGHT_TILE)):
vst1_f32(o${Y}_tmp, vo${Y}c${ABC[0:2]}); o${Y}_tmp += 2;
vo${Y}c${ABC[0:2]} = vget_high_f32(vo${Y}c${ABC[0:4]});
$elif LOG2_CHANNEL_TILE == 0:
$for Y in reversed(range(HEIGHT_TILE)):
vst1_lane_f32(o${Y}_tmp, vo${Y}c${ABC[0:2]}, 0);
}
$for Y in range(HEIGHT_TILE):
o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
}
}
if XNN_UNLIKELY(iw & 1) {
float32x4_t vo0c${ABC[0:4]} = vld1q_f32(w);
$for C in range(4, CHANNEL_TILE, 4):
float32x4_t vo0c${ABC[C:C+4]} = vld1q_f32(w + ${C});
$for Y in range(1, HEIGHT_TILE):
$for C in range(0, CHANNEL_TILE, 4):
float32x4_t vo${Y}c${ABC[C:C+4]} = vo0c${ABC[C:C+4]};
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk00c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk10c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 2});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk20c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 3});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk00c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 4});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk10c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 5});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk20c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 6});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c1x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk00c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 7});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk00c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk10c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 8});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk10c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x0), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk20c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 9});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk20c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x0), 1);
// viMx1 = ( iM2c0, iM1c2, iM1c1, iM1c0 )
$for Y in range(HEIGHT_TILE + 3):
const float32x4_t vi${Y}x1 = vld1q_f32(i${Y}); i${Y} += 3;
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk01c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 10});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk11c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 11});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk21c0x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 12});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c0x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk01c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 13});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk11c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 14});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+1}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk21c1x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 15});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c1x${ABC[C:C+4]}, vget_low_f32(vi${Y*2+2}x1), 1);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk01c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 16});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk01c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk11c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 17});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk11c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+1}x1), 0);
$for C in range(0, CHANNEL_TILE, 4):
const float32x4_t vk21c2x${ABC[C:C+4]} = vld1q_f32(w + ${C + CHANNEL_TILE * 18});
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = ${VMULADDQ_LANE_F32}(vo${Y}c${ABC[C:C+4]}, vk21c2x${ABC[C:C+4]}, vget_high_f32(vi${Y*2+2}x1), 0);
$if not FMA:
const float32x4_t vmin = vld1q_dup_f32(&params->scalar.min);
const float32x4_t vmax = vld1q_dup_f32(&params->scalar.max);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = vmaxq_f32(vo${Y}c${ABC[C:C+4]}, vmin);
$for C in range(0, CHANNEL_TILE, 4):
$for Y in range(HEIGHT_TILE):
vo${Y}c${ABC[C:C+4]} = vminq_f32(vo${Y}c${ABC[C:C+4]}, vmax);
if XNN_LIKELY(c >= ${CHANNEL_TILE}) {
$for Y in reversed(range(HEIGHT_TILE)):
vst1q_f32(o${Y}, vo${Y}c${ABC[0:4]});
$for C in range(4, CHANNEL_TILE, 4):
vst1q_f32(o${Y} + 4, vo${Y}c${ABC[C:C+4]});
o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
} else {
$for Y in range(HEIGHT_TILE):
float* o${Y}_tmp = o${Y};
$for LOG2_CHANNEL_TILE in reversed(range(CHANNEL_TILE.bit_length())):
$if CHANNEL_TILE != 1 << LOG2_CHANNEL_TILE:
$if LOG2_CHANNEL_TILE == 1:
$for Y in range(HEIGHT_TILE):
float32x2_t vo${Y}c${ABC[0:2]} = vget_low_f32(vo${Y}c${ABC[0:4]});
if (c & ${1 << LOG2_CHANNEL_TILE}) {
$if LOG2_CHANNEL_TILE >= 2:
$for C in range(0, 1 << (LOG2_CHANNEL_TILE - 1), 4):
$for Y in reversed(range(HEIGHT_TILE)):
vst1q_f32(o${Y}_tmp, vo${Y}c${ABC[C:C+4]}); o${Y}_tmp += 4;
vo${Y}c${ABC[C:C+4]} = vo${Y}c${ABC[C+(1<<LOG2_CHANNEL_TILE):C+(1<<LOG2_CHANNEL_TILE)+4]};
$elif LOG2_CHANNEL_TILE == 1:
$for Y in reversed(range(HEIGHT_TILE)):
vst1_f32(o${Y}_tmp, vo${Y}c${ABC[0:2]}); o${Y}_tmp += 2;
vo${Y}c${ABC[0:2]} = vget_high_f32(vo${Y}c${ABC[0:4]});
$elif LOG2_CHANNEL_TILE == 0:
$for Y in reversed(range(HEIGHT_TILE)):
vst1_lane_f32(o${Y}_tmp, vo${Y}c${ABC[0:2]}, 0);
}
$for Y in range(HEIGHT_TILE):
o${Y} = (float*) ((uintptr_t) o${Y} + output_width_stride);
}
}
// Move output pointers back to the position of the first pixel in a row,
// and forward to the next block of output channels
o0 = (float*) ((uintptr_t) o0 - output_channel_decrement);
o1 = (float*) ((uintptr_t) o1 - output_channel_decrement);
// Revert input pointers to the position of the first pixel in a row
i0 = (const float*) ((uintptr_t) i0 - input_width_decrement);
i1 = (const float*) ((uintptr_t) i1 - input_width_decrement);
i2 = (const float*) ((uintptr_t) i2 - input_width_decrement);
i3 = (const float*) ((uintptr_t) i3 - input_width_decrement);
i4 = (const float*) ((uintptr_t) i4 - input_width_decrement);
// Move to the block of weights for the next ${CHANNEL_TILE} output channels
w += ${CHANNEL_TILE * 28};
c = doz(c, ${CHANNEL_TILE});
} while (c != 0);
// Move output pointers back to the position of the first channel, and forward to the next block of rows
o0 = (float*) ((uintptr_t) o0 + output_height_increment);
o1 = (float*) ((uintptr_t) o1 + output_height_increment);
// Move input pointers forward to the next four rows
i0 = i4;
i1 = (const float*) ((uintptr_t) i0 + input_height_stride);
i2 = (const float*) ((uintptr_t) i1 + input_height_stride);
i3 = (const float*) ((uintptr_t) i2 + input_height_stride);
i4 = (const float*) ((uintptr_t) i3 + input_height_stride);
}
}