| /* | |
| * Copyright (C) 2014 The Android Open Source Project | |
| * | |
| * Licensed under the Apache License, Version 2.0 (the "License"); | |
| * you may not use this file except in compliance with the License. | |
| * You may obtain a copy of the License at | |
| * | |
| * http://www.apache.org/licenses/LICENSE-2.0 | |
| * | |
| * Unless required by applicable law or agreed to in writing, software | |
| * distributed under the License is distributed on an "AS IS" BASIS, | |
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| * See the License for the specific language governing permissions and | |
| * limitations under the License. | |
| */ | |
| #define ENTRY(f) .text; .align 4; .globl f; .type f,#function; f: .fnstart | |
| #define PRIVATE(f) .text; .align 4; .type f,#function; f: .fnstart | |
| #define END(f) .fnend; .size f, .-f; | |
| #define ARCH_ARM_USE_BLUR_PRELOAD | |
| .eabi_attribute 25,1 @Tag_ABI_align8_preserved | |
| .arm | |
| /* Number of fractional bits to preserve in intermediate results. The | |
| * intermediate storage is 16-bit, and we started with 8 bit data (the integer | |
| * part), so this should be between 0 and 8. | |
| */ | |
| .set FRACTION_BITS, 7 | |
| .set MAX_R, 25 | |
| /* A quick way of making a line of code conditional on some other condition. | |
| * Use `.set cc, 1` or `.set cc, 0` to enable or disable lines prefixed with | |
| * `ifcc`: | |
| */ | |
| .macro ifcc zzz:vararg | |
| .if cc | |
| \zzz | |
| .endif | |
| .endm | |
| /* It's not always clear that prefetching is beneficial and this needs further | |
| * testing on different cores, so it's made switchable here. | |
| */ | |
| #if defined(ARCH_ARM_USE_BLUR_PRELOAD) | |
| #define VERTPLD(...) pld [__VA_ARGS__] | |
| #else | |
| #define VERTPLD(...) nop | |
| #endif | |
| /* Fetch 16 columns of bytes (regardless of image format), convolve these | |
| * vertically, and leave them in the register file. If working near the top or | |
| * bottom of an image then clamp the addressing while loading the data in. | |
| * | |
| * The convolution is fully unrolled for windows up to max_r, with the | |
| * outermost edges calculated first. This way it's possible to branch directly | |
| * into the relevant part of the code for an arbitrary convolution radius. Two | |
| * variants of the loop are produced; one eliminates the clamping code for a | |
| * slight speed advantage. | |
| * | |
| * Where the macro is called with reg=x, the specified register is taken to | |
| * contain a pre-calculated pointer into one of the two loops. | |
| * | |
| * Input: | |
| * r1 -- src | |
| * r2 -- pitch | |
| * r5 -- r | |
| * r6 -- rup | |
| * r7 -- rdn | |
| * r12 -- switch index | |
| * q0-q3 -- coefficient table | |
| * Output: | |
| * r1 += 16 | |
| * q10,q11 -- 16 convolved columns | |
| * Modifies: | |
| * r10 = upper row pointer | |
| * r11 = lower row pointer | |
| * q12-q15 = temporary sums | |
| */ | |
| .macro fetch, max_r=MAX_R, labelc=1, labelnc=2, reg=r12 /*{{{*/ | |
| .ifc \reg,r12 ; .set cc, 1 ; .else ; .set cc, 0 ; .endif | |
| vld1.8 {d30,d31}, [r1] | |
| mls r10, r2, r6, r1 | |
| vmovl.u8 q14, d30 | |
| VERTPLD(r1, #32) | |
| vmovl.u8 q15, d31 | |
| .if \max_r < 16 // approximate | |
| ifcc adr \reg, 1f | |
| .else | |
| ifcc ldr \reg, 2f | |
| 1: ifcc add \reg, \reg, pc | |
| .endif | |
| vmull.u16 q12, d28, d0[0] | |
| ifcc sub \reg, r5, LSL #6 | |
| vmull.u16 q13, d29, d0[0] | |
| mla r11, r2, r7, r1 | |
| vmull.u16 q14, d30, d0[0] | |
| add r1, r1, #16 | |
| vmull.u16 q15, d31, d0[0] | |
| bx \reg | |
| ifcc .align 2 | |
| 2: ifcc .word 1f-1b-8 | |
| /* This version of the vertical fetch loop body is used away from the edges | |
| * of the source image. The pointers start at the top and bottom source rows | |
| * and work their way towards the centre on each iteration. This way the | |
| * number of taps used can be controlled by jumping directly into the middle | |
| * of the loop and running to completion. | |
| * If the loop body changes size then the code which caculates the address of | |
| * the initial iteration must be updated to accordingly. | |
| */ | |
| .macro vertfetch_noclamp i, dreg | |
| .if 0 < \i && \i <= \max_r | |
| vld1.8 {d20,d21}, [r10], r2 | |
| vld1.8 {d22,d23}, [r11] | |
| sub r11, r11, r2 | |
| vswp d21, d22 | |
| VERTPLD(r10, #32) | |
| vaddl.u8 q10, d20, d21 | |
| vaddl.u8 q11, d22, d23 | |
| vmlal.u16 q12, d20, \dreg | |
| VERTPLD(r11, #32) | |
| vmlal.u16 q13, d21, \dreg | |
| vmlal.u16 q14, d22, \dreg | |
| vmlal.u16 q15, d23, \dreg | |
| .endif | |
| .endm | |
| /* This version of the vertical fetch loop body is used near the edges of the | |
| * source image, where one or both of the accesses may start with a clamped | |
| * value, and the row addresses only begin to change after some number of | |
| * iterations before the end. | |
| * If the loop body changes size then the code which caculates the address of | |
| * the initial iteration must be updated to accordingly. | |
| */ | |
| .macro vertfetch_clamped i, dreg | |
| .if 0 < \i && \i <= \max_r | |
| vld1.8 {d20,d21}, [r10] | |
| vld1.8 {d22,d23}, [r11] | |
| cmp r6, #\i | |
| vswp d21, d22 | |
| VERTPLD(r10, #32) | |
| vaddl.u8 q10, d20, d21 | |
| addhs r10, r10, r2 | |
| vaddl.u8 q11, d22, d23 | |
| cmp r7, #\i | |
| vmlal.u16 q12, d20, \dreg | |
| VERTPLD(r11, #32) | |
| vmlal.u16 q13, d21, \dreg | |
| subhs r11, r11, r2 | |
| vmlal.u16 q14, d22, \dreg | |
| nop | |
| vmlal.u16 q15, d23, \dreg | |
| .endif | |
| .endm | |
| /* Entry into this unrolled loop is computed as a negative index from | |
| * \labelc at the end of the block. | |
| */ | |
| .align 4 | |
| vertfetch_clamped 27, d6[3] | |
| vertfetch_clamped 26, d6[2] | |
| vertfetch_clamped 25, d6[1] | |
| vertfetch_clamped 24, d6[0] | |
| vertfetch_clamped 23, d5[3] | |
| vertfetch_clamped 22, d5[2] | |
| vertfetch_clamped 21, d5[1] | |
| vertfetch_clamped 20, d5[0] | |
| vertfetch_clamped 19, d4[3] | |
| vertfetch_clamped 18, d4[2] | |
| vertfetch_clamped 17, d4[1] | |
| vertfetch_clamped 16, d4[0] | |
| vertfetch_clamped 15, d3[3] | |
| vertfetch_clamped 14, d3[2] | |
| vertfetch_clamped 13, d3[1] | |
| vertfetch_clamped 12, d3[0] | |
| vertfetch_clamped 11, d2[3] | |
| vertfetch_clamped 10, d2[2] | |
| vertfetch_clamped 9, d2[1] | |
| vertfetch_clamped 8, d2[0] | |
| vertfetch_clamped 7, d1[3] | |
| vertfetch_clamped 6, d1[2] | |
| vertfetch_clamped 5, d1[1] | |
| vertfetch_clamped 4, d1[0] | |
| vertfetch_clamped 3, d0[3] | |
| vertfetch_clamped 2, d0[2] | |
| vertfetch_clamped 1, d0[1] | |
| vertfetch_clamped 0, d0[0] | |
| 1: | |
| \labelc : b 2f /* done with clamped loop, skip over non-clamped loop */ | |
| /* Entry into this unrolled loop is computed as a negative index from | |
| * \labelnc at the end of the block. | |
| */ | |
| .align 4 | |
| vertfetch_noclamp 27, d6[3] | |
| vertfetch_noclamp 26, d6[2] | |
| vertfetch_noclamp 25, d6[1] | |
| vertfetch_noclamp 24, d6[0] | |
| vertfetch_noclamp 23, d5[3] | |
| vertfetch_noclamp 22, d5[2] | |
| vertfetch_noclamp 21, d5[1] | |
| vertfetch_noclamp 20, d5[0] | |
| vertfetch_noclamp 19, d4[3] | |
| vertfetch_noclamp 18, d4[2] | |
| vertfetch_noclamp 17, d4[1] | |
| vertfetch_noclamp 16, d4[0] | |
| vertfetch_noclamp 15, d3[3] | |
| vertfetch_noclamp 14, d3[2] | |
| vertfetch_noclamp 13, d3[1] | |
| vertfetch_noclamp 12, d3[0] | |
| vertfetch_noclamp 11, d2[3] | |
| vertfetch_noclamp 10, d2[2] | |
| vertfetch_noclamp 9, d2[1] | |
| vertfetch_noclamp 8, d2[0] | |
| vertfetch_noclamp 7, d1[3] | |
| vertfetch_noclamp 6, d1[2] | |
| vertfetch_noclamp 5, d1[1] | |
| vertfetch_noclamp 4, d1[0] | |
| vertfetch_noclamp 3, d0[3] | |
| vertfetch_noclamp 2, d0[2] | |
| vertfetch_noclamp 1, d0[1] | |
| vertfetch_noclamp 0, d0[0] | |
| \labelnc : | |
| .purgem vertfetch_clamped | |
| .purgem vertfetch_noclamp | |
| 2: vqrshrn.u32 d20, q12, #16 - FRACTION_BITS | |
| vqrshrn.u32 d21, q13, #16 - FRACTION_BITS | |
| vqrshrn.u32 d22, q14, #16 - FRACTION_BITS | |
| vqrshrn.u32 d23, q15, #16 - FRACTION_BITS | |
| .endm /*}}}*/ | |
| /* Some portion of the convolution window (as much as will fit, and all of it | |
| * for the uchar1 cases) is kept in the register file to avoid unnecessary | |
| * memory accesses. This forces the horizontal loops to be unrolled because | |
| * there's no indexed addressing into the register file. | |
| * | |
| * As in the fetch macro, the operations are ordered from outside to inside, so | |
| * that jumping into the middle of the block bypasses the unwanted window taps. | |
| * | |
| * There are several variants of the macro because of the fixed offets of the | |
| * taps -- the wider the maximum radius the further the centre tap is from the | |
| * most recently fetched data. This means that pre-filling the window requires | |
| * more data that won't be used and it means that rotating the window involves | |
| * more mov operations. | |
| * | |
| * When the buffer gets too big the buffer at [r9] is used. | |
| * | |
| * Input: | |
| * q4-q11 -- convoltion window | |
| * r9 -- pointer to additional convolution window data | |
| * Output: | |
| * r9 -- updated buffer pointer (if used) | |
| * d31 -- result to be stored | |
| * Modifies: | |
| * r12 -- temp buffer pointer | |
| * q12-q13 -- temporaries for load and vext operations. | |
| * q14-q15 -- intermediate sums | |
| */ | |
| #define TUNED_LIST1 8, 16 | |
| .macro hconv1_8/*{{{*/ | |
| vmull.u16 q14, d18, d0[0] | |
| vmull.u16 q15, d19, d0[0] | |
| ldr r12, [pc, r5, LSL #2] | |
| add pc, pc, r12 | |
| bkpt | |
| 100: .word 101f-100b | |
| .word 102f-100b | |
| .word 103f-100b | |
| .word 104f-100b | |
| .word 105f-100b | |
| .word 106f-100b | |
| .word 107f-100b | |
| .word 108f-100b | |
| 108: vmlal.u16 q14, d16, d2[0] | |
| vmlal.u16 q15, d17, d2[0] | |
| vmlal.u16 q14, d20, d2[0] | |
| vmlal.u16 q15, d21, d2[0] | |
| 107: vext.u16 q12, q8, q9, #1 | |
| vext.u16 q13, q9, q10, #7 | |
| vmlal.u16 q14, d24, d1[3] | |
| vmlal.u16 q15, d25, d1[3] | |
| vmlal.u16 q14, d26, d1[3] | |
| vmlal.u16 q15, d27, d1[3] | |
| 106: vext.u16 q12, q8, q9, #2 | |
| vext.u16 q13, q9, q10, #6 | |
| vmlal.u16 q14, d24, d1[2] | |
| vmlal.u16 q15, d25, d1[2] | |
| vmlal.u16 q14, d26, d1[2] | |
| vmlal.u16 q15, d27, d1[2] | |
| 105: vext.u16 q12, q8, q9, #3 | |
| vext.u16 q13, q9, q10, #5 | |
| vmlal.u16 q14, d24, d1[1] | |
| vmlal.u16 q15, d25, d1[1] | |
| vmlal.u16 q14, d26, d1[1] | |
| vmlal.u16 q15, d27, d1[1] | |
| 104: //vext.u16 q12, q8, q9, #4 | |
| //vext.u16 q13, q9, q10, #4 | |
| vmlal.u16 q14, d17, d1[0] | |
| vmlal.u16 q15, d18, d1[0] | |
| vmlal.u16 q14, d19, d1[0] | |
| vmlal.u16 q15, d20, d1[0] | |
| 103: vext.u16 q12, q8, q9, #5 | |
| vext.u16 q13, q9, q10, #3 | |
| vmlal.u16 q14, d24, d0[3] | |
| vmlal.u16 q15, d25, d0[3] | |
| vmlal.u16 q14, d26, d0[3] | |
| vmlal.u16 q15, d27, d0[3] | |
| 102: vext.u16 q12, q8, q9, #6 | |
| vext.u16 q13, q9, q10, #2 | |
| vmlal.u16 q14, d24, d0[2] | |
| vmlal.u16 q15, d25, d0[2] | |
| vmlal.u16 q14, d26, d0[2] | |
| vmlal.u16 q15, d27, d0[2] | |
| 101: vext.u16 q12, q8, q9, #7 | |
| vext.u16 q13, q9, q10, #1 | |
| vmlal.u16 q14, d24, d0[1] | |
| vmlal.u16 q15, d25, d0[1] | |
| vmlal.u16 q14, d26, d0[1] | |
| vmlal.u16 q15, d27, d0[1] | |
| vqrshrn.u32 d28, q14, #16 | |
| vqrshrn.u32 d29, q15, #16 | |
| vqrshrn.u16 d31, q14, #FRACTION_BITS | |
| vmov q8, q9 | |
| vmov q9, q10 | |
| vmov q10, q11 | |
| .endm/*}}}*/ | |
| .macro hconv1_16/*{{{*/ | |
| vmull.u16 q14, d16, d0[0] | |
| vmull.u16 q15, d17, d0[0] | |
| ldr r12, [pc, r5, LSL #2] | |
| add pc, pc, r12 | |
| bkpt | |
| 100: .word 101f-100b | |
| .word 102f-100b | |
| .word 103f-100b | |
| .word 104f-100b | |
| .word 105f-100b | |
| .word 106f-100b | |
| .word 107f-100b | |
| .word 108f-100b | |
| .word 109f-100b | |
| .word 110f-100b | |
| .word 111f-100b | |
| .word 112f-100b | |
| .word 113f-100b | |
| .word 114f-100b | |
| .word 115f-100b | |
| .word 116f-100b | |
| 116: //vext.u16 q12, q6, q7, #0 | |
| //vext.u16 q13, q10, q11, #0 | |
| vmlal.u16 q14, d12, d4[0] | |
| vmlal.u16 q15, d13, d4[0] | |
| vmlal.u16 q14, d20, d4[0] | |
| vmlal.u16 q15, d21, d4[0] | |
| 115: vext.u16 q12, q6, q7, #1 | |
| vext.u16 q13, q9, q10, #7 | |
| vmlal.u16 q14, d24, d3[3] | |
| vmlal.u16 q15, d25, d3[3] | |
| vmlal.u16 q14, d26, d3[3] | |
| vmlal.u16 q15, d27, d3[3] | |
| 114: vext.u16 q12, q6, q7, #2 | |
| vext.u16 q13, q9, q10, #6 | |
| vmlal.u16 q14, d24, d3[2] | |
| vmlal.u16 q15, d25, d3[2] | |
| vmlal.u16 q14, d26, d3[2] | |
| vmlal.u16 q15, d27, d3[2] | |
| 113: vext.u16 q12, q6, q7, #3 | |
| vext.u16 q13, q9, q10, #5 | |
| vmlal.u16 q14, d24, d3[1] | |
| vmlal.u16 q15, d25, d3[1] | |
| vmlal.u16 q14, d26, d3[1] | |
| vmlal.u16 q15, d27, d3[1] | |
| 112: //vext.u16 q12, q6, q7, #4 | |
| //vext.u16 q13, q9, q10, #4 | |
| vmlal.u16 q14, d13, d3[0] | |
| vmlal.u16 q15, d14, d3[0] | |
| vmlal.u16 q14, d19, d3[0] | |
| vmlal.u16 q15, d20, d3[0] | |
| 111: vext.u16 q12, q6, q7, #5 | |
| vext.u16 q13, q9, q10, #3 | |
| vmlal.u16 q14, d24, d2[3] | |
| vmlal.u16 q15, d25, d2[3] | |
| vmlal.u16 q14, d26, d2[3] | |
| vmlal.u16 q15, d27, d2[3] | |
| 110: vext.u16 q12, q6, q7, #6 | |
| vext.u16 q13, q9, q10, #2 | |
| vmlal.u16 q14, d24, d2[2] | |
| vmlal.u16 q15, d25, d2[2] | |
| vmlal.u16 q14, d26, d2[2] | |
| vmlal.u16 q15, d27, d2[2] | |
| 109: vext.u16 q12, q6, q7, #7 | |
| vext.u16 q13, q9, q10, #1 | |
| vmlal.u16 q14, d24, d2[1] | |
| vmlal.u16 q15, d25, d2[1] | |
| vmlal.u16 q14, d26, d2[1] | |
| vmlal.u16 q15, d27, d2[1] | |
| 108: //vext.u16 q12, q7, q8, #0 | |
| //vext.u16 q13, q9, q10, #0 | |
| vmlal.u16 q14, d14, d2[0] | |
| vmlal.u16 q15, d15, d2[0] | |
| vmlal.u16 q14, d18, d2[0] | |
| vmlal.u16 q15, d19, d2[0] | |
| 107: vext.u16 q12, q7, q8, #1 | |
| vext.u16 q13, q8, q9, #7 | |
| vmlal.u16 q14, d24, d1[3] | |
| vmlal.u16 q15, d25, d1[3] | |
| vmlal.u16 q14, d26, d1[3] | |
| vmlal.u16 q15, d27, d1[3] | |
| 106: vext.u16 q12, q7, q8, #2 | |
| vext.u16 q13, q8, q9, #6 | |
| vmlal.u16 q14, d24, d1[2] | |
| vmlal.u16 q15, d25, d1[2] | |
| vmlal.u16 q14, d26, d1[2] | |
| vmlal.u16 q15, d27, d1[2] | |
| 105: vext.u16 q12, q7, q8, #3 | |
| vext.u16 q13, q8, q9, #5 | |
| vmlal.u16 q14, d24, d1[1] | |
| vmlal.u16 q15, d25, d1[1] | |
| vmlal.u16 q14, d26, d1[1] | |
| vmlal.u16 q15, d27, d1[1] | |
| 104: //vext.u16 q12, q7, q8, #4 | |
| //vext.u16 q13, q8, q9, #4 | |
| vmlal.u16 q14, d15, d1[0] | |
| vmlal.u16 q15, d16, d1[0] | |
| vmlal.u16 q14, d17, d1[0] | |
| vmlal.u16 q15, d18, d1[0] | |
| 103: vext.u16 q12, q7, q8, #5 | |
| vext.u16 q13, q8, q9, #3 | |
| vmlal.u16 q14, d24, d0[3] | |
| vmlal.u16 q15, d25, d0[3] | |
| vmlal.u16 q14, d26, d0[3] | |
| vmlal.u16 q15, d27, d0[3] | |
| 102: vext.u16 q12, q7, q8, #6 | |
| vext.u16 q13, q8, q9, #2 | |
| vmlal.u16 q14, d24, d0[2] | |
| vmlal.u16 q15, d25, d0[2] | |
| vmlal.u16 q14, d26, d0[2] | |
| vmlal.u16 q15, d27, d0[2] | |
| 101: vext.u16 q12, q7, q8, #7 | |
| vext.u16 q13, q8, q9, #1 | |
| vmlal.u16 q14, d24, d0[1] | |
| vmlal.u16 q15, d25, d0[1] | |
| vmlal.u16 q14, d26, d0[1] | |
| vmlal.u16 q15, d27, d0[1] | |
| vqrshrn.u32 d28, q14, #16 | |
| vqrshrn.u32 d29, q15, #16 | |
| vqrshrn.u16 d31, q14, #FRACTION_BITS | |
| vmov q6, q7 | |
| vmov q7, q8 | |
| vmov q8, q9 | |
| vmov q9, q10 | |
| vmov q10, q11 | |
| .endm/*}}}*/ | |
| .macro hconv1_25/*{{{*/ | |
| vext.u16 q12, q6, q7, #7 | |
| vmull.u16 q14, d24, d0[0] | |
| vmull.u16 q15, d25, d0[0] | |
| ldr r12, [pc, r5, LSL #2] | |
| add pc, pc, r12 | |
| bkpt | |
| 100: .word 101f-100b | |
| .word 102f-100b | |
| .word 103f-100b | |
| .word 104f-100b | |
| .word 105f-100b | |
| .word 106f-100b | |
| .word 107f-100b | |
| .word 108f-100b | |
| .word 109f-100b | |
| .word 110f-100b | |
| .word 111f-100b | |
| .word 112f-100b | |
| .word 113f-100b | |
| .word 114f-100b | |
| .word 115f-100b | |
| .word 116f-100b | |
| .word 117f-100b | |
| .word 118f-100b | |
| .word 119f-100b | |
| .word 120f-100b | |
| .word 121f-100b | |
| .word 122f-100b | |
| .word 123f-100b | |
| .word 124f-100b | |
| .word 125f-100b | |
| 125: vext.u16 q12, q3, q4, #6 | |
| vext.u16 q13, q10, q11, #0 | |
| vmlal.u16 q14, d24, d6[1] | |
| vmlal.u16 q15, d25, d6[1] | |
| vmlal.u16 q14, d26, d6[1] | |
| vmlal.u16 q15, d27, d6[1] | |
| 124: vext.u16 q12, q3, q4, #7 | |
| vext.u16 q13, q9, q10, #7 | |
| vmlal.u16 q14, d24, d6[0] | |
| vmlal.u16 q15, d25, d6[0] | |
| vmlal.u16 q14, d26, d6[0] | |
| vmlal.u16 q15, d27, d6[0] | |
| 123: vext.u16 q12, q4, q5, #0 | |
| vext.u16 q13, q9, q10, #6 | |
| vmlal.u16 q14, d24, d5[3] | |
| vmlal.u16 q15, d25, d5[3] | |
| vmlal.u16 q14, d26, d5[3] | |
| vmlal.u16 q15, d27, d5[3] | |
| 122: vext.u16 q12, q4, q5, #1 | |
| vext.u16 q13, q9, q10, #5 | |
| vmlal.u16 q14, d24, d5[2] | |
| vmlal.u16 q15, d25, d5[2] | |
| vmlal.u16 q14, d26, d5[2] | |
| vmlal.u16 q15, d27, d5[2] | |
| 121: vext.u16 q12, q4, q5, #2 | |
| vext.u16 q13, q9, q10, #4 | |
| vmlal.u16 q14, d24, d5[1] | |
| vmlal.u16 q15, d25, d5[1] | |
| vmlal.u16 q14, d26, d5[1] | |
| vmlal.u16 q15, d27, d5[1] | |
| 120: vext.u16 q12, q4, q5, #3 | |
| vext.u16 q13, q9, q10, #3 | |
| vmlal.u16 q14, d24, d5[0] | |
| vmlal.u16 q15, d25, d5[0] | |
| vmlal.u16 q14, d26, d5[0] | |
| vmlal.u16 q15, d27, d5[0] | |
| 119: vext.u16 q12, q4, q5, #4 | |
| vext.u16 q13, q9, q10, #2 | |
| vmlal.u16 q14, d24, d4[3] | |
| vmlal.u16 q15, d25, d4[3] | |
| vmlal.u16 q14, d26, d4[3] | |
| vmlal.u16 q15, d27, d4[3] | |
| 118: vext.u16 q12, q4, q5, #5 | |
| vext.u16 q13, q9, q10, #1 | |
| vmlal.u16 q14, d24, d4[2] | |
| vmlal.u16 q15, d25, d4[2] | |
| vmlal.u16 q14, d26, d4[2] | |
| vmlal.u16 q15, d27, d4[2] | |
| 117: vext.u16 q12, q4, q5, #6 | |
| vext.u16 q13, q9, q10, #0 | |
| vmlal.u16 q14, d24, d4[1] | |
| vmlal.u16 q15, d25, d4[1] | |
| vmlal.u16 q14, d26, d4[1] | |
| vmlal.u16 q15, d27, d4[1] | |
| 116: vext.u16 q12, q4, q5, #7 | |
| vext.u16 q13, q8, q9, #7 | |
| vmlal.u16 q14, d24, d4[0] | |
| vmlal.u16 q15, d25, d4[0] | |
| vmlal.u16 q14, d26, d4[0] | |
| vmlal.u16 q15, d27, d4[0] | |
| 115: vext.u16 q12, q5, q6, #0 | |
| vext.u16 q13, q8, q9, #6 | |
| vmlal.u16 q14, d24, d3[3] | |
| vmlal.u16 q15, d25, d3[3] | |
| vmlal.u16 q14, d26, d3[3] | |
| vmlal.u16 q15, d27, d3[3] | |
| 114: vext.u16 q12, q5, q6, #1 | |
| vext.u16 q13, q8, q9, #5 | |
| vmlal.u16 q14, d24, d3[2] | |
| vmlal.u16 q15, d25, d3[2] | |
| vmlal.u16 q14, d26, d3[2] | |
| vmlal.u16 q15, d27, d3[2] | |
| 113: vext.u16 q12, q5, q6, #2 | |
| vext.u16 q13, q8, q9, #4 | |
| vmlal.u16 q14, d24, d3[1] | |
| vmlal.u16 q15, d25, d3[1] | |
| vmlal.u16 q14, d26, d3[1] | |
| vmlal.u16 q15, d27, d3[1] | |
| 112: vext.u16 q12, q5, q6, #3 | |
| vext.u16 q13, q8, q9, #3 | |
| vmlal.u16 q14, d24, d3[0] | |
| vmlal.u16 q15, d25, d3[0] | |
| vmlal.u16 q14, d26, d3[0] | |
| vmlal.u16 q15, d27, d3[0] | |
| 111: vext.u16 q12, q5, q6, #4 | |
| vext.u16 q13, q8, q9, #2 | |
| vmlal.u16 q14, d24, d2[3] | |
| vmlal.u16 q15, d25, d2[3] | |
| vmlal.u16 q14, d26, d2[3] | |
| vmlal.u16 q15, d27, d2[3] | |
| 110: vext.u16 q12, q5, q6, #5 | |
| vext.u16 q13, q8, q9, #1 | |
| vmlal.u16 q14, d24, d2[2] | |
| vmlal.u16 q15, d25, d2[2] | |
| vmlal.u16 q14, d26, d2[2] | |
| vmlal.u16 q15, d27, d2[2] | |
| 109: vext.u16 q12, q5, q6, #6 | |
| vext.u16 q13, q8, q9, #0 | |
| vmlal.u16 q14, d24, d2[1] | |
| vmlal.u16 q15, d25, d2[1] | |
| vmlal.u16 q14, d26, d2[1] | |
| vmlal.u16 q15, d27, d2[1] | |
| 108: vext.u16 q12, q5, q6, #7 | |
| vext.u16 q13, q7, q8, #7 | |
| vmlal.u16 q14, d24, d2[0] | |
| vmlal.u16 q15, d25, d2[0] | |
| vmlal.u16 q14, d26, d2[0] | |
| vmlal.u16 q15, d27, d2[0] | |
| 107: vext.u16 q12, q6, q7, #0 | |
| vext.u16 q13, q7, q8, #6 | |
| vmlal.u16 q14, d24, d1[3] | |
| vmlal.u16 q15, d25, d1[3] | |
| vmlal.u16 q14, d26, d1[3] | |
| vmlal.u16 q15, d27, d1[3] | |
| 106: vext.u16 q12, q6, q7, #1 | |
| vext.u16 q13, q7, q8, #5 | |
| vmlal.u16 q14, d24, d1[2] | |
| vmlal.u16 q15, d25, d1[2] | |
| vmlal.u16 q14, d26, d1[2] | |
| vmlal.u16 q15, d27, d1[2] | |
| 105: vext.u16 q12, q6, q7, #2 | |
| vext.u16 q13, q7, q8, #4 | |
| vmlal.u16 q14, d24, d1[1] | |
| vmlal.u16 q15, d25, d1[1] | |
| vmlal.u16 q14, d26, d1[1] | |
| vmlal.u16 q15, d27, d1[1] | |
| 104: vext.u16 q12, q6, q7, #3 | |
| vext.u16 q13, q7, q8, #3 | |
| vmlal.u16 q14, d24, d1[0] | |
| vmlal.u16 q15, d25, d1[0] | |
| vmlal.u16 q14, d26, d1[0] | |
| vmlal.u16 q15, d27, d1[0] | |
| 103: vext.u16 q12, q6, q7, #4 | |
| vext.u16 q13, q7, q8, #2 | |
| vmlal.u16 q14, d24, d0[3] | |
| vmlal.u16 q15, d25, d0[3] | |
| vmlal.u16 q14, d26, d0[3] | |
| vmlal.u16 q15, d27, d0[3] | |
| 102: vext.u16 q12, q6, q7, #5 | |
| vext.u16 q13, q7, q8, #1 | |
| vmlal.u16 q14, d24, d0[2] | |
| vmlal.u16 q15, d25, d0[2] | |
| vmlal.u16 q14, d26, d0[2] | |
| vmlal.u16 q15, d27, d0[2] | |
| 101: vext.u16 q12, q6, q7, #6 | |
| vext.u16 q13, q7, q8, #0 | |
| vmlal.u16 q14, d24, d0[1] | |
| vmlal.u16 q15, d25, d0[1] | |
| vmlal.u16 q14, d26, d0[1] | |
| vmlal.u16 q15, d27, d0[1] | |
| vqrshrn.u32 d28, q14, #16 | |
| vqrshrn.u32 d29, q15, #16 | |
| vqrshrn.u16 d31, q14, #FRACTION_BITS | |
| vmov d7, d9 | |
| vmov q4, q5 | |
| vmov q5, q6 | |
| vmov q6, q7 | |
| vmov q7, q8 | |
| vmov q8, q9 | |
| vmov q9, q10 | |
| vmov q10, q11 | |
| .endm/*}}}*/ | |
| #define TUNED_LIST4 6, 12 | |
| .macro hconv4_6/*{{{*/ | |
| vmull.u16 q14, d14, d0[0] | |
| vmull.u16 q15, d15, d0[0] | |
| ldr r12, [pc, r5, LSL #2] | |
| add pc, pc, r12 | |
| bkpt | |
| 100: .word 101f-100b | |
| .word 102f-100b | |
| .word 103f-100b | |
| .word 104f-100b | |
| .word 105f-100b | |
| .word 106f-100b | |
| 106: vmlal.u16 q14, d8, d1[2] | |
| vmlal.u16 q15, d9, d1[2] | |
| vmlal.u16 q14, d20, d1[2] | |
| vmlal.u16 q15, d21, d1[2] | |
| 105: vmlal.u16 q14, d9, d1[1] | |
| vmlal.u16 q15, d10, d1[1] | |
| vmlal.u16 q14, d19, d1[1] | |
| vmlal.u16 q15, d20, d1[1] | |
| 104: vmlal.u16 q14, d10, d1[0] | |
| vmlal.u16 q15, d11, d1[0] | |
| vmlal.u16 q14, d18, d1[0] | |
| vmlal.u16 q15, d19, d1[0] | |
| 103: vmlal.u16 q14, d11, d0[3] | |
| vmlal.u16 q15, d12, d0[3] | |
| vmlal.u16 q14, d17, d0[3] | |
| vmlal.u16 q15, d18, d0[3] | |
| 102: vmlal.u16 q14, d12, d0[2] | |
| vmlal.u16 q15, d13, d0[2] | |
| vmlal.u16 q14, d16, d0[2] | |
| vmlal.u16 q15, d17, d0[2] | |
| 101: vmlal.u16 q14, d13, d0[1] | |
| vmlal.u16 q15, d14, d0[1] | |
| vmlal.u16 q14, d15, d0[1] | |
| vmlal.u16 q15, d16, d0[1] | |
| vqrshrn.u32 d28, q14, #16 | |
| vqrshrn.u32 d29, q15, #16 | |
| vqrshrn.u16 d31, q14, #FRACTION_BITS | |
| vmov q4, q5 | |
| vmov q5, q6 | |
| vmov q6, q7 | |
| vmov q7, q8 | |
| vmov q8, q9 | |
| vmov q9, q10 | |
| vmov q10, q11 | |
| .endm/*}}}*/ | |
| .macro hconv4_12/*{{{*/ | |
| vmull.u16 q14, d8, d0[0] | |
| vmull.u16 q15, d9, d0[0] | |
| ldr r12, [pc, r5, LSL #2] | |
| add pc, pc, r12 | |
| bkpt | |
| 100: .word 101f-100b | |
| .word 102f-100b | |
| .word 103f-100b | |
| .word 104f-100b | |
| .word 105f-100b | |
| .word 106f-100b | |
| .word 107f-100b | |
| .word 108f-100b | |
| .word 109f-100b | |
| .word 110f-100b | |
| .word 111f-100b | |
| .word 112f-100b | |
| 112: add r12, r9, #0x1a0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d3[0] | |
| vmlal.u16 q15, d25, d3[0] | |
| vmlal.u16 q14, d20, d3[0] | |
| vmlal.u16 q15, d21, d3[0] | |
| 111: add r12, r9, #0x1a8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12:64] | |
| vmlal.u16 q14, d24, d2[3] | |
| vmlal.u16 q15, d25, d2[3] | |
| vmlal.u16 q14, d19, d2[3] | |
| vmlal.u16 q15, d20, d2[3] | |
| 110: add r12, r9, #0x1b0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d2[2] | |
| vmlal.u16 q15, d25, d2[2] | |
| vmlal.u16 q14, d18, d2[2] | |
| vmlal.u16 q15, d19, d2[2] | |
| 109: add r12, r9, #0x1b8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12:64] | |
| vmlal.u16 q14, d24, d2[1] | |
| vmlal.u16 q15, d25, d2[1] | |
| vmlal.u16 q14, d17, d2[1] | |
| vmlal.u16 q15, d18, d2[1] | |
| 108: add r12, r9, #0x1c0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d2[0] | |
| vmlal.u16 q15, d25, d2[0] | |
| vmlal.u16 q14, d16, d2[0] | |
| vmlal.u16 q15, d17, d2[0] | |
| 107: add r12, r9, #0x1c8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12:64] | |
| vmlal.u16 q14, d24, d1[3] | |
| vmlal.u16 q15, d25, d1[3] | |
| vmlal.u16 q14, d15, d1[3] | |
| vmlal.u16 q15, d16, d1[3] | |
| 106: add r12, r9, #0x1d0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d1[2] | |
| vmlal.u16 q15, d25, d1[2] | |
| vmlal.u16 q14, d14, d1[2] | |
| vmlal.u16 q15, d15, d1[2] | |
| 105: add r12, r9, #0x1d8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12:64] | |
| vmlal.u16 q14, d24, d1[1] | |
| vmlal.u16 q15, d25, d1[1] | |
| vmlal.u16 q14, d13, d1[1] | |
| vmlal.u16 q15, d14, d1[1] | |
| 104: add r12, r9, #0x1e0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d1[0] | |
| vmlal.u16 q15, d25, d1[0] | |
| vmlal.u16 q14, d12, d1[0] | |
| vmlal.u16 q15, d13, d1[0] | |
| 103: add r12, r9, #0x1e8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12:64] | |
| vmlal.u16 q14, d24, d0[3] | |
| vmlal.u16 q15, d25, d0[3] | |
| vmlal.u16 q14, d11, d0[3] | |
| vmlal.u16 q15, d12, d0[3] | |
| 102: add r12, r9, #0x1f0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d0[2] | |
| vmlal.u16 q15, d25, d0[2] | |
| vmlal.u16 q14, d10, d0[2] | |
| vmlal.u16 q15, d11, d0[2] | |
| 101: add r12, r9, #0x1f8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64] | |
| vmlal.u16 q14, d24, d0[1] | |
| vmlal.u16 q15, d8, d0[1] | |
| vmlal.u16 q14, d9, d0[1] | |
| vmlal.u16 q15, d10, d0[1] | |
| vqrshrn.u32 d28, q14, #16 | |
| vqrshrn.u32 d29, q15, #16 | |
| vqrshrn.u16 d31, q14, #FRACTION_BITS | |
| vst1.u8 {q4}, [r9:128]! | |
| bic r9, r9, #0x200 | |
| vmov q4, q5 | |
| vmov q5, q6 | |
| vmov q6, q7 | |
| vmov q7, q8 | |
| vmov q8, q9 | |
| vmov q9, q10 | |
| vmov q10, q11 | |
| .endm/*}}}*/ | |
| .macro hconv4_25/*{{{*/ | |
| add r12, r9, #0x198 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12:64] | |
| vmull.u16 q14, d24, d0[0] | |
| vmull.u16 q15, d25, d0[0] | |
| ldr r12, [pc, r5, LSL #2] | |
| add pc, pc, r12 | |
| bkpt | |
| 100: .word 101f-100b | |
| .word 102f-100b | |
| .word 103f-100b | |
| .word 104f-100b | |
| .word 105f-100b | |
| .word 106f-100b | |
| .word 107f-100b | |
| .word 108f-100b | |
| .word 109f-100b | |
| .word 110f-100b | |
| .word 111f-100b | |
| .word 112f-100b | |
| .word 113f-100b | |
| .word 114f-100b | |
| .word 115f-100b | |
| .word 116f-100b | |
| .word 117f-100b | |
| .word 118f-100b | |
| .word 119f-100b | |
| .word 120f-100b | |
| .word 121f-100b | |
| .word 122f-100b | |
| .word 123f-100b | |
| .word 124f-100b | |
| .word 125f-100b | |
| 125: add r12, r9, #0x0d0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d6[1] | |
| vmlal.u16 q15, d25, d6[1] | |
| vmlal.u16 q14, d20, d6[1] | |
| vmlal.u16 q15, d21, d6[1] | |
| 124: add r12, r9, #0x0d8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| vmlal.u16 q14, d24, d6[0] | |
| vmlal.u16 q15, d25, d6[0] | |
| vmlal.u16 q14, d19, d6[0] | |
| vmlal.u16 q15, d20, d6[0] | |
| 123: add r12, r9, #0x0e0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d5[3] | |
| vmlal.u16 q15, d25, d5[3] | |
| vmlal.u16 q14, d18, d5[3] | |
| vmlal.u16 q15, d19, d5[3] | |
| 122: add r12, r9, #0x0e8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| vmlal.u16 q14, d24, d5[2] | |
| vmlal.u16 q15, d25, d5[2] | |
| vmlal.u16 q14, d17, d5[2] | |
| vmlal.u16 q15, d18, d5[2] | |
| 121: add r12, r9, #0x0f0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d5[1] | |
| vmlal.u16 q15, d25, d5[1] | |
| vmlal.u16 q14, d16, d5[1] | |
| vmlal.u16 q15, d17, d5[1] | |
| 120: add r12, r9, #0x0f8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| vmlal.u16 q14, d24, d5[0] | |
| vmlal.u16 q15, d25, d5[0] | |
| vmlal.u16 q14, d15, d5[0] | |
| vmlal.u16 q15, d16, d5[0] | |
| 119: add r12, r9, #0x100 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d4[3] | |
| vmlal.u16 q15, d25, d4[3] | |
| vmlal.u16 q14, d14, d4[3] | |
| vmlal.u16 q15, d15, d4[3] | |
| 118: add r12, r9, #0x108 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| vmlal.u16 q14, d24, d4[2] | |
| vmlal.u16 q15, d25, d4[2] | |
| vmlal.u16 q14, d13, d4[2] | |
| vmlal.u16 q15, d14, d4[2] | |
| 117: add r12, r9, #0x110 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d4[1] | |
| vmlal.u16 q15, d25, d4[1] | |
| vmlal.u16 q14, d12, d4[1] | |
| vmlal.u16 q15, d13, d4[1] | |
| 116: add r12, r9, #0x118 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| vmlal.u16 q14, d24, d4[0] | |
| vmlal.u16 q15, d25, d4[0] | |
| vmlal.u16 q14, d11, d4[0] | |
| vmlal.u16 q15, d12, d4[0] | |
| 115: add r12, r9, #0x120 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d3[3] | |
| vmlal.u16 q15, d25, d3[3] | |
| vmlal.u16 q14, d10, d3[3] | |
| vmlal.u16 q15, d11, d3[3] | |
| 114: add r12, r9, #0x128 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| vmlal.u16 q14, d24, d3[2] | |
| vmlal.u16 q15, d25, d3[2] | |
| vmlal.u16 q14, d9, d3[2] | |
| vmlal.u16 q15, d10, d3[2] | |
| 113: add r12, r9, #0x130 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| vmlal.u16 q14, d24, d3[1] | |
| vmlal.u16 q15, d25, d3[1] | |
| vmlal.u16 q14, d8, d3[1] | |
| vmlal.u16 q15, d9, d3[1] | |
| 112: add r12, r9, #0x138 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| add r12, r9, #0x1f8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26}, [r12:64] | |
| vmlal.u16 q14, d24, d3[0] | |
| vmlal.u16 q15, d25, d3[0] | |
| vmlal.u16 q14, d26, d3[0] @ Could be d7, without the load, right? | |
| vmlal.u16 q15, d8, d3[0] | |
| 111: add r12, r9, #0x140 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| add r12, r9, #0x1f0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26,d27}, [r12:128] | |
| vmlal.u16 q14, d24, d2[3] | |
| vmlal.u16 q15, d25, d2[3] | |
| vmlal.u16 q14, d26, d2[3] | |
| vmlal.u16 q15, d27, d2[3] | |
| 110: add r12, r9, #0x148 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| add r12, r9, #0x1e8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d27}, [r12:64] | |
| vmlal.u16 q14, d24, d2[2] | |
| vmlal.u16 q15, d25, d2[2] | |
| vmlal.u16 q14, d26, d2[2] | |
| vmlal.u16 q15, d27, d2[2] | |
| 109: add r12, r9, #0x150 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| add r12, r9, #0x1e0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26,d27}, [r12:128] | |
| vmlal.u16 q14, d24, d2[1] | |
| vmlal.u16 q15, d25, d2[1] | |
| vmlal.u16 q14, d26, d2[1] | |
| vmlal.u16 q15, d27, d2[1] | |
| 108: add r12, r9, #0x158 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| add r12, r9, #0x1d8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d27}, [r12:64] | |
| vmlal.u16 q14, d24, d2[0] | |
| vmlal.u16 q15, d25, d2[0] | |
| vmlal.u16 q14, d26, d2[0] | |
| vmlal.u16 q15, d27, d2[0] | |
| 107: add r12, r9, #0x160 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| add r12, r9, #0x1d0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26,d27}, [r12:128] | |
| vmlal.u16 q14, d24, d1[3] | |
| vmlal.u16 q15, d25, d1[3] | |
| vmlal.u16 q14, d26, d1[3] | |
| vmlal.u16 q15, d27, d1[3] | |
| 106: add r12, r9, #0x168 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| add r12, r9, #0x1c8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d27}, [r12:64] | |
| vmlal.u16 q14, d24, d1[2] | |
| vmlal.u16 q15, d25, d1[2] | |
| vmlal.u16 q14, d26, d1[2] | |
| vmlal.u16 q15, d27, d1[2] | |
| 105: add r12, r9, #0x170 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| add r12, r9, #0x1c0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26,d27}, [r12:128] | |
| vmlal.u16 q14, d24, d1[1] | |
| vmlal.u16 q15, d25, d1[1] | |
| vmlal.u16 q14, d26, d1[1] | |
| vmlal.u16 q15, d27, d1[1] | |
| 104: add r12, r9, #0x178 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| add r12, r9, #0x1b8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d27}, [r12:64] | |
| vmlal.u16 q14, d24, d1[0] | |
| vmlal.u16 q15, d25, d1[0] | |
| vmlal.u16 q14, d26, d1[0] | |
| vmlal.u16 q15, d27, d1[0] | |
| 103: add r12, r9, #0x180 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128] | |
| add r12, r9, #0x1b0 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26,d27}, [r12:128] | |
| vmlal.u16 q14, d24, d0[3] | |
| vmlal.u16 q15, d25, d0[3] | |
| vmlal.u16 q14, d26, d0[3] | |
| vmlal.u16 q15, d27, d0[3] | |
| 102: add r12, r9, #0x188 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d25}, [r12] | |
| add r12, r9, #0x1a8 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26}, [r12:64]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d27}, [r12:64] | |
| vmlal.u16 q14, d24, d0[2] | |
| vmlal.u16 q15, d25, d0[2] | |
| vmlal.u16 q14, d26, d0[2] | |
| vmlal.u16 q15, d27, d0[2] | |
| 101: add r12, r9, #0x190 | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d24,d25}, [r12:128]! | |
| bic r12, r12, #0x200 | |
| vld1.u16 {d26,d27}, [r12:128] | |
| vmlal.u16 q14, d24, d0[1] | |
| vmlal.u16 q15, d25, d0[1] | |
| vmlal.u16 q14, d26, d0[1] | |
| vmlal.u16 q15, d27, d0[1] | |
| vqrshrn.u32 d28, q14, #16 | |
| vqrshrn.u32 d29, q15, #16 | |
| vqrshrn.u16 d31, q14, #FRACTION_BITS | |
| vst1.u8 {q4}, [r9:128]! | |
| bic r9, r9, #0x200 | |
| vmov q4, q5 | |
| vmov q5, q6 | |
| vmov q6, q7 | |
| vmov q7, q8 | |
| vmov q8, q9 | |
| vmov q9, q10 | |
| vmov q10, q11 | |
| .endm/*}}}*/ | |
| /* Dedicated function wrapper for the fetch macro, for the cases where | |
| * performance isn't that important, to keep code size down. | |
| */ | |
| PRIVATE(fetch_generic_asm) | |
| push {r10,r11} | |
| fetch | |
| pop {r10,r11} | |
| bx lr | |
| END(fetch_generic_asm) | |
| /* Given values in q10 and q11, and an index in r11, sweep the (r11&15)th value | |
| * across to fill the rest of the register pair. Used for filling the right | |
| * hand edge of the window when starting too close to the right hand edge of | |
| * the image. | |
| * Also returns a dup-ed copy of the last element in q12 for the tail-fill | |
| * case (this happens incidentally in common path, but must be done | |
| * deliberately in the fast-out path). | |
| */ | |
| PRIVATE(prefetch_clampright1) | |
| ands r12, r11, #15 | |
| beq 1f | |
| sub r12, r12, #1 | |
| sub sp, sp, #64 | |
| vst1.u16 {q10,q11}, [sp] | |
| add r12, sp, r12, LSL #1 | |
| vld1.u16 {d24[]}, [r12] | |
| vld1.u16 {d25[]}, [r12] | |
| vst1.u16 {q12}, [r12]! | |
| vst1.u16 {q12}, [r12] | |
| vld1.u16 {q10,q11}, [sp] | |
| add sp, sp, #64 | |
| bx lr | |
| 1: vdup.u16 q12, d23[3] | |
| bx lr | |
| END(prefetch_clampright1) | |
| PRIVATE(prefetch_clampright4) | |
| ands r12, r11, #15 | |
| beq 1f | |
| sub r12, r12, #4 | |
| sub sp, sp, #64 | |
| vst1.u16 {q10,q11}, [sp] | |
| add r12, sp, r12, LSL #1 | |
| vld1.u64 {d24}, [r12] | |
| vld1.u64 {d25}, [r12] | |
| vst1.u16 {q12}, [r12]! | |
| vst1.u16 {q12}, [r12] | |
| vld1.u16 {q10,q11}, [sp] | |
| add sp, sp, #64 | |
| bx lr | |
| 1: vmov.u16 d24, d23 | |
| vmov.u16 d25, d23 | |
| bx lr | |
| END(prefetch_clampright4) | |
| /* Helpers for prefetch, below. | |
| */ | |
| .macro prefetch_out qa, qb, store, qsa, qsb, qsb_hi | |
| .if \store > 0 | |
| .ifc \qsa,\qsb | |
| vst1.u16 {\qsa}, [r9:128]! | |
| vst1.u16 {\qsb}, [r9:128]! | |
| .else | |
| vst1.u16 {\qsa,\qsb}, [r9:256]! | |
| .endif | |
| .elseif \store == 0 | |
| vmov.u16 \qa, \qsa | |
| vmov.u16 \qb, \qsb | |
| .else | |
| vmov.u16 \qb, \qsb_hi | |
| .endif | |
| .endm | |
| .macro prefetch_one qa, qb, rem, c, store=0, step=1 | |
| .set i, (need - 16) - \rem | |
| .if i >= 0 | |
| 1: cmp r10, #i+16 | |
| blo 2f | |
| prefetch_out \qa, \qb, \store, q9, q9, d19 | |
| b 1f | |
| 2: cmp r11, #i+16 | |
| bls 3f | |
| prefetch_out \qa, \qb, \store, q10, q11, d23 | |
| bl fetch_generic_asm | |
| b 2f | |
| 3: bl prefetch_clampright\step | |
| prefetch_out \qa, \qb, \store, q10, q11, d23 | |
| 4: b 4f+4 | |
| @q12 contains pad word from prefetch_clampright call | |
| prefetch_out \qa, \qb, \store, q12, q12, d25 | |
| .if \rem > 0 | |
| b 4f+4 | |
| .else | |
| 1: | |
| 2: | |
| 3: | |
| 4: nop | |
| .endif | |
| .endif | |
| .endm | |
| /* Fill the convolution window with context data. The aim here is to load | |
| * exactly rlf + rrt columns, and in the main loop to read as many columns as | |
| * will be written. This is complicated by the need to handle cases when the | |
| * input starts very close to the left or right (or both) edges of the image, | |
| * and where these do not fall on 16-byte boundaries. | |
| * | |
| * Input: | |
| * r1 -- src | |
| * r2 -- pitch | |
| * r3 -- count | |
| * r4 -- inlen | |
| * r5 -- r | |
| * r6 -- rup | |
| * r7 -- rdn | |
| * r8 -- rlf | |
| * r9 -- buffer (if needed) | |
| * Output: | |
| * r1 += rlf + min(count, rrt) | |
| * Modifies: | |
| * r10 -- fill start index in the window | |
| * r11 -- fill stop index in the window | |
| * r12 -- scratch | |
| */ | |
| .macro prefetch step=1, max_r=25 | |
| .set need, ((\max_r + \max_r) * \step + 15) & ~15 | |
| .if \step == 1 | |
| rsb r10, r8, #need - (\max_r * \step) | |
| .else | |
| mov r10, r8, LSL #2 | |
| rsb r10, r10, #need - (\max_r * \step) | |
| .endif | |
| add r11, r10, r4 | |
| cmp r11, #need | |
| movhi r11, #need | |
| bl fetch_generic_asm | |
| .if \step == 1 | |
| vdup.u16 q9, d20[0] | |
| .else | |
| vmov.u16 d18, d20 | |
| vmov.u16 d19, d20 | |
| .endif | |
| ands r12, r10, #15 | |
| beq 2f | |
| sub sp, sp, #32 | |
| vst1.u16 {q10,q11}, [sp] | |
| sub r12, sp, r12, LSL #1 | |
| sub sp, sp, #16 | |
| vst1.u16 {q9}, [sp] | |
| sub sp, sp, #16 | |
| vst1.u16 {q9}, [sp] | |
| vld1.u16 {q10,q11}, [r12] | |
| add sp, sp, #64 | |
| sub r1, r1, r10 | |
| bic r10, r10, #15 | |
| add r1, r1, r10 | |
| 2: | |
| .if \step > 1 | |
| /* it's only in the uchar2 and uchar4 cases where the register file | |
| * is insufficient (given MAX_R <= 25). | |
| */ | |
| prefetch_one xx, xx, 192, c=\max_r, step=\step, store=1 | |
| prefetch_one xx, xx, 176, c=\max_r, step=\step, store=1 | |
| prefetch_one xx, xx, 160, c=\max_r, step=\step, store=1 | |
| prefetch_one xx, xx, 144, c=\max_r, step=\step, store=1 | |
| prefetch_one xx, xx, 128, c=\max_r, step=\step, store=1 | |
| prefetch_one xx, xx, 112, c=\max_r, step=\step, store=1 | |
| prefetch_one xx, xx, 96, c=\max_r, step=\step, store=1 | |
| prefetch_one xx, xx, 80, c=\max_r, step=\step, store=1 | |
| prefetch_one xx, xx, 64, c=\max_r, step=\step, store=1 | |
| prefetch_one xx, xx, 48, c=\max_r, step=\step, store=1 | |
| .else | |
| /* q3 normally contains the coefficient table, but it's not fully | |
| * used. In the uchar1, r=25 case the other half of q3 is used for | |
| * the last two window taps to avoid falling out to memory. | |
| */ | |
| prefetch_one xx, d7, 48, c=\max_r, step=\step, store=-1 | |
| .endif | |
| prefetch_one q4, q5, 32, c=\max_r, step=\step, store=0 | |
| prefetch_one q6, q7, 16, c=\max_r, step=\step, store=0 | |
| prefetch_one q8, q9, 0, c=\max_r, step=\step, store=0 | |
| .if \step == 1 | |
| add r10, r8, #\max_r * \step | |
| .else | |
| mov r10, r8, LSL #2 | |
| add r10, r10, #\max_r * \step | |
| .endif | |
| subs r4, r4, r10 | |
| movlo r4, #0 | |
| .endm | |
| /* The main loop. | |
| * | |
| * Input: | |
| * r0 = dst | |
| * r1 = src | |
| * r2 = pitch | |
| * r3 = count | |
| * r4 = inlen | |
| * r5 = r | |
| * r6 = rup | |
| * r7 = rdn | |
| * r9 = buffer | |
| * Modifies | |
| * r8 = fetch code pointer | |
| */ | |
| .macro mainloop core, step=1, max_r=25, labelc="", labelnc="" | |
| ldr r8, 3f | |
| 1: add r8, r8, pc | |
| sub r8, r5, LSL #5 | |
| sub r8, r5, LSL #4 | |
| cmp r5, r6 | |
| cmpeq r5, r7 | |
| beq 5f | |
| /* if (r != rup || r != rdn) then the address-clamping table should | |
| * be used rather than the short-cut version. | |
| */ | |
| ldr r8, 3f+4 | |
| 2: add r8, r8, pc | |
| sub r8, r5, LSL #6 | |
| b 5f | |
| .align 3 | |
| 3: .word \labelnc-1b-8 | |
| .word \labelc-2b-8 | |
| .align 4 | |
| 3: fetch max_r=\max_r, labelc=\labelc, labelnc=\labelnc, reg=r8 | |
| /* For each call to fetch two are made to \core. It would be | |
| * preferable to have twice the work done in \core, but the | |
| * register file is too small for this to be straightforward. | |
| */ | |
| \core | |
| vst1.u8 {d31}, [r0]! | |
| \core | |
| vst1.u8 {d31}, [r0]! | |
| sub r3, r3, #16 | |
| 5: subs r4, r4, #16 | |
| bhs 3b | |
| adds r4, r4, #16 | |
| bne 1f | |
| .if \step==1 | |
| vdup.u16 q10, d19[3] | |
| vdup.u16 q11, d19[3] | |
| .else | |
| vmov.u64 d20, d19 | |
| vmov.u64 d21, d19 | |
| vmov.u64 d22, d19 | |
| vmov.u64 d23, d19 | |
| .endif | |
| b 4f | |
| 1: sub r1, r1, #16 | |
| add r1, r1, r4 | |
| bl fetch_generic_asm | |
| .if \step==1 | |
| vdup.u16 q12, d23[3] | |
| .else | |
| vmov.u64 d24, d23 | |
| vmov.u64 d25, d23 | |
| .endif | |
| rsb r4, r4, #0 | |
| tst r4, #8 | |
| beq 1f | |
| vmov q10, q11 | |
| vmov q11, q12 | |
| 1: tst r4, #4 | |
| beq 1f | |
| vext.u16 q10, q10, q11, #4 | |
| vext.u16 q11, q11, q12, #4 | |
| 1: tst r4, #2 | |
| beq 1f | |
| vext.u16 q10, q10, q11, #2 | |
| vext.u16 q11, q11, q12, #2 | |
| 1: tst r4, #1 | |
| beq 4f | |
| vext.u16 q10, q10, q11, #1 | |
| vext.u16 q11, q11, q12, #1 | |
| 4: cmp r3, #0 | |
| beq 5f | |
| 3: \core | |
| .if \step==1 | |
| vdup.u16 q11, d23[3] | |
| .else | |
| vmov.u64 d22, d23 | |
| .endif | |
| subs r3, r3, #8 | |
| blo 4f | |
| vst1.u8 {d31}, [r0]! | |
| beq 5f | |
| b 3b | |
| 4: tst r3, #4 | |
| beq 1f | |
| vst1.u32 {d31[0]}, [r0]! | |
| vext.u8 d31, d31, d31, #4 | |
| 1: tst r3, #2 | |
| beq 1f | |
| vst1.u16 {d31[0]}, [r0]! | |
| vext.u8 d31, d31, d31, #2 | |
| 1: tst r3, #1 | |
| beq 5f | |
| vst1.u8 {d31[0]}, [r0]! | |
| vext.u8 d31, d31, d31, #1 | |
| 5: nop | |
| .endm | |
| .irp r, TUNED_LIST1, 25 | |
| PRIVATE(convolve1_\r) | |
| push {r12,lr} | |
| sub r1, r1, r8 | |
| prefetch step=1, max_r=\r | |
| mainloop core=hconv1_\r, step=1, max_r=\r, labelc=.Lcnv1_\r, labelnc=.Lcnvnc1_\r | |
| pop {r12,pc} | |
| END(convolve1_\r) | |
| .endr | |
| .irp r, TUNED_LIST4, 25 | |
| PRIVATE(convolve4_\r) | |
| sub r12, sp, #0x200 | |
| bic r9, r12, #0x3fc | |
| mov sp, r9 | |
| push {r12,lr} | |
| /* r9 now points to a buffer on the stack whose address has the low | |
| * 10 bits clear. This allows easy address calculation in the | |
| * wrap-around cases. | |
| */ | |
| sub r1, r1, r8, LSL #2 | |
| prefetch step=4, max_r=\r | |
| mainloop core=hconv4_\r, step=4, max_r=\r, labelc=.Lcnv4_\r, labelnc=.Lcnvnc4_\r | |
| pop {r12,lr} | |
| add sp, r12, #0x200 | |
| bx lr | |
| END(convolve4_\r) | |
| .endr | |
| /* void rsdIntrinsicBlurU1_K( | |
| * void *out, // r0 | |
| * void *in, // r1 | |
| * size_t w, // r2 | |
| * size_t h, // r3 | |
| * size_t p, // [sp] | |
| * size_t x, // [sp,#4] | |
| * size_t y, // [sp,#8] | |
| * size_t count, // [sp,#12] | |
| * size_t r, // [sp,#16] | |
| * uint16_t *tab); // [sp,#20] | |
| */ | |
| ENTRY(rsdIntrinsicBlurU1_K) | |
| push {r4,r5,r6,r7,r8,r9,r10,r11,r12,lr} | |
| vpush {d8-d15} | |
| ldr r5, [sp,#120] | |
| ldr r8, [sp,#108] | |
| ldr r6, [sp,#112] | |
| sub r9, r2, r8 | |
| sub r7, r3, r6 | |
| ldr r2, [sp,#104] | |
| ldr r3, [sp,#116] | |
| sub r9, r9, r3 | |
| sub r7, r7, #1 | |
| ldr r12, [sp,#124] | |
| add r1, r1, r8 | |
| cmp r6, r5 | |
| movhi r6, r5 | |
| cmp r7, r5 | |
| movhi r7, r5 | |
| cmp r8, r5 | |
| movhi r8, r5 | |
| cmp r9, r5 | |
| movhi r9, r5 | |
| add r4, r8, r9 | |
| add r4, r4, r3 | |
| vld1.u16 {d0,d1,d2,d3}, [r12]! | |
| vld1.u16 {d4,d5,d6}, [r12]! | |
| adr lr, 1f | |
| .irp r, TUNED_LIST1 | |
| cmp r5, #\r | |
| bls convolve1_\r | |
| .endr | |
| b convolve1_25 | |
| 1: vpop {d8-d15} | |
| pop {r4,r5,r6,r7,r8,r9,r10,r11,r12,pc} | |
| END(rsdIntrinsicBlurU1_K) | |
| /* void rsdIntrinsicBlurU4_K( | |
| * void *out, // r0 | |
| * void *in, // r1 | |
| * size_t w, // r2 | |
| * size_t h, // r3 | |
| * size_t p, // [sp] | |
| * size_t x, // [sp,#4] | |
| * size_t y, // [sp,#8] | |
| * size_t count, // [sp,#12] | |
| * size_t r, // [sp,#16] | |
| * uint16_t *tab); // [sp,#20] | |
| */ | |
| ENTRY(rsdIntrinsicBlurU4_K) | |
| push {r4,r5,r6,r7,r8,r9,r10,r11,r12,lr} | |
| vpush {d8-d15} | |
| ldr r5, [sp,#120] | |
| ldr r8, [sp,#108] | |
| ldr r6, [sp,#112] | |
| sub r9, r2, r8 | |
| sub r7, r3, r6 | |
| ldr r2, [sp,#104] | |
| ldr r3, [sp,#116] | |
| sub r9, r9, r3 | |
| sub r7, r7, #1 | |
| ldr r12, [sp,#124] | |
| add r1, r1, r8, LSL #2 | |
| cmp r6, r5 | |
| movhi r6, r5 | |
| cmp r7, r5 | |
| movhi r7, r5 | |
| cmp r8, r5 | |
| movhi r8, r5 | |
| cmp r9, r5 | |
| movhi r9, r5 | |
| mov r3, r3, LSL #2 | |
| add r4, r8, r9 | |
| add r4, r3, r4, LSL #2 | |
| vld1.u16 {d0,d1,d2,d3}, [r12]! | |
| vld1.u16 {d4,d5,d6}, [r12]! | |
| adr lr, 1f | |
| .irp r, TUNED_LIST4 | |
| cmp r5, #\r | |
| bls convolve4_\r | |
| .endr | |
| b convolve4_25 | |
| 1: vpop {d8-d15} | |
| pop {r4,r5,r6,r7,r8,r9,r10,r11,r12,pc} | |
| END(rsdIntrinsicBlurU4_K) |