1*4bdc9457SAndroid Build Coastguard Worker// Copyright 2020 Google LLC 2*4bdc9457SAndroid Build Coastguard Worker// 3*4bdc9457SAndroid Build Coastguard Worker// This source code is licensed under the BSD-style license found in the 4*4bdc9457SAndroid Build Coastguard Worker// LICENSE file in the root directory of this source tree. 5*4bdc9457SAndroid Build Coastguard Worker 6*4bdc9457SAndroid Build Coastguard Worker$assert ROW_TILE >= 1 7*4bdc9457SAndroid Build Coastguard Worker$assert ACCUMULATORS >= 1 8*4bdc9457SAndroid Build Coastguard Worker#include <assert.h> 9*4bdc9457SAndroid Build Coastguard Worker 10*4bdc9457SAndroid Build Coastguard Worker#include <arm_neon.h> 11*4bdc9457SAndroid Build Coastguard Worker 12*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/dwconv.h> 13*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/math.h> 14*4bdc9457SAndroid Build Coastguard Worker 15*4bdc9457SAndroid Build Coastguard Worker 16*4bdc9457SAndroid Build Coastguard Workervoid xnn_f16_dwconv2d_chw_ukernel_3x3p1__neonfp16arith_${ROW_TILE}x8${"_acc%d" % ACCUMULATORS if ACCUMULATORS > 1 else ""}( 17*4bdc9457SAndroid Build Coastguard Worker size_t input_height, 18*4bdc9457SAndroid Build Coastguard Worker size_t input_width, 19*4bdc9457SAndroid Build Coastguard Worker const void* input, 20*4bdc9457SAndroid Build Coastguard Worker const void* weights, 21*4bdc9457SAndroid Build Coastguard Worker const void* zero, 22*4bdc9457SAndroid Build Coastguard Worker void* output, 23*4bdc9457SAndroid Build Coastguard Worker uint32_t padding_top, 24*4bdc9457SAndroid Build Coastguard Worker const union xnn_f16_chw_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 25*4bdc9457SAndroid Build Coastguard Worker{ 26*4bdc9457SAndroid Build Coastguard Worker assert(input_height != 0); 27*4bdc9457SAndroid Build Coastguard Worker assert(input_width != 0); 28*4bdc9457SAndroid Build Coastguard Worker assert(input_width % sizeof(__fp16) == 0); 29*4bdc9457SAndroid Build Coastguard Worker assert(padding_top == 1); 30*4bdc9457SAndroid Build Coastguard Worker 31*4bdc9457SAndroid Build Coastguard Worker const uint16x8_t vmask = vld1q_u16(params->neonfp16arith.maskx8); 32*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vmax = vld1q_dup_f16(¶ms->neonfp16arith.max); 33*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vmin = vld1q_dup_f16(¶ms->neonfp16arith.min); 34*4bdc9457SAndroid Build Coastguard Worker 35*4bdc9457SAndroid Build Coastguard Worker const __fp16* w0 = (const __fp16*)weights; 36*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vw01234567 = vld1q_f16(w0); 37*4bdc9457SAndroid Build Coastguard Worker const float16x4_t vw89 = vreinterpret_f16_u32(vld1_lane_u32((const void*)(w0 + 8), vmov_n_u32(0), 0)); 38*4bdc9457SAndroid Build Coastguard Worker 39*4bdc9457SAndroid Build Coastguard Worker const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(__fp16)); 40*4bdc9457SAndroid Build Coastguard Worker 41*4bdc9457SAndroid Build Coastguard Worker const __fp16* i0 = zero; 42*4bdc9457SAndroid Build Coastguard Worker const __fp16* i1 = input; 43*4bdc9457SAndroid Build Coastguard Worker $for M in range(2, 2 + ROW_TILE): 44*4bdc9457SAndroid Build Coastguard Worker const __fp16* i${M} = (const __fp16*) ((uintptr_t) i${M-1} + input_width); 45*4bdc9457SAndroid Build Coastguard Worker 46*4bdc9457SAndroid Build Coastguard Worker __fp16* o0 = output; 47*4bdc9457SAndroid Build Coastguard Worker $for M in range(1, ROW_TILE): 48*4bdc9457SAndroid Build Coastguard Worker __fp16* o${M} = (__fp16*) ((uintptr_t) o${M-1} + input_width); 49*4bdc9457SAndroid Build Coastguard Worker 50*4bdc9457SAndroid Build Coastguard Worker size_t output_height = input_height; 51*4bdc9457SAndroid Build Coastguard Worker do { 52*4bdc9457SAndroid Build Coastguard Worker $for M in range(2, 2 + ROW_TILE): 53*4bdc9457SAndroid Build Coastguard Worker if XNN_UNPREDICTABLE(output_height < ${M}) { 54*4bdc9457SAndroid Build Coastguard Worker i${M} = zero; 55*4bdc9457SAndroid Build Coastguard Worker $if M <= ROW_TILE: 56*4bdc9457SAndroid Build Coastguard Worker o${M-1} = o${M-2}; 57*4bdc9457SAndroid Build Coastguard Worker } 58*4bdc9457SAndroid Build Coastguard Worker 59*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 60*4bdc9457SAndroid Build Coastguard Worker float16x8_t vi${M}x01234567 = vmovq_n_f16(0); 61*4bdc9457SAndroid Build Coastguard Worker 62*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 63*4bdc9457SAndroid Build Coastguard Worker float16x8_t vi${M}x89ABCDEF = vld1q_f16(i${M}); i${M} += 8; 64*4bdc9457SAndroid Build Coastguard Worker 65*4bdc9457SAndroid Build Coastguard Worker size_t w = input_width; 66*4bdc9457SAndroid Build Coastguard Worker for (; w > 8 * sizeof(__fp16); w -= 8 * sizeof(__fp16)) { 67*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 68*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M}p0 = vdupq_lane_f16(vget_low_f16(vw01234567), 0); 69*4bdc9457SAndroid Build Coastguard Worker 70*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 71*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vi${M}xGHIJKLMN = vld1q_f16(i${M}); i${M} += 8; 72*4bdc9457SAndroid Build Coastguard Worker 73*4bdc9457SAndroid Build Coastguard Worker // Center column 74*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 75*4bdc9457SAndroid Build Coastguard Worker vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M}x89ABCDEF, vget_low_f16(vw01234567), 2); 76*4bdc9457SAndroid Build Coastguard Worker 77*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 78*4bdc9457SAndroid Build Coastguard Worker $if ACCUMULATORS >= 2: 79*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M}p1 = vmulq_lane_f16(vi${M+1}x89ABCDEF, vget_high_f16(vw01234567), 1); 80*4bdc9457SAndroid Build Coastguard Worker $else: 81*4bdc9457SAndroid Build Coastguard Worker vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M+1}x89ABCDEF, vget_high_f16(vw01234567), 1); 82*4bdc9457SAndroid Build Coastguard Worker 83*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 84*4bdc9457SAndroid Build Coastguard Worker $if ACCUMULATORS >= 3: 85*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M}p2 = vmulq_lane_f16(vi${M+2}x89ABCDEF, vw89, 0); 86*4bdc9457SAndroid Build Coastguard Worker $else: 87*4bdc9457SAndroid Build Coastguard Worker vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M+2}x89ABCDEF, vw89, 0); 88*4bdc9457SAndroid Build Coastguard Worker 89*4bdc9457SAndroid Build Coastguard Worker // Left column 90*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 91*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vi${M}x789ABCDE = vextq_f16(vi${M}x01234567, vi${M}x89ABCDEF, 7); 92*4bdc9457SAndroid Build Coastguard Worker 93*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 94*4bdc9457SAndroid Build Coastguard Worker $if ACCUMULATORS >= 4: 95*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M}p3 = vmulq_lane_f16(vi${M}x789ABCDE, vget_low_f16(vw01234567), 1); 96*4bdc9457SAndroid Build Coastguard Worker $else: 97*4bdc9457SAndroid Build Coastguard Worker vo${M}p${3 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${3 % ACCUMULATORS}, vi${M}x789ABCDE, vget_low_f16(vw01234567), 1); 98*4bdc9457SAndroid Build Coastguard Worker 99*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 100*4bdc9457SAndroid Build Coastguard Worker vo${M}p${4 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${4 % ACCUMULATORS}, vi${M+1}x789ABCDE, vget_high_f16(vw01234567), 0); 101*4bdc9457SAndroid Build Coastguard Worker 102*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 103*4bdc9457SAndroid Build Coastguard Worker vo${M}p${5 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${5 % ACCUMULATORS}, vi${M+2}x789ABCDE, vget_high_f16(vw01234567), 3); 104*4bdc9457SAndroid Build Coastguard Worker 105*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 106*4bdc9457SAndroid Build Coastguard Worker vi${M}x01234567 = vi${M}x89ABCDEF; 107*4bdc9457SAndroid Build Coastguard Worker 108*4bdc9457SAndroid Build Coastguard Worker // Right column 109*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 110*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vi${M}x9ABCDEFG = vextq_f16(vi${M}x89ABCDEF, vi${M}xGHIJKLMN, 1); 111*4bdc9457SAndroid Build Coastguard Worker 112*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 113*4bdc9457SAndroid Build Coastguard Worker vo${M}p${6 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${6 % ACCUMULATORS}, vi${M}x9ABCDEFG, vget_low_f16(vw01234567), 3); 114*4bdc9457SAndroid Build Coastguard Worker 115*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 116*4bdc9457SAndroid Build Coastguard Worker vo${M}p${7 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${7 % ACCUMULATORS}, vi${M+1}x9ABCDEFG, vget_high_f16(vw01234567), 2); 117*4bdc9457SAndroid Build Coastguard Worker 118*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 119*4bdc9457SAndroid Build Coastguard Worker vo${M}p${8 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${8 % ACCUMULATORS}, vi${M+2}x9ABCDEFG, vw89, 1); 120*4bdc9457SAndroid Build Coastguard Worker 121*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 122*4bdc9457SAndroid Build Coastguard Worker vi${M}x89ABCDEF = vi${M}xGHIJKLMN; 123*4bdc9457SAndroid Build Coastguard Worker 124*4bdc9457SAndroid Build Coastguard Worker $if ACCUMULATORS > 1: 125*4bdc9457SAndroid Build Coastguard Worker $ACC_SLICE = 1 126*4bdc9457SAndroid Build Coastguard Worker $while ACC_SLICE < ACCUMULATORS: 127*4bdc9457SAndroid Build Coastguard Worker $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): 128*4bdc9457SAndroid Build Coastguard Worker $if A + ACC_SLICE < ACCUMULATORS: 129*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 130*4bdc9457SAndroid Build Coastguard Worker vo${M}p${A} = vaddq_f16(vo${M}p${A}, vo${M}p${A + ACC_SLICE}); 131*4bdc9457SAndroid Build Coastguard Worker $ACC_SLICE *= 2 132*4bdc9457SAndroid Build Coastguard Worker 133*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 134*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M} = vmaxq_f16(vo${M}p0, vmin); 135*4bdc9457SAndroid Build Coastguard Worker 136*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 137*4bdc9457SAndroid Build Coastguard Worker vo${M} = vminq_f16(vo${M}, vmax); 138*4bdc9457SAndroid Build Coastguard Worker 139*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(ROW_TILE)): 140*4bdc9457SAndroid Build Coastguard Worker vst1q_f16(o${M}, vo${M}); o${M} += 8; 141*4bdc9457SAndroid Build Coastguard Worker } 142*4bdc9457SAndroid Build Coastguard Worker // Always process the last block of 1..8 pixels. 143*4bdc9457SAndroid Build Coastguard Worker assert(w >= 1 * sizeof(__fp16)); 144*4bdc9457SAndroid Build Coastguard Worker assert(w <= 8 * sizeof(__fp16)); 145*4bdc9457SAndroid Build Coastguard Worker { 146*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 147*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M}p0 = vdupq_lane_f16(vget_low_f16(vw01234567), 0); 148*4bdc9457SAndroid Build Coastguard Worker 149*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 150*4bdc9457SAndroid Build Coastguard Worker vi${M}x89ABCDEF = vreinterpretq_f16_u16(vandq_u16(vmask, vreinterpretq_u16_f16(vi${M}x89ABCDEF))); 151*4bdc9457SAndroid Build Coastguard Worker 152*4bdc9457SAndroid Build Coastguard Worker // Center column 153*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 154*4bdc9457SAndroid Build Coastguard Worker vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M}x89ABCDEF, vget_low_f16(vw01234567), 2); 155*4bdc9457SAndroid Build Coastguard Worker 156*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 157*4bdc9457SAndroid Build Coastguard Worker $if ACCUMULATORS >= 2: 158*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M}p1 = vmulq_lane_f16(vi${M+1}x89ABCDEF, vget_high_f16(vw01234567), 1); 159*4bdc9457SAndroid Build Coastguard Worker $else: 160*4bdc9457SAndroid Build Coastguard Worker vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M+1}x89ABCDEF, vget_high_f16(vw01234567), 1); 161*4bdc9457SAndroid Build Coastguard Worker 162*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 163*4bdc9457SAndroid Build Coastguard Worker $if ACCUMULATORS >= 3: 164*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M}p2 = vmulq_lane_f16(vi${M+2}x89ABCDEF, vw89, 0); 165*4bdc9457SAndroid Build Coastguard Worker $else: 166*4bdc9457SAndroid Build Coastguard Worker vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M+2}x89ABCDEF, vw89, 0); 167*4bdc9457SAndroid Build Coastguard Worker 168*4bdc9457SAndroid Build Coastguard Worker // Left column 169*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 170*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vi${M}x789ABCDE = vextq_f16(vi${M}x01234567, vi${M}x89ABCDEF, 7); 171*4bdc9457SAndroid Build Coastguard Worker 172*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 173*4bdc9457SAndroid Build Coastguard Worker $if ACCUMULATORS >= 4: 174*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M}p3 = vmulq_lane_f16(vi${M}x789ABCDE, vget_low_f16(vw01234567), 1); 175*4bdc9457SAndroid Build Coastguard Worker $else: 176*4bdc9457SAndroid Build Coastguard Worker vo${M}p${3 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${3 % ACCUMULATORS}, vi${M}x789ABCDE, vget_low_f16(vw01234567), 1); 177*4bdc9457SAndroid Build Coastguard Worker 178*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 179*4bdc9457SAndroid Build Coastguard Worker vo${M}p${4 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${4 % ACCUMULATORS}, vi${M+1}x789ABCDE, vget_high_f16(vw01234567), 0); 180*4bdc9457SAndroid Build Coastguard Worker 181*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 182*4bdc9457SAndroid Build Coastguard Worker vo${M}p${5 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${5 % ACCUMULATORS}, vi${M+2}x789ABCDE, vget_high_f16(vw01234567), 3); 183*4bdc9457SAndroid Build Coastguard Worker 184*4bdc9457SAndroid Build Coastguard Worker // Right column 185*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vzero = vmovq_n_f16(0); 186*4bdc9457SAndroid Build Coastguard Worker $for M in range(2 + ROW_TILE): 187*4bdc9457SAndroid Build Coastguard Worker const float16x8_t vi${M}x9ABCDEFG = vextq_f16(vi${M}x89ABCDEF, vzero, 1); 188*4bdc9457SAndroid Build Coastguard Worker 189*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 190*4bdc9457SAndroid Build Coastguard Worker vo${M}p${6 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${6 % ACCUMULATORS}, vi${M}x9ABCDEFG, vget_low_f16(vw01234567), 3); 191*4bdc9457SAndroid Build Coastguard Worker 192*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 193*4bdc9457SAndroid Build Coastguard Worker vo${M}p${7 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${7 % ACCUMULATORS}, vi${M+1}x9ABCDEFG, vget_high_f16(vw01234567), 2); 194*4bdc9457SAndroid Build Coastguard Worker 195*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 196*4bdc9457SAndroid Build Coastguard Worker vo${M}p${8 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${8 % ACCUMULATORS}, vi${M+2}x9ABCDEFG, vw89, 1); 197*4bdc9457SAndroid Build Coastguard Worker 198*4bdc9457SAndroid Build Coastguard Worker $if ACCUMULATORS > 1: 199*4bdc9457SAndroid Build Coastguard Worker $ACC_SLICE = 1 200*4bdc9457SAndroid Build Coastguard Worker $while ACC_SLICE < ACCUMULATORS: 201*4bdc9457SAndroid Build Coastguard Worker $for A in range(0, ACCUMULATORS, ACC_SLICE * 2): 202*4bdc9457SAndroid Build Coastguard Worker $if A + ACC_SLICE < ACCUMULATORS: 203*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 204*4bdc9457SAndroid Build Coastguard Worker vo${M}p${A} = vaddq_f16(vo${M}p${A}, vo${M}p${A + ACC_SLICE}); 205*4bdc9457SAndroid Build Coastguard Worker $ACC_SLICE *= 2 206*4bdc9457SAndroid Build Coastguard Worker 207*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 208*4bdc9457SAndroid Build Coastguard Worker float16x8_t vo${M} = vmaxq_f16(vo${M}p0, vmin); 209*4bdc9457SAndroid Build Coastguard Worker 210*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 211*4bdc9457SAndroid Build Coastguard Worker vo${M} = vminq_f16(vo${M}, vmax); 212*4bdc9457SAndroid Build Coastguard Worker 213*4bdc9457SAndroid Build Coastguard Worker if XNN_LIKELY(w == 8 * sizeof(__fp16)) { 214*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(ROW_TILE)): 215*4bdc9457SAndroid Build Coastguard Worker vst1q_f16(o${M}, vo${M}); o${M} += 8; 216*4bdc9457SAndroid Build Coastguard Worker } else { 217*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(ROW_TILE)): 218*4bdc9457SAndroid Build Coastguard Worker float16x4_t vo${M}_lo = vget_low_f16(vo${M}); 219*4bdc9457SAndroid Build Coastguard Worker 220*4bdc9457SAndroid Build Coastguard Worker if (w & (4 * sizeof(__fp16))) { 221*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(ROW_TILE)): 222*4bdc9457SAndroid Build Coastguard Worker vst1_f16(o${M}, vo${M}_lo); o${M} += 4; 223*4bdc9457SAndroid Build Coastguard Worker 224*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(ROW_TILE)): 225*4bdc9457SAndroid Build Coastguard Worker vo${M}_lo = vget_high_f16(vo${M}); 226*4bdc9457SAndroid Build Coastguard Worker } 227*4bdc9457SAndroid Build Coastguard Worker if (w & (2 * sizeof(__fp16))) { 228*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(ROW_TILE)): 229*4bdc9457SAndroid Build Coastguard Worker vst1_lane_u32((void*) o${M}, vreinterpret_u32_f16(vo${M}_lo), 0); o${M} += 2; 230*4bdc9457SAndroid Build Coastguard Worker 231*4bdc9457SAndroid Build Coastguard Worker $for M in range(ROW_TILE): 232*4bdc9457SAndroid Build Coastguard Worker vo${M}_lo = vext_f16(vo${M}_lo, vo${M}_lo, 2); 233*4bdc9457SAndroid Build Coastguard Worker } 234*4bdc9457SAndroid Build Coastguard Worker if (w & (1 * sizeof(__fp16))) { 235*4bdc9457SAndroid Build Coastguard Worker $for M in reversed(range(ROW_TILE)): 236*4bdc9457SAndroid Build Coastguard Worker vst1_lane_f16(o${M}, vo${M}_lo, 0); o${M} += 1; 237*4bdc9457SAndroid Build Coastguard Worker } 238*4bdc9457SAndroid Build Coastguard Worker } 239*4bdc9457SAndroid Build Coastguard Worker } 240*4bdc9457SAndroid Build Coastguard Worker 241*4bdc9457SAndroid Build Coastguard Worker i0 = (const __fp16*) ((uintptr_t) i${ROW_TILE} - input_decrement); 242*4bdc9457SAndroid Build Coastguard Worker i1 = (const __fp16*) ((uintptr_t) i${ROW_TILE+1} - input_decrement); 243*4bdc9457SAndroid Build Coastguard Worker $for M in range(2, 2 + ROW_TILE): 244*4bdc9457SAndroid Build Coastguard Worker i${M} = (const __fp16*) ((uintptr_t) i${M-1} + input_width); 245*4bdc9457SAndroid Build Coastguard Worker 246*4bdc9457SAndroid Build Coastguard Worker $if ROW_TILE > 1: 247*4bdc9457SAndroid Build Coastguard Worker o0 = o${ROW_TILE - 1}; 248*4bdc9457SAndroid Build Coastguard Worker $for M in range(1, ROW_TILE): 249*4bdc9457SAndroid Build Coastguard Worker o${M} = (__fp16*) ((uintptr_t) o${M-1} + input_width); 250*4bdc9457SAndroid Build Coastguard Worker 251*4bdc9457SAndroid Build Coastguard Worker $if ROW_TILE > 1: 252*4bdc9457SAndroid Build Coastguard Worker output_height = doz(output_height, ${ROW_TILE}); 253*4bdc9457SAndroid Build Coastguard Worker } while (${"--" if ROW_TILE == 1 else ""}output_height != 0); 254*4bdc9457SAndroid Build Coastguard Worker} 255