xref: /aosp_15_r20/external/XNNPACK/src/f16-dwconv2d-chw/3x3p1-neonfp16arith.c.in (revision 4bdc94577ba0e567308109d787f7fec7b531ce36)
1*4bdc9457SAndroid Build Coastguard Worker// Copyright 2020 Google LLC
2*4bdc9457SAndroid Build Coastguard Worker//
3*4bdc9457SAndroid Build Coastguard Worker// This source code is licensed under the BSD-style license found in the
4*4bdc9457SAndroid Build Coastguard Worker// LICENSE file in the root directory of this source tree.
5*4bdc9457SAndroid Build Coastguard Worker
6*4bdc9457SAndroid Build Coastguard Worker$assert ROW_TILE >= 1
7*4bdc9457SAndroid Build Coastguard Worker$assert ACCUMULATORS >= 1
8*4bdc9457SAndroid Build Coastguard Worker#include <assert.h>
9*4bdc9457SAndroid Build Coastguard Worker
10*4bdc9457SAndroid Build Coastguard Worker#include <arm_neon.h>
11*4bdc9457SAndroid Build Coastguard Worker
12*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/dwconv.h>
13*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/math.h>
14*4bdc9457SAndroid Build Coastguard Worker
15*4bdc9457SAndroid Build Coastguard Worker
16*4bdc9457SAndroid Build Coastguard Workervoid xnn_f16_dwconv2d_chw_ukernel_3x3p1__neonfp16arith_${ROW_TILE}x8${"_acc%d" % ACCUMULATORS if ACCUMULATORS > 1 else ""}(
17*4bdc9457SAndroid Build Coastguard Worker    size_t input_height,
18*4bdc9457SAndroid Build Coastguard Worker    size_t input_width,
19*4bdc9457SAndroid Build Coastguard Worker    const void* input,
20*4bdc9457SAndroid Build Coastguard Worker    const void* weights,
21*4bdc9457SAndroid Build Coastguard Worker    const void* zero,
22*4bdc9457SAndroid Build Coastguard Worker    void* output,
23*4bdc9457SAndroid Build Coastguard Worker    uint32_t padding_top,
24*4bdc9457SAndroid Build Coastguard Worker    const union xnn_f16_chw_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
25*4bdc9457SAndroid Build Coastguard Worker{
26*4bdc9457SAndroid Build Coastguard Worker  assert(input_height != 0);
27*4bdc9457SAndroid Build Coastguard Worker  assert(input_width != 0);
28*4bdc9457SAndroid Build Coastguard Worker  assert(input_width % sizeof(__fp16) == 0);
29*4bdc9457SAndroid Build Coastguard Worker  assert(padding_top == 1);
30*4bdc9457SAndroid Build Coastguard Worker
31*4bdc9457SAndroid Build Coastguard Worker  const uint16x8_t vmask = vld1q_u16(params->neonfp16arith.maskx8);
32*4bdc9457SAndroid Build Coastguard Worker  const float16x8_t vmax = vld1q_dup_f16(&params->neonfp16arith.max);
33*4bdc9457SAndroid Build Coastguard Worker  const float16x8_t vmin = vld1q_dup_f16(&params->neonfp16arith.min);
34*4bdc9457SAndroid Build Coastguard Worker
35*4bdc9457SAndroid Build Coastguard Worker  const __fp16* w0 = (const __fp16*)weights;
36*4bdc9457SAndroid Build Coastguard Worker  const float16x8_t vw01234567 = vld1q_f16(w0);
37*4bdc9457SAndroid Build Coastguard Worker  const float16x4_t vw89 = vreinterpret_f16_u32(vld1_lane_u32((const void*)(w0 + 8), vmov_n_u32(0), 0));
38*4bdc9457SAndroid Build Coastguard Worker
39*4bdc9457SAndroid Build Coastguard Worker  const size_t input_decrement = round_up_po2(input_width, 8 * sizeof(__fp16));
40*4bdc9457SAndroid Build Coastguard Worker
41*4bdc9457SAndroid Build Coastguard Worker  const __fp16* i0 = zero;
42*4bdc9457SAndroid Build Coastguard Worker  const __fp16* i1 = input;
43*4bdc9457SAndroid Build Coastguard Worker  $for M in range(2, 2 + ROW_TILE):
44*4bdc9457SAndroid Build Coastguard Worker    const __fp16* i${M} = (const __fp16*) ((uintptr_t) i${M-1} + input_width);
45*4bdc9457SAndroid Build Coastguard Worker
46*4bdc9457SAndroid Build Coastguard Worker  __fp16* o0 = output;
47*4bdc9457SAndroid Build Coastguard Worker  $for M in range(1, ROW_TILE):
48*4bdc9457SAndroid Build Coastguard Worker    __fp16* o${M} = (__fp16*) ((uintptr_t) o${M-1} + input_width);
49*4bdc9457SAndroid Build Coastguard Worker
50*4bdc9457SAndroid Build Coastguard Worker  size_t output_height = input_height;
51*4bdc9457SAndroid Build Coastguard Worker  do {
52*4bdc9457SAndroid Build Coastguard Worker    $for M in range(2, 2 + ROW_TILE):
53*4bdc9457SAndroid Build Coastguard Worker      if XNN_UNPREDICTABLE(output_height < ${M}) {
54*4bdc9457SAndroid Build Coastguard Worker        i${M} = zero;
55*4bdc9457SAndroid Build Coastguard Worker        $if M <= ROW_TILE:
56*4bdc9457SAndroid Build Coastguard Worker          o${M-1} = o${M-2};
57*4bdc9457SAndroid Build Coastguard Worker      }
58*4bdc9457SAndroid Build Coastguard Worker
59*4bdc9457SAndroid Build Coastguard Worker    $for M in range(2 + ROW_TILE):
60*4bdc9457SAndroid Build Coastguard Worker      float16x8_t vi${M}x01234567 = vmovq_n_f16(0);
61*4bdc9457SAndroid Build Coastguard Worker
62*4bdc9457SAndroid Build Coastguard Worker    $for M in range(2 + ROW_TILE):
63*4bdc9457SAndroid Build Coastguard Worker      float16x8_t vi${M}x89ABCDEF = vld1q_f16(i${M}); i${M} += 8;
64*4bdc9457SAndroid Build Coastguard Worker
65*4bdc9457SAndroid Build Coastguard Worker    size_t w = input_width;
66*4bdc9457SAndroid Build Coastguard Worker    for (; w > 8 * sizeof(__fp16); w -= 8 * sizeof(__fp16)) {
67*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
68*4bdc9457SAndroid Build Coastguard Worker        float16x8_t vo${M}p0 = vdupq_lane_f16(vget_low_f16(vw01234567), 0);
69*4bdc9457SAndroid Build Coastguard Worker
70*4bdc9457SAndroid Build Coastguard Worker      $for M in range(2 + ROW_TILE):
71*4bdc9457SAndroid Build Coastguard Worker        const float16x8_t vi${M}xGHIJKLMN = vld1q_f16(i${M}); i${M} += 8;
72*4bdc9457SAndroid Build Coastguard Worker
73*4bdc9457SAndroid Build Coastguard Worker      // Center column
74*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
75*4bdc9457SAndroid Build Coastguard Worker        vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M}x89ABCDEF, vget_low_f16(vw01234567), 2);
76*4bdc9457SAndroid Build Coastguard Worker
77*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
78*4bdc9457SAndroid Build Coastguard Worker        $if ACCUMULATORS >= 2:
79*4bdc9457SAndroid Build Coastguard Worker          float16x8_t vo${M}p1 = vmulq_lane_f16(vi${M+1}x89ABCDEF, vget_high_f16(vw01234567), 1);
80*4bdc9457SAndroid Build Coastguard Worker        $else:
81*4bdc9457SAndroid Build Coastguard Worker          vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M+1}x89ABCDEF, vget_high_f16(vw01234567), 1);
82*4bdc9457SAndroid Build Coastguard Worker
83*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
84*4bdc9457SAndroid Build Coastguard Worker        $if ACCUMULATORS >= 3:
85*4bdc9457SAndroid Build Coastguard Worker          float16x8_t vo${M}p2 = vmulq_lane_f16(vi${M+2}x89ABCDEF, vw89, 0);
86*4bdc9457SAndroid Build Coastguard Worker        $else:
87*4bdc9457SAndroid Build Coastguard Worker          vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M+2}x89ABCDEF, vw89, 0);
88*4bdc9457SAndroid Build Coastguard Worker
89*4bdc9457SAndroid Build Coastguard Worker      // Left column
90*4bdc9457SAndroid Build Coastguard Worker      $for M in range(2 + ROW_TILE):
91*4bdc9457SAndroid Build Coastguard Worker        const float16x8_t vi${M}x789ABCDE = vextq_f16(vi${M}x01234567, vi${M}x89ABCDEF, 7);
92*4bdc9457SAndroid Build Coastguard Worker
93*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
94*4bdc9457SAndroid Build Coastguard Worker        $if ACCUMULATORS >= 4:
95*4bdc9457SAndroid Build Coastguard Worker          float16x8_t vo${M}p3 = vmulq_lane_f16(vi${M}x789ABCDE, vget_low_f16(vw01234567), 1);
96*4bdc9457SAndroid Build Coastguard Worker        $else:
97*4bdc9457SAndroid Build Coastguard Worker          vo${M}p${3 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${3 % ACCUMULATORS}, vi${M}x789ABCDE, vget_low_f16(vw01234567), 1);
98*4bdc9457SAndroid Build Coastguard Worker
99*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
100*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${4 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${4 % ACCUMULATORS}, vi${M+1}x789ABCDE, vget_high_f16(vw01234567), 0);
101*4bdc9457SAndroid Build Coastguard Worker
102*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
103*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${5 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${5 % ACCUMULATORS}, vi${M+2}x789ABCDE, vget_high_f16(vw01234567), 3);
104*4bdc9457SAndroid Build Coastguard Worker
105*4bdc9457SAndroid Build Coastguard Worker      $for M in range(2 + ROW_TILE):
106*4bdc9457SAndroid Build Coastguard Worker        vi${M}x01234567 = vi${M}x89ABCDEF;
107*4bdc9457SAndroid Build Coastguard Worker
108*4bdc9457SAndroid Build Coastguard Worker      // Right column
109*4bdc9457SAndroid Build Coastguard Worker      $for M in range(2 + ROW_TILE):
110*4bdc9457SAndroid Build Coastguard Worker        const float16x8_t vi${M}x9ABCDEFG = vextq_f16(vi${M}x89ABCDEF, vi${M}xGHIJKLMN, 1);
111*4bdc9457SAndroid Build Coastguard Worker
112*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
113*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${6 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${6 % ACCUMULATORS}, vi${M}x9ABCDEFG, vget_low_f16(vw01234567), 3);
114*4bdc9457SAndroid Build Coastguard Worker
115*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
116*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${7 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${7 % ACCUMULATORS}, vi${M+1}x9ABCDEFG, vget_high_f16(vw01234567), 2);
117*4bdc9457SAndroid Build Coastguard Worker
118*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
119*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${8 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${8 % ACCUMULATORS}, vi${M+2}x9ABCDEFG, vw89, 1);
120*4bdc9457SAndroid Build Coastguard Worker
121*4bdc9457SAndroid Build Coastguard Worker      $for M in range(2 + ROW_TILE):
122*4bdc9457SAndroid Build Coastguard Worker        vi${M}x89ABCDEF = vi${M}xGHIJKLMN;
123*4bdc9457SAndroid Build Coastguard Worker
124*4bdc9457SAndroid Build Coastguard Worker      $if ACCUMULATORS > 1:
125*4bdc9457SAndroid Build Coastguard Worker        $ACC_SLICE = 1
126*4bdc9457SAndroid Build Coastguard Worker        $while ACC_SLICE < ACCUMULATORS:
127*4bdc9457SAndroid Build Coastguard Worker          $for A in range(0, ACCUMULATORS, ACC_SLICE * 2):
128*4bdc9457SAndroid Build Coastguard Worker            $if A + ACC_SLICE < ACCUMULATORS:
129*4bdc9457SAndroid Build Coastguard Worker              $for M in range(ROW_TILE):
130*4bdc9457SAndroid Build Coastguard Worker                vo${M}p${A} = vaddq_f16(vo${M}p${A}, vo${M}p${A + ACC_SLICE});
131*4bdc9457SAndroid Build Coastguard Worker          $ACC_SLICE *= 2
132*4bdc9457SAndroid Build Coastguard Worker
133*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
134*4bdc9457SAndroid Build Coastguard Worker        float16x8_t vo${M} = vmaxq_f16(vo${M}p0, vmin);
135*4bdc9457SAndroid Build Coastguard Worker
136*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
137*4bdc9457SAndroid Build Coastguard Worker        vo${M} = vminq_f16(vo${M}, vmax);
138*4bdc9457SAndroid Build Coastguard Worker
139*4bdc9457SAndroid Build Coastguard Worker      $for M in reversed(range(ROW_TILE)):
140*4bdc9457SAndroid Build Coastguard Worker        vst1q_f16(o${M}, vo${M}); o${M} += 8;
141*4bdc9457SAndroid Build Coastguard Worker    }
142*4bdc9457SAndroid Build Coastguard Worker    // Always process the last block of 1..8 pixels.
143*4bdc9457SAndroid Build Coastguard Worker    assert(w >= 1 * sizeof(__fp16));
144*4bdc9457SAndroid Build Coastguard Worker    assert(w <= 8 * sizeof(__fp16));
145*4bdc9457SAndroid Build Coastguard Worker    {
146*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
147*4bdc9457SAndroid Build Coastguard Worker        float16x8_t vo${M}p0 = vdupq_lane_f16(vget_low_f16(vw01234567), 0);
148*4bdc9457SAndroid Build Coastguard Worker
149*4bdc9457SAndroid Build Coastguard Worker      $for M in range(2 + ROW_TILE):
150*4bdc9457SAndroid Build Coastguard Worker        vi${M}x89ABCDEF = vreinterpretq_f16_u16(vandq_u16(vmask, vreinterpretq_u16_f16(vi${M}x89ABCDEF)));
151*4bdc9457SAndroid Build Coastguard Worker
152*4bdc9457SAndroid Build Coastguard Worker      // Center column
153*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
154*4bdc9457SAndroid Build Coastguard Worker        vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M}x89ABCDEF, vget_low_f16(vw01234567), 2);
155*4bdc9457SAndroid Build Coastguard Worker
156*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
157*4bdc9457SAndroid Build Coastguard Worker        $if ACCUMULATORS >= 2:
158*4bdc9457SAndroid Build Coastguard Worker          float16x8_t vo${M}p1 = vmulq_lane_f16(vi${M+1}x89ABCDEF, vget_high_f16(vw01234567), 1);
159*4bdc9457SAndroid Build Coastguard Worker        $else:
160*4bdc9457SAndroid Build Coastguard Worker          vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M+1}x89ABCDEF, vget_high_f16(vw01234567), 1);
161*4bdc9457SAndroid Build Coastguard Worker
162*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
163*4bdc9457SAndroid Build Coastguard Worker        $if ACCUMULATORS >= 3:
164*4bdc9457SAndroid Build Coastguard Worker          float16x8_t vo${M}p2 = vmulq_lane_f16(vi${M+2}x89ABCDEF, vw89, 0);
165*4bdc9457SAndroid Build Coastguard Worker        $else:
166*4bdc9457SAndroid Build Coastguard Worker          vo${M}p0 = vfmaq_lane_f16(vo${M}p0, vi${M+2}x89ABCDEF, vw89, 0);
167*4bdc9457SAndroid Build Coastguard Worker
168*4bdc9457SAndroid Build Coastguard Worker      // Left column
169*4bdc9457SAndroid Build Coastguard Worker      $for M in range(2 + ROW_TILE):
170*4bdc9457SAndroid Build Coastguard Worker        const float16x8_t vi${M}x789ABCDE = vextq_f16(vi${M}x01234567, vi${M}x89ABCDEF, 7);
171*4bdc9457SAndroid Build Coastguard Worker
172*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
173*4bdc9457SAndroid Build Coastguard Worker        $if ACCUMULATORS >= 4:
174*4bdc9457SAndroid Build Coastguard Worker          float16x8_t vo${M}p3 = vmulq_lane_f16(vi${M}x789ABCDE, vget_low_f16(vw01234567), 1);
175*4bdc9457SAndroid Build Coastguard Worker        $else:
176*4bdc9457SAndroid Build Coastguard Worker          vo${M}p${3 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${3 % ACCUMULATORS}, vi${M}x789ABCDE, vget_low_f16(vw01234567), 1);
177*4bdc9457SAndroid Build Coastguard Worker
178*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
179*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${4 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${4 % ACCUMULATORS}, vi${M+1}x789ABCDE, vget_high_f16(vw01234567), 0);
180*4bdc9457SAndroid Build Coastguard Worker
181*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
182*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${5 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${5 % ACCUMULATORS}, vi${M+2}x789ABCDE, vget_high_f16(vw01234567), 3);
183*4bdc9457SAndroid Build Coastguard Worker
184*4bdc9457SAndroid Build Coastguard Worker      // Right column
185*4bdc9457SAndroid Build Coastguard Worker      const float16x8_t vzero = vmovq_n_f16(0);
186*4bdc9457SAndroid Build Coastguard Worker      $for M in range(2 + ROW_TILE):
187*4bdc9457SAndroid Build Coastguard Worker        const float16x8_t vi${M}x9ABCDEFG = vextq_f16(vi${M}x89ABCDEF, vzero, 1);
188*4bdc9457SAndroid Build Coastguard Worker
189*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
190*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${6 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${6 % ACCUMULATORS}, vi${M}x9ABCDEFG, vget_low_f16(vw01234567), 3);
191*4bdc9457SAndroid Build Coastguard Worker
192*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
193*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${7 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${7 % ACCUMULATORS}, vi${M+1}x9ABCDEFG, vget_high_f16(vw01234567), 2);
194*4bdc9457SAndroid Build Coastguard Worker
195*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
196*4bdc9457SAndroid Build Coastguard Worker        vo${M}p${8 % ACCUMULATORS} = vfmaq_lane_f16(vo${M}p${8 % ACCUMULATORS}, vi${M+2}x9ABCDEFG, vw89, 1);
197*4bdc9457SAndroid Build Coastguard Worker
198*4bdc9457SAndroid Build Coastguard Worker      $if ACCUMULATORS > 1:
199*4bdc9457SAndroid Build Coastguard Worker        $ACC_SLICE = 1
200*4bdc9457SAndroid Build Coastguard Worker        $while ACC_SLICE < ACCUMULATORS:
201*4bdc9457SAndroid Build Coastguard Worker          $for A in range(0, ACCUMULATORS, ACC_SLICE * 2):
202*4bdc9457SAndroid Build Coastguard Worker            $if A + ACC_SLICE < ACCUMULATORS:
203*4bdc9457SAndroid Build Coastguard Worker              $for M in range(ROW_TILE):
204*4bdc9457SAndroid Build Coastguard Worker                vo${M}p${A} = vaddq_f16(vo${M}p${A}, vo${M}p${A + ACC_SLICE});
205*4bdc9457SAndroid Build Coastguard Worker          $ACC_SLICE *= 2
206*4bdc9457SAndroid Build Coastguard Worker
207*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
208*4bdc9457SAndroid Build Coastguard Worker        float16x8_t vo${M} = vmaxq_f16(vo${M}p0, vmin);
209*4bdc9457SAndroid Build Coastguard Worker
210*4bdc9457SAndroid Build Coastguard Worker      $for M in range(ROW_TILE):
211*4bdc9457SAndroid Build Coastguard Worker        vo${M} = vminq_f16(vo${M}, vmax);
212*4bdc9457SAndroid Build Coastguard Worker
213*4bdc9457SAndroid Build Coastguard Worker      if XNN_LIKELY(w == 8 * sizeof(__fp16)) {
214*4bdc9457SAndroid Build Coastguard Worker        $for M in reversed(range(ROW_TILE)):
215*4bdc9457SAndroid Build Coastguard Worker          vst1q_f16(o${M}, vo${M}); o${M} += 8;
216*4bdc9457SAndroid Build Coastguard Worker      } else {
217*4bdc9457SAndroid Build Coastguard Worker        $for M in reversed(range(ROW_TILE)):
218*4bdc9457SAndroid Build Coastguard Worker          float16x4_t vo${M}_lo = vget_low_f16(vo${M});
219*4bdc9457SAndroid Build Coastguard Worker
220*4bdc9457SAndroid Build Coastguard Worker        if (w & (4 * sizeof(__fp16))) {
221*4bdc9457SAndroid Build Coastguard Worker         $for M in reversed(range(ROW_TILE)):
222*4bdc9457SAndroid Build Coastguard Worker            vst1_f16(o${M}, vo${M}_lo); o${M} += 4;
223*4bdc9457SAndroid Build Coastguard Worker
224*4bdc9457SAndroid Build Coastguard Worker          $for M in reversed(range(ROW_TILE)):
225*4bdc9457SAndroid Build Coastguard Worker            vo${M}_lo = vget_high_f16(vo${M});
226*4bdc9457SAndroid Build Coastguard Worker        }
227*4bdc9457SAndroid Build Coastguard Worker        if (w & (2 * sizeof(__fp16))) {
228*4bdc9457SAndroid Build Coastguard Worker          $for M in reversed(range(ROW_TILE)):
229*4bdc9457SAndroid Build Coastguard Worker            vst1_lane_u32((void*) o${M}, vreinterpret_u32_f16(vo${M}_lo), 0); o${M} += 2;
230*4bdc9457SAndroid Build Coastguard Worker
231*4bdc9457SAndroid Build Coastguard Worker          $for M in range(ROW_TILE):
232*4bdc9457SAndroid Build Coastguard Worker            vo${M}_lo = vext_f16(vo${M}_lo, vo${M}_lo, 2);
233*4bdc9457SAndroid Build Coastguard Worker        }
234*4bdc9457SAndroid Build Coastguard Worker        if (w & (1 * sizeof(__fp16))) {
235*4bdc9457SAndroid Build Coastguard Worker          $for M in reversed(range(ROW_TILE)):
236*4bdc9457SAndroid Build Coastguard Worker            vst1_lane_f16(o${M}, vo${M}_lo, 0); o${M} += 1;
237*4bdc9457SAndroid Build Coastguard Worker        }
238*4bdc9457SAndroid Build Coastguard Worker      }
239*4bdc9457SAndroid Build Coastguard Worker    }
240*4bdc9457SAndroid Build Coastguard Worker
241*4bdc9457SAndroid Build Coastguard Worker    i0 = (const __fp16*) ((uintptr_t) i${ROW_TILE} - input_decrement);
242*4bdc9457SAndroid Build Coastguard Worker    i1 = (const __fp16*) ((uintptr_t) i${ROW_TILE+1} - input_decrement);
243*4bdc9457SAndroid Build Coastguard Worker    $for M in range(2, 2 + ROW_TILE):
244*4bdc9457SAndroid Build Coastguard Worker      i${M} = (const __fp16*) ((uintptr_t) i${M-1} + input_width);
245*4bdc9457SAndroid Build Coastguard Worker
246*4bdc9457SAndroid Build Coastguard Worker    $if ROW_TILE > 1:
247*4bdc9457SAndroid Build Coastguard Worker      o0 = o${ROW_TILE - 1};
248*4bdc9457SAndroid Build Coastguard Worker      $for M in range(1, ROW_TILE):
249*4bdc9457SAndroid Build Coastguard Worker        o${M} = (__fp16*) ((uintptr_t) o${M-1} + input_width);
250*4bdc9457SAndroid Build Coastguard Worker
251*4bdc9457SAndroid Build Coastguard Worker    $if ROW_TILE > 1:
252*4bdc9457SAndroid Build Coastguard Worker      output_height = doz(output_height, ${ROW_TILE});
253*4bdc9457SAndroid Build Coastguard Worker  } while (${"--" if ROW_TILE == 1 else ""}output_height != 0);
254*4bdc9457SAndroid Build Coastguard Worker}
255