xref: /aosp_15_r20/external/XNNPACK/src/f16-vrnd/neonfp16arith.c.in (revision 4bdc94577ba0e567308109d787f7fec7b531ce36)
1*4bdc9457SAndroid Build Coastguard Worker// Copyright 2022 Google LLC
2*4bdc9457SAndroid Build Coastguard Worker//
3*4bdc9457SAndroid Build Coastguard Worker// This source code is licensed under the BSD-style license found in the
4*4bdc9457SAndroid Build Coastguard Worker// LICENSE file in the root directory of this source tree.
5*4bdc9457SAndroid Build Coastguard Worker
6*4bdc9457SAndroid Build Coastguard Worker$assert BATCH_TILE % 8 == 0
7*4bdc9457SAndroid Build Coastguard Worker$assert BATCH_TILE >= 8
8*4bdc9457SAndroid Build Coastguard Worker$SIMD_TILE = BATCH_TILE // 8
9*4bdc9457SAndroid Build Coastguard Worker$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
10*4bdc9457SAndroid Build Coastguard Worker$assert OP in ["RNDNE", "RNDZ", "RNDU", "RNDD"]
11*4bdc9457SAndroid Build Coastguard Worker#include <assert.h>
12*4bdc9457SAndroid Build Coastguard Worker
13*4bdc9457SAndroid Build Coastguard Worker#include <arm_neon.h>
14*4bdc9457SAndroid Build Coastguard Worker
15*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/common.h>
16*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/math.h>
17*4bdc9457SAndroid Build Coastguard Worker#include <xnnpack/vunary.h>
18*4bdc9457SAndroid Build Coastguard Worker
19*4bdc9457SAndroid Build Coastguard Worker
20*4bdc9457SAndroid Build Coastguard Worker$VRNDQ_F16 = {
21*4bdc9457SAndroid Build Coastguard Worker$  "RNDNE": "vrndnq_f16",
22*4bdc9457SAndroid Build Coastguard Worker$  "RNDZ": "vrndq_f16",
23*4bdc9457SAndroid Build Coastguard Worker$  "RNDU": "vrndpq_f16",
24*4bdc9457SAndroid Build Coastguard Worker$  "RNDD": "vrndmq_f16",
25*4bdc9457SAndroid Build Coastguard Worker$}[OP]
26*4bdc9457SAndroid Build Coastguard Workervoid xnn_f16_v${OP.lower()}_ukernel__neonfp16arith_x${BATCH_TILE}(
27*4bdc9457SAndroid Build Coastguard Worker    size_t n,
28*4bdc9457SAndroid Build Coastguard Worker    const void* input,
29*4bdc9457SAndroid Build Coastguard Worker    void* output,
30*4bdc9457SAndroid Build Coastguard Worker    const union xnn_f16_rnd_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
31*4bdc9457SAndroid Build Coastguard Worker{
32*4bdc9457SAndroid Build Coastguard Worker  assert(n != 0);
33*4bdc9457SAndroid Build Coastguard Worker  assert(n % sizeof(__fp16) == 0);
34*4bdc9457SAndroid Build Coastguard Worker
35*4bdc9457SAndroid Build Coastguard Worker  const __fp16* i = (const __fp16*) input;
36*4bdc9457SAndroid Build Coastguard Worker  __fp16* o = (__fp16*) output;
37*4bdc9457SAndroid Build Coastguard Worker  $if BATCH_TILE > 8:
38*4bdc9457SAndroid Build Coastguard Worker    for (; n >= ${BATCH_TILE} * sizeof(__fp16); n -= ${BATCH_TILE} * sizeof(__fp16)) {
39*4bdc9457SAndroid Build Coastguard Worker      $for N in range(SIMD_TILE):
40*4bdc9457SAndroid Build Coastguard Worker        float16x8_t vacc${ABC[N]} = vld1q_f16(i); i += 8;
41*4bdc9457SAndroid Build Coastguard Worker
42*4bdc9457SAndroid Build Coastguard Worker      $for N in range(SIMD_TILE):
43*4bdc9457SAndroid Build Coastguard Worker        vacc${ABC[N]} = ${VRNDQ_F16}(vacc${ABC[N]});
44*4bdc9457SAndroid Build Coastguard Worker
45*4bdc9457SAndroid Build Coastguard Worker      $for N in range(SIMD_TILE):
46*4bdc9457SAndroid Build Coastguard Worker        vst1q_f16(o, vacc${ABC[N]}); o += 8;
47*4bdc9457SAndroid Build Coastguard Worker    }
48*4bdc9457SAndroid Build Coastguard Worker  for (; n >= 8 * sizeof(__fp16); n -= 8 * sizeof(__fp16)) {
49*4bdc9457SAndroid Build Coastguard Worker    float16x8_t vacc = vld1q_f16(i); i += 8;
50*4bdc9457SAndroid Build Coastguard Worker    vacc = ${VRNDQ_F16}(vacc);
51*4bdc9457SAndroid Build Coastguard Worker    vst1q_f16(o, vacc); o += 8;
52*4bdc9457SAndroid Build Coastguard Worker  }
53*4bdc9457SAndroid Build Coastguard Worker  if XNN_UNLIKELY(n != 0) {
54*4bdc9457SAndroid Build Coastguard Worker    float16x8_t vacc = vld1q_f16(i);
55*4bdc9457SAndroid Build Coastguard Worker    vacc = ${VRNDQ_F16}(vacc);
56*4bdc9457SAndroid Build Coastguard Worker    float16x4_t vacc_lo = vget_low_f16(vacc);
57*4bdc9457SAndroid Build Coastguard Worker    if (n & (4 * sizeof(__fp16))) {
58*4bdc9457SAndroid Build Coastguard Worker      vst1_f16(o, vacc_lo); o += 4;
59*4bdc9457SAndroid Build Coastguard Worker      vacc_lo = vget_high_f16(vacc);
60*4bdc9457SAndroid Build Coastguard Worker    }
61*4bdc9457SAndroid Build Coastguard Worker    if (n & (2 * sizeof(__fp16))) {
62*4bdc9457SAndroid Build Coastguard Worker      vst1_lane_u32((void*) o, vreinterpret_u32_f16(vacc_lo), 0); o += 2;
63*4bdc9457SAndroid Build Coastguard Worker      vacc_lo = vext_f16(vacc_lo, vacc_lo, 2);
64*4bdc9457SAndroid Build Coastguard Worker    }
65*4bdc9457SAndroid Build Coastguard Worker    if (n & (1 * sizeof(__fp16))) {
66*4bdc9457SAndroid Build Coastguard Worker      vst1_lane_f16(o, vacc_lo, 0);
67*4bdc9457SAndroid Build Coastguard Worker    }
68*4bdc9457SAndroid Build Coastguard Worker  }
69*4bdc9457SAndroid Build Coastguard Worker}
70