1 // Copyright 2022 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <arm_neon.h>
9
10 #include <xnnpack/common.h>
11 #include <xnnpack/rmax.h>
12
13
xnn_f16_rmax_ukernel__neonfp16arith(size_t batch,const void * input,void * output)14 void xnn_f16_rmax_ukernel__neonfp16arith(
15 size_t batch,
16 const void* input,
17 void* output) XNN_OOB_READS
18 {
19 assert(batch != 0);
20 assert(batch % sizeof(__fp16) == 0);
21
22 const __fp16* i = (const __fp16*) input;
23 __fp16* o = (__fp16*) output;
24 float16x8_t vmax0 = vld1q_dup_f16(i);
25 float16x8_t vmax1 = vmax0;
26 float16x8_t vmax2 = vmax0;
27 float16x8_t vmax3 = vmax0;
28 for (; batch >= 32 * sizeof(__fp16); batch -= 32 * sizeof(__fp16)) {
29 const float16x8_t vx0 = vld1q_f16(i); i += 8;
30 const float16x8_t vx1 = vld1q_f16(i); i += 8;
31 const float16x8_t vx2 = vld1q_f16(i); i += 8;
32 const float16x8_t vx3 = vld1q_f16(i); i += 8;
33
34 vmax0 = vmaxq_f16(vmax0, vx0);
35 vmax1 = vmaxq_f16(vmax1, vx1);
36 vmax2 = vmaxq_f16(vmax2, vx2);
37 vmax3 = vmaxq_f16(vmax3, vx3);
38 }
39 float16x8_t vmax = vmaxq_f16(vmaxq_f16(vmax0, vmax1), vmaxq_f16(vmax2, vmax3));
40 for (; batch >= 8 * sizeof(__fp16); batch -= 8 * sizeof(__fp16)) {
41 const float16x8_t vx = vld1q_f16(i); i += 8;
42 vmax = vmaxq_f16(vmax, vx);
43 }
44 float16x4_t vmax_lo = vmax_f16(vget_low_f16(vmax), vget_high_f16(vmax));
45 if XNN_UNLIKELY(batch != 0) {
46 const float16x8_t vx = vld1q_f16(i);
47 float16x4_t vx_lo = vget_low_f16(vx);
48 if (batch & (4 * sizeof(__fp16))) {
49 vmax_lo = vmax_f16(vmax_lo, vx_lo);
50 vx_lo = vget_high_f16(vx);
51 }
52 if (batch & (2 * sizeof(__fp16))) {
53 vmax_lo = vmax_f16(vmax_lo, vext_f16(vmax_lo, vx_lo, 2));
54 vx_lo = vext_f16(vx_lo, vx_lo, 2);
55 }
56 if (batch & (1 * sizeof(__fp16))) {
57 vmax_lo = vmax_f16(vmax_lo, vext_f16(vmax_lo, vx_lo, 1));
58 }
59 }
60 #if XNN_ARCH_ARM64
61 *o = vmaxv_f16(vmax_lo);
62 #else
63 vmax_lo = vpmax_f16(vmax_lo, vmax_lo);
64 vmax_lo = vpmax_f16(vmax_lo, vmax_lo);
65 vst1_lane_f16(o, vmax_lo, 0);
66 #endif
67 }
68