1// Copyright 2022 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE % 8 == 0 7$assert BATCH_TILE >= 8 8$SIMD_TILE = BATCH_TILE // 8 9#include <assert.h> 10#include <stddef.h> 11#include <stdint.h> 12 13#include <arm_neon.h> 14 15#include <xnnpack/math.h> 16#include <xnnpack/rmaxabs.h> 17 18 19void xnn_s16_rmaxabs_ukernel__neon_x${BATCH_TILE}( 20 size_t batch, 21 const int16_t* input, 22 uint16_t* output) { 23 24 assert(batch > 0); 25 assert(input != NULL); 26 assert(output != NULL); 27 28 const uint16x8_t vzero = vdupq_n_u16(0); 29 $for N in range(SIMD_TILE): 30 uint16x8_t vmax${N} = vzero; 31 32 $if BATCH_TILE > 8: 33 for (; batch >= ${BATCH_TILE}; batch -= ${BATCH_TILE}) { 34 $for N in range(SIMD_TILE): 35 const int16x8_t vi${N} = vld1q_s16(input); input += 8; 36 37 $for N in range(SIMD_TILE): 38 const uint16x8_t vabs${N} = vreinterpretq_u16_s16(vabsq_s16(vi${N})); 39 40 $for N in range(SIMD_TILE): 41 vmax${N} = vmaxq_u16(vmax${N}, vabs${N}); 42 } 43 44 $SIMD_SLICE = 1 45 $while SIMD_SLICE < SIMD_TILE: 46 $for S in range(0, SIMD_TILE, SIMD_SLICE * 2): 47 $if S + SIMD_SLICE < SIMD_TILE: 48 vmax${S} = vmaxq_u16(vmax${S}, vmax${S + SIMD_SLICE}); 49 $SIMD_SLICE *= 2 50 51 // Remainder of full vectors 52 for (; batch >= 8; batch -= 8) { 53 const int16x8_t vi = vld1q_s16(input); input += 8; 54 const uint16x8_t vabs = vreinterpretq_u16_s16(vabsq_s16(vi)); 55 vmax0 = vmaxq_u16(vmax0, vabs); 56 } 57 58 // Remainder 59 if (batch != 0) { 60 do { 61 const int16x8_t vi = vld1q_dup_s16(input); input += 1; 62 const uint16x8_t vabs = vreinterpretq_u16_s16(vabsq_s16(vi)); 63 vmax0 = vmaxq_u16(vmax0, vabs); 64 } while (--batch != 0); 65 } 66 67 #if XNN_ARCH_ARM64 68 *output = vmaxvq_u16(vmax0); 69 #else 70 uint16x4_t vmax_lo = vmax_u16(vget_low_u16(vmax0), vget_high_u16(vmax0)); 71 vmax_lo = vpmax_u16(vmax_lo, vmax_lo); 72 vmax_lo = vpmax_u16(vmax_lo, vmax_lo); 73 vst1_lane_u16(output, vmax_lo, 0); 74 #endif 75} 76