1// Copyright 2021 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE % 8 == 0 7$assert BATCH_TILE >= 8 8$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 9#include <assert.h> 10 11#include <arm_neon.h> 12 13#include <xnnpack/common.h> 14#include <xnnpack/intrinsics-polyfill.h> 15#include <xnnpack/vcvt.h> 16 17 18$XINT8_T = {"QS8": "int8_t", "QU8": "uint8_t"}[DATATYPE] 19$XINT8X8_T = {"QS8": "int8x8_t", "QU8": "uint8x8_t"}[DATATYPE] 20$VLD1_X8 = {"QS8": "vld1_s8", "QU8": "vld1_u8"}[DATATYPE] 21$VLD1_X16 = {"QS8": "vld1_s16", "QU8": "vld1_u16"}[DATATYPE] 22void xnn_${DATATYPE.lower()}_f32_vcvt_ukernel__neon_x${BATCH_TILE}( 23 size_t n, 24 const ${XINT8_T}* x, 25 float* y, 26 const union xnn_${DATATYPE.lower()}_f32_cvt_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 27{ 28 assert(n != 0); 29 assert(n % sizeof(${XINT8_T}) == 0); 30 assert(x != NULL); 31 assert(y != NULL); 32 33 const int16x8_t vminus_zero_point = vreinterpretq_s16_u32(vld1q_dup_u32((const void*) params->neon.minus_zero_point)); 34 const float32x4_t vscale = vld1q_dup_f32(¶ms->neon.scale); 35 $if BATCH_TILE > 8: 36 for (; n >= ${BATCH_TILE} * sizeof(${XINT8_T}); n -= ${BATCH_TILE} * sizeof(${XINT8_T})) { 37 $for N in range(0, BATCH_TILE, 8): 38 const ${XINT8X8_T} vx${ABC[N:N+8]} = ${VLD1_X8}(x); x += 8; 39 40 $for N in range(0, BATCH_TILE, 8): 41 $if DATATYPE == "QU8": 42 const int16x8_t vhx${ABC[N:N+8]} = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(vminus_zero_point), vx${ABC[N:N+8]})); 43 $else: 44 const int16x8_t vhx${ABC[N:N+8]} = vaddw_s8(vminus_zero_point, vx${ABC[N:N+8]}); 45 46 $for N in range(0, BATCH_TILE, 8): 47 const int32x4_t vwx${ABC[N:N+4]} = vmovl_s16(vget_low_s16(vhx${ABC[N:N+8]})); 48 const int32x4_t vwx${ABC[N+4:N+8]} = vmovl_s16(vget_high_s16(vhx${ABC[N:N+8]})); 49 50 $for N in range(0, BATCH_TILE, 4): 51 float32x4_t vy${ABC[N:N+4]} = vcvtq_f32_s32(vwx${ABC[N:N+4]}); 52 53 $for N in range(0, BATCH_TILE, 4): 54 vy${ABC[N:N+4]} = vmulq_f32(vy${ABC[N:N+4]}, vscale); 55 56 $for N in range(0, BATCH_TILE, 4): 57 vst1q_f32(y, vy${ABC[N:N+4]}); y += 4; 58 } 59 for (; n >= 8 * sizeof(${XINT8_T}); n -= 8 * sizeof(${XINT8_T})) { 60 const ${XINT8X8_T} vx = ${VLD1_X8}(x); x += 8; 61 62 $if DATATYPE == "QU8": 63 const int16x8_t vhx = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(vminus_zero_point), vx)); 64 $else: 65 const int16x8_t vhx = vaddw_s8(vminus_zero_point, vx); 66 67 const int32x4_t vwx_lo = vmovl_s16(vget_low_s16(vhx)); 68 const int32x4_t vwx_hi = vmovl_s16(vget_high_s16(vhx)); 69 70 float32x4_t vy_lo = vcvtq_f32_s32(vwx_lo); 71 float32x4_t vy_hi = vcvtq_f32_s32(vwx_hi); 72 73 vy_lo = vmulq_f32(vy_lo, vscale); 74 vy_hi = vmulq_f32(vy_hi, vscale); 75 76 vst1q_f32(y, vy_lo); y += 4; 77 vst1q_f32(y, vy_hi); y += 4; 78 } 79 if XNN_UNLIKELY(n != 0) { 80 assert(n >= 1 * sizeof(${XINT8_T})); 81 assert(n <= 7 * sizeof(${XINT8_T})); 82 83 const ${XINT8X8_T} vx = ${VLD1_X8}(x); 84 85 $if DATATYPE == "QU8": 86 const int16x8_t vhx = vreinterpretq_s16_u16(vaddw_u8(vreinterpretq_u16_s16(vminus_zero_point), vx)); 87 $else: 88 const int16x8_t vhx = vaddw_s8(vminus_zero_point, vx); 89 90 const int32x4_t vwx_lo = vmovl_s16(vget_low_s16(vhx)); 91 const int32x4_t vwx_hi = vmovl_s16(vget_high_s16(vhx)); 92 93 float32x4_t vy = vcvtq_f32_s32(vwx_lo); 94 vy = vmulq_f32(vy, vscale); 95 96 if (n & (4 * sizeof(${XINT8_T}))) { 97 vst1q_f32(y, vy); y += 4; 98 vy = vcvtq_f32_s32(vwx_hi); 99 vy = vmulq_f32(vy, vscale); 100 } 101 float32x2_t vy_lo = vget_low_f32(vy); 102 if (n & (2 * sizeof(${XINT8_T}))) { 103 vst1_f32(y, vy_lo); y += 2; 104 vy_lo = vget_high_f32(vy); 105 } 106 if (n & (1 * sizeof(${XINT8_T}))) { 107 vst1_lane_f32(y, vy_lo, 0); 108 } 109 } 110} 111