1// Copyright 2021 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE % 8 == 0 7$assert BATCH_TILE >= 8 8$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 9#include <assert.h> 10 11#include <arm_neon.h> 12 13#include <xnnpack/common.h> 14#include <xnnpack/intrinsics-polyfill.h> 15#include <xnnpack/vcvt.h> 16 17 18$XINT8_T = {"QS8": "int8_t", "QU8": "uint8_t"}[DATATYPE] 19$WASM_X8X16_NARROW_I16X8 = {"QS8": "wasm_i8x16_narrow_i16x8", "QU8": "wasm_u8x16_narrow_i16x8"}[DATATYPE] 20$WASM_X8X16_MIN = {"QS8": "wasm_i8x16_min", "QU8": "wasm_u8x16_min"}[DATATYPE] 21$WASM_X8X16_MAX = {"QS8": "wasm_i8x16_max", "QU8": "wasm_u8x16_max"}[DATATYPE] 22void xnn_f32_${DATATYPE.lower()}_vcvt_ukernel__wasmsimd_cvt_x${BATCH_TILE}( 23 size_t n, 24 const float* x, 25 ${XINT8_T}* y, 26 const union xnn_f32_${DATATYPE.lower()}_cvt_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 27{ 28 assert(n != 0); 29 assert(n % sizeof(float) == 0); 30 assert(x != NULL); 31 assert(y != NULL); 32 33 const v128_t vscale = wasm_v128_load64_splat(params->wasmsimd_cvt.scale); 34 const v128_t voutput_zero_point = wasm_v128_load64_splat(params->wasmsimd_cvt.output_zero_point); 35 const v128_t voutput_min = wasm_v128_load64_splat(params->wasmsimd_cvt.output_min); 36 const v128_t voutput_max = wasm_v128_load64_splat(params->wasmsimd_cvt.output_max); 37 $if BATCH_TILE > 8: 38 for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) { 39 v128_t vx${ABC[0:4]} = wasm_v128_load(x); 40 $for N in range(4, BATCH_TILE, 4): 41 v128_t vx${ABC[N:N+4]} = wasm_v128_load(x + ${N}); 42 x += ${BATCH_TILE}; 43 44 $for N in range(0, BATCH_TILE, 4): 45 vx${ABC[N:N+4]} = wasm_f32x4_mul(vx${ABC[N:N+4]}, vscale); 46 47 $for N in range(0, BATCH_TILE, 4): 48 vx${ABC[N:N+4]} = wasm_f32x4_nearest(vx${ABC[N:N+4]}); 49 50 $for N in range(0, BATCH_TILE, 4): 51 v128_t vacc${ABC[N:N+4]} = wasm_i32x4_trunc_sat_f32x4(vx${ABC[N:N+4]}); 52 53 $for N in range(0, BATCH_TILE, 8): 54 v128_t vacc${ABC[N:N+8]} = wasm_i16x8_narrow_i32x4(vacc${ABC[N:N+4]}, vacc${ABC[N+4:N+8]}); 55 56 $for N in range(0, BATCH_TILE, 8): 57 vacc${ABC[N:N+8]} = wasm_i16x8_add_sat(vacc${ABC[N:N+8]}, voutput_zero_point); 58 59 $for N in range(0, BATCH_TILE, 16): 60 $if N + 8 < BATCH_TILE: 61 v128_t vy${ABC[N:N+16]} = ${WASM_X8X16_NARROW_I16X8}(vacc${ABC[N:N+8]}, vacc${ABC[N+8:N+16]}); 62 $else: 63 v128_t vy${ABC[N:N+8]} = ${WASM_X8X16_NARROW_I16X8}(vacc${ABC[N:N+8]}, vacc${ABC[N:N+8]}); 64 65 $for N in range(0, BATCH_TILE, 16): 66 $if N + 8 < BATCH_TILE: 67 vy${ABC[N:N+16]} = ${WASM_X8X16_MAX}(vy${ABC[N:N+16]}, voutput_min); 68 $else: 69 vy${ABC[N:N+8]} = ${WASM_X8X16_MAX}(vy${ABC[N:N+8]}, voutput_min); 70 71 $for N in range(0, BATCH_TILE, 16): 72 $if N + 8 < BATCH_TILE: 73 vy${ABC[N:N+16]} = ${WASM_X8X16_MIN}(vy${ABC[N:N+16]}, voutput_max); 74 $else: 75 vy${ABC[N:N+8]} = ${WASM_X8X16_MIN}(vy${ABC[N:N+8]}, voutput_max); 76 77 wasm_v128_store(y, vy${ABC[0:16]}); 78 $for N in range(16, BATCH_TILE, 16): 79 $if N + 8 < BATCH_TILE: 80 wasm_v128_store(y + ${N}, vy${ABC[N:N+16]}); 81 $else: 82 *((double*) (y + ${N})) = wasm_f64x2_extract_lane(vy${ABC[N:N+8]}, 0); 83 y += ${BATCH_TILE}; 84 } 85 for (; n >= 8 * sizeof(float); n -= 8 * sizeof(float)) { 86 v128_t vx_lo = wasm_v128_load(x); 87 v128_t vx_hi = wasm_v128_load(x + 4); 88 x += 8; 89 90 vx_lo = wasm_f32x4_mul(vx_lo, vscale); 91 vx_hi = wasm_f32x4_mul(vx_hi, vscale); 92 93 vx_lo = wasm_f32x4_nearest(vx_lo); 94 vx_hi = wasm_f32x4_nearest(vx_hi); 95 96 v128_t vacc_lo = wasm_i32x4_trunc_sat_f32x4(vx_lo); 97 v128_t vacc_hi = wasm_i32x4_trunc_sat_f32x4(vx_hi); 98 99 v128_t vacc = wasm_i16x8_narrow_i32x4(vacc_lo, vacc_hi); 100 vacc = wasm_i16x8_add_sat(vacc, voutput_zero_point); 101 102 v128_t vy = ${WASM_X8X16_NARROW_I16X8}(vacc, vacc); 103 vy = ${WASM_X8X16_MAX}(vy, voutput_min); 104 vy = ${WASM_X8X16_MIN}(vy, voutput_max); 105 106 *((double*) y) = wasm_f64x2_extract_lane(vy, 0); 107 y += 8; 108 } 109 if XNN_UNLIKELY(n != 0) { 110 assert(n >= 1 * sizeof(float)); 111 assert(n <= 7 * sizeof(float)); 112 v128_t vx_lo = wasm_v128_load(x); 113 const float* x_hi = (const float*) ((uintptr_t) x + (n & (4 * sizeof(float)))); 114 v128_t vx_hi = wasm_v128_load(x_hi); 115 116 vx_lo = wasm_f32x4_mul(vx_lo, vscale); 117 vx_hi = wasm_f32x4_mul(vx_hi, vscale); 118 119 vx_lo = wasm_f32x4_nearest(vx_lo); 120 vx_hi = wasm_f32x4_nearest(vx_hi); 121 122 v128_t vacc_lo = wasm_i32x4_trunc_sat_f32x4(vx_lo); 123 v128_t vacc_hi = wasm_i32x4_trunc_sat_f32x4(vx_hi); 124 125 v128_t vacc = wasm_i16x8_narrow_i32x4(vacc_lo, vacc_hi); 126 vacc = wasm_i16x8_add_sat(vacc, voutput_zero_point); 127 128 v128_t vy = ${WASM_X8X16_NARROW_I16X8}(vacc, vacc); 129 vy = ${WASM_X8X16_MAX}(vy, voutput_min); 130 vy = ${WASM_X8X16_MIN}(vy, voutput_max); 131 132 if (n & (4 * sizeof(float))) { 133 *((float*) y) = wasm_f32x4_extract_lane(vy, 0); 134 y += 4; 135 vy = wasm_u64x2_shr(vy, 32); 136 } 137 uint32_t vy_lo = (uint32_t) wasm_i32x4_extract_lane(vy, 0); 138 if (n & (2 * sizeof(float))) { 139 *((uint16_t*) y) = (uint16_t) vy_lo; 140 y += 2; 141 vy_lo >>= 16; 142 } 143 if (n & (1 * sizeof(float))) { 144 *y = (${XINT8_T}) vy_lo; 145 } 146 } 147} 148