1// Copyright 2021 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE % 8 == 0 7$assert BATCH_TILE >= 8 8$SIMD_TILE = BATCH_TILE // 8 9$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 10#include <assert.h> 11 12#include <wasm_simd128.h> 13 14#include <xnnpack/common.h> 15#include <xnnpack/vcvt.h> 16 17 18void xnn_f16_f32_vcvt_ukernel__wasmsimd_int32_x${BATCH_TILE}( 19 size_t n, 20 const void* input, 21 float* output, 22 const union xnn_f16_f32_cvt_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS 23{ 24 assert(n != 0); 25 assert(n % sizeof(uint16_t) == 0); 26 assert(input != NULL); 27 assert(output != NULL); 28 29 const v128_t vsign_mask = wasm_v128_load64_splat(params->wasmsimd_int32.sign_mask); 30 const v128_t vexp_offset = wasm_v128_load64_splat(params->wasmsimd_int32.exp_offset); 31 const v128_t vexp_scale = wasm_v128_load64_splat(params->wasmsimd_int32.exp_scale); 32 const v128_t vmagic_bias = wasm_v128_load64_splat(params->wasmsimd_int32.magic_bias); 33 const v128_t vdenorm_cutoff = wasm_v128_load64_splat(params->wasmsimd_int32.denorm_cutoff); 34 35 const uint16_t* i = (const uint16_t*) input; 36 $if BATCH_TILE > 8: 37 for (; n >= ${BATCH_TILE} * sizeof(uint16_t); n -= ${BATCH_TILE} * sizeof(uint16_t)) { 38 const v128_t vh0 = wasm_v128_load(i); 39 $for N in range(1, SIMD_TILE): 40 const v128_t vh${N} = wasm_v128_load(i + ${N * 8}); 41 i += ${BATCH_TILE}; 42 43 const v128_t vzero = wasm_i16x8_const_splat(0); 44 $for N in range(SIMD_TILE): 45 const v128_t vw${N*2} = wasm_v16x8_shuffle(vzero, vh${N}, 0, 8, 1, 9, 2, 10, 3, 11); 46 const v128_t vw${N*2+1} = wasm_v16x8_shuffle(vzero, vh${N}, 4, 12, 5, 13, 6, 14, 7, 15); 47 48 $for N in range(2*SIMD_TILE): 49 const v128_t vsign${N} = wasm_v128_and(vw${N}, vsign_mask); 50 51 $for N in range(2*SIMD_TILE): 52 const v128_t vnonsign${N} = wasm_v128_xor(vw${N}, vsign${N}); 53 54 $for N in range(2*SIMD_TILE): 55 const v128_t vnorm${N} = wasm_f32x4_mul(wasm_i32x4_add(wasm_u32x4_shr(vnonsign${N}, 3), vexp_offset), vexp_scale); 56 57 $for N in range(2*SIMD_TILE): 58 const v128_t vdenorm${N} = wasm_f32x4_sub(wasm_v128_or(wasm_u32x4_shr(vnonsign${N}, 16), vmagic_bias), vmagic_bias); 59 60 $for N in range(2*SIMD_TILE): 61 const v128_t vxmask${N} = wasm_i32x4_gt(vnonsign${N}, vdenorm_cutoff); 62 63 $for N in range(2*SIMD_TILE): 64 const v128_t vf${N} = wasm_v128_or(vsign${N}, wasm_v128_bitselect(vnorm${N}, vdenorm${N}, vxmask${N})); 65 66 wasm_v128_store(output, vf0); 67 $for N in range(1, 2*SIMD_TILE): 68 wasm_v128_store(output + ${N*4}, vf${N}); 69 output += ${BATCH_TILE}; 70 } 71 for (; n >= 8 * sizeof(uint16_t); n -= 8 * sizeof(uint16_t)) { 72 const v128_t vh = wasm_v128_load(i); 73 i += 8; 74 75 const v128_t vzero = wasm_i16x8_const_splat(0); 76 const v128_t vw_lo = wasm_v16x8_shuffle(vzero, vh, 0, 8, 1, 9, 2, 10, 3, 11); 77 const v128_t vw_hi = wasm_v16x8_shuffle(vzero, vh, 4, 12, 5, 13, 6, 14, 7, 15); 78 79 const v128_t vsign_lo = wasm_v128_and(vw_lo, vsign_mask); 80 const v128_t vsign_hi = wasm_v128_and(vw_hi, vsign_mask); 81 82 const v128_t vnonsign_lo = wasm_v128_xor(vw_lo, vsign_lo); 83 const v128_t vnonsign_hi = wasm_v128_xor(vw_hi, vsign_hi); 84 85 const v128_t vnorm_lo = wasm_f32x4_mul(wasm_i32x4_add(wasm_u32x4_shr(vnonsign_lo, 3), vexp_offset), vexp_scale); 86 const v128_t vnorm_hi = wasm_f32x4_mul(wasm_i32x4_add(wasm_u32x4_shr(vnonsign_hi, 3), vexp_offset), vexp_scale); 87 88 const v128_t vdenorm_lo = wasm_f32x4_sub(wasm_v128_or(wasm_u32x4_shr(vnonsign_lo, 16), vmagic_bias), vmagic_bias); 89 const v128_t vdenorm_hi = wasm_f32x4_sub(wasm_v128_or(wasm_u32x4_shr(vnonsign_hi, 16), vmagic_bias), vmagic_bias); 90 91 const v128_t vxmask_lo = wasm_i32x4_gt(vnonsign_lo, vdenorm_cutoff); 92 const v128_t vxmask_hi = wasm_i32x4_gt(vnonsign_hi, vdenorm_cutoff); 93 94 const v128_t vf_lo = wasm_v128_or(vsign_lo, wasm_v128_bitselect(vnorm_lo, vdenorm_lo, vxmask_lo)); 95 const v128_t vf_hi = wasm_v128_or(vsign_hi, wasm_v128_bitselect(vnorm_hi, vdenorm_hi, vxmask_hi)); 96 97 wasm_v128_store(output, vf_lo); 98 wasm_v128_store(output + 4, vf_hi); 99 output += 8; 100 } 101 if XNN_UNLIKELY(n != 0) { 102 assert(n >= 1 * sizeof(uint16_t)); 103 assert(n <= 7 * sizeof(uint16_t)); 104 const v128_t vh = wasm_v128_load(i); 105 106 const v128_t vzero = wasm_i16x8_const_splat(0); 107 const v128_t vw_lo = wasm_v16x8_shuffle(vzero, vh, 0, 8, 1, 9, 2, 10, 3, 11); 108 const v128_t vw_hi = wasm_v16x8_shuffle(vzero, vh, 4, 12, 5, 13, 6, 14, 7, 15); 109 110 const v128_t vsign_lo = wasm_v128_and(vw_lo, vsign_mask); 111 const v128_t vsign_hi = wasm_v128_and(vw_hi, vsign_mask); 112 113 const v128_t vnonsign_lo = wasm_v128_xor(vw_lo, vsign_lo); 114 const v128_t vnonsign_hi = wasm_v128_xor(vw_hi, vsign_hi); 115 116 const v128_t vnorm_lo = wasm_f32x4_mul(wasm_i32x4_add(wasm_u32x4_shr(vnonsign_lo, 3), vexp_offset), vexp_scale); 117 const v128_t vnorm_hi = wasm_f32x4_mul(wasm_i32x4_add(wasm_u32x4_shr(vnonsign_hi, 3), vexp_offset), vexp_scale); 118 119 const v128_t vdenorm_lo = wasm_f32x4_sub(wasm_v128_or(wasm_u32x4_shr(vnonsign_lo, 16), vmagic_bias), vmagic_bias); 120 const v128_t vdenorm_hi = wasm_f32x4_sub(wasm_v128_or(wasm_u32x4_shr(vnonsign_hi, 16), vmagic_bias), vmagic_bias); 121 122 const v128_t vxmask_lo = wasm_i32x4_gt(vnonsign_lo, vdenorm_cutoff); 123 v128_t vf = wasm_v128_or(vsign_lo, wasm_v128_bitselect(vnorm_lo, vdenorm_lo, vxmask_lo)); 124 125 if (n & (4 * sizeof(uint16_t))) { 126 wasm_v128_store(output, vf); 127 output += 4; 128 129 const v128_t vxmask_hi = wasm_i32x4_gt(vnonsign_hi, vdenorm_cutoff); 130 vf = wasm_v128_or(vsign_hi, wasm_v128_bitselect(vnorm_hi, vdenorm_hi, vxmask_hi)); 131 } 132 if (n & (2 * sizeof(uint16_t))) { 133 *((double*) output) = wasm_f64x2_extract_lane(vf, 0); 134 output += 2; 135 136 vf = wasm_v64x2_shuffle(vf, vf, 1, 1); 137 } 138 if (n & (1 * sizeof(uint16_t))) { 139 *((float*) output) = wasm_f32x4_extract_lane(vf, 0); 140 } 141 } 142} 143