1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE >= 1 7$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 8#include <assert.h> 9#include <math.h> 10 11#include <xnnpack/common.h> 12#include <xnnpack/math.h> 13#include <xnnpack/vunary.h> 14 15 16// Note redefine as uint32[] to avoid redundant bitcasts. 17extern XNN_INTERNAL const uint32_t xnn_table_exp2minus_k_over_2048[2048]; 18 19void xnn_f32_vsigmoid_ukernel__scalar_rr2_lut2048_p1_div_x${BATCH_TILE}( 20 size_t n, 21 const float* x, 22 float* y, 23 const union xnn_f32_sigmoid_params params[restrict XNN_MIN_ELEMENTS(1)]) 24{ 25 assert(n % sizeof(float) == 0); 26 27 const float vmagic_bias = params->scalar_rr2_lut2048_p1.magic_bias; 28 const float vminus_log2e = params->scalar_rr2_lut2048_p1.minus_log2e; 29 const uint32_t vindex_mask = UINT32_C(0x7FF); 30 const float vln2_hi = params->scalar_rr2_lut2048_p1.ln2_hi; 31 const float vln2_lo = params->scalar_rr2_lut2048_p1.ln2_lo; 32 const float vc1 = params->scalar_rr2_lut2048_p1.c1; 33 const float vone = params->scalar_rr2_lut2048_p1.one; 34 const float vdenorm_cutoff = params->scalar_rr2_lut2048_p1.denorm_cutoff; 35 36 $if BATCH_TILE > 1: 37 for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) { 38 $for N in range(BATCH_TILE): 39 const float vx${N} = x[${N}]; 40 x += ${BATCH_TILE}; 41 42 $for N in range(BATCH_TILE): 43 const float vz${N} = fabsf(vx${N}); 44 45 $for N in range(BATCH_TILE): 46 float vn${N} = vz${N} * vminus_log2e + vmagic_bias; 47 48 $for N in range(BATCH_TILE): 49 const uint32_t ve${N} = float_as_uint32(vn${N}) << 12; 50 51 $for N in range(BATCH_TILE): 52 const uint32_t vidx${N} = float_as_uint32(vn${N}) & vindex_mask; 53 const float vs${N} = uint32_as_float(xnn_table_exp2minus_k_over_2048[vidx${N}] + ve${N}); 54 55 $for N in range(BATCH_TILE): 56 vn${N} -= vmagic_bias; 57 58 $for N in range(BATCH_TILE): 59 float vt${N} = vn${N} * vln2_hi + vz${N}; 60 61 $for N in range(BATCH_TILE): 62 vt${N} = vn${N} * vln2_lo + vt${N}; 63 64 $for N in range(BATCH_TILE): 65 const float vp${N} = vt${N} * vc1; 66 67 $for N in range(BATCH_TILE): 68 const float vy${N} = vp${N} * vs${N} + vs${N}; 69 70 $for N in range(BATCH_TILE): 71 const float vd${N} = vy${N} + vone; 72 73 $for N in range(BATCH_TILE): 74 float vf${N} = vy${N} / vd${N}; 75 76 $for N in range(BATCH_TILE): 77 if XNN_UNPREDICTABLE(vz${N} > vdenorm_cutoff) { 78 vf${N} = 0.0f; 79 } 80 81 $for N in range(BATCH_TILE): 82 if XNN_UNPREDICTABLE(vx${N} > 0.0f) { 83 vf${N} = vone - vf${N}; 84 } 85 86 $for N in range(BATCH_TILE): 87 y[${N}] = vf${N}; 88 y += ${BATCH_TILE}; 89 } 90 $if BATCH_TILE == 1: 91 do { 92 const float vx = *x++; 93 94 const float vz = fabsf(vx); 95 96 float vn = vz * vminus_log2e + vmagic_bias; 97 const uint32_t ve = float_as_uint32(vn) << 12; 98 const uint32_t vidx = float_as_uint32(vn) & vindex_mask; 99 const float vs = uint32_as_float(xnn_table_exp2minus_k_over_2048[vidx] + ve); 100 vn -= vmagic_bias; 101 102 float vt = vn * vln2_hi + vz; 103 vt = vn * vln2_lo + vt; 104 105 const float vp = vt * vc1; 106 const float vy = vp * vs + vs; 107 108 const float vd = vy + vone; 109 float vf = vy / vd; 110 if XNN_UNPREDICTABLE(vz > vdenorm_cutoff) { 111 vf = 0.0f; 112 } 113 if XNN_UNPREDICTABLE(vx > 0.0f) { 114 vf = vone - vf; 115 } 116 117 *y++ = vf; 118 119 n -= sizeof(float); 120 } while (n != 0); 121 $elif BATCH_TILE == 2: 122 if XNN_UNLIKELY(n != 0) { 123 const float vx = *x; 124 125 const float vz = fabsf(vx); 126 127 float vn = vz * vminus_log2e + vmagic_bias; 128 const uint32_t ve = float_as_uint32(vn) << 12; 129 const uint32_t vidx = float_as_uint32(vn) & vindex_mask; 130 const float vs = uint32_as_float(xnn_table_exp2minus_k_over_2048[vidx] + ve); 131 vn -= vmagic_bias; 132 133 float vt = vn * vln2_hi + vz; 134 vt = vn * vln2_lo + vt; 135 136 const float vp = vt * vc1; 137 const float vy = vp * vs + vs; 138 139 const float vd = vy + vone; 140 float vf = vy / vd; 141 if XNN_UNPREDICTABLE(vz > vdenorm_cutoff) { 142 vf = 0.0f; 143 } 144 if XNN_UNPREDICTABLE(vx > 0.0f) { 145 vf = vone - vf; 146 } 147 148 *y = vf; 149 } 150 $else: 151 if XNN_UNLIKELY(n != 0) { 152 do { 153 const float vx = *x++; 154 155 const float vz = fabsf(vx); 156 157 float vn = vz * vminus_log2e + vmagic_bias; 158 const uint32_t ve = float_as_uint32(vn) << 12; 159 const uint32_t vidx = float_as_uint32(vn) & vindex_mask; 160 const float vs = uint32_as_float(xnn_table_exp2minus_k_over_2048[vidx] + ve); 161 vn -= vmagic_bias; 162 163 float vt = vn * vln2_hi + vz; 164 vt = vn * vln2_lo + vt; 165 166 const float vp = vt * vc1; 167 const float vy = vp * vs + vs; 168 169 const float vd = vy + vone; 170 float vf = vy / vd; 171 if XNN_UNPREDICTABLE(vz > vdenorm_cutoff) { 172 vf = 0.0f; 173 } 174 if XNN_UNPREDICTABLE(vx > 0.0f) { 175 vf = vone - vf; 176 } 177 178 *y++ = vf; 179 180 n -= sizeof(float); 181 } while (n != 0); 182 } 183} 184