1 /****************************************************************************** 2 * 3 * Copyright 2022 Google LLC 4 * 5 * Licensed under the Apache License, Version 2.0 (the "License"); 6 * you may not use this file except in compliance with the License. 7 * You may obtain a copy of the License at: 8 * 9 * http://www.apache.org/licenses/LICENSE-2.0 10 * 11 * Unless required by applicable law or agreed to in writing, software 12 * distributed under the License is distributed on an "AS IS" BASIS, 13 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 * See the License for the specific language governing permissions and 15 * limitations under the License. 16 * 17 ******************************************************************************/ 18 19 #if __ARM_NEON && __ARM_ARCH_ISA_A64 20 21 #ifndef TEST_NEON 22 #include <arm_neon.h> 23 #endif /* TEST_NEON */ 24 25 26 /** 27 * Import 28 */ 29 30 static inline int32_t filter_hp50(struct lc3_ltpf_hp50_state *, int32_t); 31 32 33 /** 34 * Resample from 16 Khz to 12.8 KHz 35 */ 36 #ifndef resample_16k_12k8 37 #define resample_16k_12k8 neon_resample_16k_12k8 38 LC3_HOT static void neon_resample_16k_12k8( 39 struct lc3_ltpf_hp50_state *hp50, const int16_t *x, int16_t *y, int n) 40 { 41 static const int16_t h[4][20] = { 42 43 { -61, 214, -398, 417, 0, -1052, 2686, -4529, 5997, 26233, 44 5997, -4529, 2686, -1052, 0, 417, -398, 214, -61, 0 }, 45 46 { -79, 180, -213, 0, 598, -1522, 2389, -2427, 0, 24506, 47 13068, -5289, 1873, 0, -752, 763, -457, 156, 0, -28 }, 48 49 { -61, 92, 0, -323, 861, -1361, 1317, 0, -3885, 19741, 50 19741, -3885, 0, 1317, -1361, 861, -323, 0, 92, -61 }, 51 52 { -28, 0, 156, -457, 763, -752, 0, 1873, -5289, 13068, 53 24506, 0, -2427, 2389, -1522, 598, 0, -213, 180, -79 }, 54 55 }; 56 57 x -= 20 - 1; 58 59 for (int i = 0; i < 5*n; i += 5) { 60 const int16_t *hn = h[i & 3]; 61 const int16_t *xn = x + (i >> 2); 62 int32x4_t un; 63 64 un = vmull_s16( vld1_s16(xn), vld1_s16(hn)), xn += 4, hn += 4; 65 un = vmlal_s16(un, vld1_s16(xn), vld1_s16(hn)), xn += 4, hn += 4; 66 un = vmlal_s16(un, vld1_s16(xn), vld1_s16(hn)), xn += 4, hn += 4; 67 un = vmlal_s16(un, vld1_s16(xn), vld1_s16(hn)), xn += 4, hn += 4; 68 un = vmlal_s16(un, vld1_s16(xn), vld1_s16(hn)), xn += 4, hn += 4; 69 70 int32_t yn = filter_hp50(hp50, vaddvq_s32(un)); 71 *(y++) = (yn + (1 << 15)) >> 16; 72 } 73 } 74 #endif /* resample_16k_12k8 */ 75 76 /** 77 * Resample from 32 Khz to 12.8 KHz 78 */ 79 #ifndef resample_32k_12k8 80 #define resample_32k_12k8 neon_resample_32k_12k8 81 LC3_HOT static void neon_resample_32k_12k8( 82 struct lc3_ltpf_hp50_state *hp50, const int16_t *x, int16_t *y, int n) 83 { 84 x -= 40 - 1; 85 86 static const int16_t h[2][40] = { 87 88 { -30, -31, 46, 107, 0, -199, -162, 209, 430, 0, 89 -681, -526, 658, 1343, 0, -2264, -1943, 2999, 9871, 13116, 90 9871, 2999, -1943, -2264, 0, 1343, 658, -526, -681, 0, 91 430, 209, -162, -199, 0, 107, 46, -31, -30, 0 }, 92 93 { -14, -39, 0, 90, 78, -106, -229, 0, 382, 299, 94 -376, -761, 0, 1194, 937, -1214, -2644, 0, 6534, 12253, 95 12253, 6534, 0, -2644, -1214, 937, 1194, 0, -761, -376, 96 299, 382, 0, -229, -106, 78, 90, 0, -39, -14 }, 97 98 }; 99 100 for (int i = 0; i < 5*n; i += 5) { 101 const int16_t *hn = h[i & 1]; 102 const int16_t *xn = x + (i >> 1); 103 104 int32x4_t un = vmull_s16(vld1_s16(xn), vld1_s16(hn)); 105 xn += 4, hn += 4; 106 107 for (int i = 1; i < 10; i++) 108 un = vmlal_s16(un, vld1_s16(xn), vld1_s16(hn)), xn += 4, hn += 4; 109 110 int32_t yn = filter_hp50(hp50, vaddvq_s32(un)); 111 *(y++) = (yn + (1 << 15)) >> 16; 112 } 113 } 114 #endif /* resample_32k_12k8 */ 115 116 /** 117 * Resample from 48 Khz to 12.8 KHz 118 */ 119 #ifndef resample_48k_12k8 120 #define resample_48k_12k8 neon_resample_48k_12k8 121 LC3_HOT static void neon_resample_48k_12k8( 122 struct lc3_ltpf_hp50_state *hp50, const int16_t *x, int16_t *y, int n) 123 { 124 static const int16_t alignas(16) h[4][64] = { 125 126 { -13, -25, -20, 10, 51, 71, 38, -47, -133, -145, 127 -42, 139, 277, 242, 0, -329, -511, -351, 144, 698, 128 895, 450, -535, -1510, -1697, -521, 1999, 5138, 7737, 8744, 129 7737, 5138, 1999, -521, -1697, -1510, -535, 450, 895, 698, 130 144, -351, -511, -329, 0, 242, 277, 139, -42, -145, 131 -133, -47, 38, 71, 51, 10, -20, -25, -13, 0 }, 132 133 { -9, -23, -24, 0, 41, 71, 52, -23, -115, -152, 134 -78, 92, 254, 272, 76, -251, -493, -427, 0, 576, 135 900, 624, -262, -1309, -1763, -954, 1272, 4356, 7203, 8679, 136 8169, 5886, 2767, 0, -1542, -1660, -809, 240, 848, 796, 137 292, -252, -507, -398, -82, 199, 288, 183, 0, -130, 138 -145, -71, 20, 69, 60, 20, -15, -26, -17, -3 }, 139 140 { -6, -20, -26, -8, 31, 67, 62, 0, -94, -152, 141 -108, 45, 223, 287, 143, -167, -454, -480, -134, 439, 142 866, 758, 0, -1071, -1748, -1295, 601, 3559, 6580, 8485, 143 8485, 6580, 3559, 601, -1295, -1748, -1071, 0, 758, 866, 144 439, -134, -480, -454, -167, 143, 287, 223, 45, -108, 145 -152, -94, 0, 62, 67, 31, -8, -26, -20, -6 }, 146 147 { -3, -17, -26, -15, 20, 60, 69, 20, -71, -145, 148 -130, 0, 183, 288, 199, -82, -398, -507, -252, 292, 149 796, 848, 240, -809, -1660, -1542, 0, 2767, 5886, 8169, 150 8679, 7203, 4356, 1272, -954, -1763, -1309, -262, 624, 900, 151 576, 0, -427, -493, -251, 76, 272, 254, 92, -78, 152 -152, -115, -23, 52, 71, 41, 0, -24, -23, -9 }, 153 154 }; 155 156 x -= 60 - 1; 157 158 for (int i = 0; i < 15*n; i += 15) { 159 const int16_t *hn = h[i & 3]; 160 const int16_t *xn = x + (i >> 2); 161 162 int32x4_t un = vmull_s16(vld1_s16(xn), vld1_s16(hn)); 163 xn += 4, hn += 4; 164 165 for (int i = 1; i < 15; i++) 166 un = vmlal_s16(un, vld1_s16(xn), vld1_s16(hn)), xn += 4, hn += 4; 167 168 int32_t yn = filter_hp50(hp50, vaddvq_s32(un)); 169 *(y++) = (yn + (1 << 15)) >> 16; 170 } 171 } 172 #endif /* resample_48k_12k8 */ 173 174 /** 175 * Return dot product of 2 vectors 176 */ 177 #ifndef dot 178 #define dot neon_dot 179 LC3_HOT static inline float neon_dot(const int16_t *a, const int16_t *b, int n) 180 { 181 int64x2_t v = vmovq_n_s64(0); 182 183 for (int i = 0; i < (n >> 4); i++) { 184 int32x4_t u; 185 186 u = vmull_s16( vld1_s16(a), vld1_s16(b)), a += 4, b += 4; 187 u = vmlal_s16(u, vld1_s16(a), vld1_s16(b)), a += 4, b += 4; 188 v = vpadalq_s32(v, u); 189 190 u = vmull_s16( vld1_s16(a), vld1_s16(b)), a += 4, b += 4; 191 u = vmlal_s16(u, vld1_s16(a), vld1_s16(b)), a += 4, b += 4; 192 v = vpadalq_s32(v, u); 193 } 194 195 int32_t v32 = (vaddvq_s64(v) + (1 << 5)) >> 6; 196 return (float)v32; 197 } 198 #endif /* dot */ 199 200 /** 201 * Return vector of correlations 202 */ 203 #ifndef correlate 204 #define correlate neon_correlate 205 LC3_HOT static void neon_correlate( 206 const int16_t *a, const int16_t *b, int n, float *y, int nc) 207 { 208 for ( ; nc >= 4; nc -= 4, b -= 4) { 209 const int16_t *an = (const int16_t *)a; 210 const int16_t *bn = (const int16_t *)b; 211 212 int64x2_t v0 = vmovq_n_s64(0), v1 = v0, v2 = v0, v3 = v0; 213 int16x4_t ax, b0, b1; 214 215 b0 = vld1_s16(bn-4); 216 217 for (int i=0; i < (n >> 4); i++ ) 218 for (int j = 0; j < 2; j++) { 219 int32x4_t u0, u1, u2, u3; 220 221 b1 = b0; 222 b0 = vld1_s16(bn), bn += 4; 223 ax = vld1_s16(an), an += 4; 224 225 u0 = vmull_s16(ax, b0); 226 u1 = vmull_s16(ax, vext_s16(b1, b0, 3)); 227 u2 = vmull_s16(ax, vext_s16(b1, b0, 2)); 228 u3 = vmull_s16(ax, vext_s16(b1, b0, 1)); 229 230 b1 = b0; 231 b0 = vld1_s16(bn), bn += 4; 232 ax = vld1_s16(an), an += 4; 233 234 u0 = vmlal_s16(u0, ax, b0); 235 u1 = vmlal_s16(u1, ax, vext_s16(b1, b0, 3)); 236 u2 = vmlal_s16(u2, ax, vext_s16(b1, b0, 2)); 237 u3 = vmlal_s16(u3, ax, vext_s16(b1, b0, 1)); 238 239 v0 = vpadalq_s32(v0, u0); 240 v1 = vpadalq_s32(v1, u1); 241 v2 = vpadalq_s32(v2, u2); 242 v3 = vpadalq_s32(v3, u3); 243 } 244 245 *(y++) = (float)((int32_t)((vaddvq_s64(v0) + (1 << 5)) >> 6)); 246 *(y++) = (float)((int32_t)((vaddvq_s64(v1) + (1 << 5)) >> 6)); 247 *(y++) = (float)((int32_t)((vaddvq_s64(v2) + (1 << 5)) >> 6)); 248 *(y++) = (float)((int32_t)((vaddvq_s64(v3) + (1 << 5)) >> 6)); 249 } 250 251 for ( ; nc > 0; nc--) 252 *(y++) = neon_dot(a, b--, n); 253 } 254 #endif /* correlate */ 255 256 #endif /* __ARM_NEON && __ARM_ARCH_ISA_A64 */ 257