1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert NR % 4 == 0 7$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 8$VMULADDQ_F32 = "vfmaq_f32" if FMA else "vmlaq_f32" 9 10#include <assert.h> 11 12#include <arm_neon.h> 13 14#include <xnnpack/gemm.h> 15 16 17void xnn_f32_gemm${"inc" if INC else ""}_minmax_ukernel_${MR}x${NR}s4__${"neonfma" if FMA else "neon"}( 18 size_t mr, 19 size_t nc, 20 size_t kc, 21 const float* restrict a, 22 size_t a_stride, 23 const float* restrict w, 24 float* restrict c, 25 size_t cm_stride, 26 size_t cn_stride, 27 $if INC: 28 const float*restrict acc, 29 const union xnn_f32_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) 30{ 31 assert(mr != 0); 32 assert(mr <= ${MR}); 33 assert(nc != 0); 34 assert(kc != 0); 35 assert(kc % sizeof(float) == 0); 36 assert(a != NULL); 37 assert(w != NULL); 38 assert(c != NULL); 39 $if INC: 40 assert(acc != NULL); 41 42 const float* a0 = a; 43 float* c0 = c; 44 $for M in range(1, MR): 45 const float* a${M} = (const float*) ((uintptr_t) a${M-1} + a_stride); 46 float* c${M} = (float*) ((uintptr_t) c${M-1} + cm_stride); 47 $if M % 2 == 0: 48 if XNN_UNPREDICTABLE(mr <= ${M}) { 49 a${M} = a${M-1}; 50 c${M} = c${M-1}; 51 } 52 $elif M + 1 == MR: 53 if XNN_UNPREDICTABLE(mr != ${M+1}) { 54 a${M} = a${M-1}; 55 c${M} = c${M-1}; 56 } 57 $else: 58 if XNN_UNPREDICTABLE(mr < ${M+1}) { 59 a${M} = a${M-1}; 60 c${M} = c${M-1}; 61 } 62 63 do { 64 $if INC: 65 $for M in range(MR): 66 $for N in range(0, NR, 4): 67 float32x4_t vacc${M}x${ABC[N:N+4]} = vld1q_f32(acc); acc += 4; 68 $else: 69 $for N in range(0, NR, 4): 70 float32x4_t vacc0x${ABC[N:N+4]} = vld1q_f32(w); w += 4; 71 $for M in range(1, MR): 72 $for N in range(0, NR, 4): 73 float32x4_t vacc${M}x${ABC[N:N+4]} = vacc0x${ABC[N:N+4]}; 74 75 size_t k = kc; 76 while (k >= 4 * sizeof(float)) { 77 $for M in range(MR): 78 float32x4_t va${M} = vld1q_f32(a${M}); a${M} += 4; 79 80 $for L in range(4): 81 82 $for N in range(0, NR, 4): 83 const float32x4_t vb${ABC[N:N+4]}c${L} = vld1q_f32(w); w += 4; 84 85 $for N in range(0, NR, 4): 86 $for M in range(MR): 87 vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, va${M}, vb${ABC[N:N+4]}c${L}); 88 89 $if L + 1 != 4: 90 $for M in range(MR): 91 va${M} = vextq_f32(va${M}, va${M}, 1); 92 93 k -= 4 * sizeof(float); 94 } 95 if XNN_UNLIKELY(k != 0) { 96 $for M in range(MR): 97 float32x4_t va${M} = vld1q_f32(a${M}); a${M} = (const float*) ((uintptr_t) a${M} + k); 98 99 $for L in range(4): 100 101 $for N in range(0, NR, 4): 102 const float32x4_t vb${ABC[N:N+4]}c${L} = vld1q_f32(w); w += 4; 103 104 $for N in range(0, NR, 4): 105 $for M in range(MR): 106 const float32x4_t vmska${M}x${ABC[N:N+4]}c${L} = vreinterpretq_f32_u32(vbicq_u32(vreinterpretq_u32_f32(va${M}), vceqq_f32(vb${ABC[N:N+4]}c${L}, vmovq_n_f32(0.0f)))); 107 vacc${M}x${ABC[N:N+4]} = ${VMULADDQ_F32}(vacc${M}x${ABC[N:N+4]}, vmska${M}x${ABC[N:N+4]}c${L}, vb${ABC[N:N+4]}c${L}); 108 109 $if L + 1 != 4: 110 $for M in range(MR): 111 va${M} = vextq_f32(va${M}, va${M}, 1); 112 } 113 const float32x4_t vmax = vld1q_dup_f32(¶ms->scalar.max); 114 $for N in range(0, NR, 4): 115 $for M in range(MR): 116 vacc${M}x${ABC[N:N+4]} = vminq_f32(vacc${M}x${ABC[N:N+4]}, vmax); 117 118 const float32x4_t vmin = vld1q_dup_f32(¶ms->scalar.min); 119 $for N in range(0, NR, 4): 120 $for M in range(MR): 121 vacc${M}x${ABC[N:N+4]} = vmaxq_f32(vacc${M}x${ABC[N:N+4]}, vmin); 122 123 if XNN_LIKELY(nc >= ${NR}) { 124 $for M in reversed(range(MR)): 125 vst1q_f32(c${M}, vacc${M}x${ABC[0:4]}); 126 $for N in range(4, NR, 4): 127 vst1q_f32(c${M} + ${N}, vacc${M}x${ABC[N:N+4]}); 128 c${M} = (float*) ((uintptr_t) c${M} + cn_stride); 129 130 $for M in reversed(range(MR)): 131 a${M} = (const float*) ((uintptr_t) a${M} - kc); 132 133 nc -= ${NR}; 134 135 } else { 136 $for LOG2N in reversed(range(NR.bit_length())): 137 $if NR != 1 << LOG2N: 138 if (nc & ${1 << LOG2N}) { 139 $if LOG2N >= 2: 140 $for N in range(0, 1 << LOG2N, 4): 141 $for M in reversed(range(MR)): 142 vst1q_f32(c${M}, vacc${M}x${ABC[N:N+4]}); c${M} += 4; 143 144 $for M in reversed(range(MR)): 145 $for N in range(0, 1 << (LOG2N - 1), 4): 146 vacc${M}x${ABC[N:N+4]} = vacc${M}x${ABC[N + (1 << LOG2N):N + (1 << LOG2N)+4]}; 147 $elif LOG2N == 1: 148 $for M in reversed(range(MR)): 149 vst1_f32(c${M}, vacc${M}x${ABC[0:2]}); c${M} += 2; 150 151 $for M in reversed(range(MR)): 152 vacc${M}x${ABC[0:2]} = vget_high_f32(vacc${M}x${ABC[0:4]}); 153 $elif LOG2N == 0: 154 $for M in reversed(range(MR)): 155 vst1_lane_f32(c${M}, vacc${M}x${ABC[0:2]}, 0); 156 } 157 $if LOG2N == 2: 158 $for M in reversed(range(MR)): 159 float32x2_t vacc${M}x${ABC[0:2]} = vget_low_f32(vacc${M}x${ABC[0:4]}); 160 161 nc = 0; 162 } 163 } while (nc != 0); 164}