1 /* 2 * Copyright (c) 2021 Arm Limited. 3 * 4 * SPDX-License-Identifier: MIT 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a copy 7 * of this software and associated documentation files (the "Software"), to 8 * deal in the Software without restriction, including without limitation the 9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 10 * sell copies of the Software, and to permit persons to whom the Software is 11 * furnished to do so, subject to the following conditions: 12 * 13 * The above copyright notice and this permission notice shall be included in all 14 * copies or substantial portions of the Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 * SOFTWARE. 23 */ 24 #include "helpers.h" 25 26 /** (EXPERIMENTAL_POST_OPS) Macros for (binary) elementwise operations */ 27 28 /** List of (binary) elementwise operators, accounting for the argument position of argument X 29 * @note X_Pos denotes the position of argument X. e.g. X_POS_0 means X is in the first place whereas X_POS_1 means X is in the second place 30 * @name elementwise_post_ops 31 * @{ 32 */ 33 #if defined(N0) && !defined(VEC_SIZE) 34 #define VEC_SIZE N0 35 #endif // defined(N0) && !defined(VEC_SIZE) 36 37 #if defined(VEC_SIZE) && defined(DATA_TYPE) 38 39 #define ADD_X_POS_0(x, y) (x) + (y) 40 #define SUB_X_POS_0(x, y) (x) - (y) 41 #define MAX_X_POS_0(x, y) max(x, y) 42 #define MIN_X_POS_0(x, y) min(x, y) 43 #define SQUARED_DIFF_X_POS_0(x, y) (x - y) * (x - y) 44 #define POWER_X_POS_0(x, y) pow(x, y) 45 #if VEC_SIZE == 1 46 #define PRELU_X_POS_0(x, y) (x > 0 ? x : x * y) 47 #else // VEC_SIZE == 1 48 49 #if defined(MIXED_PRECISION) 50 #define PRELU_X_POS_0(x, y) (select(y * x, x, CONVERT((x > (DATA_TYPE_ACCUMULATOR)0), SELECT_VEC_DATA_TYPE(DATA_TYPE_ACCUMULATOR, VEC_SIZE)))) 51 #else // MIXED_PRECISION 52 #define PRELU_X_POS_0(x, y) (select(y * x, x, CONVERT((x > (DATA_TYPE)0), SELECT_VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)))) 53 #endif // MIXED_PRECISION 54 55 #endif // VEC_SIZE == 1 56 #define DIV_X_POS_0(x, y) (x / y) 57 #define AND_X_POS_0(x, y) (CONVERT((x && y), VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)) & ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))1)) 58 #define OR_X_POS_0(x, y) (CONVERT((x || y), VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)) & ((VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE))1)) 59 60 #define ADD_X_POS_1(x, y) ADD_X_POS_0(x, y) 61 #define SUB_X_POS_1(x, y) (y) - (x) 62 #define MAX_X_POS_1(x, y) MAX_X_POS_0(x, y) 63 #define MIN_X_POS_1(x, y) MIN_X_POS_0(x, y) 64 #define SQUARED_DIFF_X_POS_1(x, y) SQUARED_DIFF_X_POS_0(x, y) 65 #define POWER_X_POS_1(x, y) pow(y, x) 66 #if VEC_SIZE == 1 67 #define PRELU_X_POS_1(x, y) (y > 0 ? y : y * x) 68 #else // VEC_SIZE == 1 69 70 #if defined(MIXED_PRECISION) 71 #define PRELU_X_POS_1(x, y) (select(x * y, y, CONVERT((y > (DATA_TYPE_ACCUMULATOR)0), SELECT_VEC_DATA_TYPE(DATA_TYPE_ACCUMULATOR, VEC_SIZE)))) 72 #else // MIXED_PRECISION 73 #define PRELU_X_POS_1(x, y) (select(x * y, y, CONVERT((y > (DATA_TYPE)0), SELECT_VEC_DATA_TYPE(DATA_TYPE, VEC_SIZE)))) 74 #endif // MIXED_PRECISION 75 76 #endif // VEC_SIZE == 1 77 #define DIV_X_POS_1(x, y) (y / x) 78 #define AND_X_POS_1(x, y) AND_X_POS_0(x, y) 79 #define OR_X_POS_1(x, y) OR_X_POS_0(x, y) 80 81 // By default use the order of the arguments as they are passed in, ie. _X_POS_0 82 #define ADD(x, y) ADD_X_POS_0(x, y) 83 #define SUB(x, y) SUB_X_POS_0(x, y) 84 #define MAX(x, y) MAX_X_POS_0(x, y) 85 #define MIN(x, y) MIN_X_POS_0(x, y) 86 #define SQUARED_DIFF(x, y) SQUARED_DIFF_X_POS_0(x, y) 87 #define POWER(x, y) POWER_X_POS_0(x, y) 88 #define PRELU(x, y) PRELU_X_POS_0(x, y) 89 #define DIV(x, y) DIV_X_POS_0(x, y) 90 #define AND(x, y) AND_X_POS_0(x, y) 91 #define OR(x, y) OR_X_POS_0(x, y) 92 93 #endif // defined(VEC_SIZE) && defined(DATA_TYPE) 94 /** @} */ // end of group elementwise_post_ops 95 96 /** Performs OPERAND1 = OP(OPERAND1, OPERAND2) 97 * @name ELTWISE_OP_ROW_n 98 * 99 * @param[in] OP The elementwise post op 100 * @param[in, out] OPERAND1 The basename of the destination and operand 1 variables 101 * @param[in] OPERAND2 The basename of the operand 2 variables 102 * @{ 103 */ 104 #define ELTWISE_OP_ROW_1(OP, OPERAND1, OPERAND2) \ 105 OPERAND1##0 = OP(OPERAND1##0, OPERAND2##0); 106 107 #define ELTWISE_OP_ROW_2(OP, OPERAND1, OPERAND2) \ 108 ELTWISE_OP_ROW_1(OP, OPERAND1, OPERAND2) \ 109 OPERAND1##1 = OP(OPERAND1##1, OPERAND2##1); 110 111 #define ELTWISE_OP_ROW_3(OP, OPERAND1, OPERAND2) \ 112 ELTWISE_OP_ROW_2(OP, OPERAND1, OPERAND2) \ 113 OPERAND1##2 = OP(OPERAND1##2, OPERAND2##2); 114 115 #define ELTWISE_OP_ROW_4(OP, OPERAND1, OPERAND2) \ 116 ELTWISE_OP_ROW_3(OP, OPERAND1, OPERAND2) \ 117 OPERAND1##3 = OP(OPERAND1##3, OPERAND2##3); 118 119 #define ELTWISE_OP_ROW_5(OP, OPERAND1, OPERAND2) \ 120 ELTWISE_OP_ROW_4(OP, OPERAND1, OPERAND2) \ 121 OPERAND1##4 = OP(OPERAND1##4, OPERAND2##4); 122 123 #define ELTWISE_OP_ROW_6(OP, OPERAND1, OPERAND2) \ 124 ELTWISE_OP_ROW_5(OP, OPERAND1, OPERAND2) \ 125 OPERAND1##5 = OP(OPERAND1##5, OPERAND2##5); 126 127 #define ELTWISE_OP_ROW_7(OP, OPERAND1, OPERAND2) \ 128 ELTWISE_OP_ROW_6(OP, OPERAND1, OPERAND2) \ 129 OPERAND1##6 = OP(OPERAND1##6, OPERAND2##6); 130 131 #define ELTWISE_OP_ROW_8(OP, OPERAND1, OPERAND2) \ 132 ELTWISE_OP_ROW_7(OP, OPERAND1, OPERAND2) \ 133 OPERAND1##7 = OP(OPERAND1##7, OPERAND2##7); 134 135 #define ELTWISE_OP_ROW_9(OP, OPERAND1, OPERAND2) \ 136 ELTWISE_OP_ROW_8(OP, OPERAND1, OPERAND2) \ 137 OPERAND1##8 = OP(OPERAND1##8, OPERAND2##8); 138 139 #define ELTWISE_OP_ROW_10(OP, OPERAND1, OPERAND2) \ 140 ELTWISE_OP_ROW_9(OP, OPERAND1, OPERAND2) \ 141 OPERAND1##9 = OP(OPERAND1##9, OPERAND2##9); 142 143 #define ELTWISE_OP_ROW_11(OP, OPERAND1, OPERAND2) \ 144 ELTWISE_OP_ROW_10(OP, OPERAND1, OPERAND2) \ 145 OPERAND1##A = OP(OPERAND1##A, OPERAND2##A); 146 147 #define ELTWISE_OP_ROW_12(OP, OPERAND1, OPERAND2) \ 148 ELTWISE_OP_ROW_11(OP, OPERAND1, OPERAND2) \ 149 OPERAND1##B = OP(OPERAND1##B, OPERAND2##B); 150 151 #define ELTWISE_OP_ROW_13(OP, OPERAND1, OPERAND2) \ 152 ELTWISE_OP_ROW_12(OP, OPERAND1, OPERAND2) \ 153 OPERAND1##C = OP(OPERAND1##C, OPERAND2##C); 154 155 #define ELTWISE_OP_ROW_14(OP, OPERAND1, OPERAND2) \ 156 ELTWISE_OP_ROW_13(OP, OPERAND1, OPERAND2) \ 157 OPERAND1##D = OP(OPERAND1##D, OPERAND2##D); 158 159 #define ELTWISE_OP_ROW_15(OP, OPERAND1, OPERAND2) \ 160 ELTWISE_OP_ROW_14(OP, OPERAND1, OPERAND2) \ 161 OPERAND1##E = OP(OPERAND1##E, OPERAND2##E); 162 163 #define ELTWISE_OP_ROW_16(OP, OPERAND1, OPERAND2) \ 164 ELTWISE_OP_ROW_15(OP, OPERAND1, OPERAND2) \ 165 OPERAND1##F = OP(OPERAND1##F, OPERAND2##F); 166 167 /** @} */ // end of group ELTWISE_OP_ROW_n 168 169 /** Performs OPERAND1 = OP(OPERAND1, OPERAND2) 170 * @name ELTWISE_OP_BLOCK 171 * 172 * Supported cases are N=1,2,3,...,16 173 * 174 * @param[in] OP The elementwise post op 175 * @param[in] N The number of vectors in the block 176 * @param[in] OPERAND1 The basename of the destination and operand 1 variables 177 * @param[in] OPERAND2 The basename of the operand 2 variables 178 * @{ 179 */ 180 #define ELTWISE_OP_BLOCK_STR(OP, N, OPERAND1, OPERAND2) ELTWISE_OP_ROW_##N(OP, OPERAND1, OPERAND2) 181 #define ELTWISE_OP_BLOCK(OP, N, OPERAND1, OPERAND2) ELTWISE_OP_BLOCK_STR(OP, N, OPERAND1, OPERAND2) 182 /** @} */ // end of group ELTWISE_OP_BLOCK 183 184 /** Performs OPERAND1 = OP(OPERAND1, OPERAND2) with broadcasting 185 * @name ELTWISE_OP_ROW_BROADCAST_n 186 * 187 * @param[in] OP The elementwise post op 188 * @param[in, out] OPERAND1 The basename of the destination and operand 1 variables 189 * @param[in] OPERAND2 The basename of the broadcast operand 2 variables 190 * @{ 191 */ 192 #define ELTWISE_OP_ROW_BROADCAST_1(OP, OPERAND1, OPERAND2) \ 193 OPERAND1##0 = OP(OPERAND1##0, OPERAND2); 194 195 #define ELTWISE_OP_ROW_BROADCAST_2(OP, OPERAND1, OPERAND2) \ 196 ELTWISE_OP_ROW_BROADCAST_1(OP, OPERAND1, OPERAND2) \ 197 OPERAND1##1 = OP(OPERAND1##1, OPERAND2); 198 199 #define ELTWISE_OP_ROW_BROADCAST_3(OP, OPERAND1, OPERAND2) \ 200 ELTWISE_OP_ROW_BROADCAST_2(OP, OPERAND1, OPERAND2) \ 201 OPERAND1##2 = OP(OPERAND1##2, OPERAND2); 202 203 #define ELTWISE_OP_ROW_BROADCAST_4(OP, OPERAND1, OPERAND2) \ 204 ELTWISE_OP_ROW_BROADCAST_3(OP, OPERAND1, OPERAND2) \ 205 OPERAND1##3 = OP(OPERAND1##3, OPERAND2); 206 207 #define ELTWISE_OP_ROW_BROADCAST_5(OP, OPERAND1, OPERAND2) \ 208 ELTWISE_OP_ROW_BROADCAST_4(OP, OPERAND1, OPERAND2) \ 209 OPERAND1##4 = OP(OPERAND1##4, OPERAND2); 210 211 #define ELTWISE_OP_ROW_BROADCAST_6(OP, OPERAND1, OPERAND2) \ 212 ELTWISE_OP_ROW_BROADCAST_5(OP, OPERAND1, OPERAND2) \ 213 OPERAND1##5 = OP(OPERAND1##5, OPERAND2); 214 215 #define ELTWISE_OP_ROW_BROADCAST_7(OP, OPERAND1, OPERAND2) \ 216 ELTWISE_OP_ROW_BROADCAST_6(OP, OPERAND1, OPERAND2) \ 217 OPERAND1##6 = OP(OPERAND1##6, OPERAND2); 218 219 #define ELTWISE_OP_ROW_BROADCAST_8(OP, OPERAND1, OPERAND2) \ 220 ELTWISE_OP_ROW_BROADCAST_7(OP, OPERAND1, OPERAND2) \ 221 OPERAND1##7 = OP(OPERAND1##7, OPERAND2); 222 223 #define ELTWISE_OP_ROW_BROADCAST_9(OP, OPERAND1, OPERAND2) \ 224 ELTWISE_OP_ROW_BROADCAST_8(OP, OPERAND1, OPERAND2) \ 225 OPERAND1##8 = OP(OPERAND1##8, OPERAND2); 226 227 #define ELTWISE_OP_ROW_BROADCAST_10(OP, OPERAND1, OPERAND2) \ 228 ELTWISE_OP_ROW_BROADCAST_9(OP, OPERAND1, OPERAND2) \ 229 OPERAND1##9 = OP(OPERAND1##9, OPERAND2); 230 231 #define ELTWISE_OP_ROW_BROADCAST_11(OP, OPERAND1, OPERAND2) \ 232 ELTWISE_OP_ROW_BROADCAST_10(OP, OPERAND1, OPERAND2) \ 233 OPERAND1##A = OP(OPERAND1##A, OPERAND2); 234 235 #define ELTWISE_OP_ROW_BROADCAST_12(OP, OPERAND1, OPERAND2) \ 236 ELTWISE_OP_ROW_BROADCAST_11(OP, OPERAND1, OPERAND2) \ 237 OPERAND1##B = OP(OPERAND1##B, OPERAND2); 238 239 #define ELTWISE_OP_ROW_BROADCAST_13(OP, OPERAND1, OPERAND2) \ 240 ELTWISE_OP_ROW_BROADCAST_12(OP, OPERAND1, OPERAND2) \ 241 OPERAND1##C = OP(OPERAND1##C, OPERAND2); 242 243 #define ELTWISE_OP_ROW_BROADCAST_14(OP, OPERAND1, OPERAND2) \ 244 ELTWISE_OP_ROW_BROADCAST_13(OP, OPERAND1, OPERAND2) \ 245 OPERAND1##D = OP(OPERAND1##D, OPERAND2); 246 247 #define ELTWISE_OP_ROW_BROADCAST_15(OP, OPERAND1, OPERAND2) \ 248 ELTWISE_OP_ROW_BROADCAST_14(OP, OPERAND1, OPERAND2) \ 249 OPERAND1##E = OP(OPERAND1##E, OPERAND2); 250 251 #define ELTWISE_OP_ROW_BROADCAST_16(OP, OPERAND1, OPERAND2) \ 252 ELTWISE_OP_ROW_BROADCAST_15(OP, OPERAND1, OPERAND2) \ 253 OPERAND1##F = OP(OPERAND1##F, OPERAND2); 254 255 /** @} */ // end of group ELTWISE_OP_ROW_BROADCAST_n 256 257 /** Performs OPERAND1 = OP(OPERAND1, OPERAND2) with broadcasting 258 * @name ELTWISE_OP_BLOCK_BROADCAST 259 * @note Only support: 260 * case 1 broadcast in Y dimension : Operand1 [YxX] + Operand2 [1xX]; 261 * case 2 broadcast in both Y and X dimensions : Operand1 [YxX] + Operand2 [1x1] (scalar); 262 * Does NOT support broad cast in X dimension: Operand1 [YxX] + Operand2 [Yx1]; 263 * 264 * Supported cases are N=1,2,3,...,16 265 * 266 * @param[in] OP The elementwise post op 267 * @param[in] N The number of vectors in the block 268 * @param[in] OPERAND1 The basename of the destination and operand 1 variables 269 * @param[in] OPERAND2 The basename of the operand 2 variables 270 * @{ 271 */ 272 #define ELTWISE_OP_BLOCK_BROADCAST_STR(OP, N, OPERAND1, OPERAND2) ELTWISE_OP_ROW_BROADCAST_##N(OP, OPERAND1, OPERAND2) 273 #define ELTWISE_OP_BLOCK_BROADCAST(OP, N, OPERAND1, OPERAND2) ELTWISE_OP_BLOCK_BROADCAST_STR(OP, N, OPERAND1, OPERAND2) 274 /** @} */ // end of group ELTWISE_OP_BLOCK_BROADCAST