1//===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines all of the AARCH64-specific intrinsics. 10// 11//===----------------------------------------------------------------------===// 12 13let TargetPrefix = "aarch64" in { 14 15def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty], 16 [IntrNoFree, IntrWillReturn]>; 17def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty], 18 [IntrNoFree, IntrWillReturn]>; 19def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty], 20 [IntrNoFree, IntrWillReturn]>; 21def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty], 22 [IntrNoFree, IntrWillReturn]>; 23 24def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty], 25 [IntrNoFree, IntrWillReturn]>; 26def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty], 27 [IntrNoFree, IntrWillReturn]>; 28def int_aarch64_stxp : Intrinsic<[llvm_i32_ty], 29 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], 30 [IntrNoFree, IntrWillReturn]>; 31def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty], 32 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty], 33 [IntrNoFree, IntrWillReturn]>; 34 35def int_aarch64_clrex : Intrinsic<[]>; 36 37def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 38 LLVMMatchType<0>], [IntrNoMem]>; 39def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 40 LLVMMatchType<0>], [IntrNoMem]>; 41 42def int_aarch64_fjcvtzs : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>; 43 44def int_aarch64_cls: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 45def int_aarch64_cls64: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>; 46 47def int_aarch64_frint32z 48 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 49 [ IntrNoMem ]>; 50def int_aarch64_frint64z 51 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 52 [ IntrNoMem ]>; 53def int_aarch64_frint32x 54 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 55 [ IntrNoMem ]>; 56def int_aarch64_frint64x 57 : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ], 58 [ IntrNoMem ]>; 59 60//===----------------------------------------------------------------------===// 61// HINT 62 63def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>; 64 65def int_aarch64_break : Intrinsic<[], [llvm_i32_ty], 66 [IntrNoMem, IntrHasSideEffects, IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>; 67 68 69def int_aarch64_prefetch : Intrinsic<[], 70 [llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty, llvm_i32_ty], 71 [IntrInaccessibleMemOrArgMemOnly, IntrWillReturn, ReadOnly<ArgIndex<0>>, 72 ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>> 73 ]>, 74 ClangBuiltin<"__builtin_arm_prefetch">; 75 76//===----------------------------------------------------------------------===// 77// Data Barrier Instructions 78 79def int_aarch64_dmb : ClangBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, 80 Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; 81def int_aarch64_dsb : ClangBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, 82 Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; 83def int_aarch64_isb : ClangBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, 84 Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>; 85 86// A space-consuming intrinsic primarily for testing block and jump table 87// placements. The first argument is the number of bytes this "instruction" 88// takes up, the second and return value are essentially chains, used to force 89// ordering during ISel. 90def int_aarch64_space : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>; 91 92} 93 94//===----------------------------------------------------------------------===// 95// Advanced SIMD (NEON) 96 97let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 98 class AdvSIMD_2Scalar_Float_Intrinsic 99 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 100 [IntrNoMem]>; 101 102 class AdvSIMD_FPToIntRounding_Intrinsic 103 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>; 104 105 class AdvSIMD_1IntArg_Intrinsic 106 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>; 107 class AdvSIMD_1FloatArg_Intrinsic 108 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; 109 class AdvSIMD_1VectorArg_Intrinsic 110 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>; 111 class AdvSIMD_1VectorArg_Expand_Intrinsic 112 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>; 113 class AdvSIMD_1VectorArg_Long_Intrinsic 114 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>; 115 class AdvSIMD_1IntArg_Narrow_Intrinsic 116 : DefaultAttrsIntrinsic<[llvm_any_ty], [llvm_any_ty], [IntrNoMem]>; 117 class AdvSIMD_1VectorArg_Narrow_Intrinsic 118 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>; 119 class AdvSIMD_1VectorArg_Int_Across_Intrinsic 120 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>; 121 class AdvSIMD_1VectorArg_Float_Across_Intrinsic 122 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>; 123 124 class AdvSIMD_2IntArg_Intrinsic 125 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 126 [IntrNoMem]>; 127 class AdvSIMD_2FloatArg_Intrinsic 128 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 129 [IntrNoMem]>; 130 class AdvSIMD_2VectorArg_Intrinsic 131 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 132 [IntrNoMem]>; 133 class AdvSIMD_2VectorArg_Compare_Intrinsic 134 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>], 135 [IntrNoMem]>; 136 class AdvSIMD_2Arg_FloatCompare_Intrinsic 137 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>], 138 [IntrNoMem]>; 139 class AdvSIMD_2VectorArg_Long_Intrinsic 140 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 141 [LLVMTruncatedType<0>, LLVMTruncatedType<0>], 142 [IntrNoMem]>; 143 class AdvSIMD_2VectorArg_Wide_Intrinsic 144 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 145 [LLVMMatchType<0>, LLVMTruncatedType<0>], 146 [IntrNoMem]>; 147 class AdvSIMD_2VectorArg_Narrow_Intrinsic 148 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 149 [LLVMExtendedType<0>, LLVMExtendedType<0>], 150 [IntrNoMem]>; 151 class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic 152 : DefaultAttrsIntrinsic<[llvm_anyint_ty], 153 [LLVMExtendedType<0>, llvm_i32_ty], 154 [IntrNoMem]>; 155 class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic 156 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 157 [llvm_anyvector_ty], 158 [IntrNoMem]>; 159 class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic 160 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 161 [LLVMTruncatedType<0>], 162 [IntrNoMem]>; 163 class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic 164 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 165 [LLVMTruncatedType<0>, llvm_i32_ty], 166 [IntrNoMem]>; 167 class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic 168 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 169 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty], 170 [IntrNoMem]>; 171 class AdvSIMD_2VectorArg_Lane_Intrinsic 172 : DefaultAttrsIntrinsic<[llvm_anyint_ty], 173 [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty], 174 [IntrNoMem]>; 175 176 class AdvSIMD_3IntArg_Intrinsic 177 : DefaultAttrsIntrinsic<[llvm_anyint_ty], 178 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 179 [IntrNoMem]>; 180 class AdvSIMD_3VectorArg_Intrinsic 181 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 182 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 183 [IntrNoMem]>; 184 class AdvSIMD_3VectorArg_Scalar_Intrinsic 185 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 186 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], 187 [IntrNoMem]>; 188 class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic 189 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 190 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, 191 LLVMMatchType<1>], [IntrNoMem]>; 192 class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic 193 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 194 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty], 195 [IntrNoMem]>; 196 class AdvSIMD_CvtFxToFP_Intrinsic 197 : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty], 198 [IntrNoMem]>; 199 class AdvSIMD_CvtFPToFx_Intrinsic 200 : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty], 201 [IntrNoMem]>; 202 203 class AdvSIMD_1Arg_Intrinsic 204 : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>; 205 206 class AdvSIMD_Dot_Intrinsic 207 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 208 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 209 [IntrNoMem]>; 210 211 class AdvSIMD_FP16FML_Intrinsic 212 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 213 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 214 [IntrNoMem]>; 215 216 class AdvSIMD_MatMul_Intrinsic 217 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 218 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 219 [IntrNoMem]>; 220 221 class AdvSIMD_FML_Intrinsic 222 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 223 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 224 [IntrNoMem]>; 225 226 class AdvSIMD_BF16FML_Intrinsic 227 : DefaultAttrsIntrinsic<[llvm_v4f32_ty], 228 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty], 229 [IntrNoMem]>; 230} 231 232// Arithmetic ops 233 234let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { 235 // Vector Add Across Lanes 236 def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 237 def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 238 def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 239 240 // Vector Long Add Across Lanes 241 def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 242 def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 243 244 // Vector Halving Add 245 def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic; 246 def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic; 247 248 // Vector Rounding Halving Add 249 def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic; 250 def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic; 251 252 // Vector Saturating Add 253 def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic; 254 def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic; 255 def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic; 256 def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic; 257 258 // Vector Add High-Half 259 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 260 // header is no longer supported. 261 def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 262 263 // Vector Rounding Add High-Half 264 def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 265 266 // Vector Saturating Doubling Multiply High 267 def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic; 268 def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 269 def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 270 271 // Vector Saturating Rounding Doubling Multiply High 272 def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic; 273 def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 274 def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 275 276 def int_aarch64_neon_sqrdmlah : AdvSIMD_3IntArg_Intrinsic; 277 def int_aarch64_neon_sqrdmlsh : AdvSIMD_3IntArg_Intrinsic; 278 279 // Vector Polynominal Multiply 280 def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic; 281 282 // Vector Long Multiply 283 def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic; 284 def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic; 285 def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic; 286 287 // 64-bit polynomial multiply really returns an i128, which is not legal. Fake 288 // it with a v16i8. 289 def int_aarch64_neon_pmull64 : 290 DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; 291 292 // Vector Extending Multiply 293 def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic { 294 let IntrProperties = [IntrNoMem, Commutative]; 295 } 296 297 // Vector Saturating Doubling Long Multiply 298 def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic; 299 def int_aarch64_neon_sqdmulls_scalar 300 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; 301 302 // Vector Halving Subtract 303 def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic; 304 def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic; 305 306 // Vector Saturating Subtract 307 def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic; 308 def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic; 309 310 // Vector Subtract High-Half 311 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 312 // header is no longer supported. 313 def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 314 315 // Vector Rounding Subtract High-Half 316 def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 317 318 // Vector Compare Absolute Greater-than-or-equal 319 def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic; 320 321 // Vector Compare Absolute Greater-than 322 def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic; 323 324 // Vector Absolute Difference 325 def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic; 326 def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic; 327 def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic; 328 329 // Scalar Absolute Difference 330 def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic; 331 332 // Vector Max 333 def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic; 334 def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic; 335 def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic; 336 def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic; 337 338 // Vector Max Across Lanes 339 def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 340 def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 341 def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 342 def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 343 344 // Vector Min 345 def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic; 346 def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic; 347 def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic; 348 def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic; 349 350 // Vector Min/Max Number 351 def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic; 352 def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic; 353 354 // Vector Min Across Lanes 355 def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 356 def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 357 def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 358 def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 359 360 // Pairwise Add 361 def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic; 362 def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic; 363 364 // Long Pairwise Add 365 // FIXME: In theory, we shouldn't need intrinsics for saddlp or 366 // uaddlp, but tblgen's type inference currently can't handle the 367 // pattern fragments this ends up generating. 368 def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 369 def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 370 371 // Folding Maximum 372 def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic; 373 def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic; 374 def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic; 375 376 // Folding Minimum 377 def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic; 378 def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic; 379 def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic; 380 381 // Reciprocal Estimate/Step 382 def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic; 383 def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic; 384 385 // Reciprocal Exponent 386 def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic; 387 388 // Vector Saturating Shift Left 389 def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic; 390 def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic; 391 392 // Vector Rounding Shift Left 393 def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic; 394 def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic; 395 396 // Vector Saturating Rounding Shift Left 397 def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic; 398 def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic; 399 400 // Vector Signed->Unsigned Shift Left by Constant 401 def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic; 402 403 // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant 404 def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 405 406 // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const 407 def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 408 409 // Vector Narrowing Shift Right by Constant 410 def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 411 def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 412 413 // Vector Rounding Narrowing Shift Right by Constant 414 def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 415 416 // Vector Rounding Narrowing Saturating Shift Right by Constant 417 def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 418 def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 419 420 // Vector Shift Left 421 def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic; 422 def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic; 423 424 // Vector Widening Shift Left by Constant 425 def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic; 426 def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 427 def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 428 429 // Vector Shift Right by Constant and Insert 430 def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic; 431 432 // Vector Shift Left by Constant and Insert 433 def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic; 434 435 // Vector Saturating Narrow 436 def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic; 437 def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic; 438 def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 439 def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 440 441 // Vector Saturating Extract and Unsigned Narrow 442 def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic; 443 def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic; 444 445 // Vector Absolute Value 446 def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic; 447 448 // Vector Saturating Absolute Value 449 def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic; 450 451 // Vector Saturating Negation 452 def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic; 453 454 // Vector Count Leading Sign Bits 455 def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic; 456 457 // Vector Reciprocal Estimate 458 def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic; 459 def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic; 460 461 // Vector Square Root Estimate 462 def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic; 463 def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic; 464 465 // Vector Conversions Between Half-Precision and Single-Precision. 466 def int_aarch64_neon_vcvtfp2hf 467 : DefaultAttrsIntrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 468 def int_aarch64_neon_vcvthf2fp 469 : DefaultAttrsIntrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>; 470 471 // Vector Conversions Between Floating-point and Fixed-point. 472 def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic; 473 def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic; 474 def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic; 475 def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic; 476 477 // Vector FP->Int Conversions 478 def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic; 479 def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic; 480 def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic; 481 def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic; 482 def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic; 483 def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic; 484 def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic; 485 def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic; 486 def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic; 487 def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic; 488 489 // v8.5-A Vector FP Rounding 490 def int_aarch64_neon_frint32x : AdvSIMD_1FloatArg_Intrinsic; 491 def int_aarch64_neon_frint32z : AdvSIMD_1FloatArg_Intrinsic; 492 def int_aarch64_neon_frint64x : AdvSIMD_1FloatArg_Intrinsic; 493 def int_aarch64_neon_frint64z : AdvSIMD_1FloatArg_Intrinsic; 494 495 // Scalar FP->Int conversions 496 497 // Vector FP Inexact Narrowing 498 def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic; 499 500 // Scalar FP Inexact Narrowing 501 def int_aarch64_sisd_fcvtxn : DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty], 502 [IntrNoMem]>; 503 504 // v8.2-A Dot Product 505 def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic; 506 def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic; 507 508 // v8.6-A Matrix Multiply Intrinsics 509 def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic; 510 def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic; 511 def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic; 512 def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic; 513 def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic; 514 def int_aarch64_neon_bfmmla 515 : DefaultAttrsIntrinsic<[llvm_v4f32_ty], 516 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty], 517 [IntrNoMem]>; 518 def int_aarch64_neon_bfmlalb : AdvSIMD_BF16FML_Intrinsic; 519 def int_aarch64_neon_bfmlalt : AdvSIMD_BF16FML_Intrinsic; 520 521 522 // v8.6-A Bfloat Intrinsics 523 def int_aarch64_neon_bfcvt 524 : DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>; 525 def int_aarch64_neon_bfcvtn 526 : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 527 def int_aarch64_neon_bfcvtn2 528 : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], 529 [llvm_v8bf16_ty, llvm_v4f32_ty], 530 [IntrNoMem]>; 531 532 // v8.2-A FP16 Fused Multiply-Add Long 533 def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic; 534 def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic; 535 def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic; 536 def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic; 537 538 // v8.3-A Floating-point complex add 539 def int_aarch64_neon_vcadd_rot90 : AdvSIMD_2VectorArg_Intrinsic; 540 def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic; 541 542 def int_aarch64_neon_vcmla_rot0 : AdvSIMD_3VectorArg_Intrinsic; 543 def int_aarch64_neon_vcmla_rot90 : AdvSIMD_3VectorArg_Intrinsic; 544 def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic; 545 def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic; 546} 547 548let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 549 class AdvSIMD_2Vector2Index_Intrinsic 550 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 551 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty], 552 [IntrNoMem]>; 553} 554 555// Vector element to element moves 556def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic; 557 558let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 559 class AdvSIMD_1Vec_Load_Intrinsic 560 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyptr_ty], 561 [IntrReadMem, IntrArgMemOnly]>; 562 class AdvSIMD_1Vec_Store_Lane_Intrinsic 563 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty], 564 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 565 566 class AdvSIMD_2Vec_Load_Intrinsic 567 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty], 568 [llvm_anyptr_ty], 569 [IntrReadMem, IntrArgMemOnly]>; 570 class AdvSIMD_2Vec_Load_Lane_Intrinsic 571 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>], 572 [LLVMMatchType<0>, llvm_anyvector_ty, 573 llvm_i64_ty, llvm_anyptr_ty], 574 [IntrReadMem, IntrArgMemOnly]>; 575 class AdvSIMD_2Vec_Store_Intrinsic 576 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 577 llvm_anyptr_ty], 578 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 579 class AdvSIMD_2Vec_Store_Lane_Intrinsic 580 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 581 llvm_i64_ty, llvm_anyptr_ty], 582 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 583 584 class AdvSIMD_3Vec_Load_Intrinsic 585 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty], 586 [llvm_anyptr_ty], 587 [IntrReadMem, IntrArgMemOnly]>; 588 class AdvSIMD_3Vec_Load_Lane_Intrinsic 589 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 590 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, 591 llvm_i64_ty, llvm_anyptr_ty], 592 [IntrReadMem, IntrArgMemOnly]>; 593 class AdvSIMD_3Vec_Store_Intrinsic 594 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 595 LLVMMatchType<0>, llvm_anyptr_ty], 596 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 597 class AdvSIMD_3Vec_Store_Lane_Intrinsic 598 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, 599 LLVMMatchType<0>, LLVMMatchType<0>, 600 llvm_i64_ty, llvm_anyptr_ty], 601 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 602 603 class AdvSIMD_4Vec_Load_Intrinsic 604 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 605 LLVMMatchType<0>, llvm_anyvector_ty], 606 [llvm_anyptr_ty], 607 [IntrReadMem, IntrArgMemOnly]>; 608 class AdvSIMD_4Vec_Load_Lane_Intrinsic 609 : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 610 LLVMMatchType<0>, LLVMMatchType<0>], 611 [LLVMMatchType<0>, LLVMMatchType<0>, 612 LLVMMatchType<0>, llvm_anyvector_ty, 613 llvm_i64_ty, llvm_anyptr_ty], 614 [IntrReadMem, IntrArgMemOnly]>; 615 class AdvSIMD_4Vec_Store_Intrinsic 616 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 617 LLVMMatchType<0>, LLVMMatchType<0>, 618 llvm_anyptr_ty], 619 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 620 class AdvSIMD_4Vec_Store_Lane_Intrinsic 621 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 622 LLVMMatchType<0>, LLVMMatchType<0>, 623 llvm_i64_ty, llvm_anyptr_ty], 624 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 625} 626 627// Memory ops 628 629def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic; 630def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic; 631def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic; 632 633def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic; 634def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic; 635def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic; 636 637def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic; 638def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic; 639def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic; 640 641def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic; 642def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic; 643def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic; 644 645def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic; 646def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic; 647def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic; 648 649def int_aarch64_neon_st2 : AdvSIMD_2Vec_Store_Intrinsic; 650def int_aarch64_neon_st3 : AdvSIMD_3Vec_Store_Intrinsic; 651def int_aarch64_neon_st4 : AdvSIMD_4Vec_Store_Intrinsic; 652 653def int_aarch64_neon_st2lane : AdvSIMD_2Vec_Store_Lane_Intrinsic; 654def int_aarch64_neon_st3lane : AdvSIMD_3Vec_Store_Lane_Intrinsic; 655def int_aarch64_neon_st4lane : AdvSIMD_4Vec_Store_Lane_Intrinsic; 656 657let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 658 class AdvSIMD_Tbl1_Intrinsic 659 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>], 660 [IntrNoMem]>; 661 class AdvSIMD_Tbl2_Intrinsic 662 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 663 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>; 664 class AdvSIMD_Tbl3_Intrinsic 665 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 666 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 667 LLVMMatchType<0>], 668 [IntrNoMem]>; 669 class AdvSIMD_Tbl4_Intrinsic 670 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 671 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 672 LLVMMatchType<0>], 673 [IntrNoMem]>; 674 675 class AdvSIMD_Tbx1_Intrinsic 676 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 677 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>], 678 [IntrNoMem]>; 679 class AdvSIMD_Tbx2_Intrinsic 680 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 681 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 682 LLVMMatchType<0>], 683 [IntrNoMem]>; 684 class AdvSIMD_Tbx3_Intrinsic 685 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 686 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 687 llvm_v16i8_ty, LLVMMatchType<0>], 688 [IntrNoMem]>; 689 class AdvSIMD_Tbx4_Intrinsic 690 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 691 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 692 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], 693 [IntrNoMem]>; 694} 695def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic; 696def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic; 697def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic; 698def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic; 699 700def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic; 701def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic; 702def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic; 703def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic; 704 705let TargetPrefix = "aarch64" in { 706 class FPENV_Get_Intrinsic 707 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>; 708 class FPENV_Set_Intrinsic 709 : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>; 710 class RNDR_Intrinsic 711 : DefaultAttrsIntrinsic<[llvm_i64_ty, llvm_i1_ty], [], [IntrNoMem, IntrHasSideEffects]>; 712} 713 714// FP environment registers. 715def int_aarch64_get_fpcr : FPENV_Get_Intrinsic; 716def int_aarch64_set_fpcr : FPENV_Set_Intrinsic; 717def int_aarch64_get_fpsr : FPENV_Get_Intrinsic; 718def int_aarch64_set_fpsr : FPENV_Set_Intrinsic; 719 720// Armv8.5-A Random number generation intrinsics 721def int_aarch64_rndr : RNDR_Intrinsic; 722def int_aarch64_rndrrs : RNDR_Intrinsic; 723 724let TargetPrefix = "aarch64" in { 725 class Crypto_AES_DataKey_Intrinsic 726 : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; 727 728 class Crypto_AES_Data_Intrinsic 729 : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; 730 731 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 732 // (v4i32). 733 class Crypto_SHA_5Hash4Schedule_Intrinsic 734 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty], 735 [IntrNoMem]>; 736 737 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 738 // (v4i32). 739 class Crypto_SHA_1Hash_Intrinsic 740 : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 741 742 // SHA intrinsic taking 8 words of the schedule 743 class Crypto_SHA_8Schedule_Intrinsic 744 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; 745 746 // SHA intrinsic taking 12 words of the schedule 747 class Crypto_SHA_12Schedule_Intrinsic 748 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 749 [IntrNoMem]>; 750 751 // SHA intrinsic taking 8 words of the hash and 4 of the schedule. 752 class Crypto_SHA_8Hash4Schedule_Intrinsic 753 : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 754 [IntrNoMem]>; 755 756 // SHA512 intrinsic taking 2 arguments 757 class Crypto_SHA512_2Arg_Intrinsic 758 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>; 759 760 // SHA512 intrinsic taking 3 Arguments 761 class Crypto_SHA512_3Arg_Intrinsic 762 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty], 763 [IntrNoMem]>; 764 765 // SHA3 Intrinsics taking 3 arguments 766 class Crypto_SHA3_3Arg_Intrinsic 767 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 768 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 769 [IntrNoMem]>; 770 771 // SHA3 Intrinsic taking 2 arguments 772 class Crypto_SHA3_2Arg_Intrinsic 773 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], 774 [IntrNoMem]>; 775 776 // SHA3 Intrinsic taking 3 Arguments 1 immediate 777 class Crypto_SHA3_2ArgImm_Intrinsic 778 : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i64_ty], 779 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 780 781 class Crypto_SM3_3Vector_Intrinsic 782 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 783 [IntrNoMem]>; 784 785 class Crypto_SM3_3VectorIndexed_Intrinsic 786 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i64_ty], 787 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 788 789 class Crypto_SM4_2Vector_Intrinsic 790 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; 791} 792 793// AES 794def int_aarch64_crypto_aese : Crypto_AES_DataKey_Intrinsic; 795def int_aarch64_crypto_aesd : Crypto_AES_DataKey_Intrinsic; 796def int_aarch64_crypto_aesmc : Crypto_AES_Data_Intrinsic; 797def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic; 798 799// SHA1 800def int_aarch64_crypto_sha1c : Crypto_SHA_5Hash4Schedule_Intrinsic; 801def int_aarch64_crypto_sha1p : Crypto_SHA_5Hash4Schedule_Intrinsic; 802def int_aarch64_crypto_sha1m : Crypto_SHA_5Hash4Schedule_Intrinsic; 803def int_aarch64_crypto_sha1h : Crypto_SHA_1Hash_Intrinsic; 804 805def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic; 806def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic; 807 808// SHA256 809def int_aarch64_crypto_sha256h : Crypto_SHA_8Hash4Schedule_Intrinsic; 810def int_aarch64_crypto_sha256h2 : Crypto_SHA_8Hash4Schedule_Intrinsic; 811def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic; 812def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic; 813 814//SHA3 815def int_aarch64_crypto_eor3s : Crypto_SHA3_3Arg_Intrinsic; 816def int_aarch64_crypto_eor3u : Crypto_SHA3_3Arg_Intrinsic; 817def int_aarch64_crypto_bcaxs : Crypto_SHA3_3Arg_Intrinsic; 818def int_aarch64_crypto_bcaxu : Crypto_SHA3_3Arg_Intrinsic; 819def int_aarch64_crypto_rax1 : Crypto_SHA3_2Arg_Intrinsic; 820def int_aarch64_crypto_xar : Crypto_SHA3_2ArgImm_Intrinsic; 821 822// SHA512 823def int_aarch64_crypto_sha512h : Crypto_SHA512_3Arg_Intrinsic; 824def int_aarch64_crypto_sha512h2 : Crypto_SHA512_3Arg_Intrinsic; 825def int_aarch64_crypto_sha512su0 : Crypto_SHA512_2Arg_Intrinsic; 826def int_aarch64_crypto_sha512su1 : Crypto_SHA512_3Arg_Intrinsic; 827 828//SM3 & SM4 829def int_aarch64_crypto_sm3partw1 : Crypto_SM3_3Vector_Intrinsic; 830def int_aarch64_crypto_sm3partw2 : Crypto_SM3_3Vector_Intrinsic; 831def int_aarch64_crypto_sm3ss1 : Crypto_SM3_3Vector_Intrinsic; 832def int_aarch64_crypto_sm3tt1a : Crypto_SM3_3VectorIndexed_Intrinsic; 833def int_aarch64_crypto_sm3tt1b : Crypto_SM3_3VectorIndexed_Intrinsic; 834def int_aarch64_crypto_sm3tt2a : Crypto_SM3_3VectorIndexed_Intrinsic; 835def int_aarch64_crypto_sm3tt2b : Crypto_SM3_3VectorIndexed_Intrinsic; 836def int_aarch64_crypto_sm4e : Crypto_SM4_2Vector_Intrinsic; 837def int_aarch64_crypto_sm4ekey : Crypto_SM4_2Vector_Intrinsic; 838 839//===----------------------------------------------------------------------===// 840// CRC32 841 842let TargetPrefix = "aarch64" in { 843 844def int_aarch64_crc32b : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 845 [IntrNoMem]>; 846def int_aarch64_crc32cb : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 847 [IntrNoMem]>; 848def int_aarch64_crc32h : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 849 [IntrNoMem]>; 850def int_aarch64_crc32ch : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 851 [IntrNoMem]>; 852def int_aarch64_crc32w : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 853 [IntrNoMem]>; 854def int_aarch64_crc32cw : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 855 [IntrNoMem]>; 856def int_aarch64_crc32x : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 857 [IntrNoMem]>; 858def int_aarch64_crc32cx : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 859 [IntrNoMem]>; 860} 861 862//===----------------------------------------------------------------------===// 863// Memory Tagging Extensions (MTE) Intrinsics 864let TargetPrefix = "aarch64" in { 865def int_aarch64_irg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 866 [IntrNoMem, IntrHasSideEffects]>; 867def int_aarch64_addg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 868 [IntrNoMem]>; 869def int_aarch64_gmi : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], 870 [IntrNoMem]>; 871def int_aarch64_ldg : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty], 872 [IntrReadMem]>; 873def int_aarch64_stg : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], 874 [IntrWriteMem]>; 875def int_aarch64_subp : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty], 876 [IntrNoMem]>; 877 878// The following are codegen-only intrinsics for stack instrumentation. 879 880// Generate a randomly tagged stack base pointer. 881def int_aarch64_irg_sp : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_i64_ty], 882 [IntrNoMem, IntrHasSideEffects]>; 883 884// Transfer pointer tag with offset. 885// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where 886// * address is the address in ptr0 887// * tag is a function of (tag in baseptr, tag_offset). 888// ** Beware, this is not the same function as implemented by the ADDG instruction! 889// Backend optimizations may change tag_offset; the only guarantee is that calls 890// to tagp with the same pair of (baseptr, tag_offset) will produce pointers 891// with the same tag value, assuming the set of excluded tags has not changed. 892// Address bits in baseptr and tag bits in ptr0 are ignored. 893// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as 894// ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset 895// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp. 896def int_aarch64_tagp : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty], 897 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 898 899// Update allocation tags for the memory range to match the tag in the pointer argument. 900def int_aarch64_settag : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 901 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 902 903// Update allocation tags for the memory range to match the tag in the pointer argument, 904// and set memory contents to zero. 905def int_aarch64_settag_zero : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 906 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 907 908// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values. 909def int_aarch64_stgp : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], 910 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 911} 912 913//===----------------------------------------------------------------------===// 914// Memory Operations (MOPS) Intrinsics 915let TargetPrefix = "aarch64" in { 916 // Sizes are chosen to correspond to the llvm.memset intrinsic: ptr, i8, i64 917 def int_aarch64_mops_memset_tag : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i8_ty, llvm_i64_ty], 918 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 919} 920 921// Transactional Memory Extension (TME) Intrinsics 922let TargetPrefix = "aarch64" in { 923def int_aarch64_tstart : ClangBuiltin<"__builtin_arm_tstart">, 924 Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>; 925 926def int_aarch64_tcommit : ClangBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>; 927 928def int_aarch64_tcancel : ClangBuiltin<"__builtin_arm_tcancel">, 929 Intrinsic<[], [llvm_i64_ty], [IntrWillReturn, ImmArg<ArgIndex<0>>]>; 930 931def int_aarch64_ttest : ClangBuiltin<"__builtin_arm_ttest">, 932 Intrinsic<[llvm_i64_ty], [], 933 [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>; 934 935// Armv8.7-A load/store 64-byte intrinsics 936defvar data512 = !listsplat(llvm_i64_ty, 8); 937def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>; 938def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>; 939def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>; 940def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>; 941 942} 943 944def llvm_nxv1i1_ty : LLVMType<nxv1i1>; 945def llvm_nxv2i1_ty : LLVMType<nxv2i1>; 946def llvm_nxv4i1_ty : LLVMType<nxv4i1>; 947def llvm_nxv8i1_ty : LLVMType<nxv8i1>; 948def llvm_nxv16i1_ty : LLVMType<nxv16i1>; 949def llvm_nxv16i8_ty : LLVMType<nxv16i8>; 950def llvm_nxv4i32_ty : LLVMType<nxv4i32>; 951def llvm_nxv2i64_ty : LLVMType<nxv2i64>; 952def llvm_nxv8f16_ty : LLVMType<nxv8f16>; 953def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>; 954def llvm_nxv4f32_ty : LLVMType<nxv4f32>; 955def llvm_nxv2f64_ty : LLVMType<nxv2f64>; 956 957let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 958 959 class AdvSIMD_1Vec_PredLoad_Intrinsic 960 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 961 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 962 [IntrReadMem, IntrArgMemOnly]>; 963 964 class AdvSIMD_2Vec_PredLoad_Intrinsic 965 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 966 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 967 [IntrReadMem, IntrArgMemOnly]>; 968 969 class AdvSIMD_3Vec_PredLoad_Intrinsic 970 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>], 971 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 972 [IntrReadMem, IntrArgMemOnly]>; 973 974 class AdvSIMD_4Vec_PredLoad_Intrinsic 975 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 976 LLVMMatchType<0>], 977 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 978 [IntrReadMem, IntrArgMemOnly]>; 979 980 class AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic 981 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 982 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 983 [IntrInaccessibleMemOrArgMemOnly]>; 984 985 class AdvSIMD_1Vec_PredStore_Intrinsic 986 : DefaultAttrsIntrinsic<[], 987 [llvm_anyvector_ty, 988 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 989 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 990 991 class AdvSIMD_2Vec_PredStore_Intrinsic 992 : DefaultAttrsIntrinsic<[], 993 [llvm_anyvector_ty, LLVMMatchType<0>, 994 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 995 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 996 997 class AdvSIMD_3Vec_PredStore_Intrinsic 998 : DefaultAttrsIntrinsic<[], 999 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 1000 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 1001 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 1002 1003 class AdvSIMD_4Vec_PredStore_Intrinsic 1004 : DefaultAttrsIntrinsic<[], 1005 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 1006 LLVMMatchType<0>, 1007 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_ptr_ty], 1008 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 1009 1010 class AdvSIMD_SVE_Index_Intrinsic 1011 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1012 [LLVMVectorElementType<0>, 1013 LLVMVectorElementType<0>], 1014 [IntrNoMem]>; 1015 1016 class AdvSIMD_Merged1VectorArg_Intrinsic 1017 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1018 [LLVMMatchType<0>, 1019 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1020 LLVMMatchType<0>], 1021 [IntrNoMem]>; 1022 1023 class AdvSIMD_2VectorArgIndexed_Intrinsic 1024 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1025 [LLVMMatchType<0>, 1026 LLVMMatchType<0>, 1027 llvm_i32_ty], 1028 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1029 1030 class AdvSIMD_3VectorArgIndexed_Intrinsic 1031 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1032 [LLVMMatchType<0>, 1033 LLVMMatchType<0>, 1034 LLVMMatchType<0>, 1035 llvm_i32_ty], 1036 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1037 1038 class AdvSIMD_Pred1VectorArg_Intrinsic 1039 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1040 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1041 LLVMMatchType<0>], 1042 [IntrNoMem]>; 1043 1044 class AdvSIMD_Pred2VectorArg_Intrinsic 1045 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1046 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1047 LLVMMatchType<0>, 1048 LLVMMatchType<0>], 1049 [IntrNoMem]>; 1050 1051 class AdvSIMD_Pred3VectorArg_Intrinsic 1052 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1053 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1054 LLVMMatchType<0>, 1055 LLVMMatchType<0>, 1056 LLVMMatchType<0>], 1057 [IntrNoMem]>; 1058 1059 class AdvSIMD_SVE_Compare_Intrinsic 1060 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1061 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1062 llvm_anyvector_ty, 1063 LLVMMatchType<0>], 1064 [IntrNoMem]>; 1065 1066 class AdvSIMD_SVE_CompareWide_Intrinsic 1067 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1068 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1069 llvm_anyvector_ty, 1070 llvm_nxv2i64_ty], 1071 [IntrNoMem]>; 1072 1073 class AdvSIMD_SVE_Saturating_Intrinsic 1074 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1075 [LLVMMatchType<0>, 1076 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1077 [IntrNoMem]>; 1078 1079 class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic 1080 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1081 [LLVMMatchType<0>, 1082 llvm_i32_ty, 1083 llvm_i32_ty], 1084 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 1085 1086 class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T> 1087 : DefaultAttrsIntrinsic<[T], 1088 [T, llvm_anyvector_ty], 1089 [IntrNoMem]>; 1090 1091 class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T> 1092 : DefaultAttrsIntrinsic<[T], 1093 [T, llvm_i32_ty, llvm_i32_ty], 1094 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 1095 1096 class AdvSIMD_SVE_CNT_Intrinsic 1097 : DefaultAttrsIntrinsic<[LLVMVectorOfBitcastsToInt<0>], 1098 [LLVMVectorOfBitcastsToInt<0>, 1099 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1100 llvm_anyvector_ty], 1101 [IntrNoMem]>; 1102 1103 class AdvSIMD_SVE_ReduceWithInit_Intrinsic 1104 : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], 1105 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1106 LLVMVectorElementType<0>, 1107 llvm_anyvector_ty], 1108 [IntrNoMem]>; 1109 1110 class AdvSIMD_SVE_ShiftByImm_Intrinsic 1111 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1112 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1113 LLVMMatchType<0>, 1114 llvm_i32_ty], 1115 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1116 1117 class AdvSIMD_SVE_ShiftWide_Intrinsic 1118 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1119 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1120 LLVMMatchType<0>, 1121 llvm_nxv2i64_ty], 1122 [IntrNoMem]>; 1123 1124 class AdvSIMD_SVE_Unpack_Intrinsic 1125 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1126 [LLVMSubdivide2VectorType<0>], 1127 [IntrNoMem]>; 1128 1129 class AdvSIMD_SVE_CADD_Intrinsic 1130 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1131 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1132 LLVMMatchType<0>, 1133 LLVMMatchType<0>, 1134 llvm_i32_ty], 1135 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1136 1137 class AdvSIMD_SVE_CMLA_Intrinsic 1138 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1139 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1140 LLVMMatchType<0>, 1141 LLVMMatchType<0>, 1142 LLVMMatchType<0>, 1143 llvm_i32_ty], 1144 [IntrNoMem, ImmArg<ArgIndex<4>>]>; 1145 1146 class AdvSIMD_SVE_CMLA_LANE_Intrinsic 1147 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1148 [LLVMMatchType<0>, 1149 LLVMMatchType<0>, 1150 LLVMMatchType<0>, 1151 llvm_i32_ty, 1152 llvm_i32_ty], 1153 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1154 1155 class AdvSIMD_SVE_DUP_Intrinsic 1156 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1157 [LLVMMatchType<0>, 1158 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1159 LLVMVectorElementType<0>], 1160 [IntrNoMem]>; 1161 1162 class AdvSIMD_SVE_DUP_Unpred_Intrinsic 1163 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>], 1164 [IntrNoMem]>; 1165 1166 class AdvSIMD_SVE_DUPQ_Intrinsic 1167 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1168 [LLVMMatchType<0>, 1169 llvm_i64_ty], 1170 [IntrNoMem]>; 1171 1172 class AdvSIMD_SVE_EXPA_Intrinsic 1173 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1174 [LLVMVectorOfBitcastsToInt<0>], 1175 [IntrNoMem]>; 1176 1177 class AdvSIMD_SVE_FCVT_Intrinsic 1178 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1179 [LLVMMatchType<0>, 1180 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1181 llvm_anyvector_ty], 1182 [IntrNoMem]>; 1183 1184 class AdvSIMD_SVE_FCVTZS_Intrinsic 1185 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1186 [LLVMVectorOfBitcastsToInt<0>, 1187 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1188 llvm_anyvector_ty], 1189 [IntrNoMem]>; 1190 1191 class AdvSIMD_SVE_INSR_Intrinsic 1192 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1193 [LLVMMatchType<0>, 1194 LLVMVectorElementType<0>], 1195 [IntrNoMem]>; 1196 1197 class AdvSIMD_SVE_PTRUE_Intrinsic 1198 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1199 [llvm_i32_ty], 1200 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1201 1202 class AdvSIMD_SVE_PUNPKHI_Intrinsic 1203 : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType<0>], 1204 [llvm_anyvector_ty], 1205 [IntrNoMem]>; 1206 1207 class AdvSIMD_SVE_SCALE_Intrinsic 1208 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1209 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1210 LLVMMatchType<0>, 1211 LLVMVectorOfBitcastsToInt<0>], 1212 [IntrNoMem]>; 1213 1214 class AdvSIMD_SVE_SCVTF_Intrinsic 1215 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1216 [LLVMMatchType<0>, 1217 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1218 llvm_anyvector_ty], 1219 [IntrNoMem]>; 1220 1221 class AdvSIMD_SVE_TSMUL_Intrinsic 1222 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1223 [LLVMMatchType<0>, 1224 LLVMVectorOfBitcastsToInt<0>], 1225 [IntrNoMem]>; 1226 1227 class AdvSIMD_SVE_CNTB_Intrinsic 1228 : DefaultAttrsIntrinsic<[llvm_i64_ty], 1229 [llvm_i32_ty], 1230 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1231 1232 class AdvSIMD_SVE_CNTP_Intrinsic 1233 : DefaultAttrsIntrinsic<[llvm_i64_ty], 1234 [llvm_anyvector_ty, LLVMMatchType<0>], 1235 [IntrNoMem]>; 1236 1237 class AdvSIMD_SVE_DOT_Intrinsic 1238 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1239 [LLVMMatchType<0>, 1240 LLVMSubdivide4VectorType<0>, 1241 LLVMSubdivide4VectorType<0>], 1242 [IntrNoMem]>; 1243 1244 class AdvSIMD_SVE_DOT_Indexed_Intrinsic 1245 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1246 [LLVMMatchType<0>, 1247 LLVMSubdivide4VectorType<0>, 1248 LLVMSubdivide4VectorType<0>, 1249 llvm_i32_ty], 1250 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1251 1252 class AdvSIMD_SVE_PTEST_Intrinsic 1253 : DefaultAttrsIntrinsic<[llvm_i1_ty], 1254 [llvm_anyvector_ty, 1255 LLVMMatchType<0>], 1256 [IntrNoMem]>; 1257 1258 class AdvSIMD_SVE_TBL_Intrinsic 1259 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1260 [LLVMMatchType<0>, 1261 LLVMVectorOfBitcastsToInt<0>], 1262 [IntrNoMem]>; 1263 1264 class AdvSIMD_SVE2_TBX_Intrinsic 1265 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1266 [LLVMMatchType<0>, 1267 LLVMMatchType<0>, 1268 LLVMVectorOfBitcastsToInt<0>], 1269 [IntrNoMem]>; 1270 1271 class SVE2_1VectorArg_Long_Intrinsic 1272 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1273 [LLVMSubdivide2VectorType<0>, 1274 llvm_i32_ty], 1275 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1276 1277 class SVE2_2VectorArg_Long_Intrinsic 1278 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1279 [LLVMSubdivide2VectorType<0>, 1280 LLVMSubdivide2VectorType<0>], 1281 [IntrNoMem]>; 1282 1283 class SVE2_2VectorArgIndexed_Long_Intrinsic 1284 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1285 [LLVMSubdivide2VectorType<0>, 1286 LLVMSubdivide2VectorType<0>, 1287 llvm_i32_ty], 1288 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1289 1290 class SVE2_2VectorArg_Wide_Intrinsic 1291 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1292 [LLVMMatchType<0>, 1293 LLVMSubdivide2VectorType<0>], 1294 [IntrNoMem]>; 1295 1296 class SVE2_2VectorArg_Pred_Long_Intrinsic 1297 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1298 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1299 LLVMMatchType<0>, 1300 LLVMSubdivide2VectorType<0>], 1301 [IntrNoMem]>; 1302 1303 class SVE2_3VectorArg_Long_Intrinsic 1304 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1305 [LLVMMatchType<0>, 1306 LLVMSubdivide2VectorType<0>, 1307 LLVMSubdivide2VectorType<0>], 1308 [IntrNoMem]>; 1309 1310 class SVE2_3VectorArgIndexed_Long_Intrinsic 1311 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1312 [LLVMMatchType<0>, 1313 LLVMSubdivide2VectorType<0>, 1314 LLVMSubdivide2VectorType<0>, 1315 llvm_i32_ty], 1316 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1317 1318 class SVE2_1VectorArg_Narrowing_Intrinsic 1319 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1320 [llvm_anyvector_ty], 1321 [IntrNoMem]>; 1322 1323 class SVE2_Merged1VectorArg_Narrowing_Intrinsic 1324 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1325 [LLVMSubdivide2VectorType<0>, 1326 llvm_anyvector_ty], 1327 [IntrNoMem]>; 1328 class SVE2_2VectorArg_Narrowing_Intrinsic 1329 : DefaultAttrsIntrinsic< 1330 [LLVMSubdivide2VectorType<0>], 1331 [llvm_anyvector_ty, LLVMMatchType<0>], 1332 [IntrNoMem]>; 1333 1334 class SVE2_Merged2VectorArg_Narrowing_Intrinsic 1335 : DefaultAttrsIntrinsic< 1336 [LLVMSubdivide2VectorType<0>], 1337 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>], 1338 [IntrNoMem]>; 1339 1340 class SVE2_1VectorArg_Imm_Narrowing_Intrinsic 1341 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1342 [llvm_anyvector_ty, llvm_i32_ty], 1343 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1344 1345 class SVE2_2VectorArg_Imm_Narrowing_Intrinsic 1346 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 1347 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, 1348 llvm_i32_ty], 1349 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1350 1351 class SVE2_CONFLICT_DETECT_Intrinsic 1352 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1353 [llvm_anyptr_ty, LLVMMatchType<1>], 1354 [IntrNoMem]>; 1355 1356 class SVE2_3VectorArg_Indexed_Intrinsic 1357 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1358 [LLVMMatchType<0>, 1359 LLVMSubdivide2VectorType<0>, 1360 LLVMSubdivide2VectorType<0>, 1361 llvm_i32_ty], 1362 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1363 1364 class SVE2_1VectorArgIndexed_Intrinsic 1365 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1366 [LLVMMatchType<0>, 1367 llvm_i32_ty], 1368 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1369 1370 class AdvSIMD_SVE_CDOT_LANE_Intrinsic 1371 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1372 [LLVMMatchType<0>, 1373 LLVMSubdivide4VectorType<0>, 1374 LLVMSubdivide4VectorType<0>, 1375 llvm_i32_ty, 1376 llvm_i32_ty], 1377 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1378 1379 class SVE2_1VectorArg_Pred_Intrinsic 1380 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1381 [llvm_anyvector_ty], 1382 [IntrNoMem]>; 1383 1384 class SVE2_1VectorArgIndexed_Pred_Intrinsic 1385 : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1386 [llvm_anyvector_ty, llvm_i32_ty], 1387 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1388 1389 class SVE2_Pred_1VectorArgIndexed_Intrinsic 1390 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1391 [LLVMMatchType<0>, 1392 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i32_ty], 1393 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1394 1395 class SVE2_Pred_1VectorArg_Intrinsic 1396 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1397 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 1398 [IntrNoMem]>; 1399 1400 // NOTE: There is no relationship between these intrinsics beyond an attempt 1401 // to reuse currently identical class definitions. 1402 class AdvSIMD_SVE_LOGB_Intrinsic : AdvSIMD_SVE_CNT_Intrinsic; 1403 class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic; 1404 class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic; 1405 1406 // This class of intrinsics are not intended to be useful within LLVM IR but 1407 // are instead here to support some of the more regid parts of the ACLE. 1408 class Builtin_SVCVT<LLVMType OUT, LLVMType PRED, LLVMType IN> 1409 : DefaultAttrsIntrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>; 1410} 1411 1412//===----------------------------------------------------------------------===// 1413// SVE 1414 1415let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 1416 1417class AdvSIMD_SVE_2SVBoolArg_Intrinsic 1418 : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], 1419 [llvm_nxv16i1_ty], 1420 [IntrNoMem]>; 1421 1422class AdvSIMD_SVE_3SVBoolArg_Intrinsic 1423 : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], 1424 [llvm_nxv16i1_ty, llvm_nxv16i1_ty], 1425 [IntrNoMem]>; 1426 1427class AdvSIMD_SVE_Reduce_Intrinsic 1428 : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>], 1429 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1430 llvm_anyvector_ty], 1431 [IntrNoMem]>; 1432 1433class AdvSIMD_SVE_V128_Reduce_Intrinsic 1434 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1435 [LLVMScalarOrSameVectorWidth<1, llvm_i1_ty>, 1436 llvm_anyvector_ty], 1437 [IntrNoMem]>; 1438 1439 1440class AdvSIMD_SVE_SADDV_Reduce_Intrinsic 1441 : DefaultAttrsIntrinsic<[llvm_i64_ty], 1442 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1443 llvm_anyvector_ty], 1444 [IntrNoMem]>; 1445 1446class AdvSIMD_SVE_WHILE_Intrinsic 1447 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1448 [llvm_anyint_ty, LLVMMatchType<1>], 1449 [IntrNoMem]>; 1450 1451class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic 1452 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1453 [ 1454 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1455 llvm_ptr_ty, 1456 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1457 ], 1458 [IntrReadMem, IntrArgMemOnly]>; 1459 1460class AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic 1461 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1462 [ 1463 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1464 llvm_ptr_ty, 1465 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1466 ], 1467 [IntrInaccessibleMemOrArgMemOnly]>; 1468 1469class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic 1470 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1471 [ 1472 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1473 llvm_ptr_ty, 1474 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1475 ], 1476 [IntrReadMem, IntrArgMemOnly]>; 1477 1478class AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic 1479 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1480 [ 1481 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1482 llvm_ptr_ty, 1483 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1484 ], 1485 [IntrInaccessibleMemOrArgMemOnly]>; 1486 1487class AdvSIMD_GatherLoad_VS_Intrinsic 1488 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1489 [ 1490 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1491 llvm_anyvector_ty, 1492 llvm_i64_ty 1493 ], 1494 [IntrReadMem]>; 1495 1496class AdvSIMD_GatherLoadQ_VS_Intrinsic 1497 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1498 [ 1499 llvm_nxv1i1_ty, 1500 llvm_anyvector_ty, 1501 llvm_i64_ty 1502 ], 1503 [IntrReadMem]>; 1504 1505class AdvSIMD_GatherLoadQ_SV_Intrinsic 1506 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1507 [ 1508 llvm_nxv1i1_ty, 1509 llvm_ptr_ty, 1510 llvm_nxv2i64_ty 1511 ], 1512 [IntrReadMem, IntrArgMemOnly]>; 1513 1514class AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic 1515 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1516 [ 1517 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1518 llvm_anyvector_ty, 1519 llvm_i64_ty 1520 ], 1521 [IntrInaccessibleMemOrArgMemOnly]>; 1522 1523class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic 1524 : DefaultAttrsIntrinsic<[], 1525 [ 1526 llvm_anyvector_ty, 1527 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1528 llvm_ptr_ty, 1529 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1530 ], 1531 [IntrWriteMem, IntrArgMemOnly]>; 1532 1533class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic 1534 : DefaultAttrsIntrinsic<[], 1535 [ 1536 llvm_anyvector_ty, 1537 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1538 llvm_ptr_ty, 1539 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1540 ], 1541 [IntrWriteMem, IntrArgMemOnly]>; 1542 1543class AdvSIMD_ScatterStore_VS_Intrinsic 1544 : DefaultAttrsIntrinsic<[], 1545 [ 1546 llvm_anyvector_ty, 1547 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1548 llvm_anyvector_ty, llvm_i64_ty 1549 ], 1550 [IntrWriteMem]>; 1551 1552class AdvSIMD_ScatterStoreQ_VS_Intrinsic 1553 : DefaultAttrsIntrinsic<[], 1554 [ 1555 llvm_anyvector_ty, 1556 llvm_nxv1i1_ty, 1557 llvm_anyvector_ty, 1558 llvm_i64_ty 1559 ], 1560 [IntrWriteMem]>; 1561 1562class AdvSIMD_ScatterStoreQ_SV_Intrinsic 1563 : DefaultAttrsIntrinsic<[], 1564 [ 1565 llvm_anyvector_ty, 1566 llvm_nxv1i1_ty, 1567 llvm_ptr_ty, 1568 llvm_nxv2i64_ty 1569 ], 1570 [IntrWriteMem, IntrArgMemOnly]>; 1571 1572class SVE_gather_prf_SV 1573 : DefaultAttrsIntrinsic<[], 1574 [ 1575 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1576 llvm_ptr_ty, // Base address 1577 llvm_anyvector_ty, // Offsets 1578 llvm_i32_ty // Prfop 1579 ], 1580 [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>; 1581 1582class SVE_gather_prf_VS 1583 : DefaultAttrsIntrinsic<[], 1584 [ 1585 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1586 llvm_anyvector_ty, // Base addresses 1587 llvm_i64_ty, // Scalar offset 1588 llvm_i32_ty // Prfop 1589 ], 1590 [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>; 1591 1592class SVE_MatMul_Intrinsic 1593 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 1594 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>], 1595 [IntrNoMem]>; 1596 1597class SVE_4Vec_BF16 1598 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 1599 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty], 1600 [IntrNoMem]>; 1601 1602class SVE_4Vec_BF16_Indexed 1603 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 1604 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i32_ty], 1605 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1606 1607// 1608// Loads 1609// 1610 1611def int_aarch64_sve_ld1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1612 1613def int_aarch64_sve_ld2_sret : AdvSIMD_2Vec_PredLoad_Intrinsic; 1614def int_aarch64_sve_ld3_sret : AdvSIMD_3Vec_PredLoad_Intrinsic; 1615def int_aarch64_sve_ld4_sret : AdvSIMD_4Vec_PredLoad_Intrinsic; 1616 1617def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1618def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic; 1619def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic; 1620 1621def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic; 1622def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic; 1623 1624// 1625// Stores 1626// 1627 1628def int_aarch64_sve_st1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1629def int_aarch64_sve_st2 : AdvSIMD_2Vec_PredStore_Intrinsic; 1630def int_aarch64_sve_st3 : AdvSIMD_3Vec_PredStore_Intrinsic; 1631def int_aarch64_sve_st4 : AdvSIMD_4Vec_PredStore_Intrinsic; 1632 1633def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1634 1635// 1636// Prefetches 1637// 1638 1639def int_aarch64_sve_prf 1640 : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty], 1641 [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>; 1642 1643// Scalar + 32-bit scaled offset vector, zero extend, packed and 1644// unpacked. 1645def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV; 1646def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV; 1647def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV; 1648def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV; 1649 1650// Scalar + 32-bit scaled offset vector, sign extend, packed and 1651// unpacked. 1652def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV; 1653def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV; 1654def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV; 1655def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV; 1656 1657// Scalar + 64-bit scaled offset vector. 1658def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV; 1659def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV; 1660def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV; 1661def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV; 1662 1663// Vector + scalar. 1664def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS; 1665def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS; 1666def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS; 1667def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS; 1668 1669// 1670// Scalar to vector operations 1671// 1672 1673def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic; 1674def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic; 1675 1676def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic; 1677 1678// 1679// Address calculation 1680// 1681 1682def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic; 1683def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic; 1684def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic; 1685def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic; 1686 1687// 1688// Integer arithmetic 1689// 1690 1691def int_aarch64_sve_add : AdvSIMD_Pred2VectorArg_Intrinsic; 1692def int_aarch64_sve_add_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1693def int_aarch64_sve_sub : AdvSIMD_Pred2VectorArg_Intrinsic; 1694def int_aarch64_sve_sub_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1695def int_aarch64_sve_subr : AdvSIMD_Pred2VectorArg_Intrinsic; 1696 1697def int_aarch64_sve_pmul : AdvSIMD_2VectorArg_Intrinsic; 1698 1699def int_aarch64_sve_mul : AdvSIMD_Pred2VectorArg_Intrinsic; 1700def int_aarch64_sve_mul_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1701def int_aarch64_sve_mul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 1702def int_aarch64_sve_smulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1703def int_aarch64_sve_smulh_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1704def int_aarch64_sve_umulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1705def int_aarch64_sve_umulh_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1706 1707def int_aarch64_sve_sdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1708def int_aarch64_sve_sdiv_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1709def int_aarch64_sve_udiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1710def int_aarch64_sve_udiv_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1711def int_aarch64_sve_sdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1712def int_aarch64_sve_udivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1713 1714def int_aarch64_sve_smax : AdvSIMD_Pred2VectorArg_Intrinsic; 1715def int_aarch64_sve_smax_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1716def int_aarch64_sve_umax : AdvSIMD_Pred2VectorArg_Intrinsic; 1717def int_aarch64_sve_umax_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1718def int_aarch64_sve_smin : AdvSIMD_Pred2VectorArg_Intrinsic; 1719def int_aarch64_sve_smin_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1720def int_aarch64_sve_umin : AdvSIMD_Pred2VectorArg_Intrinsic; 1721def int_aarch64_sve_umin_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1722def int_aarch64_sve_sabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1723def int_aarch64_sve_sabd_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1724def int_aarch64_sve_uabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1725def int_aarch64_sve_uabd_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1726 1727def int_aarch64_sve_mad : AdvSIMD_Pred3VectorArg_Intrinsic; 1728def int_aarch64_sve_msb : AdvSIMD_Pred3VectorArg_Intrinsic; 1729def int_aarch64_sve_mla : AdvSIMD_Pred3VectorArg_Intrinsic; 1730def int_aarch64_sve_mla_u : AdvSIMD_Pred3VectorArg_Intrinsic; 1731def int_aarch64_sve_mla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1732def int_aarch64_sve_mls : AdvSIMD_Pred3VectorArg_Intrinsic; 1733def int_aarch64_sve_mls_u : AdvSIMD_Pred3VectorArg_Intrinsic; 1734def int_aarch64_sve_mls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1735 1736def int_aarch64_sve_saddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1737def int_aarch64_sve_uaddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1738 1739def int_aarch64_sve_smaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1740def int_aarch64_sve_umaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1741def int_aarch64_sve_sminv : AdvSIMD_SVE_Reduce_Intrinsic; 1742def int_aarch64_sve_uminv : AdvSIMD_SVE_Reduce_Intrinsic; 1743 1744def int_aarch64_sve_orv : AdvSIMD_SVE_Reduce_Intrinsic; 1745def int_aarch64_sve_eorv : AdvSIMD_SVE_Reduce_Intrinsic; 1746def int_aarch64_sve_andv : AdvSIMD_SVE_Reduce_Intrinsic; 1747 1748def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic; 1749def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic; 1750 1751def int_aarch64_sve_sdot : AdvSIMD_SVE_DOT_Intrinsic; 1752def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1753 1754def int_aarch64_sve_udot : AdvSIMD_SVE_DOT_Intrinsic; 1755def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1756 1757def int_aarch64_sve_sqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1758def int_aarch64_sve_sqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1759def int_aarch64_sve_uqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1760def int_aarch64_sve_uqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1761 1762def int_aarch64_sve_orqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1763def int_aarch64_sve_eorqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1764def int_aarch64_sve_andqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1765def int_aarch64_sve_addqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1766def int_aarch64_sve_smaxqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1767def int_aarch64_sve_umaxqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1768def int_aarch64_sve_sminqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1769def int_aarch64_sve_uminqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 1770 1771 1772// Shifts 1773 1774def int_aarch64_sve_asr : AdvSIMD_Pred2VectorArg_Intrinsic; 1775def int_aarch64_sve_asr_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1776def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1777def int_aarch64_sve_asrd : AdvSIMD_SVE_ShiftByImm_Intrinsic; 1778def int_aarch64_sve_insr : AdvSIMD_SVE_INSR_Intrinsic; 1779def int_aarch64_sve_lsl : AdvSIMD_Pred2VectorArg_Intrinsic; 1780def int_aarch64_sve_lsl_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1781def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1782def int_aarch64_sve_lsr : AdvSIMD_Pred2VectorArg_Intrinsic; 1783def int_aarch64_sve_lsr_u : AdvSIMD_Pred2VectorArg_Intrinsic; 1784def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1785 1786// 1787// Integer comparisons 1788// 1789 1790def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic; 1791def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic; 1792def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic; 1793def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic; 1794def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic; 1795def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic; 1796 1797def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1798def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1799def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1800def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1801def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1802def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1803def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1804def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1805def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1806def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1807 1808// 1809// Counting bits 1810// 1811 1812def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic; 1813def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic; 1814def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic; 1815 1816// 1817// Counting elements 1818// 1819 1820def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic; 1821def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic; 1822def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic; 1823def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic; 1824 1825def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic; 1826 1827// 1828// FFR manipulation 1829// 1830 1831def int_aarch64_sve_rdffr : ClangBuiltin<"__builtin_sve_svrdffr">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [], [IntrReadMem, IntrInaccessibleMemOnly]>; 1832def int_aarch64_sve_rdffr_z : ClangBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty], [IntrReadMem, IntrInaccessibleMemOnly]>; 1833def int_aarch64_sve_setffr : ClangBuiltin<"__builtin_sve_svsetffr">, DefaultAttrsIntrinsic<[], [], [IntrWriteMem, IntrInaccessibleMemOnly]>; 1834def int_aarch64_sve_wrffr : ClangBuiltin<"__builtin_sve_svwrffr">, DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>; 1835 1836// 1837// Saturating scalar arithmetic 1838// 1839 1840def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1841def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1842def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1843def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1844 1845def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1846def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1847def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1848def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1849def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1850def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1851def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1852def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1853def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1854def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1855 1856def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1857def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1858def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1859def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1860 1861def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1862def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1863def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1864def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1865def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1866def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1867def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1868def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1869def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1870def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1871 1872def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1873def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1874def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1875def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1876 1877def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1878def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1879def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1880def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1881def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1882def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1883def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1884def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1885def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1886def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1887 1888def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1889def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1890def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1891def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1892 1893def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1894def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1895def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1896def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1897def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1898def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1899def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1900def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1901def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1902def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1903 1904// 1905// Reversal 1906// 1907 1908def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic; 1909def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic; 1910def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic; 1911def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic; 1912 1913// 1914// Permutations and selection 1915// 1916 1917def int_aarch64_sve_clasta : AdvSIMD_Pred2VectorArg_Intrinsic; 1918def int_aarch64_sve_clasta_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1919def int_aarch64_sve_clastb : AdvSIMD_Pred2VectorArg_Intrinsic; 1920def int_aarch64_sve_clastb_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1921def int_aarch64_sve_compact : AdvSIMD_Pred1VectorArg_Intrinsic; 1922def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic; 1923def int_aarch64_sve_dup_laneq : SVE2_1VectorArgIndexed_Intrinsic; 1924def int_aarch64_sve_ext : AdvSIMD_2VectorArgIndexed_Intrinsic; 1925def int_aarch64_sve_sel : AdvSIMD_Pred2VectorArg_Intrinsic; 1926def int_aarch64_sve_lasta : AdvSIMD_SVE_Reduce_Intrinsic; 1927def int_aarch64_sve_lastb : AdvSIMD_SVE_Reduce_Intrinsic; 1928def int_aarch64_sve_rev : AdvSIMD_1VectorArg_Intrinsic; 1929def int_aarch64_sve_rev_b16 : AdvSIMD_SVE_2SVBoolArg_Intrinsic; 1930def int_aarch64_sve_rev_b32 : AdvSIMD_SVE_2SVBoolArg_Intrinsic; 1931def int_aarch64_sve_rev_b64 : AdvSIMD_SVE_2SVBoolArg_Intrinsic; 1932def int_aarch64_sve_splice : AdvSIMD_Pred2VectorArg_Intrinsic; 1933def int_aarch64_sve_sunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1934def int_aarch64_sve_sunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1935def int_aarch64_sve_tbl : AdvSIMD_SVE_TBL_Intrinsic; 1936def int_aarch64_sve_trn1 : AdvSIMD_2VectorArg_Intrinsic; 1937def int_aarch64_sve_trn1_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1938def int_aarch64_sve_trn1_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1939def int_aarch64_sve_trn1_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1940def int_aarch64_sve_trn2 : AdvSIMD_2VectorArg_Intrinsic; 1941def int_aarch64_sve_trn2_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1942def int_aarch64_sve_trn2_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1943def int_aarch64_sve_trn2_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1944def int_aarch64_sve_trn1q : AdvSIMD_2VectorArg_Intrinsic; 1945def int_aarch64_sve_trn2q : AdvSIMD_2VectorArg_Intrinsic; 1946def int_aarch64_sve_uunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1947def int_aarch64_sve_uunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1948def int_aarch64_sve_uzp1 : AdvSIMD_2VectorArg_Intrinsic; 1949def int_aarch64_sve_uzp1_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1950def int_aarch64_sve_uzp1_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1951def int_aarch64_sve_uzp1_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1952def int_aarch64_sve_uzp2 : AdvSIMD_2VectorArg_Intrinsic; 1953def int_aarch64_sve_uzp2_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1954def int_aarch64_sve_uzp2_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1955def int_aarch64_sve_uzp2_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1956def int_aarch64_sve_uzp1q : AdvSIMD_2VectorArg_Intrinsic; 1957def int_aarch64_sve_uzp2q : AdvSIMD_2VectorArg_Intrinsic; 1958def int_aarch64_sve_zip1 : AdvSIMD_2VectorArg_Intrinsic; 1959def int_aarch64_sve_zip1_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1960def int_aarch64_sve_zip1_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1961def int_aarch64_sve_zip1_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1962def int_aarch64_sve_zip2 : AdvSIMD_2VectorArg_Intrinsic; 1963def int_aarch64_sve_zip2_b16 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1964def int_aarch64_sve_zip2_b32 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1965def int_aarch64_sve_zip2_b64 : AdvSIMD_SVE_3SVBoolArg_Intrinsic; 1966def int_aarch64_sve_zip1q : AdvSIMD_2VectorArg_Intrinsic; 1967def int_aarch64_sve_zip2q : AdvSIMD_2VectorArg_Intrinsic; 1968 1969// 1970// Logical operations 1971// 1972 1973def int_aarch64_sve_and : AdvSIMD_Pred2VectorArg_Intrinsic; 1974def int_aarch64_sve_and_u: AdvSIMD_Pred2VectorArg_Intrinsic; 1975def int_aarch64_sve_bic : AdvSIMD_Pred2VectorArg_Intrinsic; 1976def int_aarch64_sve_bic_u: AdvSIMD_Pred2VectorArg_Intrinsic; 1977def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic; 1978def int_aarch64_sve_eor : AdvSIMD_Pred2VectorArg_Intrinsic; 1979def int_aarch64_sve_eor_u: AdvSIMD_Pred2VectorArg_Intrinsic; 1980def int_aarch64_sve_not : AdvSIMD_Merged1VectorArg_Intrinsic; 1981def int_aarch64_sve_orr : AdvSIMD_Pred2VectorArg_Intrinsic; 1982def int_aarch64_sve_orr_u: AdvSIMD_Pred2VectorArg_Intrinsic; 1983 1984// 1985// Conversion 1986// 1987 1988def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 1989def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic; 1990def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 1991def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 1992def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic; 1993def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 1994 1995// 1996// While comparisons 1997// 1998 1999def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic; 2000def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic; 2001def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic; 2002def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic; 2003def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic; 2004def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic; 2005def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic; 2006def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic; 2007 2008// 2009// Floating-point arithmetic 2010// 2011 2012def int_aarch64_sve_fabd : AdvSIMD_Pred2VectorArg_Intrinsic; 2013def int_aarch64_sve_fabd_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2014def int_aarch64_sve_fabs : AdvSIMD_Merged1VectorArg_Intrinsic; 2015def int_aarch64_sve_fadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2016def int_aarch64_sve_fadd_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2017def int_aarch64_sve_fcadd : AdvSIMD_SVE_CADD_Intrinsic; 2018def int_aarch64_sve_fcmla : AdvSIMD_SVE_CMLA_Intrinsic; 2019def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2020def int_aarch64_sve_fdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 2021def int_aarch64_sve_fdiv_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2022def int_aarch64_sve_fdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 2023def int_aarch64_sve_fexpa_x : AdvSIMD_SVE_EXPA_Intrinsic; 2024def int_aarch64_sve_fmad : AdvSIMD_Pred3VectorArg_Intrinsic; 2025def int_aarch64_sve_fmax : AdvSIMD_Pred2VectorArg_Intrinsic; 2026def int_aarch64_sve_fmax_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2027def int_aarch64_sve_fmaxnm : AdvSIMD_Pred2VectorArg_Intrinsic; 2028def int_aarch64_sve_fmaxnm_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2029def int_aarch64_sve_fmin : AdvSIMD_Pred2VectorArg_Intrinsic; 2030def int_aarch64_sve_fmin_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2031def int_aarch64_sve_fminnm : AdvSIMD_Pred2VectorArg_Intrinsic; 2032def int_aarch64_sve_fminnm_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2033def int_aarch64_sve_fmla : AdvSIMD_Pred3VectorArg_Intrinsic; 2034def int_aarch64_sve_fmla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2035def int_aarch64_sve_fmla_u : AdvSIMD_Pred3VectorArg_Intrinsic; 2036def int_aarch64_sve_fmls : AdvSIMD_Pred3VectorArg_Intrinsic; 2037def int_aarch64_sve_fmls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2038def int_aarch64_sve_fmls_u : AdvSIMD_Pred3VectorArg_Intrinsic; 2039def int_aarch64_sve_fmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 2040def int_aarch64_sve_fmul : AdvSIMD_Pred2VectorArg_Intrinsic; 2041def int_aarch64_sve_fmul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2042def int_aarch64_sve_fmul_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2043def int_aarch64_sve_fmulx : AdvSIMD_Pred2VectorArg_Intrinsic; 2044def int_aarch64_sve_fmulx_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2045def int_aarch64_sve_fneg : AdvSIMD_Merged1VectorArg_Intrinsic; 2046def int_aarch64_sve_fnmad : AdvSIMD_Pred3VectorArg_Intrinsic; 2047def int_aarch64_sve_fnmla : AdvSIMD_Pred3VectorArg_Intrinsic; 2048def int_aarch64_sve_fnmla_u : AdvSIMD_Pred3VectorArg_Intrinsic; 2049def int_aarch64_sve_fnmls : AdvSIMD_Pred3VectorArg_Intrinsic; 2050def int_aarch64_sve_fnmls_u : AdvSIMD_Pred3VectorArg_Intrinsic; 2051def int_aarch64_sve_fnmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 2052def int_aarch64_sve_frecpe_x : AdvSIMD_1VectorArg_Intrinsic; 2053def int_aarch64_sve_frecps_x : AdvSIMD_2VectorArg_Intrinsic; 2054def int_aarch64_sve_frecpx : AdvSIMD_Merged1VectorArg_Intrinsic; 2055def int_aarch64_sve_frinta : AdvSIMD_Merged1VectorArg_Intrinsic; 2056def int_aarch64_sve_frinti : AdvSIMD_Merged1VectorArg_Intrinsic; 2057def int_aarch64_sve_frintm : AdvSIMD_Merged1VectorArg_Intrinsic; 2058def int_aarch64_sve_frintn : AdvSIMD_Merged1VectorArg_Intrinsic; 2059def int_aarch64_sve_frintp : AdvSIMD_Merged1VectorArg_Intrinsic; 2060def int_aarch64_sve_frintx : AdvSIMD_Merged1VectorArg_Intrinsic; 2061def int_aarch64_sve_frintz : AdvSIMD_Merged1VectorArg_Intrinsic; 2062def int_aarch64_sve_frsqrte_x : AdvSIMD_1VectorArg_Intrinsic; 2063def int_aarch64_sve_frsqrts_x : AdvSIMD_2VectorArg_Intrinsic; 2064def int_aarch64_sve_fscale : AdvSIMD_SVE_SCALE_Intrinsic; 2065def int_aarch64_sve_fsqrt : AdvSIMD_Merged1VectorArg_Intrinsic; 2066def int_aarch64_sve_fsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2067def int_aarch64_sve_fsub_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2068def int_aarch64_sve_fsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2069def int_aarch64_sve_ftmad_x : AdvSIMD_2VectorArgIndexed_Intrinsic; 2070def int_aarch64_sve_ftsmul_x : AdvSIMD_SVE_TSMUL_Intrinsic; 2071def int_aarch64_sve_ftssel_x : AdvSIMD_SVE_TSMUL_Intrinsic; 2072 2073// 2074// Floating-point reductions 2075// 2076 2077def int_aarch64_sve_fadda : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 2078def int_aarch64_sve_faddv : AdvSIMD_SVE_Reduce_Intrinsic; 2079def int_aarch64_sve_fmaxv : AdvSIMD_SVE_Reduce_Intrinsic; 2080def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic; 2081def int_aarch64_sve_fminv : AdvSIMD_SVE_Reduce_Intrinsic; 2082def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic; 2083 2084def int_aarch64_sve_faddqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2085def int_aarch64_sve_fmaxnmqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2086def int_aarch64_sve_fminnmqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2087def int_aarch64_sve_fmaxqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2088def int_aarch64_sve_fminqv : AdvSIMD_SVE_V128_Reduce_Intrinsic; 2089 2090// 2091// Floating-point conversions 2092// 2093 2094def int_aarch64_sve_fcvt : AdvSIMD_SVE_FCVT_Intrinsic; 2095def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic; 2096def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic; 2097def int_aarch64_sve_scvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 2098def int_aarch64_sve_ucvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 2099 2100// 2101// Floating-point comparisons 2102// 2103 2104def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic; 2105def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic; 2106 2107def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic; 2108def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic; 2109def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic; 2110def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic; 2111def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic; 2112 2113def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2114def int_aarch64_sve_fcvtzs_i32f64 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2115def int_aarch64_sve_fcvtzs_i64f16 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 2116def int_aarch64_sve_fcvtzs_i64f32 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2117 2118def int_aarch64_sve_fcvt_bf16f32 : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 2119def int_aarch64_sve_fcvtnt_bf16f32 : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 2120 2121def int_aarch64_sve_fcvtzu_i32f16 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2122def int_aarch64_sve_fcvtzu_i32f64 : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2123def int_aarch64_sve_fcvtzu_i64f16 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 2124def int_aarch64_sve_fcvtzu_i64f32 : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2125 2126def int_aarch64_sve_fcvt_f16f32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 2127def int_aarch64_sve_fcvt_f16f64 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2128def int_aarch64_sve_fcvt_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2129 2130def int_aarch64_sve_fcvt_f32f16 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2131def int_aarch64_sve_fcvt_f64f16 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 2132def int_aarch64_sve_fcvt_f64f32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2133 2134def int_aarch64_sve_fcvtlt_f32f16 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 2135def int_aarch64_sve_fcvtlt_f64f32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 2136def int_aarch64_sve_fcvtnt_f16f32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 2137def int_aarch64_sve_fcvtnt_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2138 2139def int_aarch64_sve_fcvtx_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2140def int_aarch64_sve_fcvtxnt_f32f64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 2141 2142def int_aarch64_sve_scvtf_f16i32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 2143def int_aarch64_sve_scvtf_f16i64 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2144def int_aarch64_sve_scvtf_f32i64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2145def int_aarch64_sve_scvtf_f64i32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 2146 2147def int_aarch64_sve_ucvtf_f16i32 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 2148def int_aarch64_sve_ucvtf_f16i64 : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2149def int_aarch64_sve_ucvtf_f32i64 : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 2150def int_aarch64_sve_ucvtf_f64i32 : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 2151 2152// 2153// Predicate creation 2154// 2155 2156def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic; 2157 2158// 2159// Predicate operations 2160// 2161 2162def int_aarch64_sve_and_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2163def int_aarch64_sve_bic_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2164def int_aarch64_sve_brka : AdvSIMD_Merged1VectorArg_Intrinsic; 2165def int_aarch64_sve_brka_z : AdvSIMD_Pred1VectorArg_Intrinsic; 2166def int_aarch64_sve_brkb : AdvSIMD_Merged1VectorArg_Intrinsic; 2167def int_aarch64_sve_brkb_z : AdvSIMD_Pred1VectorArg_Intrinsic; 2168def int_aarch64_sve_brkn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2169def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2170def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2171def int_aarch64_sve_eor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2172def int_aarch64_sve_nand_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2173def int_aarch64_sve_nor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2174def int_aarch64_sve_orn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2175def int_aarch64_sve_orr_z : AdvSIMD_Pred2VectorArg_Intrinsic; 2176def int_aarch64_sve_pfirst : AdvSIMD_Pred1VectorArg_Intrinsic; 2177def int_aarch64_sve_pnext : AdvSIMD_Pred1VectorArg_Intrinsic; 2178def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic; 2179def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic; 2180 2181// 2182// Testing predicates 2183// 2184 2185def int_aarch64_sve_ptest_any : AdvSIMD_SVE_PTEST_Intrinsic; 2186def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic; 2187def int_aarch64_sve_ptest_last : AdvSIMD_SVE_PTEST_Intrinsic; 2188 2189// 2190// Reinterpreting data 2191// 2192 2193def int_aarch64_sve_convert_from_svbool : DefaultAttrsIntrinsic<[llvm_any_ty], 2194 [llvm_nxv16i1_ty], 2195 [IntrNoMem]>; 2196 2197def int_aarch64_sve_convert_to_svbool : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], 2198 [llvm_any_ty], 2199 [IntrNoMem]>; 2200 2201// 2202// Gather loads: scalar base + vector offsets 2203// 2204 2205// 64 bit unscaled offsets 2206def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2207 2208// 64 bit scaled offsets 2209def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2210 2211// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2212def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2213def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2214 2215// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2216def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2217def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2218 2219// 128-bit loads, scaled offsets (indices) 2220def int_aarch64_sve_ld1q_gather_index : AdvSIMD_GatherLoadQ_SV_Intrinsic; 2221 2222// 128-bit loads, unscaled offsets 2223def int_aarch64_sve_ld1q_gather_vector_offset : AdvSIMD_GatherLoadQ_SV_Intrinsic; 2224 2225// 2226// Gather loads: vector base + scalar offset 2227// 2228 2229def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 2230 2231// 128-bit loads, unscaled offsets 2232def int_aarch64_sve_ld1q_gather_scalar_offset : AdvSIMD_GatherLoadQ_VS_Intrinsic; 2233 2234// 2235// First-faulting gather loads: scalar base + vector offsets 2236// 2237 2238// 64 bit unscaled offsets 2239def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic; 2240 2241// 64 bit scaled offsets 2242def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic; 2243 2244// 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 2245def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2246def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2247 2248// 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 2249def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2250def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic; 2251 2252// 2253// First-faulting gather loads: vector base + scalar offset 2254// 2255 2256def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic; 2257 2258 2259// 2260// Non-temporal gather loads: scalar base + vector offsets 2261// 2262 2263// 64 bit unscaled offsets 2264def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2265 2266// 64 bit indices 2267def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 2268 2269// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 2270def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 2271 2272// 2273// Non-temporal gather loads: vector base + scalar offset 2274// 2275 2276def int_aarch64_sve_ldnt1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 2277 2278// 2279// Scatter stores: scalar base + vector offsets 2280// 2281 2282// 64 bit unscaled offsets 2283def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2284 2285// 64 bit scaled offsets 2286def int_aarch64_sve_st1_scatter_index 2287 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2288 2289// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2290def int_aarch64_sve_st1_scatter_sxtw 2291 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2292 2293def int_aarch64_sve_st1_scatter_uxtw 2294 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2295 2296// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 2297def int_aarch64_sve_st1_scatter_sxtw_index 2298 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2299 2300def int_aarch64_sve_st1_scatter_uxtw_index 2301 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2302 2303// 128-bit stores, scaled offsets (indices) 2304def int_aarch64_sve_st1q_scatter_index : AdvSIMD_ScatterStoreQ_SV_Intrinsic; 2305 2306// 128-bit stores, unscaled offsets 2307def int_aarch64_sve_st1q_scatter_vector_offset : AdvSIMD_ScatterStoreQ_SV_Intrinsic; 2308 2309// 2310// Scatter stores: vector base + scalar offset 2311// 2312 2313def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 2314 2315// 128-bit stores, unscaled offsets 2316def int_aarch64_sve_st1q_scatter_scalar_offset : AdvSIMD_ScatterStoreQ_VS_Intrinsic; 2317 2318// 2319// Non-temporal scatter stores: scalar base + vector offsets 2320// 2321 2322// 64 bit unscaled offsets 2323def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2324 2325// 64 bit indices 2326def int_aarch64_sve_stnt1_scatter_index 2327 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2328 2329// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 2330def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2331 2332// 2333// Non-temporal scatter stores: vector base + scalar offset 2334// 2335 2336def int_aarch64_sve_stnt1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 2337 2338// 2339// SVE2 - Uniform DSP operations 2340// 2341 2342def int_aarch64_sve_saba : AdvSIMD_3VectorArg_Intrinsic; 2343def int_aarch64_sve_shadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2344def int_aarch64_sve_shsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2345def int_aarch64_sve_shsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2346def int_aarch64_sve_sli : AdvSIMD_2VectorArgIndexed_Intrinsic; 2347def int_aarch64_sve_sqabs : AdvSIMD_Merged1VectorArg_Intrinsic; 2348def int_aarch64_sve_sqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2349def int_aarch64_sve_sqdmulh : AdvSIMD_2VectorArg_Intrinsic; 2350def int_aarch64_sve_sqdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2351def int_aarch64_sve_sqneg : AdvSIMD_Merged1VectorArg_Intrinsic; 2352def int_aarch64_sve_sqrdmlah : AdvSIMD_3VectorArg_Intrinsic; 2353def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2354def int_aarch64_sve_sqrdmlsh : AdvSIMD_3VectorArg_Intrinsic; 2355def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2356def int_aarch64_sve_sqrdmulh : AdvSIMD_2VectorArg_Intrinsic; 2357def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2358def int_aarch64_sve_sqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2359def int_aarch64_sve_sqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2360def int_aarch64_sve_sqshlu : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2361def int_aarch64_sve_sqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2362def int_aarch64_sve_sqsub_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2363def int_aarch64_sve_sqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2364def int_aarch64_sve_srhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2365def int_aarch64_sve_sri : AdvSIMD_2VectorArgIndexed_Intrinsic; 2366def int_aarch64_sve_srshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2367def int_aarch64_sve_srshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2368def int_aarch64_sve_srsra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2369def int_aarch64_sve_ssra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2370def int_aarch64_sve_suqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2371def int_aarch64_sve_uaba : AdvSIMD_3VectorArg_Intrinsic; 2372def int_aarch64_sve_uhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2373def int_aarch64_sve_uhsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2374def int_aarch64_sve_uhsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2375def int_aarch64_sve_uqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2376def int_aarch64_sve_uqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2377def int_aarch64_sve_uqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2378def int_aarch64_sve_uqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2379def int_aarch64_sve_uqsub_u : AdvSIMD_Pred2VectorArg_Intrinsic; 2380def int_aarch64_sve_uqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2381def int_aarch64_sve_urecpe : AdvSIMD_Merged1VectorArg_Intrinsic; 2382def int_aarch64_sve_urhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2383def int_aarch64_sve_urshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2384def int_aarch64_sve_urshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2385def int_aarch64_sve_ursqrte : AdvSIMD_Merged1VectorArg_Intrinsic; 2386def int_aarch64_sve_ursra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2387def int_aarch64_sve_usqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2388def int_aarch64_sve_usra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2389 2390// 2391// SVE2 - Widening DSP operations 2392// 2393 2394def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic; 2395def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic; 2396def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic; 2397def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic; 2398def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic; 2399def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic; 2400def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic; 2401def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic; 2402def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic; 2403def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic; 2404def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic; 2405def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic; 2406def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic; 2407def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic; 2408def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic; 2409def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic; 2410def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic; 2411def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic; 2412def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic; 2413def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic; 2414def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic; 2415def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic; 2416def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic; 2417def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic; 2418def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic; 2419def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic; 2420def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic; 2421def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic; 2422 2423// 2424// SVE2 - Non-widening pairwise arithmetic 2425// 2426 2427def int_aarch64_sve_addp : AdvSIMD_Pred2VectorArg_Intrinsic; 2428def int_aarch64_sve_faddp : AdvSIMD_Pred2VectorArg_Intrinsic; 2429def int_aarch64_sve_fmaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2430def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2431def int_aarch64_sve_fminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2432def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2433def int_aarch64_sve_smaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2434def int_aarch64_sve_sminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2435def int_aarch64_sve_umaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2436def int_aarch64_sve_uminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2437 2438// 2439// SVE2 - Widening pairwise arithmetic 2440// 2441 2442def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2443def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2444 2445// 2446// SVE2 - Uniform complex integer arithmetic 2447// 2448 2449def int_aarch64_sve_cadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2450def int_aarch64_sve_sqcadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2451def int_aarch64_sve_cmla_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2452def int_aarch64_sve_cmla_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2453def int_aarch64_sve_sqrdcmlah_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2454def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2455 2456// 2457// SVE2 - Widening complex integer arithmetic 2458// 2459 2460def int_aarch64_sve_saddlbt : SVE2_2VectorArg_Long_Intrinsic; 2461def int_aarch64_sve_ssublbt : SVE2_2VectorArg_Long_Intrinsic; 2462def int_aarch64_sve_ssubltb : SVE2_2VectorArg_Long_Intrinsic; 2463 2464// 2465// SVE2 - Widening complex integer dot product 2466// 2467 2468def int_aarch64_sve_cdot : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2469def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic; 2470 2471// 2472// SVE2 - Floating-point widening multiply-accumulate 2473// 2474 2475def int_aarch64_sve_fmlalb : SVE2_3VectorArg_Long_Intrinsic; 2476def int_aarch64_sve_fmlalb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2477def int_aarch64_sve_fmlalt : SVE2_3VectorArg_Long_Intrinsic; 2478def int_aarch64_sve_fmlalt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2479def int_aarch64_sve_fmlslb : SVE2_3VectorArg_Long_Intrinsic; 2480def int_aarch64_sve_fmlslb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2481def int_aarch64_sve_fmlslt : SVE2_3VectorArg_Long_Intrinsic; 2482def int_aarch64_sve_fmlslt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2483 2484// 2485// SVE2 - Floating-point integer binary logarithm 2486// 2487 2488def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic; 2489 2490// 2491// SVE2 - Vector histogram count 2492// 2493 2494def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic; 2495def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic; 2496 2497// 2498// SVE2 - Character match 2499// 2500 2501def int_aarch64_sve_match : AdvSIMD_SVE_Compare_Intrinsic; 2502def int_aarch64_sve_nmatch : AdvSIMD_SVE_Compare_Intrinsic; 2503 2504// 2505// SVE2 - Unary narrowing operations 2506// 2507 2508def int_aarch64_sve_sqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2509def int_aarch64_sve_sqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2510def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic; 2511def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2512def int_aarch64_sve_uqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2513def int_aarch64_sve_uqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2514 2515// 2516// SVE2 - Binary narrowing DSP operations 2517// 2518def int_aarch64_sve_addhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2519def int_aarch64_sve_addhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2520 2521def int_aarch64_sve_raddhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2522def int_aarch64_sve_raddhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2523 2524def int_aarch64_sve_subhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2525def int_aarch64_sve_subhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2526 2527def int_aarch64_sve_rsubhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2528def int_aarch64_sve_rsubhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2529 2530// Narrowing shift right 2531def int_aarch64_sve_shrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2532def int_aarch64_sve_shrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2533 2534def int_aarch64_sve_rshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2535def int_aarch64_sve_rshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2536 2537// Saturating shift right - signed input/output 2538def int_aarch64_sve_sqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2539def int_aarch64_sve_sqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2540 2541def int_aarch64_sve_sqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2542def int_aarch64_sve_sqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2543 2544// Saturating shift right - unsigned input/output 2545def int_aarch64_sve_uqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2546def int_aarch64_sve_uqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2547 2548def int_aarch64_sve_uqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2549def int_aarch64_sve_uqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2550 2551// Saturating shift right - signed input, unsigned output 2552def int_aarch64_sve_sqshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2553def int_aarch64_sve_sqshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2554 2555def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2556def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2557 2558// SVE2 MLA LANE. 2559def int_aarch64_sve_smlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2560def int_aarch64_sve_smlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2561def int_aarch64_sve_umlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2562def int_aarch64_sve_umlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2563def int_aarch64_sve_smlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2564def int_aarch64_sve_smlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2565def int_aarch64_sve_umlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2566def int_aarch64_sve_umlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2567def int_aarch64_sve_smullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2568def int_aarch64_sve_smullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2569def int_aarch64_sve_umullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2570def int_aarch64_sve_umullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2571def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2572def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2573def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2574def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2575def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2576def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2577 2578// SVE2 MLA Unpredicated. 2579def int_aarch64_sve_smlalb : SVE2_3VectorArg_Long_Intrinsic; 2580def int_aarch64_sve_smlalt : SVE2_3VectorArg_Long_Intrinsic; 2581def int_aarch64_sve_umlalb : SVE2_3VectorArg_Long_Intrinsic; 2582def int_aarch64_sve_umlalt : SVE2_3VectorArg_Long_Intrinsic; 2583def int_aarch64_sve_smlslb : SVE2_3VectorArg_Long_Intrinsic; 2584def int_aarch64_sve_smlslt : SVE2_3VectorArg_Long_Intrinsic; 2585def int_aarch64_sve_umlslb : SVE2_3VectorArg_Long_Intrinsic; 2586def int_aarch64_sve_umlslt : SVE2_3VectorArg_Long_Intrinsic; 2587def int_aarch64_sve_smullb : SVE2_2VectorArg_Long_Intrinsic; 2588def int_aarch64_sve_smullt : SVE2_2VectorArg_Long_Intrinsic; 2589def int_aarch64_sve_umullb : SVE2_2VectorArg_Long_Intrinsic; 2590def int_aarch64_sve_umullt : SVE2_2VectorArg_Long_Intrinsic; 2591 2592def int_aarch64_sve_sqdmlalb : SVE2_3VectorArg_Long_Intrinsic; 2593def int_aarch64_sve_sqdmlalt : SVE2_3VectorArg_Long_Intrinsic; 2594def int_aarch64_sve_sqdmlslb : SVE2_3VectorArg_Long_Intrinsic; 2595def int_aarch64_sve_sqdmlslt : SVE2_3VectorArg_Long_Intrinsic; 2596def int_aarch64_sve_sqdmullb : SVE2_2VectorArg_Long_Intrinsic; 2597def int_aarch64_sve_sqdmullt : SVE2_2VectorArg_Long_Intrinsic; 2598def int_aarch64_sve_sqdmlalbt : SVE2_3VectorArg_Long_Intrinsic; 2599def int_aarch64_sve_sqdmlslbt : SVE2_3VectorArg_Long_Intrinsic; 2600 2601// SVE2 ADDSUB Long Unpredicated. 2602def int_aarch64_sve_adclb : AdvSIMD_3VectorArg_Intrinsic; 2603def int_aarch64_sve_adclt : AdvSIMD_3VectorArg_Intrinsic; 2604def int_aarch64_sve_sbclb : AdvSIMD_3VectorArg_Intrinsic; 2605def int_aarch64_sve_sbclt : AdvSIMD_3VectorArg_Intrinsic; 2606 2607// 2608// SVE2 - Polynomial arithmetic 2609// 2610def int_aarch64_sve_eorbt : AdvSIMD_3VectorArg_Intrinsic; 2611def int_aarch64_sve_eortb : AdvSIMD_3VectorArg_Intrinsic; 2612def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic; 2613def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic; 2614 2615// 2616// SVE2 bitwise ternary operations. 2617// 2618def int_aarch64_sve_eor3 : AdvSIMD_3VectorArg_Intrinsic; 2619def int_aarch64_sve_bcax : AdvSIMD_3VectorArg_Intrinsic; 2620def int_aarch64_sve_bsl : AdvSIMD_3VectorArg_Intrinsic; 2621def int_aarch64_sve_bsl1n : AdvSIMD_3VectorArg_Intrinsic; 2622def int_aarch64_sve_bsl2n : AdvSIMD_3VectorArg_Intrinsic; 2623def int_aarch64_sve_nbsl : AdvSIMD_3VectorArg_Intrinsic; 2624def int_aarch64_sve_xar : AdvSIMD_2VectorArgIndexed_Intrinsic; 2625 2626// 2627// SVE2 - Optional AES, SHA-3 and SM4 2628// 2629 2630def int_aarch64_sve_aesd : ClangBuiltin<"__builtin_sve_svaesd_u8">, 2631 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2632 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2633 [IntrNoMem]>; 2634def int_aarch64_sve_aesimc : ClangBuiltin<"__builtin_sve_svaesimc_u8">, 2635 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2636 [llvm_nxv16i8_ty], 2637 [IntrNoMem]>; 2638def int_aarch64_sve_aese : ClangBuiltin<"__builtin_sve_svaese_u8">, 2639 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2640 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2641 [IntrNoMem]>; 2642def int_aarch64_sve_aesmc : ClangBuiltin<"__builtin_sve_svaesmc_u8">, 2643 DefaultAttrsIntrinsic<[llvm_nxv16i8_ty], 2644 [llvm_nxv16i8_ty], 2645 [IntrNoMem]>; 2646def int_aarch64_sve_rax1 : ClangBuiltin<"__builtin_sve_svrax1_u64">, 2647 DefaultAttrsIntrinsic<[llvm_nxv2i64_ty], 2648 [llvm_nxv2i64_ty, llvm_nxv2i64_ty], 2649 [IntrNoMem]>; 2650def int_aarch64_sve_sm4e : ClangBuiltin<"__builtin_sve_svsm4e_u32">, 2651 DefaultAttrsIntrinsic<[llvm_nxv4i32_ty], 2652 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2653 [IntrNoMem]>; 2654def int_aarch64_sve_sm4ekey : ClangBuiltin<"__builtin_sve_svsm4ekey_u32">, 2655 DefaultAttrsIntrinsic<[llvm_nxv4i32_ty], 2656 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2657 [IntrNoMem]>; 2658// 2659// SVE2 - Extended table lookup/permute 2660// 2661 2662def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic; 2663def int_aarch64_sve_tbx : AdvSIMD_SVE2_TBX_Intrinsic; 2664 2665// 2666// SVE2 - Optional bit permutation 2667// 2668 2669def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic; 2670def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic; 2671def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic; 2672 2673 2674// 2675// SVE ACLE: 7.3. INT8 matrix multiply extensions 2676// 2677def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic; 2678def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic; 2679def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic; 2680 2681def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic; 2682def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2683def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2684 2685// 2686// SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions 2687// 2688def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic; 2689 2690// 2691// SVE ACLE: 7.2. BFloat16 extensions 2692// 2693 2694def int_aarch64_sve_bfdot : SVE_4Vec_BF16; 2695def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16; 2696def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16; 2697 2698def int_aarch64_sve_bfmmla : SVE_4Vec_BF16; 2699 2700def int_aarch64_sve_bfdot_lane_v2 : SVE_4Vec_BF16_Indexed; 2701def int_aarch64_sve_bfmlalb_lane_v2 : SVE_4Vec_BF16_Indexed; 2702def int_aarch64_sve_bfmlalt_lane_v2 : SVE_4Vec_BF16_Indexed; 2703 2704// 2705// SVE2.1 - Contiguous loads to multiple consecutive vectors 2706// 2707 2708 class SVE2p1_Load_PN_X2_Intrinsic 2709 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 2710 [llvm_aarch64_svcount_ty, llvm_ptr_ty], 2711 [IntrReadMem, IntrArgMemOnly]>; 2712 2713 class SVE2p1_Load_PN_X4_Intrinsic 2714 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 2715 LLVMMatchType<0>, LLVMMatchType<0>], 2716 [llvm_aarch64_svcount_ty, llvm_ptr_ty], 2717 [IntrReadMem, IntrArgMemOnly]>; 2718 2719def int_aarch64_sve_ld1_pn_x2 : SVE2p1_Load_PN_X2_Intrinsic; 2720def int_aarch64_sve_ld1_pn_x4 : SVE2p1_Load_PN_X4_Intrinsic; 2721def int_aarch64_sve_ldnt1_pn_x2 : SVE2p1_Load_PN_X2_Intrinsic; 2722def int_aarch64_sve_ldnt1_pn_x4 : SVE2p1_Load_PN_X4_Intrinsic; 2723 2724// 2725// SVE2.1 - Contiguous loads to quadword (single vector) 2726// 2727 2728class SVE2p1_Single_Load_Quadword 2729 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 2730 [llvm_nxv1i1_ty, llvm_ptr_ty], 2731 [IntrReadMem, IntrArgMemOnly]>; 2732def int_aarch64_sve_ld1uwq : SVE2p1_Single_Load_Quadword; 2733def int_aarch64_sve_ld1udq : SVE2p1_Single_Load_Quadword; 2734 2735// 2736// SVE2.1 - Contiguous store from quadword (single vector) 2737// 2738 2739class SVE2p1_Single_Store_Quadword 2740 : DefaultAttrsIntrinsic<[], 2741 [llvm_anyvector_ty, llvm_nxv1i1_ty, llvm_ptr_ty], 2742 [IntrWriteMem, IntrArgMemOnly]>; 2743def int_aarch64_sve_st1wq : SVE2p1_Single_Store_Quadword; 2744def int_aarch64_sve_st1dq : SVE2p1_Single_Store_Quadword; 2745 2746 2747def int_aarch64_sve_ld2q_sret : AdvSIMD_2Vec_PredLoad_Intrinsic; 2748def int_aarch64_sve_ld3q_sret : AdvSIMD_3Vec_PredLoad_Intrinsic; 2749def int_aarch64_sve_ld4q_sret : AdvSIMD_4Vec_PredLoad_Intrinsic; 2750 2751def int_aarch64_sve_st2q : AdvSIMD_2Vec_PredStore_Intrinsic; 2752def int_aarch64_sve_st3q : AdvSIMD_3Vec_PredStore_Intrinsic; 2753def int_aarch64_sve_st4q : AdvSIMD_4Vec_PredStore_Intrinsic; 2754 2755// 2756// SVE2.1 - Contiguous stores to multiple consecutive vectors 2757// 2758 2759 class SVE2p1_Store_PN_X2_Intrinsic 2760 : DefaultAttrsIntrinsic<[], [ llvm_anyvector_ty, LLVMMatchType<0>, 2761 llvm_aarch64_svcount_ty, llvm_ptr_ty ], 2762 [IntrWriteMem, IntrArgMemOnly]>; 2763 2764 class SVE2p1_Store_PN_X4_Intrinsic 2765 : DefaultAttrsIntrinsic<[], [ llvm_anyvector_ty, LLVMMatchType<0>, 2766 LLVMMatchType<0>, LLVMMatchType<0>, 2767 llvm_aarch64_svcount_ty, llvm_ptr_ty], 2768 [IntrWriteMem, IntrArgMemOnly]>; 2769 2770def int_aarch64_sve_st1_pn_x2 : SVE2p1_Store_PN_X2_Intrinsic; 2771def int_aarch64_sve_st1_pn_x4 : SVE2p1_Store_PN_X4_Intrinsic; 2772def int_aarch64_sve_stnt1_pn_x2 : SVE2p1_Store_PN_X2_Intrinsic; 2773def int_aarch64_sve_stnt1_pn_x4 : SVE2p1_Store_PN_X4_Intrinsic; 2774} 2775 2776// 2777// SVE2 - Contiguous conflict detection 2778// 2779 2780def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic; 2781def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic; 2782def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic; 2783def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic; 2784def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic; 2785def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic; 2786def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic; 2787def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic; 2788 2789// Scalable Matrix Extension (SME) Intrinsics 2790let TargetPrefix = "aarch64" in { 2791 class SME_Load_Store_Intrinsic<LLVMType pred_ty> 2792 : DefaultAttrsIntrinsic<[], 2793 [pred_ty, llvm_ptr_ty, llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; 2794 2795 // Loads 2796 def int_aarch64_sme_ld1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2797 def int_aarch64_sme_ld1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2798 def int_aarch64_sme_ld1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2799 def int_aarch64_sme_ld1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2800 def int_aarch64_sme_ld1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2801 def int_aarch64_sme_ld1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2802 def int_aarch64_sme_ld1h_vert : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2803 def int_aarch64_sme_ld1w_vert : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2804 def int_aarch64_sme_ld1d_vert : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2805 def int_aarch64_sme_ld1q_vert : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2806 2807 // Stores 2808 def int_aarch64_sme_st1b_horiz : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2809 def int_aarch64_sme_st1h_horiz : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2810 def int_aarch64_sme_st1w_horiz : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2811 def int_aarch64_sme_st1d_horiz : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2812 def int_aarch64_sme_st1q_horiz : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2813 def int_aarch64_sme_st1b_vert : SME_Load_Store_Intrinsic<llvm_nxv16i1_ty>; 2814 def int_aarch64_sme_st1h_vert : SME_Load_Store_Intrinsic<llvm_nxv8i1_ty>; 2815 def int_aarch64_sme_st1w_vert : SME_Load_Store_Intrinsic<llvm_nxv4i1_ty>; 2816 def int_aarch64_sme_st1d_vert : SME_Load_Store_Intrinsic<llvm_nxv2i1_ty>; 2817 def int_aarch64_sme_st1q_vert : SME_Load_Store_Intrinsic<llvm_nxv1i1_ty>; 2818 2819 // Spill + fill 2820 class SME_LDR_STR_ZA_Intrinsic 2821 : DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_ptr_ty, llvm_i32_ty]>; 2822 def int_aarch64_sme_ldr : SME_LDR_STR_ZA_Intrinsic; 2823 def int_aarch64_sme_str : SME_LDR_STR_ZA_Intrinsic; 2824 2825 class SME_TileToVector_Intrinsic 2826 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 2827 [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2828 llvm_i32_ty, llvm_i32_ty], [ImmArg<ArgIndex<2>>]>; 2829 class SME_VectorToTile_Intrinsic 2830 : DefaultAttrsIntrinsic<[], 2831 [llvm_i32_ty, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2832 llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>; 2833 2834 def int_aarch64_sme_read_horiz : SME_TileToVector_Intrinsic; 2835 def int_aarch64_sme_read_vert : SME_TileToVector_Intrinsic; 2836 def int_aarch64_sme_write_horiz : SME_VectorToTile_Intrinsic; 2837 def int_aarch64_sme_write_vert : SME_VectorToTile_Intrinsic; 2838 2839 def int_aarch64_sme_readq_horiz : SME_TileToVector_Intrinsic; 2840 def int_aarch64_sme_readq_vert : SME_TileToVector_Intrinsic; 2841 def int_aarch64_sme_writeq_horiz : SME_VectorToTile_Intrinsic; 2842 def int_aarch64_sme_writeq_vert : SME_VectorToTile_Intrinsic; 2843 2844 def int_aarch64_sme_zero : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>; 2845 2846 class SME_OuterProduct_Intrinsic 2847 : DefaultAttrsIntrinsic<[], 2848 [llvm_i32_ty, 2849 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2850 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2851 LLVMMatchType<0>, 2852 llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>; 2853 2854 def int_aarch64_sme_mopa : SME_OuterProduct_Intrinsic; 2855 def int_aarch64_sme_mops : SME_OuterProduct_Intrinsic; 2856 2857 def int_aarch64_sme_mopa_wide : SME_OuterProduct_Intrinsic; 2858 def int_aarch64_sme_mops_wide : SME_OuterProduct_Intrinsic; 2859 2860 def int_aarch64_sme_smopa_wide : SME_OuterProduct_Intrinsic; 2861 def int_aarch64_sme_smops_wide : SME_OuterProduct_Intrinsic; 2862 def int_aarch64_sme_umopa_wide : SME_OuterProduct_Intrinsic; 2863 def int_aarch64_sme_umops_wide : SME_OuterProduct_Intrinsic; 2864 def int_aarch64_sme_sumopa_wide : SME_OuterProduct_Intrinsic; 2865 def int_aarch64_sme_sumops_wide : SME_OuterProduct_Intrinsic; 2866 def int_aarch64_sme_usmopa_wide : SME_OuterProduct_Intrinsic; 2867 def int_aarch64_sme_usmops_wide : SME_OuterProduct_Intrinsic; 2868 2869 class SME_AddVectorToTile_Intrinsic 2870 : DefaultAttrsIntrinsic<[], 2871 [llvm_i32_ty, 2872 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2873 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 2874 llvm_anyvector_ty], [ImmArg<ArgIndex<0>>]>; 2875 2876 def int_aarch64_sme_addha : SME_AddVectorToTile_Intrinsic; 2877 def int_aarch64_sme_addva : SME_AddVectorToTile_Intrinsic; 2878 2879 // 2880 // Counting elements 2881 // 2882 2883 class AdvSIMD_SME_CNTSB_Intrinsic 2884 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem]>; 2885 2886 def int_aarch64_sme_cntsb : AdvSIMD_SME_CNTSB_Intrinsic; 2887 def int_aarch64_sme_cntsh : AdvSIMD_SME_CNTSB_Intrinsic; 2888 def int_aarch64_sme_cntsw : AdvSIMD_SME_CNTSB_Intrinsic; 2889 def int_aarch64_sme_cntsd : AdvSIMD_SME_CNTSB_Intrinsic; 2890 2891 // 2892 // PSTATE Functions 2893 // 2894 2895 def int_aarch64_sme_get_tpidr2 2896 : DefaultAttrsIntrinsic<[llvm_i64_ty], [], 2897 [IntrNoMem, IntrHasSideEffects]>; 2898 def int_aarch64_sme_set_tpidr2 2899 : DefaultAttrsIntrinsic<[], [llvm_i64_ty], 2900 [IntrNoMem, IntrHasSideEffects]>; 2901 2902 def int_aarch64_sme_za_enable 2903 : DefaultAttrsIntrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; 2904 def int_aarch64_sme_za_disable 2905 : DefaultAttrsIntrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>; 2906 2907 // Clamp 2908 // 2909 2910 def int_aarch64_sve_sclamp : AdvSIMD_3VectorArg_Intrinsic; 2911 def int_aarch64_sve_uclamp : AdvSIMD_3VectorArg_Intrinsic; 2912 def int_aarch64_sve_fclamp : AdvSIMD_3VectorArg_Intrinsic; 2913 2914 2915 // 2916 // Reversal 2917 // 2918 2919 def int_aarch64_sve_revd : AdvSIMD_Merged1VectorArg_Intrinsic; 2920 2921 // 2922 // Predicate selection 2923 // 2924 2925 def int_aarch64_sve_psel 2926 : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], 2927 [llvm_nxv16i1_ty, 2928 llvm_anyvector_ty, llvm_i32_ty], 2929 [IntrNoMem]>; 2930 2931 // 2932 // Predicate-pair intrinsics 2933 // 2934 foreach cmp = ["ge", "gt", "hi", "hs", "le", "lo", "ls", "lt"] in { 2935 def int_aarch64_sve_while # cmp # _x2 2936 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 2937 [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; 2938 } 2939 2940 // 2941 // Predicate-as-counter intrinsics 2942 // 2943 2944 def int_aarch64_sve_pext 2945 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], 2946 [llvm_aarch64_svcount_ty, llvm_i32_ty], 2947 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 2948 2949 def int_aarch64_sve_pext_x2 2950 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 2951 [llvm_aarch64_svcount_ty, llvm_i32_ty], 2952 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 2953 2954 def int_aarch64_sve_ptrue_c8 2955 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], [], [IntrNoMem]>; 2956 def int_aarch64_sve_ptrue_c16 2957 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], [], [IntrNoMem]>; 2958 def int_aarch64_sve_ptrue_c32 2959 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], [], [IntrNoMem]>; 2960 def int_aarch64_sve_ptrue_c64 2961 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], [], [IntrNoMem]>; 2962 2963 def int_aarch64_sve_cntp_c8 2964 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty], 2965 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 2966 def int_aarch64_sve_cntp_c16 2967 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty], 2968 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 2969 def int_aarch64_sve_cntp_c32 2970 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty], 2971 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 2972 def int_aarch64_sve_cntp_c64 2973 : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_aarch64_svcount_ty, llvm_i32_ty], 2974 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 2975 2976 // While (predicate-as-counter) intrinsics 2977 foreach cmp = ["ge", "gt", "hi", "hs", "le", "lo", "ls", "lt"] in { 2978 foreach ty = ["c8", "c16", "c32", "c64"] in { 2979 def int_aarch64_sve_while # cmp # _ # ty 2980 : DefaultAttrsIntrinsic<[llvm_aarch64_svcount_ty], 2981 [llvm_i64_ty, llvm_i64_ty, llvm_i32_ty], 2982 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 2983 } 2984 } 2985 2986 // 2987 // SME2 Intrinsics 2988 // 2989 2990 class SME2_Matrix_ArrayVector_Single_Single_Intrinsic 2991 : DefaultAttrsIntrinsic<[], 2992 [llvm_i32_ty, 2993 llvm_anyvector_ty, LLVMMatchType<0>], 2994 []>; 2995 2996 class SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic 2997 : DefaultAttrsIntrinsic<[], 2998 [llvm_i32_ty, 2999 llvm_anyvector_ty, LLVMMatchType<0>, 3000 LLVMMatchType<0>], 3001 []>; 3002 3003 class SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic 3004 : DefaultAttrsIntrinsic<[], 3005 [llvm_i32_ty, 3006 llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, 3007 LLVMMatchType<0>], 3008 []>; 3009 3010 class SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic 3011 : DefaultAttrsIntrinsic<[], 3012 [llvm_i32_ty, 3013 llvm_anyvector_ty, LLVMMatchType<0>, 3014 LLVMMatchType<0>, LLVMMatchType<0>], 3015 []>; 3016 3017 class SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic 3018 : DefaultAttrsIntrinsic<[], 3019 [llvm_i32_ty, 3020 llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, 3021 LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3022 []>; 3023 3024 class SME2_Matrix_ArrayVector_Single_Index_Intrinsic 3025 : DefaultAttrsIntrinsic<[], 3026 [llvm_i32_ty, 3027 llvm_anyvector_ty, 3028 LLVMMatchType<0>, llvm_i32_ty], 3029 [ImmArg<ArgIndex<3>>]>; 3030 3031 class SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic 3032 : DefaultAttrsIntrinsic<[], 3033 [llvm_i32_ty, 3034 llvm_anyvector_ty, LLVMMatchType<0>, 3035 LLVMMatchType<0>, llvm_i32_ty], 3036 [ImmArg<ArgIndex<4>>]>; 3037 3038 class SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic 3039 : DefaultAttrsIntrinsic<[], 3040 [llvm_i32_ty, 3041 llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>, 3042 LLVMMatchType<0>, llvm_i32_ty], 3043 [ImmArg<ArgIndex<6>>]>; 3044 3045 class SME2_VG2_Multi_Imm_Intrinsic 3046 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 3047 [llvm_anyvector_ty, LLVMMatchType<0>, 3048 llvm_i32_ty], 3049 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 3050 3051 class SME2_VG4_Multi_Imm_Intrinsic 3052 : DefaultAttrsIntrinsic<[LLVMSubdivide4VectorType<0>], 3053 [llvm_anyvector_ty, LLVMMatchType<0>, 3054 LLVMMatchType<0>, LLVMMatchType<0>, 3055 llvm_i32_ty], 3056 [IntrNoMem, ImmArg<ArgIndex<4>>]>; 3057 3058 class SME2_ZA_Write_VG2_Intrinsic 3059 : DefaultAttrsIntrinsic<[], 3060 [llvm_i32_ty, 3061 llvm_anyvector_ty, LLVMMatchType<0>], 3062 []>; 3063 3064 class SME2_ZA_Write_VG4_Intrinsic 3065 : DefaultAttrsIntrinsic<[], 3066 [llvm_i32_ty, 3067 llvm_anyvector_ty, LLVMMatchType<0>, 3068 LLVMMatchType<0>, LLVMMatchType<0>], 3069 []>; 3070 3071 class SME2_VG2_Multi_Single_Intrinsic 3072 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3073 [LLVMMatchType<0>, LLVMMatchType<0>, 3074 LLVMMatchType<0>], 3075 [IntrNoMem]>; 3076 3077 class SME2_VG4_Multi_Single_Intrinsic 3078 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3079 LLVMMatchType<0>, LLVMMatchType<0>], 3080 [LLVMMatchType<0>, LLVMMatchType<0>, 3081 LLVMMatchType<0>, LLVMMatchType<0>, 3082 LLVMMatchType<0>], 3083 [IntrNoMem]>; 3084 3085 class SME2_VG2_Multi_Multi_Intrinsic 3086 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3087 [LLVMMatchType<0>, LLVMMatchType<0>, 3088 LLVMMatchType<0>, LLVMMatchType<0>], 3089 [IntrNoMem]>; 3090 3091 class SME2_VG4_Multi_Multi_Intrinsic 3092 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3093 LLVMMatchType<0>, LLVMMatchType<0>], 3094 [LLVMMatchType<0>, LLVMMatchType<0>, 3095 LLVMMatchType<0>, LLVMMatchType<0>, 3096 LLVMMatchType<0>, LLVMMatchType<0>, 3097 LLVMMatchType<0>, LLVMMatchType<0>], 3098 [IntrNoMem]>; 3099 3100 class SVE2_VG2_Sel_Intrinsic 3101 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3102 [llvm_aarch64_svcount_ty, LLVMMatchType<0>, 3103 LLVMMatchType<0>, LLVMMatchType<0>, 3104 LLVMMatchType<0>], [IntrNoMem]>; 3105 3106 class SVE2_VG4_Sel_Intrinsic 3107 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3108 LLVMMatchType<0>, LLVMMatchType<0>], 3109 [llvm_aarch64_svcount_ty, LLVMMatchType<0>, 3110 LLVMMatchType<0>, LLVMMatchType<0>, 3111 LLVMMatchType<0>, LLVMMatchType<0>, 3112 LLVMMatchType<0>, LLVMMatchType<0>, 3113 LLVMMatchType<0>], [IntrNoMem]>; 3114 3115 class SME2_CVT_VG2_SINGLE_Intrinsic 3116 : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>], 3117 [llvm_anyvector_ty, LLVMMatchType<0>], 3118 [IntrNoMem]>; 3119 3120 class SME2_CVT_VG2_SINGLE_BF16_Intrinsic 3121 : DefaultAttrsIntrinsic<[llvm_nxv8bf16_ty], 3122 [llvm_nxv4f32_ty, llvm_nxv4f32_ty], 3123 [IntrNoMem]>; 3124 3125 class SME2_CVT_VG4_SINGLE_Intrinsic 3126 : DefaultAttrsIntrinsic<[LLVMSubdivide4VectorType<0>], 3127 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3128 [IntrNoMem]>; 3129 3130 class SME2_CVT_X2_Intrinsic 3131 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3132 [llvm_anyvector_ty, LLVMMatchType<1>], 3133 [IntrNoMem]>; 3134 3135 class SME2_CVT_X4_Intrinsic 3136 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3137 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, LLVMMatchType<1>], 3138 [IntrNoMem]>; 3139 3140 class SME2_BFMLS_Intrinsic 3141 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 3142 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty], 3143 [IntrNoMem]>; 3144 3145 class SME2_BFMLS_Lane_Intrinsic 3146 : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty], 3147 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i32_ty], 3148 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 3149 3150 class SME2_ZA_ArrayVector_Read_VG2_Intrinsic 3151 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3152 [llvm_i32_ty], 3153 []>; 3154 3155 class SME2_ZA_ArrayVector_Read_VG4_Intrinsic 3156 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3157 LLVMMatchType<0>, LLVMMatchType<0>], 3158 [llvm_i32_ty], 3159 []>; 3160 3161 class SME2_Matrix_TileVector_Read_VG2_Intrinsic 3162 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3163 [llvm_i32_ty, llvm_i32_ty], 3164 []>; 3165 3166 class SME2_Matrix_TileVector_Read_VG4_Intrinsic 3167 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3168 LLVMMatchType<0>, LLVMMatchType<0>], 3169 [llvm_i32_ty, llvm_i32_ty], 3170 []>; 3171 3172 class SME2_ZA_ArrayVector_Write_VG2_Intrinsic 3173 : DefaultAttrsIntrinsic<[], 3174 [llvm_i32_ty, 3175 llvm_anyvector_ty, LLVMMatchType<0>], 3176 []>; 3177 3178 class SME2_ZA_ArrayVector_Write_VG4_Intrinsic 3179 : DefaultAttrsIntrinsic<[], 3180 [llvm_i32_ty, 3181 llvm_anyvector_ty, LLVMMatchType<0>, 3182 LLVMMatchType<0>, LLVMMatchType<0>], 3183 []>; 3184 3185 class SME2_Matrix_TileVector_Write_VG2_Intrinsic 3186 : DefaultAttrsIntrinsic<[], 3187 [llvm_i32_ty, llvm_i32_ty, 3188 llvm_anyvector_ty, LLVMMatchType<0>], 3189 [ImmArg<ArgIndex<0>>]>; 3190 3191 class SME2_Matrix_TileVector_Write_VG4_Intrinsic 3192 : DefaultAttrsIntrinsic<[], 3193 [llvm_i32_ty, llvm_i32_ty, 3194 llvm_anyvector_ty, LLVMMatchType<0>, 3195 LLVMMatchType<0>, LLVMMatchType<0>], 3196 [ImmArg<ArgIndex<0>>]>; 3197 3198 class SME2_VG2_Multi_Single_Single_Intrinsic 3199 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3200 [LLVMMatchType<0>, LLVMMatchType<0>, 3201 LLVMMatchType<0>, LLVMMatchType<0>], 3202 [IntrNoMem]>; 3203 3204 class SME2_VG4_Multi_Single_Single_Intrinsic 3205 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3206 LLVMMatchType<0>, LLVMMatchType<0>], 3207 [LLVMMatchType<0>, LLVMMatchType<0>, 3208 LLVMMatchType<0>, LLVMMatchType<0>, 3209 LLVMMatchType<0>, LLVMMatchType<0>], 3210 [IntrNoMem]>; 3211 3212 class SVE2_VG2_ZipUzp_Intrinsic 3213 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3214 [LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; 3215 3216 class SVE2_VG4_ZipUzp_Intrinsic 3217 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3218 LLVMMatchType<0>, LLVMMatchType<0>], 3219 [LLVMMatchType<0>, LLVMMatchType<0>, 3220 LLVMMatchType<0>, LLVMMatchType<0>], [IntrNoMem]>; 3221 3222 class SME2_VG2_Unpk_Intrinsic 3223 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], 3224 [LLVMSubdivide2VectorType<0>], [IntrNoMem]>; 3225 3226 class SME2_VG4_Unpk_Intrinsic 3227 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, 3228 LLVMMatchType<0>, LLVMMatchType<0>], 3229 [LLVMSubdivide2VectorType<0>, LLVMSubdivide2VectorType<0>], 3230 [IntrNoMem]>; 3231 3232 // 3233 // Multi-vector fused multiply-add/subtract 3234 // 3235 3236 def int_aarch64_sme_fmla_single_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3237 def int_aarch64_sme_fmls_single_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3238 def int_aarch64_sme_fmla_single_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3239 def int_aarch64_sme_fmls_single_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3240 3241 def int_aarch64_sme_fmla_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3242 def int_aarch64_sme_fmls_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3243 def int_aarch64_sme_fmla_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3244 def int_aarch64_sme_fmls_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3245 3246 def int_aarch64_sme_fmla_lane_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3247 def int_aarch64_sme_fmls_lane_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3248 def int_aarch64_sme_fmla_lane_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3249 def int_aarch64_sme_fmls_lane_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3250 3251 // 3252 // Outer product and accumulate/subtract intrinsics 3253 // 3254 3255 def int_aarch64_sme_smopa_za32 : SME_OuterProduct_Intrinsic; 3256 def int_aarch64_sme_umopa_za32 : SME_OuterProduct_Intrinsic; 3257 def int_aarch64_sme_smops_za32 : SME_OuterProduct_Intrinsic; 3258 def int_aarch64_sme_umops_za32 : SME_OuterProduct_Intrinsic; 3259 3260 def int_aarch64_sme_bmopa_za32 : SME_OuterProduct_Intrinsic; 3261 def int_aarch64_sme_bmops_za32 : SME_OuterProduct_Intrinsic; 3262 3263 // 3264 // Multi-vector rounding shift left intrinsics 3265 // 3266 3267 def int_aarch64_sve_srshl_single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3268 def int_aarch64_sve_urshl_single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3269 def int_aarch64_sve_srshl_single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3270 def int_aarch64_sve_urshl_single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3271 3272 def int_aarch64_sve_srshl_x2 : SME2_VG2_Multi_Multi_Intrinsic; 3273 def int_aarch64_sve_urshl_x2 : SME2_VG2_Multi_Multi_Intrinsic; 3274 def int_aarch64_sve_srshl_x4 : SME2_VG4_Multi_Multi_Intrinsic; 3275 def int_aarch64_sve_urshl_x4 : SME2_VG4_Multi_Multi_Intrinsic; 3276 3277 // Multi-vector saturating rounding shift right intrinsics 3278 3279 def int_aarch64_sve_sqrshr_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3280 def int_aarch64_sve_uqrshr_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3281 def int_aarch64_sve_sqrshr_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3282 def int_aarch64_sve_uqrshr_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3283 3284 def int_aarch64_sve_sqrshrn_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3285 def int_aarch64_sve_uqrshrn_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3286 def int_aarch64_sve_sqrshrn_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3287 def int_aarch64_sve_uqrshrn_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3288 3289 def int_aarch64_sve_sqrshru_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3290 def int_aarch64_sve_sqrshru_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3291 3292 def int_aarch64_sve_sqrshrun_x2 : SME2_VG2_Multi_Imm_Intrinsic; 3293 def int_aarch64_sve_sqrshrun_x4 : SME2_VG4_Multi_Imm_Intrinsic; 3294 3295 // 3296 // Multi-vector multiply-add/subtract long 3297 // 3298 3299 foreach ty = ["f", "s", "u"] in { 3300 foreach instr = ["mlal", "mlsl"] in { 3301 def int_aarch64_sme_ # ty # instr # _single_vg2x1 : SME2_Matrix_ArrayVector_Single_Single_Intrinsic; 3302 def int_aarch64_sme_ # ty # instr # _single_vg2x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3303 def int_aarch64_sme_ # ty # instr # _single_vg2x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3304 3305 def int_aarch64_sme_ # ty # instr # _vg2x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3306 def int_aarch64_sme_ # ty # instr # _vg2x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3307 3308 def int_aarch64_sme_ # ty # instr # _lane_vg2x1 : SME2_Matrix_ArrayVector_Single_Index_Intrinsic; 3309 def int_aarch64_sme_ # ty # instr # _lane_vg2x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3310 def int_aarch64_sme_ # ty # instr # _lane_vg2x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3311 } 3312 } 3313 3314 // 3315 // Multi-vector multiply-add long long 3316 // 3317 3318 foreach ty = ["s", "u"] in { 3319 foreach instr = ["mla", "mls"] in { 3320 foreach za = ["za32", "za64"] in { 3321 def int_aarch64_sme_ # ty # instr # _ # za # _single_vg4x1 : SME2_Matrix_ArrayVector_Single_Single_Intrinsic; 3322 def int_aarch64_sme_ # ty # instr # _ # za # _single_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3323 def int_aarch64_sme_ # ty # instr # _ # za # _single_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3324 3325 def int_aarch64_sme_ # ty # instr # _ # za # _vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3326 def int_aarch64_sme_ # ty # instr # _ # za # _vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3327 3328 def int_aarch64_sme_ # ty # instr # _ # za # _lane_vg4x1 : SME2_Matrix_ArrayVector_Single_Index_Intrinsic; 3329 def int_aarch64_sme_ # ty # instr # _ # za # _lane_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3330 def int_aarch64_sme_ # ty # instr # _ # za # _lane_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3331 } 3332 } 3333 } 3334 3335 def int_aarch64_sme_sumla_za32_single_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3336 def int_aarch64_sme_sumla_za32_single_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3337 3338 def int_aarch64_sme_sumla_za32_lane_vg4x1 : SME2_Matrix_ArrayVector_Single_Index_Intrinsic; 3339 def int_aarch64_sme_sumla_za32_lane_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3340 def int_aarch64_sme_sumla_za32_lane_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3341 3342 def int_aarch64_sme_usmla_za32_single_vg4x1 : SME2_Matrix_ArrayVector_Single_Single_Intrinsic; 3343 def int_aarch64_sme_usmla_za32_single_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3344 def int_aarch64_sme_usmla_za32_single_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3345 3346 def int_aarch64_sme_usmla_za32_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3347 def int_aarch64_sme_usmla_za32_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3348 3349 def int_aarch64_sme_usmla_za32_lane_vg4x1 : SME2_Matrix_ArrayVector_Single_Index_Intrinsic; 3350 def int_aarch64_sme_usmla_za32_lane_vg4x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3351 def int_aarch64_sme_usmla_za32_lane_vg4x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3352 3353 def int_aarch64_sve_bfmlslb : SME2_BFMLS_Intrinsic; 3354 def int_aarch64_sve_bfmlslb_lane : SME2_BFMLS_Lane_Intrinsic; 3355 3356 def int_aarch64_sve_bfmlslt : SME2_BFMLS_Intrinsic; 3357 def int_aarch64_sve_bfmlslt_lane : SME2_BFMLS_Lane_Intrinsic; 3358 3359 // Multi-vector signed saturating doubling multiply high 3360 3361 def int_aarch64_sve_sqdmulh_single_vgx2 : SME2_VG2_Multi_Single_Intrinsic; 3362 def int_aarch64_sve_sqdmulh_single_vgx4 : SME2_VG4_Multi_Single_Intrinsic; 3363 3364 def int_aarch64_sve_sqdmulh_vgx2 : SME2_VG2_Multi_Multi_Intrinsic; 3365 def int_aarch64_sve_sqdmulh_vgx4 : SME2_VG4_Multi_Multi_Intrinsic; 3366 3367 // Multi-vector floating-point round to integral value 3368 3369 foreach inst = ["a", "m", "n", "p"] in { 3370 def int_aarch64_sve_frint # inst # _x2 : SVE2_VG2_ZipUzp_Intrinsic; 3371 def int_aarch64_sve_frint # inst # _x4 : SVE2_VG4_ZipUzp_Intrinsic; 3372 } 3373 3374 // 3375 // Multi-vector min/max 3376 // 3377 3378 foreach ty = ["f", "s", "u"] in { 3379 foreach instr = ["max", "min"] in { 3380 def int_aarch64_sve_ # ty # instr # _single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3381 def int_aarch64_sve_ # ty # instr # _single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3382 3383 def int_aarch64_sve_ # ty # instr # _x2 : SME2_VG2_Multi_Multi_Intrinsic; 3384 def int_aarch64_sve_ # ty # instr # _x4 : SME2_VG4_Multi_Multi_Intrinsic; 3385 } 3386 } 3387 3388 // 3389 // Multi-vector floating point min/max number 3390 // 3391 3392 foreach instr = ["fmaxnm", "fminnm"] in { 3393 def int_aarch64_sve_ # instr # _single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3394 def int_aarch64_sve_ # instr # _single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3395 3396 def int_aarch64_sve_ # instr # _x2 : SME2_VG2_Multi_Multi_Intrinsic; 3397 def int_aarch64_sve_ # instr # _x4 : SME2_VG4_Multi_Multi_Intrinsic; 3398 } 3399 3400 // 3401 // Multi-vector vertical dot-products 3402 // 3403 3404 def int_aarch64_sme_fvdot_lane_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3405 3406 foreach ty = ["s", "u"] in { 3407 def int_aarch64_sme_ #ty # vdot_lane_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3408 def int_aarch64_sme_ #ty # vdot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3409 def int_aarch64_sme_ #ty # vdot_lane_za64_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3410 } 3411 3412 def int_aarch64_sme_suvdot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3413 def int_aarch64_sme_usvdot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3414 3415 // 3416 // Multi-vector floating-point CVT from single-precision to interleaved half-precision/BFloat16 3417 // 3418 def int_aarch64_sve_fcvtn_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3419 def int_aarch64_sve_bfcvtn_x2 : SME2_CVT_VG2_SINGLE_BF16_Intrinsic; 3420 3421 // 3422 // Multi-vector convert to/from floating-point. 3423 // 3424 def int_aarch64_sve_fcvt_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3425 def int_aarch64_sve_bfcvt_x2 : SME2_CVT_VG2_SINGLE_BF16_Intrinsic; 3426 def int_aarch64_sve_fcvtzs_x2 : SME2_CVT_X2_Intrinsic; 3427 def int_aarch64_sve_fcvtzu_x2 : SME2_CVT_X2_Intrinsic; 3428 def int_aarch64_sve_scvtf_x2 : SME2_CVT_X2_Intrinsic; 3429 def int_aarch64_sve_ucvtf_x2 : SME2_CVT_X2_Intrinsic; 3430 def int_aarch64_sve_fcvtzs_x4 : SME2_CVT_X4_Intrinsic; 3431 def int_aarch64_sve_fcvtzu_x4 : SME2_CVT_X4_Intrinsic; 3432 def int_aarch64_sve_scvtf_x4 : SME2_CVT_X4_Intrinsic; 3433 def int_aarch64_sve_ucvtf_x4 : SME2_CVT_X4_Intrinsic; 3434 3435 // 3436 // Multi-vector saturating extract narrow 3437 // 3438 def int_aarch64_sve_sqcvt_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3439 def int_aarch64_sve_uqcvt_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3440 def int_aarch64_sve_sqcvtu_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3441 def int_aarch64_sve_sqcvt_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3442 def int_aarch64_sve_uqcvt_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3443 def int_aarch64_sve_sqcvtu_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3444 3445 // 3446 // Multi-vector saturating extract narrow and interleave 3447 // 3448 def int_aarch64_sve_sqcvtn_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3449 def int_aarch64_sve_uqcvtn_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3450 def int_aarch64_sve_sqcvtun_x2 : SME2_CVT_VG2_SINGLE_Intrinsic; 3451 def int_aarch64_sve_sqcvtn_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3452 def int_aarch64_sve_uqcvtn_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3453 def int_aarch64_sve_sqcvtun_x4 : SME2_CVT_VG4_SINGLE_Intrinsic; 3454 3455 // 3456 // Multi-Single add/sub 3457 // 3458 def int_aarch64_sme_add_write_single_za_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3459 def int_aarch64_sme_sub_write_single_za_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3460 def int_aarch64_sme_add_write_single_za_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3461 def int_aarch64_sme_sub_write_single_za_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3462 3463 // 3464 // Multi-Multi add/sub 3465 // 3466 def int_aarch64_sme_add_write_za_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3467 def int_aarch64_sme_sub_write_za_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3468 def int_aarch64_sme_add_write_za_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3469 def int_aarch64_sme_sub_write_za_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3470 3471 // Multi-vector clamps 3472 def int_aarch64_sve_sclamp_single_x2 : SME2_VG2_Multi_Single_Single_Intrinsic; 3473 def int_aarch64_sve_uclamp_single_x2 : SME2_VG2_Multi_Single_Single_Intrinsic; 3474 def int_aarch64_sve_fclamp_single_x2 : SME2_VG2_Multi_Single_Single_Intrinsic; 3475 3476 def int_aarch64_sve_sclamp_single_x4 : SME2_VG4_Multi_Single_Single_Intrinsic; 3477 def int_aarch64_sve_uclamp_single_x4 : SME2_VG4_Multi_Single_Single_Intrinsic; 3478 def int_aarch64_sve_fclamp_single_x4 : SME2_VG4_Multi_Single_Single_Intrinsic; 3479 3480 // 3481 // Multi-vector add/sub and accumulate into ZA 3482 // 3483 foreach intr = ["add", "sub"] in { 3484 foreach za = ["za32", "za64"] in { 3485 def int_aarch64_sme_ # intr # _ # za # _vg1x2 : SME2_ZA_Write_VG2_Intrinsic; 3486 def int_aarch64_sme_ # intr # _ # za # _vg1x4 : SME2_ZA_Write_VG4_Intrinsic; 3487 } 3488 } 3489 3490 // 3491 // Move multi-vectors to/from ZA 3492 // 3493 3494 def int_aarch64_sme_read_hor_vg2 : SME2_Matrix_TileVector_Read_VG2_Intrinsic; 3495 def int_aarch64_sme_read_hor_vg4 : SME2_Matrix_TileVector_Read_VG4_Intrinsic; 3496 3497 def int_aarch64_sme_read_ver_vg2 : SME2_Matrix_TileVector_Read_VG2_Intrinsic; 3498 def int_aarch64_sme_read_ver_vg4 : SME2_Matrix_TileVector_Read_VG4_Intrinsic; 3499 3500 def int_aarch64_sme_read_vg1x2 : SME2_ZA_ArrayVector_Read_VG2_Intrinsic; 3501 def int_aarch64_sme_read_vg1x4 : SME2_ZA_ArrayVector_Read_VG4_Intrinsic; 3502 3503 def int_aarch64_sme_write_hor_vg2 : SME2_Matrix_TileVector_Write_VG2_Intrinsic; 3504 def int_aarch64_sme_write_hor_vg4 : SME2_Matrix_TileVector_Write_VG4_Intrinsic; 3505 3506 def int_aarch64_sme_write_ver_vg2 : SME2_Matrix_TileVector_Write_VG2_Intrinsic; 3507 def int_aarch64_sme_write_ver_vg4 : SME2_Matrix_TileVector_Write_VG4_Intrinsic; 3508 3509 def int_aarch64_sme_write_vg1x2 : SME2_ZA_ArrayVector_Write_VG2_Intrinsic; 3510 def int_aarch64_sme_write_vg1x4 : SME2_ZA_ArrayVector_Write_VG4_Intrinsic; 3511 3512 // 3513 // Multi-Single Vector add 3514 // 3515 def int_aarch64_sve_add_single_x2 : SME2_VG2_Multi_Single_Intrinsic; 3516 def int_aarch64_sve_add_single_x4 : SME2_VG4_Multi_Single_Intrinsic; 3517 3518 // 2-way and 4-way multi-vector signed/unsigned integer dot-product 3519 foreach ty = ["s", "u"] in { 3520 foreach sz = ["za32", "za64"] in { 3521 def int_aarch64_sme_ # ty # dot_single_ # sz # _vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3522 def int_aarch64_sme_ # ty # dot_single_ # sz # _vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3523 3524 def int_aarch64_sme_ # ty # dot_ # sz # _vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3525 def int_aarch64_sme_ # ty # dot_ # sz # _vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3526 3527 def int_aarch64_sme_ # ty # dot_lane_ # sz # _vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3528 def int_aarch64_sme_ # ty # dot_lane_ # sz # _vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3529 } 3530 } 3531 3532 foreach ty = ["su", "us"] in { 3533 def int_aarch64_sme_ # ty # dot_single_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3534 def int_aarch64_sme_ # ty # dot_single_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3535 3536 def int_aarch64_sme_ # ty # dot_lane_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3537 def int_aarch64_sme_ # ty # dot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3538 } 3539 3540 def int_aarch64_sme_usdot_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3541 def int_aarch64_sme_usdot_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3542 3543 // Multi-vector half-precision or bfloat floating-point dot-product 3544 def int_aarch64_sme_fdot_single_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Single_Intrinsic; 3545 def int_aarch64_sme_fdot_single_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Single_Intrinsic; 3546 3547 def int_aarch64_sme_fdot_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Multi_Intrinsic; 3548 def int_aarch64_sme_fdot_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Multi_Intrinsic; 3549 3550 def int_aarch64_sme_fdot_lane_za32_vg1x2 : SME2_Matrix_ArrayVector_VG2_Multi_Index_Intrinsic; 3551 def int_aarch64_sme_fdot_lane_za32_vg1x4 : SME2_Matrix_ArrayVector_VG4_Multi_Index_Intrinsic; 3552 3553 // Multi-vector zip and unzips 3554 def int_aarch64_sve_zip_x2 : SVE2_VG2_ZipUzp_Intrinsic; 3555 def int_aarch64_sve_zipq_x2 : SVE2_VG2_ZipUzp_Intrinsic; 3556 def int_aarch64_sve_zip_x4 : SVE2_VG4_ZipUzp_Intrinsic; 3557 def int_aarch64_sve_zipq_x4 : SVE2_VG4_ZipUzp_Intrinsic; 3558 def int_aarch64_sve_uzp_x2 : SVE2_VG2_ZipUzp_Intrinsic; 3559 def int_aarch64_sve_uzpq_x2 : SVE2_VG2_ZipUzp_Intrinsic; 3560 def int_aarch64_sve_uzp_x4 : SVE2_VG4_ZipUzp_Intrinsic; 3561 def int_aarch64_sve_uzpq_x4 : SVE2_VG4_ZipUzp_Intrinsic; 3562 3563 // Vector dot-products (2-way) 3564 def int_aarch64_sve_sdot_x2 : SVE2_3VectorArg_Long_Intrinsic; 3565 def int_aarch64_sve_udot_x2 : SVE2_3VectorArg_Long_Intrinsic; 3566 def int_aarch64_sve_fdot_x2 : SVE2_3VectorArg_Long_Intrinsic; 3567 def int_aarch64_sve_sdot_lane_x2 : SVE2_3VectorArgIndexed_Long_Intrinsic; 3568 def int_aarch64_sve_udot_lane_x2 : SVE2_3VectorArgIndexed_Long_Intrinsic; 3569 def int_aarch64_sve_fdot_lane_x2 : SVE2_3VectorArgIndexed_Long_Intrinsic; 3570 3571 // 3572 // Signed/unsigned multi-vector unpacks 3573 // 3574 def int_aarch64_sve_sunpk_x2 : SME2_VG2_Unpk_Intrinsic; 3575 def int_aarch64_sve_uunpk_x2 : SME2_VG2_Unpk_Intrinsic; 3576 def int_aarch64_sve_sunpk_x4 : SME2_VG4_Unpk_Intrinsic; 3577 def int_aarch64_sve_uunpk_x4 : SME2_VG4_Unpk_Intrinsic; 3578 3579 // 2-way and 4-way vector selects 3580 def int_aarch64_sve_sel_x2 : SVE2_VG2_Sel_Intrinsic; 3581 def int_aarch64_sve_sel_x4 : SVE2_VG4_Sel_Intrinsic; 3582 3583 class SME_LDR_STR_ZT_Intrinsic 3584 : DefaultAttrsIntrinsic<[], [llvm_i32_ty, llvm_ptr_ty]>; 3585 def int_aarch64_sme_ldr_zt : SME_LDR_STR_ZT_Intrinsic; 3586 def int_aarch64_sme_str_zt : SME_LDR_STR_ZT_Intrinsic; 3587 3588 // 3589 // Zero ZT0 3590 // 3591 def int_aarch64_sme_zero_zt : DefaultAttrsIntrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>, IntrWriteMem]>; 3592 3593 // 3594 // Lookup table expand one register 3595 // 3596 def int_aarch64_sme_luti2_lane_zt 3597 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3598 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3599 def int_aarch64_sme_luti4_lane_zt 3600 : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3601 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3602 3603 // Lookup table expand two registers 3604 // 3605 def int_aarch64_sme_luti2_lane_zt_x2 3606 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3607 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3608 def int_aarch64_sme_luti4_lane_zt_x2 3609 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>], [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3610 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3611 3612 // 3613 // Lookup table expand four registers 3614 // 3615 def int_aarch64_sme_luti2_lane_zt_x4 3616 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3617 [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3618 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3619 def int_aarch64_sme_luti4_lane_zt_x4 3620 : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 3621 [llvm_i32_ty, llvm_nxv16i8_ty, llvm_i32_ty], 3622 [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<2>>, IntrReadMem]>; 3623} 3624 3625// SVE2.1 - ZIPQ1, ZIPQ2, UZPQ1, UZPQ2 3626// 3627def int_aarch64_sve_zipq1 : AdvSIMD_2VectorArg_Intrinsic; 3628def int_aarch64_sve_zipq2 : AdvSIMD_2VectorArg_Intrinsic; 3629def int_aarch64_sve_uzpq1 : AdvSIMD_2VectorArg_Intrinsic; 3630def int_aarch64_sve_uzpq2 : AdvSIMD_2VectorArg_Intrinsic; 3631 3632// SVE2.1 - Programmable table lookup within each quadword vector segment 3633// (zeroing)/(merging) 3634// 3635def int_aarch64_sve_tblq : AdvSIMD_SVE_TBL_Intrinsic; 3636def int_aarch64_sve_tbxq : AdvSIMD_SVE2_TBX_Intrinsic; 3637 3638// SVE2.1 - Extract vector segment from each pair of quadword segments. 3639// 3640def int_aarch64_sve_extq : AdvSIMD_2VectorArgIndexed_Intrinsic; 3641 3642// 3643// SVE2.1 - Move predicate to/from vector 3644// 3645def int_aarch64_sve_pmov_to_pred_lane : SVE2_1VectorArgIndexed_Pred_Intrinsic; 3646 3647def int_aarch64_sve_pmov_to_pred_lane_zero : SVE2_1VectorArg_Pred_Intrinsic; 3648 3649def int_aarch64_sve_pmov_to_vector_lane_merging : SVE2_Pred_1VectorArgIndexed_Intrinsic; 3650 3651def int_aarch64_sve_pmov_to_vector_lane_zeroing : SVE2_Pred_1VectorArg_Intrinsic; 3652