1//===-- X86InstrSSE.td - SSE Instruction Set ---------------*- tablegen -*-===// 2// 3// The LLVM Compiler Infrastructure 4// 5// This file is distributed under the University of Illinois Open Source 6// License. See LICENSE.TXT for details. 7// 8//===----------------------------------------------------------------------===// 9// 10// This file describes the X86 SSE instruction set, defining the instructions, 11// and properties of the instructions which are needed for code generation, 12// machine code emission, and analysis. 13// 14//===----------------------------------------------------------------------===// 15 16class OpndItins<InstrItinClass arg_rr, InstrItinClass arg_rm> { 17 InstrItinClass rr = arg_rr; 18 InstrItinClass rm = arg_rm; 19 // InstrSchedModel info. 20 X86FoldableSchedWrite Sched = WriteFAdd; 21} 22 23class SizeItins<OpndItins arg_s, OpndItins arg_d> { 24 OpndItins s = arg_s; 25 OpndItins d = arg_d; 26} 27 28 29class ShiftOpndItins<InstrItinClass arg_rr, InstrItinClass arg_rm, 30 InstrItinClass arg_ri> { 31 InstrItinClass rr = arg_rr; 32 InstrItinClass rm = arg_rm; 33 InstrItinClass ri = arg_ri; 34} 35 36 37// scalar 38let Sched = WriteFAdd in { 39def SSE_ALU_F32S : OpndItins< 40 IIC_SSE_ALU_F32S_RR, IIC_SSE_ALU_F32S_RM 41>; 42 43def SSE_ALU_F64S : OpndItins< 44 IIC_SSE_ALU_F64S_RR, IIC_SSE_ALU_F64S_RM 45>; 46} 47 48def SSE_ALU_ITINS_S : SizeItins< 49 SSE_ALU_F32S, SSE_ALU_F64S 50>; 51 52let Sched = WriteFMul in { 53def SSE_MUL_F32S : OpndItins< 54 IIC_SSE_MUL_F32S_RR, IIC_SSE_MUL_F64S_RM 55>; 56 57def SSE_MUL_F64S : OpndItins< 58 IIC_SSE_MUL_F64S_RR, IIC_SSE_MUL_F64S_RM 59>; 60} 61 62def SSE_MUL_ITINS_S : SizeItins< 63 SSE_MUL_F32S, SSE_MUL_F64S 64>; 65 66let Sched = WriteFDiv in { 67def SSE_DIV_F32S : OpndItins< 68 IIC_SSE_DIV_F32S_RR, IIC_SSE_DIV_F64S_RM 69>; 70 71def SSE_DIV_F64S : OpndItins< 72 IIC_SSE_DIV_F64S_RR, IIC_SSE_DIV_F64S_RM 73>; 74} 75 76def SSE_DIV_ITINS_S : SizeItins< 77 SSE_DIV_F32S, SSE_DIV_F64S 78>; 79 80// parallel 81let Sched = WriteFAdd in { 82def SSE_ALU_F32P : OpndItins< 83 IIC_SSE_ALU_F32P_RR, IIC_SSE_ALU_F32P_RM 84>; 85 86def SSE_ALU_F64P : OpndItins< 87 IIC_SSE_ALU_F64P_RR, IIC_SSE_ALU_F64P_RM 88>; 89} 90 91def SSE_ALU_ITINS_P : SizeItins< 92 SSE_ALU_F32P, SSE_ALU_F64P 93>; 94 95let Sched = WriteFMul in { 96def SSE_MUL_F32P : OpndItins< 97 IIC_SSE_MUL_F32P_RR, IIC_SSE_MUL_F64P_RM 98>; 99 100def SSE_MUL_F64P : OpndItins< 101 IIC_SSE_MUL_F64P_RR, IIC_SSE_MUL_F64P_RM 102>; 103} 104 105def SSE_MUL_ITINS_P : SizeItins< 106 SSE_MUL_F32P, SSE_MUL_F64P 107>; 108 109let Sched = WriteFDiv in { 110def SSE_DIV_F32P : OpndItins< 111 IIC_SSE_DIV_F32P_RR, IIC_SSE_DIV_F64P_RM 112>; 113 114def SSE_DIV_F64P : OpndItins< 115 IIC_SSE_DIV_F64P_RR, IIC_SSE_DIV_F64P_RM 116>; 117} 118 119def SSE_DIV_ITINS_P : SizeItins< 120 SSE_DIV_F32P, SSE_DIV_F64P 121>; 122 123let Sched = WriteVecLogic in 124def SSE_VEC_BIT_ITINS_P : OpndItins< 125 IIC_SSE_BIT_P_RR, IIC_SSE_BIT_P_RM 126>; 127 128def SSE_BIT_ITINS_P : OpndItins< 129 IIC_SSE_BIT_P_RR, IIC_SSE_BIT_P_RM 130>; 131 132let Sched = WriteVecALU in { 133def SSE_INTALU_ITINS_P : OpndItins< 134 IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM 135>; 136 137def SSE_INTALUQ_ITINS_P : OpndItins< 138 IIC_SSE_INTALUQ_P_RR, IIC_SSE_INTALUQ_P_RM 139>; 140} 141 142let Sched = WriteVecIMul in 143def SSE_INTMUL_ITINS_P : OpndItins< 144 IIC_SSE_INTMUL_P_RR, IIC_SSE_INTMUL_P_RM 145>; 146 147def SSE_INTSHIFT_ITINS_P : ShiftOpndItins< 148 IIC_SSE_INTSH_P_RR, IIC_SSE_INTSH_P_RM, IIC_SSE_INTSH_P_RI 149>; 150 151def SSE_MOVA_ITINS : OpndItins< 152 IIC_SSE_MOVA_P_RR, IIC_SSE_MOVA_P_RM 153>; 154 155def SSE_MOVU_ITINS : OpndItins< 156 IIC_SSE_MOVU_P_RR, IIC_SSE_MOVU_P_RM 157>; 158 159def SSE_DPPD_ITINS : OpndItins< 160 IIC_SSE_DPPD_RR, IIC_SSE_DPPD_RM 161>; 162 163def SSE_DPPS_ITINS : OpndItins< 164 IIC_SSE_DPPS_RR, IIC_SSE_DPPD_RM 165>; 166 167def DEFAULT_ITINS : OpndItins< 168 IIC_ALU_NONMEM, IIC_ALU_MEM 169>; 170 171def SSE_EXTRACT_ITINS : OpndItins< 172 IIC_SSE_EXTRACTPS_RR, IIC_SSE_EXTRACTPS_RM 173>; 174 175def SSE_INSERT_ITINS : OpndItins< 176 IIC_SSE_INSERTPS_RR, IIC_SSE_INSERTPS_RM 177>; 178 179let Sched = WriteMPSAD in 180def SSE_MPSADBW_ITINS : OpndItins< 181 IIC_SSE_MPSADBW_RR, IIC_SSE_MPSADBW_RM 182>; 183 184let Sched = WriteVecIMul in 185def SSE_PMULLD_ITINS : OpndItins< 186 IIC_SSE_PMULLD_RR, IIC_SSE_PMULLD_RM 187>; 188 189// Definitions for backward compatibility. 190// The instructions mapped on these definitions uses a different itinerary 191// than the actual scheduling model. 192let Sched = WriteShuffle in 193def DEFAULT_ITINS_SHUFFLESCHED : OpndItins< 194 IIC_ALU_NONMEM, IIC_ALU_MEM 195>; 196 197let Sched = WriteVecIMul in 198def DEFAULT_ITINS_VECIMULSCHED : OpndItins< 199 IIC_ALU_NONMEM, IIC_ALU_MEM 200>; 201 202let Sched = WriteShuffle in 203def SSE_INTALU_ITINS_SHUFF_P : OpndItins< 204 IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM 205>; 206 207let Sched = WriteMPSAD in 208def DEFAULT_ITINS_MPSADSCHED : OpndItins< 209 IIC_ALU_NONMEM, IIC_ALU_MEM 210>; 211 212let Sched = WriteFBlend in 213def DEFAULT_ITINS_FBLENDSCHED : OpndItins< 214 IIC_ALU_NONMEM, IIC_ALU_MEM 215>; 216 217let Sched = WriteBlend in 218def DEFAULT_ITINS_BLENDSCHED : OpndItins< 219 IIC_ALU_NONMEM, IIC_ALU_MEM 220>; 221 222let Sched = WriteVarBlend in 223def DEFAULT_ITINS_VARBLENDSCHED : OpndItins< 224 IIC_ALU_NONMEM, IIC_ALU_MEM 225>; 226 227let Sched = WriteFBlend in 228def SSE_INTALU_ITINS_FBLEND_P : OpndItins< 229 IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM 230>; 231 232let Sched = WriteBlend in 233def SSE_INTALU_ITINS_BLEND_P : OpndItins< 234 IIC_SSE_INTALU_P_RR, IIC_SSE_INTALU_P_RM 235>; 236 237//===----------------------------------------------------------------------===// 238// SSE 1 & 2 Instructions Classes 239//===----------------------------------------------------------------------===// 240 241/// sse12_fp_scalar - SSE 1 & 2 scalar instructions class 242multiclass sse12_fp_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode, 243 RegisterClass RC, X86MemOperand x86memop, 244 Domain d, OpndItins itins, bit Is2Addr = 1> { 245 let isCommutable = 1 in { 246 def rr : SI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 247 !if(Is2Addr, 248 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 249 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 250 [(set RC:$dst, (OpNode RC:$src1, RC:$src2))], itins.rr, d>, 251 Sched<[itins.Sched]>; 252 } 253 def rm : SI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 254 !if(Is2Addr, 255 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 256 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 257 [(set RC:$dst, (OpNode RC:$src1, (load addr:$src2)))], itins.rm, d>, 258 Sched<[itins.Sched.Folded, ReadAfterLd]>; 259} 260 261/// sse12_fp_scalar_int - SSE 1 & 2 scalar instructions intrinsics class 262multiclass sse12_fp_scalar_int<bits<8> opc, string OpcodeStr, RegisterClass RC, 263 string asm, string SSEVer, string FPSizeStr, 264 Operand memopr, ComplexPattern mem_cpat, 265 Domain d, OpndItins itins, bit Is2Addr = 1> { 266let isCodeGenOnly = 1 in { 267 def rr_Int : SI_Int<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 268 !if(Is2Addr, 269 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"), 270 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 271 [(set RC:$dst, (!cast<Intrinsic>( 272 !strconcat("int_x86_sse", SSEVer, "_", OpcodeStr, FPSizeStr)) 273 RC:$src1, RC:$src2))], itins.rr, d>, 274 Sched<[itins.Sched]>; 275 def rm_Int : SI_Int<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, memopr:$src2), 276 !if(Is2Addr, 277 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"), 278 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 279 [(set RC:$dst, (!cast<Intrinsic>(!strconcat("int_x86_sse", 280 SSEVer, "_", OpcodeStr, FPSizeStr)) 281 RC:$src1, mem_cpat:$src2))], itins.rm, d>, 282 Sched<[itins.Sched.Folded, ReadAfterLd]>; 283} 284} 285 286/// sse12_fp_packed - SSE 1 & 2 packed instructions class 287multiclass sse12_fp_packed<bits<8> opc, string OpcodeStr, SDNode OpNode, 288 RegisterClass RC, ValueType vt, 289 X86MemOperand x86memop, PatFrag mem_frag, 290 Domain d, OpndItins itins, bit Is2Addr = 1> { 291 let isCommutable = 1 in 292 def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 293 !if(Is2Addr, 294 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 295 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 296 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], itins.rr, d>, 297 Sched<[itins.Sched]>; 298 let mayLoad = 1 in 299 def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 300 !if(Is2Addr, 301 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 302 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 303 [(set RC:$dst, (OpNode RC:$src1, (mem_frag addr:$src2)))], 304 itins.rm, d>, 305 Sched<[itins.Sched.Folded, ReadAfterLd]>; 306} 307 308/// sse12_fp_packed_logical_rm - SSE 1 & 2 packed instructions class 309multiclass sse12_fp_packed_logical_rm<bits<8> opc, RegisterClass RC, Domain d, 310 string OpcodeStr, X86MemOperand x86memop, 311 list<dag> pat_rr, list<dag> pat_rm, 312 bit Is2Addr = 1> { 313 let isCommutable = 1, hasSideEffects = 0 in 314 def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 315 !if(Is2Addr, 316 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 317 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 318 pat_rr, NoItinerary, d>, 319 Sched<[WriteVecLogic]>; 320 def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 321 !if(Is2Addr, 322 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 323 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 324 pat_rm, NoItinerary, d>, 325 Sched<[WriteVecLogicLd, ReadAfterLd]>; 326} 327 328//===----------------------------------------------------------------------===// 329// Non-instruction patterns 330//===----------------------------------------------------------------------===// 331 332// A vector extract of the first f32/f64 position is a subregister copy 333def : Pat<(f32 (extractelt (v4f32 VR128:$src), (iPTR 0))), 334 (COPY_TO_REGCLASS (v4f32 VR128:$src), FR32)>; 335def : Pat<(f64 (extractelt (v2f64 VR128:$src), (iPTR 0))), 336 (COPY_TO_REGCLASS (v2f64 VR128:$src), FR64)>; 337 338// A 128-bit subvector extract from the first 256-bit vector position 339// is a subregister copy that needs no instruction. 340def : Pat<(v4i32 (extract_subvector (v8i32 VR256:$src), (iPTR 0))), 341 (v4i32 (EXTRACT_SUBREG (v8i32 VR256:$src), sub_xmm))>; 342def : Pat<(v4f32 (extract_subvector (v8f32 VR256:$src), (iPTR 0))), 343 (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src), sub_xmm))>; 344 345def : Pat<(v2i64 (extract_subvector (v4i64 VR256:$src), (iPTR 0))), 346 (v2i64 (EXTRACT_SUBREG (v4i64 VR256:$src), sub_xmm))>; 347def : Pat<(v2f64 (extract_subvector (v4f64 VR256:$src), (iPTR 0))), 348 (v2f64 (EXTRACT_SUBREG (v4f64 VR256:$src), sub_xmm))>; 349 350def : Pat<(v8i16 (extract_subvector (v16i16 VR256:$src), (iPTR 0))), 351 (v8i16 (EXTRACT_SUBREG (v16i16 VR256:$src), sub_xmm))>; 352def : Pat<(v16i8 (extract_subvector (v32i8 VR256:$src), (iPTR 0))), 353 (v16i8 (EXTRACT_SUBREG (v32i8 VR256:$src), sub_xmm))>; 354 355// A 128-bit subvector insert to the first 256-bit vector position 356// is a subregister copy that needs no instruction. 357let AddedComplexity = 25 in { // to give priority over vinsertf128rm 358def : Pat<(insert_subvector undef, (v2i64 VR128:$src), (iPTR 0)), 359 (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 360def : Pat<(insert_subvector undef, (v2f64 VR128:$src), (iPTR 0)), 361 (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 362def : Pat<(insert_subvector undef, (v4i32 VR128:$src), (iPTR 0)), 363 (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 364def : Pat<(insert_subvector undef, (v4f32 VR128:$src), (iPTR 0)), 365 (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 366def : Pat<(insert_subvector undef, (v8i16 VR128:$src), (iPTR 0)), 367 (INSERT_SUBREG (v16i16 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 368def : Pat<(insert_subvector undef, (v16i8 VR128:$src), (iPTR 0)), 369 (INSERT_SUBREG (v32i8 (IMPLICIT_DEF)), VR128:$src, sub_xmm)>; 370} 371 372// Implicitly promote a 32-bit scalar to a vector. 373def : Pat<(v4f32 (scalar_to_vector FR32:$src)), 374 (COPY_TO_REGCLASS FR32:$src, VR128)>; 375def : Pat<(v8f32 (scalar_to_vector FR32:$src)), 376 (COPY_TO_REGCLASS FR32:$src, VR128)>; 377// Implicitly promote a 64-bit scalar to a vector. 378def : Pat<(v2f64 (scalar_to_vector FR64:$src)), 379 (COPY_TO_REGCLASS FR64:$src, VR128)>; 380def : Pat<(v4f64 (scalar_to_vector FR64:$src)), 381 (COPY_TO_REGCLASS FR64:$src, VR128)>; 382 383// Bitcasts between 128-bit vector types. Return the original type since 384// no instruction is needed for the conversion 385def : Pat<(v2i64 (bitconvert (v4i32 VR128:$src))), (v2i64 VR128:$src)>; 386def : Pat<(v2i64 (bitconvert (v8i16 VR128:$src))), (v2i64 VR128:$src)>; 387def : Pat<(v2i64 (bitconvert (v16i8 VR128:$src))), (v2i64 VR128:$src)>; 388def : Pat<(v2i64 (bitconvert (v2f64 VR128:$src))), (v2i64 VR128:$src)>; 389def : Pat<(v2i64 (bitconvert (v4f32 VR128:$src))), (v2i64 VR128:$src)>; 390def : Pat<(v4i32 (bitconvert (v2i64 VR128:$src))), (v4i32 VR128:$src)>; 391def : Pat<(v4i32 (bitconvert (v8i16 VR128:$src))), (v4i32 VR128:$src)>; 392def : Pat<(v4i32 (bitconvert (v16i8 VR128:$src))), (v4i32 VR128:$src)>; 393def : Pat<(v4i32 (bitconvert (v2f64 VR128:$src))), (v4i32 VR128:$src)>; 394def : Pat<(v4i32 (bitconvert (v4f32 VR128:$src))), (v4i32 VR128:$src)>; 395def : Pat<(v8i16 (bitconvert (v2i64 VR128:$src))), (v8i16 VR128:$src)>; 396def : Pat<(v8i16 (bitconvert (v4i32 VR128:$src))), (v8i16 VR128:$src)>; 397def : Pat<(v8i16 (bitconvert (v16i8 VR128:$src))), (v8i16 VR128:$src)>; 398def : Pat<(v8i16 (bitconvert (v2f64 VR128:$src))), (v8i16 VR128:$src)>; 399def : Pat<(v8i16 (bitconvert (v4f32 VR128:$src))), (v8i16 VR128:$src)>; 400def : Pat<(v16i8 (bitconvert (v2i64 VR128:$src))), (v16i8 VR128:$src)>; 401def : Pat<(v16i8 (bitconvert (v4i32 VR128:$src))), (v16i8 VR128:$src)>; 402def : Pat<(v16i8 (bitconvert (v8i16 VR128:$src))), (v16i8 VR128:$src)>; 403def : Pat<(v16i8 (bitconvert (v2f64 VR128:$src))), (v16i8 VR128:$src)>; 404def : Pat<(v16i8 (bitconvert (v4f32 VR128:$src))), (v16i8 VR128:$src)>; 405def : Pat<(v4f32 (bitconvert (v2i64 VR128:$src))), (v4f32 VR128:$src)>; 406def : Pat<(v4f32 (bitconvert (v4i32 VR128:$src))), (v4f32 VR128:$src)>; 407def : Pat<(v4f32 (bitconvert (v8i16 VR128:$src))), (v4f32 VR128:$src)>; 408def : Pat<(v4f32 (bitconvert (v16i8 VR128:$src))), (v4f32 VR128:$src)>; 409def : Pat<(v4f32 (bitconvert (v2f64 VR128:$src))), (v4f32 VR128:$src)>; 410def : Pat<(v2f64 (bitconvert (v2i64 VR128:$src))), (v2f64 VR128:$src)>; 411def : Pat<(v2f64 (bitconvert (v4i32 VR128:$src))), (v2f64 VR128:$src)>; 412def : Pat<(v2f64 (bitconvert (v8i16 VR128:$src))), (v2f64 VR128:$src)>; 413def : Pat<(v2f64 (bitconvert (v16i8 VR128:$src))), (v2f64 VR128:$src)>; 414def : Pat<(v2f64 (bitconvert (v4f32 VR128:$src))), (v2f64 VR128:$src)>; 415def : Pat<(f128 (bitconvert (i128 FR128:$src))), (f128 FR128:$src)>; 416def : Pat<(i128 (bitconvert (f128 FR128:$src))), (i128 FR128:$src)>; 417 418// Bitcasts between 256-bit vector types. Return the original type since 419// no instruction is needed for the conversion 420def : Pat<(v4i64 (bitconvert (v8i32 VR256:$src))), (v4i64 VR256:$src)>; 421def : Pat<(v4i64 (bitconvert (v16i16 VR256:$src))), (v4i64 VR256:$src)>; 422def : Pat<(v4i64 (bitconvert (v32i8 VR256:$src))), (v4i64 VR256:$src)>; 423def : Pat<(v4i64 (bitconvert (v8f32 VR256:$src))), (v4i64 VR256:$src)>; 424def : Pat<(v4i64 (bitconvert (v4f64 VR256:$src))), (v4i64 VR256:$src)>; 425def : Pat<(v8i32 (bitconvert (v4i64 VR256:$src))), (v8i32 VR256:$src)>; 426def : Pat<(v8i32 (bitconvert (v16i16 VR256:$src))), (v8i32 VR256:$src)>; 427def : Pat<(v8i32 (bitconvert (v32i8 VR256:$src))), (v8i32 VR256:$src)>; 428def : Pat<(v8i32 (bitconvert (v4f64 VR256:$src))), (v8i32 VR256:$src)>; 429def : Pat<(v8i32 (bitconvert (v8f32 VR256:$src))), (v8i32 VR256:$src)>; 430def : Pat<(v16i16 (bitconvert (v4i64 VR256:$src))), (v16i16 VR256:$src)>; 431def : Pat<(v16i16 (bitconvert (v8i32 VR256:$src))), (v16i16 VR256:$src)>; 432def : Pat<(v16i16 (bitconvert (v32i8 VR256:$src))), (v16i16 VR256:$src)>; 433def : Pat<(v16i16 (bitconvert (v4f64 VR256:$src))), (v16i16 VR256:$src)>; 434def : Pat<(v16i16 (bitconvert (v8f32 VR256:$src))), (v16i16 VR256:$src)>; 435def : Pat<(v32i8 (bitconvert (v4i64 VR256:$src))), (v32i8 VR256:$src)>; 436def : Pat<(v32i8 (bitconvert (v8i32 VR256:$src))), (v32i8 VR256:$src)>; 437def : Pat<(v32i8 (bitconvert (v16i16 VR256:$src))), (v32i8 VR256:$src)>; 438def : Pat<(v32i8 (bitconvert (v4f64 VR256:$src))), (v32i8 VR256:$src)>; 439def : Pat<(v32i8 (bitconvert (v8f32 VR256:$src))), (v32i8 VR256:$src)>; 440def : Pat<(v8f32 (bitconvert (v4i64 VR256:$src))), (v8f32 VR256:$src)>; 441def : Pat<(v8f32 (bitconvert (v8i32 VR256:$src))), (v8f32 VR256:$src)>; 442def : Pat<(v8f32 (bitconvert (v16i16 VR256:$src))), (v8f32 VR256:$src)>; 443def : Pat<(v8f32 (bitconvert (v32i8 VR256:$src))), (v8f32 VR256:$src)>; 444def : Pat<(v8f32 (bitconvert (v4f64 VR256:$src))), (v8f32 VR256:$src)>; 445def : Pat<(v4f64 (bitconvert (v4i64 VR256:$src))), (v4f64 VR256:$src)>; 446def : Pat<(v4f64 (bitconvert (v8i32 VR256:$src))), (v4f64 VR256:$src)>; 447def : Pat<(v4f64 (bitconvert (v16i16 VR256:$src))), (v4f64 VR256:$src)>; 448def : Pat<(v4f64 (bitconvert (v32i8 VR256:$src))), (v4f64 VR256:$src)>; 449def : Pat<(v4f64 (bitconvert (v8f32 VR256:$src))), (v4f64 VR256:$src)>; 450 451// Alias instructions that map fld0 to xorps for sse or vxorps for avx. 452// This is expanded by ExpandPostRAPseudos. 453let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, 454 isPseudo = 1, SchedRW = [WriteZero] in { 455 def FsFLD0SS : I<0, Pseudo, (outs FR32:$dst), (ins), "", 456 [(set FR32:$dst, fp32imm0)]>, Requires<[HasSSE1]>; 457 def FsFLD0SD : I<0, Pseudo, (outs FR64:$dst), (ins), "", 458 [(set FR64:$dst, fpimm0)]>, Requires<[HasSSE2]>; 459} 460 461//===----------------------------------------------------------------------===// 462// AVX & SSE - Zero/One Vectors 463//===----------------------------------------------------------------------===// 464 465// Alias instruction that maps zero vector to pxor / xorp* for sse. 466// This is expanded by ExpandPostRAPseudos to an xorps / vxorps, and then 467// swizzled by ExecutionDepsFix to pxor. 468// We set canFoldAsLoad because this can be converted to a constant-pool 469// load of an all-zeros value if folding it would be beneficial. 470let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, 471 isPseudo = 1, Predicates = [NoVLX], SchedRW = [WriteZero] in { 472def V_SET0 : I<0, Pseudo, (outs VR128:$dst), (ins), "", 473 [(set VR128:$dst, (v4f32 immAllZerosV))]>; 474} 475 476let Predicates = [NoVLX] in 477def : Pat<(v4i32 immAllZerosV), (V_SET0)>; 478 479 480// The same as done above but for AVX. The 256-bit AVX1 ISA doesn't support PI, 481// and doesn't need it because on sandy bridge the register is set to zero 482// at the rename stage without using any execution unit, so SET0PSY 483// and SET0PDY can be used for vector int instructions without penalty 484let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, 485 isPseudo = 1, Predicates = [HasAVX, NoVLX], SchedRW = [WriteZero] in { 486def AVX_SET0 : I<0, Pseudo, (outs VR256:$dst), (ins), "", 487 [(set VR256:$dst, (v8i32 immAllZerosV))]>; 488} 489 490// We set canFoldAsLoad because this can be converted to a constant-pool 491// load of an all-ones value if folding it would be beneficial. 492let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1, 493 isPseudo = 1, SchedRW = [WriteZero] in { 494 def V_SETALLONES : I<0, Pseudo, (outs VR128:$dst), (ins), "", 495 [(set VR128:$dst, (v4i32 immAllOnesV))]>; 496 let Predicates = [HasAVX2] in 497 def AVX2_SETALLONES : I<0, Pseudo, (outs VR256:$dst), (ins), "", 498 [(set VR256:$dst, (v8i32 immAllOnesV))]>; 499} 500 501 502//===----------------------------------------------------------------------===// 503// SSE 1 & 2 - Move FP Scalar Instructions 504// 505// Move Instructions. Register-to-register movss/movsd is not used for FR32/64 506// register copies because it's a partial register update; Register-to-register 507// movss/movsd is not modeled as an INSERT_SUBREG because INSERT_SUBREG requires 508// that the insert be implementable in terms of a copy, and just mentioned, we 509// don't use movss/movsd for copies. 510//===----------------------------------------------------------------------===// 511 512multiclass sse12_move_rr<RegisterClass RC, SDNode OpNode, ValueType vt, 513 X86MemOperand x86memop, string base_opc, 514 string asm_opr, Domain d = GenericDomain> { 515 def rr : SI<0x10, MRMSrcReg, (outs VR128:$dst), 516 (ins VR128:$src1, RC:$src2), 517 !strconcat(base_opc, asm_opr), 518 [(set VR128:$dst, (vt (OpNode VR128:$src1, 519 (scalar_to_vector RC:$src2))))], 520 IIC_SSE_MOV_S_RR, d>, Sched<[WriteFShuffle]>; 521 522 // For the disassembler 523 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in 524 def rr_REV : SI<0x11, MRMDestReg, (outs VR128:$dst), 525 (ins VR128:$src1, RC:$src2), 526 !strconcat(base_opc, asm_opr), 527 [], IIC_SSE_MOV_S_RR>, Sched<[WriteFShuffle]>; 528} 529 530multiclass sse12_move<RegisterClass RC, SDNode OpNode, ValueType vt, 531 X86MemOperand x86memop, string OpcodeStr, 532 Domain d = GenericDomain> { 533 // AVX 534 defm V#NAME : sse12_move_rr<RC, OpNode, vt, x86memop, OpcodeStr, 535 "\t{$src2, $src1, $dst|$dst, $src1, $src2}", d>, 536 VEX_4V, VEX_LIG; 537 538 def V#NAME#mr : SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src), 539 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 540 [(store RC:$src, addr:$dst)], IIC_SSE_MOV_S_MR, d>, 541 VEX, VEX_LIG, Sched<[WriteStore]>; 542 // SSE1 & 2 543 let Constraints = "$src1 = $dst" in { 544 defm NAME : sse12_move_rr<RC, OpNode, vt, x86memop, OpcodeStr, 545 "\t{$src2, $dst|$dst, $src2}", d>; 546 } 547 548 def NAME#mr : SI<0x11, MRMDestMem, (outs), (ins x86memop:$dst, RC:$src), 549 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 550 [(store RC:$src, addr:$dst)], IIC_SSE_MOV_S_MR, d>, 551 Sched<[WriteStore]>; 552} 553 554// Loading from memory automatically zeroing upper bits. 555multiclass sse12_move_rm<RegisterClass RC, X86MemOperand x86memop, 556 PatFrag mem_pat, string OpcodeStr, 557 Domain d = GenericDomain> { 558 def V#NAME#rm : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 559 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 560 [(set RC:$dst, (mem_pat addr:$src))], 561 IIC_SSE_MOV_S_RM, d>, VEX, VEX_LIG, Sched<[WriteLoad]>; 562 def NAME#rm : SI<0x10, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 563 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 564 [(set RC:$dst, (mem_pat addr:$src))], 565 IIC_SSE_MOV_S_RM, d>, Sched<[WriteLoad]>; 566} 567 568defm MOVSS : sse12_move<FR32, X86Movss, v4f32, f32mem, "movss", 569 SSEPackedSingle>, XS; 570defm MOVSD : sse12_move<FR64, X86Movsd, v2f64, f64mem, "movsd", 571 SSEPackedDouble>, XD; 572 573let canFoldAsLoad = 1, isReMaterializable = 1 in { 574 defm MOVSS : sse12_move_rm<FR32, f32mem, loadf32, "movss", 575 SSEPackedSingle>, XS; 576 577 let AddedComplexity = 20 in 578 defm MOVSD : sse12_move_rm<FR64, f64mem, loadf64, "movsd", 579 SSEPackedDouble>, XD; 580} 581 582// Patterns 583let Predicates = [UseAVX] in { 584 let AddedComplexity = 20 in { 585 // MOVSSrm zeros the high parts of the register; represent this 586 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 587 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))), 588 (COPY_TO_REGCLASS (VMOVSSrm addr:$src), VR128)>; 589 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))), 590 (COPY_TO_REGCLASS (VMOVSSrm addr:$src), VR128)>; 591 def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))), 592 (COPY_TO_REGCLASS (VMOVSSrm addr:$src), VR128)>; 593 594 // MOVSDrm zeros the high parts of the register; represent this 595 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0 596 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))), 597 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 598 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))), 599 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 600 def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))), 601 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 602 def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))), 603 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 604 def : Pat<(v2f64 (X86vzload addr:$src)), 605 (COPY_TO_REGCLASS (VMOVSDrm addr:$src), VR128)>; 606 607 // Represent the same patterns above but in the form they appear for 608 // 256-bit types 609 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, 610 (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))), 611 (SUBREG_TO_REG (i32 0), (VMOVSSrm addr:$src), sub_xmm)>; 612 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, 613 (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))), 614 (SUBREG_TO_REG (i32 0), (VMOVSDrm addr:$src), sub_xmm)>; 615 def : Pat<(v4f64 (X86vzload addr:$src)), 616 (SUBREG_TO_REG (i32 0), (VMOVSDrm addr:$src), sub_xmm)>; 617 } 618 619 // Extract and store. 620 def : Pat<(store (f32 (extractelt (v4f32 VR128:$src), (iPTR 0))), 621 addr:$dst), 622 (VMOVSSmr addr:$dst, (COPY_TO_REGCLASS (v4f32 VR128:$src), FR32))>; 623 624 // Shuffle with VMOVSS 625 def : Pat<(v4i32 (X86Movss VR128:$src1, VR128:$src2)), 626 (VMOVSSrr (v4i32 VR128:$src1), 627 (COPY_TO_REGCLASS (v4i32 VR128:$src2), FR32))>; 628 def : Pat<(v4f32 (X86Movss VR128:$src1, VR128:$src2)), 629 (VMOVSSrr (v4f32 VR128:$src1), 630 (COPY_TO_REGCLASS (v4f32 VR128:$src2), FR32))>; 631 632 // 256-bit variants 633 def : Pat<(v8i32 (X86Movss VR256:$src1, VR256:$src2)), 634 (SUBREG_TO_REG (i32 0), 635 (VMOVSSrr (EXTRACT_SUBREG (v8i32 VR256:$src1), sub_xmm), 636 (EXTRACT_SUBREG (v8i32 VR256:$src2), sub_xmm)), 637 sub_xmm)>; 638 def : Pat<(v8f32 (X86Movss VR256:$src1, VR256:$src2)), 639 (SUBREG_TO_REG (i32 0), 640 (VMOVSSrr (EXTRACT_SUBREG (v8f32 VR256:$src1), sub_xmm), 641 (EXTRACT_SUBREG (v8f32 VR256:$src2), sub_xmm)), 642 sub_xmm)>; 643 644 // Shuffle with VMOVSD 645 def : Pat<(v2i64 (X86Movsd VR128:$src1, VR128:$src2)), 646 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 647 def : Pat<(v2f64 (X86Movsd VR128:$src1, VR128:$src2)), 648 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 649 def : Pat<(v4f32 (X86Movsd VR128:$src1, VR128:$src2)), 650 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 651 def : Pat<(v4i32 (X86Movsd VR128:$src1, VR128:$src2)), 652 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 653 654 // 256-bit variants 655 def : Pat<(v4i64 (X86Movsd VR256:$src1, VR256:$src2)), 656 (SUBREG_TO_REG (i32 0), 657 (VMOVSDrr (EXTRACT_SUBREG (v4i64 VR256:$src1), sub_xmm), 658 (EXTRACT_SUBREG (v4i64 VR256:$src2), sub_xmm)), 659 sub_xmm)>; 660 def : Pat<(v4f64 (X86Movsd VR256:$src1, VR256:$src2)), 661 (SUBREG_TO_REG (i32 0), 662 (VMOVSDrr (EXTRACT_SUBREG (v4f64 VR256:$src1), sub_xmm), 663 (EXTRACT_SUBREG (v4f64 VR256:$src2), sub_xmm)), 664 sub_xmm)>; 665 666 // FIXME: Instead of a X86Movlps there should be a X86Movsd here, the problem 667 // is during lowering, where it's not possible to recognize the fold cause 668 // it has two uses through a bitcast. One use disappears at isel time and the 669 // fold opportunity reappears. 670 def : Pat<(v2f64 (X86Movlpd VR128:$src1, VR128:$src2)), 671 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 672 def : Pat<(v2i64 (X86Movlpd VR128:$src1, VR128:$src2)), 673 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 674 def : Pat<(v4f32 (X86Movlps VR128:$src1, VR128:$src2)), 675 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 676 def : Pat<(v4i32 (X86Movlps VR128:$src1, VR128:$src2)), 677 (VMOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 678} 679 680let Predicates = [UseSSE1] in { 681 let Predicates = [NoSSE41], AddedComplexity = 15 in { 682 // Move scalar to XMM zero-extended, zeroing a VR128 then do a 683 // MOVSS to the lower bits. 684 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32:$src)))), 685 (MOVSSrr (v4f32 (V_SET0)), FR32:$src)>; 686 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))), 687 (MOVSSrr (v4f32 (V_SET0)), (COPY_TO_REGCLASS VR128:$src, FR32))>; 688 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))), 689 (MOVSSrr (v4i32 (V_SET0)), (COPY_TO_REGCLASS VR128:$src, FR32))>; 690 } 691 692 let AddedComplexity = 20 in { 693 // MOVSSrm already zeros the high parts of the register. 694 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))), 695 (COPY_TO_REGCLASS (MOVSSrm addr:$src), VR128)>; 696 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))), 697 (COPY_TO_REGCLASS (MOVSSrm addr:$src), VR128)>; 698 def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))), 699 (COPY_TO_REGCLASS (MOVSSrm addr:$src), VR128)>; 700 } 701 702 // Extract and store. 703 def : Pat<(store (f32 (extractelt (v4f32 VR128:$src), (iPTR 0))), 704 addr:$dst), 705 (MOVSSmr addr:$dst, (COPY_TO_REGCLASS VR128:$src, FR32))>; 706 707 // Shuffle with MOVSS 708 def : Pat<(v4i32 (X86Movss VR128:$src1, VR128:$src2)), 709 (MOVSSrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR32))>; 710 def : Pat<(v4f32 (X86Movss VR128:$src1, VR128:$src2)), 711 (MOVSSrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR32))>; 712} 713 714let Predicates = [UseSSE2] in { 715 let Predicates = [NoSSE41], AddedComplexity = 15 in { 716 // Move scalar to XMM zero-extended, zeroing a VR128 then do a 717 // MOVSD to the lower bits. 718 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64:$src)))), 719 (MOVSDrr (v2f64 (V_SET0)), FR64:$src)>; 720 } 721 722 let AddedComplexity = 20 in { 723 // MOVSDrm already zeros the high parts of the register. 724 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))), 725 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 726 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))), 727 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 728 def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))), 729 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 730 def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))), 731 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 732 def : Pat<(v2f64 (X86vzload addr:$src)), 733 (COPY_TO_REGCLASS (MOVSDrm addr:$src), VR128)>; 734 } 735 736 // Shuffle with MOVSD 737 def : Pat<(v2i64 (X86Movsd VR128:$src1, VR128:$src2)), 738 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 739 def : Pat<(v2f64 (X86Movsd VR128:$src1, VR128:$src2)), 740 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 741 def : Pat<(v4f32 (X86Movsd VR128:$src1, VR128:$src2)), 742 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 743 def : Pat<(v4i32 (X86Movsd VR128:$src1, VR128:$src2)), 744 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 745 746 // FIXME: Instead of a X86Movlps there should be a X86Movsd here, the problem 747 // is during lowering, where it's not possible to recognize the fold because 748 // it has two uses through a bitcast. One use disappears at isel time and the 749 // fold opportunity reappears. 750 def : Pat<(v2f64 (X86Movlpd VR128:$src1, VR128:$src2)), 751 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 752 def : Pat<(v2i64 (X86Movlpd VR128:$src1, VR128:$src2)), 753 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 754 def : Pat<(v4f32 (X86Movlps VR128:$src1, VR128:$src2)), 755 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 756 def : Pat<(v4i32 (X86Movlps VR128:$src1, VR128:$src2)), 757 (MOVSDrr VR128:$src1, (COPY_TO_REGCLASS VR128:$src2, FR64))>; 758} 759 760// Aliases to help the assembler pick two byte VEX encodings by swapping the 761// operands relative to the normal instructions to use VEX.R instead of VEX.B. 762def : InstAlias<"vmovss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 763 (VMOVSSrr_REV VR128L:$dst, VR128:$src1, VR128H:$src2), 0>; 764def : InstAlias<"vmovsd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 765 (VMOVSDrr_REV VR128L:$dst, VR128:$src1, VR128H:$src2), 0>; 766 767//===----------------------------------------------------------------------===// 768// SSE 1 & 2 - Move Aligned/Unaligned FP Instructions 769//===----------------------------------------------------------------------===// 770 771multiclass sse12_mov_packed<bits<8> opc, RegisterClass RC, 772 X86MemOperand x86memop, PatFrag ld_frag, 773 string asm, Domain d, 774 OpndItins itins, 775 bit IsReMaterializable = 1> { 776let hasSideEffects = 0 in 777 def rr : PI<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src), 778 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), [], itins.rr, d>, 779 Sched<[WriteFShuffle]>; 780let canFoldAsLoad = 1, isReMaterializable = IsReMaterializable in 781 def rm : PI<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 782 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 783 [(set RC:$dst, (ld_frag addr:$src))], itins.rm, d>, 784 Sched<[WriteLoad]>; 785} 786 787let Predicates = [HasAVX, NoVLX] in { 788defm VMOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32, 789 "movaps", SSEPackedSingle, SSE_MOVA_ITINS>, 790 PS, VEX; 791defm VMOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64, 792 "movapd", SSEPackedDouble, SSE_MOVA_ITINS>, 793 PD, VEX; 794defm VMOVUPS : sse12_mov_packed<0x10, VR128, f128mem, loadv4f32, 795 "movups", SSEPackedSingle, SSE_MOVU_ITINS>, 796 PS, VEX; 797defm VMOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64, 798 "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>, 799 PD, VEX; 800 801defm VMOVAPSY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv8f32, 802 "movaps", SSEPackedSingle, SSE_MOVA_ITINS>, 803 PS, VEX, VEX_L; 804defm VMOVAPDY : sse12_mov_packed<0x28, VR256, f256mem, alignedloadv4f64, 805 "movapd", SSEPackedDouble, SSE_MOVA_ITINS>, 806 PD, VEX, VEX_L; 807defm VMOVUPSY : sse12_mov_packed<0x10, VR256, f256mem, loadv8f32, 808 "movups", SSEPackedSingle, SSE_MOVU_ITINS>, 809 PS, VEX, VEX_L; 810defm VMOVUPDY : sse12_mov_packed<0x10, VR256, f256mem, loadv4f64, 811 "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>, 812 PD, VEX, VEX_L; 813} 814 815let Predicates = [UseSSE1] in { 816defm MOVAPS : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv4f32, 817 "movaps", SSEPackedSingle, SSE_MOVA_ITINS>, 818 PS; 819defm MOVUPS : sse12_mov_packed<0x10, VR128, f128mem, loadv4f32, 820 "movups", SSEPackedSingle, SSE_MOVU_ITINS>, 821 PS; 822} 823let Predicates = [UseSSE2] in { 824defm MOVAPD : sse12_mov_packed<0x28, VR128, f128mem, alignedloadv2f64, 825 "movapd", SSEPackedDouble, SSE_MOVA_ITINS>, 826 PD; 827defm MOVUPD : sse12_mov_packed<0x10, VR128, f128mem, loadv2f64, 828 "movupd", SSEPackedDouble, SSE_MOVU_ITINS, 0>, 829 PD; 830} 831 832let SchedRW = [WriteStore], Predicates = [HasAVX, NoVLX] in { 833def VMOVAPSmr : VPSI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 834 "movaps\t{$src, $dst|$dst, $src}", 835 [(alignedstore (v4f32 VR128:$src), addr:$dst)], 836 IIC_SSE_MOVA_P_MR>, VEX; 837def VMOVAPDmr : VPDI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 838 "movapd\t{$src, $dst|$dst, $src}", 839 [(alignedstore (v2f64 VR128:$src), addr:$dst)], 840 IIC_SSE_MOVA_P_MR>, VEX; 841def VMOVUPSmr : VPSI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 842 "movups\t{$src, $dst|$dst, $src}", 843 [(store (v4f32 VR128:$src), addr:$dst)], 844 IIC_SSE_MOVU_P_MR>, VEX; 845def VMOVUPDmr : VPDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 846 "movupd\t{$src, $dst|$dst, $src}", 847 [(store (v2f64 VR128:$src), addr:$dst)], 848 IIC_SSE_MOVU_P_MR>, VEX; 849def VMOVAPSYmr : VPSI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src), 850 "movaps\t{$src, $dst|$dst, $src}", 851 [(alignedstore256 (v8f32 VR256:$src), addr:$dst)], 852 IIC_SSE_MOVA_P_MR>, VEX, VEX_L; 853def VMOVAPDYmr : VPDI<0x29, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src), 854 "movapd\t{$src, $dst|$dst, $src}", 855 [(alignedstore256 (v4f64 VR256:$src), addr:$dst)], 856 IIC_SSE_MOVA_P_MR>, VEX, VEX_L; 857def VMOVUPSYmr : VPSI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src), 858 "movups\t{$src, $dst|$dst, $src}", 859 [(store (v8f32 VR256:$src), addr:$dst)], 860 IIC_SSE_MOVU_P_MR>, VEX, VEX_L; 861def VMOVUPDYmr : VPDI<0x11, MRMDestMem, (outs), (ins f256mem:$dst, VR256:$src), 862 "movupd\t{$src, $dst|$dst, $src}", 863 [(store (v4f64 VR256:$src), addr:$dst)], 864 IIC_SSE_MOVU_P_MR>, VEX, VEX_L; 865} // SchedRW 866 867// For disassembler 868let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, 869 SchedRW = [WriteFShuffle] in { 870 def VMOVAPSrr_REV : VPSI<0x29, MRMDestReg, (outs VR128:$dst), 871 (ins VR128:$src), 872 "movaps\t{$src, $dst|$dst, $src}", [], 873 IIC_SSE_MOVA_P_RR>, VEX; 874 def VMOVAPDrr_REV : VPDI<0x29, MRMDestReg, (outs VR128:$dst), 875 (ins VR128:$src), 876 "movapd\t{$src, $dst|$dst, $src}", [], 877 IIC_SSE_MOVA_P_RR>, VEX; 878 def VMOVUPSrr_REV : VPSI<0x11, MRMDestReg, (outs VR128:$dst), 879 (ins VR128:$src), 880 "movups\t{$src, $dst|$dst, $src}", [], 881 IIC_SSE_MOVU_P_RR>, VEX; 882 def VMOVUPDrr_REV : VPDI<0x11, MRMDestReg, (outs VR128:$dst), 883 (ins VR128:$src), 884 "movupd\t{$src, $dst|$dst, $src}", [], 885 IIC_SSE_MOVU_P_RR>, VEX; 886 def VMOVAPSYrr_REV : VPSI<0x29, MRMDestReg, (outs VR256:$dst), 887 (ins VR256:$src), 888 "movaps\t{$src, $dst|$dst, $src}", [], 889 IIC_SSE_MOVA_P_RR>, VEX, VEX_L; 890 def VMOVAPDYrr_REV : VPDI<0x29, MRMDestReg, (outs VR256:$dst), 891 (ins VR256:$src), 892 "movapd\t{$src, $dst|$dst, $src}", [], 893 IIC_SSE_MOVA_P_RR>, VEX, VEX_L; 894 def VMOVUPSYrr_REV : VPSI<0x11, MRMDestReg, (outs VR256:$dst), 895 (ins VR256:$src), 896 "movups\t{$src, $dst|$dst, $src}", [], 897 IIC_SSE_MOVU_P_RR>, VEX, VEX_L; 898 def VMOVUPDYrr_REV : VPDI<0x11, MRMDestReg, (outs VR256:$dst), 899 (ins VR256:$src), 900 "movupd\t{$src, $dst|$dst, $src}", [], 901 IIC_SSE_MOVU_P_RR>, VEX, VEX_L; 902} 903 904// Aliases to help the assembler pick two byte VEX encodings by swapping the 905// operands relative to the normal instructions to use VEX.R instead of VEX.B. 906def : InstAlias<"vmovaps\t{$src, $dst|$dst, $src}", 907 (VMOVAPSrr_REV VR128L:$dst, VR128H:$src), 0>; 908def : InstAlias<"vmovapd\t{$src, $dst|$dst, $src}", 909 (VMOVAPDrr_REV VR128L:$dst, VR128H:$src), 0>; 910def : InstAlias<"vmovups\t{$src, $dst|$dst, $src}", 911 (VMOVUPSrr_REV VR128L:$dst, VR128H:$src), 0>; 912def : InstAlias<"vmovupd\t{$src, $dst|$dst, $src}", 913 (VMOVUPDrr_REV VR128L:$dst, VR128H:$src), 0>; 914def : InstAlias<"vmovaps\t{$src, $dst|$dst, $src}", 915 (VMOVAPSYrr_REV VR256L:$dst, VR256H:$src), 0>; 916def : InstAlias<"vmovapd\t{$src, $dst|$dst, $src}", 917 (VMOVAPDYrr_REV VR256L:$dst, VR256H:$src), 0>; 918def : InstAlias<"vmovups\t{$src, $dst|$dst, $src}", 919 (VMOVUPSYrr_REV VR256L:$dst, VR256H:$src), 0>; 920def : InstAlias<"vmovupd\t{$src, $dst|$dst, $src}", 921 (VMOVUPDYrr_REV VR256L:$dst, VR256H:$src), 0>; 922 923let SchedRW = [WriteStore] in { 924def MOVAPSmr : PSI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 925 "movaps\t{$src, $dst|$dst, $src}", 926 [(alignedstore (v4f32 VR128:$src), addr:$dst)], 927 IIC_SSE_MOVA_P_MR>; 928def MOVAPDmr : PDI<0x29, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 929 "movapd\t{$src, $dst|$dst, $src}", 930 [(alignedstore (v2f64 VR128:$src), addr:$dst)], 931 IIC_SSE_MOVA_P_MR>; 932def MOVUPSmr : PSI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 933 "movups\t{$src, $dst|$dst, $src}", 934 [(store (v4f32 VR128:$src), addr:$dst)], 935 IIC_SSE_MOVU_P_MR>; 936def MOVUPDmr : PDI<0x11, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 937 "movupd\t{$src, $dst|$dst, $src}", 938 [(store (v2f64 VR128:$src), addr:$dst)], 939 IIC_SSE_MOVU_P_MR>; 940} // SchedRW 941 942// For disassembler 943let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, 944 SchedRW = [WriteFShuffle] in { 945 def MOVAPSrr_REV : PSI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 946 "movaps\t{$src, $dst|$dst, $src}", [], 947 IIC_SSE_MOVA_P_RR>; 948 def MOVAPDrr_REV : PDI<0x29, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 949 "movapd\t{$src, $dst|$dst, $src}", [], 950 IIC_SSE_MOVA_P_RR>; 951 def MOVUPSrr_REV : PSI<0x11, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 952 "movups\t{$src, $dst|$dst, $src}", [], 953 IIC_SSE_MOVU_P_RR>; 954 def MOVUPDrr_REV : PDI<0x11, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 955 "movupd\t{$src, $dst|$dst, $src}", [], 956 IIC_SSE_MOVU_P_RR>; 957} 958 959// Use vmovaps/vmovups for AVX integer load/store. 960let Predicates = [HasAVX, NoVLX] in { 961 // 128-bit load/store 962 def : Pat<(alignedloadv2i64 addr:$src), 963 (VMOVAPSrm addr:$src)>; 964 def : Pat<(loadv2i64 addr:$src), 965 (VMOVUPSrm addr:$src)>; 966 967 def : Pat<(alignedstore (v2i64 VR128:$src), addr:$dst), 968 (VMOVAPSmr addr:$dst, VR128:$src)>; 969 def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst), 970 (VMOVAPSmr addr:$dst, VR128:$src)>; 971 def : Pat<(store (v2i64 VR128:$src), addr:$dst), 972 (VMOVUPSmr addr:$dst, VR128:$src)>; 973 def : Pat<(store (v4i32 VR128:$src), addr:$dst), 974 (VMOVUPSmr addr:$dst, VR128:$src)>; 975 976 // 256-bit load/store 977 def : Pat<(alignedloadv4i64 addr:$src), 978 (VMOVAPSYrm addr:$src)>; 979 def : Pat<(loadv4i64 addr:$src), 980 (VMOVUPSYrm addr:$src)>; 981 def : Pat<(alignedstore256 (v4i64 VR256:$src), addr:$dst), 982 (VMOVAPSYmr addr:$dst, VR256:$src)>; 983 def : Pat<(alignedstore256 (v8i32 VR256:$src), addr:$dst), 984 (VMOVAPSYmr addr:$dst, VR256:$src)>; 985 def : Pat<(store (v4i64 VR256:$src), addr:$dst), 986 (VMOVUPSYmr addr:$dst, VR256:$src)>; 987 def : Pat<(store (v8i32 VR256:$src), addr:$dst), 988 (VMOVUPSYmr addr:$dst, VR256:$src)>; 989 990 // Special patterns for storing subvector extracts of lower 128-bits 991 // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr 992 def : Pat<(alignedstore (v2f64 (extract_subvector 993 (v4f64 VR256:$src), (iPTR 0))), addr:$dst), 994 (VMOVAPDmr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 995 def : Pat<(alignedstore (v4f32 (extract_subvector 996 (v8f32 VR256:$src), (iPTR 0))), addr:$dst), 997 (VMOVAPSmr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 998 def : Pat<(alignedstore (v2i64 (extract_subvector 999 (v4i64 VR256:$src), (iPTR 0))), addr:$dst), 1000 (VMOVAPDmr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1001 def : Pat<(alignedstore (v4i32 (extract_subvector 1002 (v8i32 VR256:$src), (iPTR 0))), addr:$dst), 1003 (VMOVAPSmr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1004 def : Pat<(alignedstore (v8i16 (extract_subvector 1005 (v16i16 VR256:$src), (iPTR 0))), addr:$dst), 1006 (VMOVAPSmr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1007 def : Pat<(alignedstore (v16i8 (extract_subvector 1008 (v32i8 VR256:$src), (iPTR 0))), addr:$dst), 1009 (VMOVAPSmr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1010 1011 def : Pat<(store (v2f64 (extract_subvector 1012 (v4f64 VR256:$src), (iPTR 0))), addr:$dst), 1013 (VMOVUPDmr addr:$dst, (v2f64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1014 def : Pat<(store (v4f32 (extract_subvector 1015 (v8f32 VR256:$src), (iPTR 0))), addr:$dst), 1016 (VMOVUPSmr addr:$dst, (v4f32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1017 def : Pat<(store (v2i64 (extract_subvector 1018 (v4i64 VR256:$src), (iPTR 0))), addr:$dst), 1019 (VMOVUPDmr addr:$dst, (v2i64 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1020 def : Pat<(store (v4i32 (extract_subvector 1021 (v8i32 VR256:$src), (iPTR 0))), addr:$dst), 1022 (VMOVUPSmr addr:$dst, (v4i32 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1023 def : Pat<(store (v8i16 (extract_subvector 1024 (v16i16 VR256:$src), (iPTR 0))), addr:$dst), 1025 (VMOVUPSmr addr:$dst, (v8i16 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1026 def : Pat<(store (v16i8 (extract_subvector 1027 (v32i8 VR256:$src), (iPTR 0))), addr:$dst), 1028 (VMOVUPSmr addr:$dst, (v16i8 (EXTRACT_SUBREG VR256:$src,sub_xmm)))>; 1029} 1030 1031let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 1032 // 128-bit load/store 1033 def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst), 1034 (VMOVAPSmr addr:$dst, VR128:$src)>; 1035 def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst), 1036 (VMOVAPSmr addr:$dst, VR128:$src)>; 1037 def : Pat<(store (v8i16 VR128:$src), addr:$dst), 1038 (VMOVUPSmr addr:$dst, VR128:$src)>; 1039 def : Pat<(store (v16i8 VR128:$src), addr:$dst), 1040 (VMOVUPSmr addr:$dst, VR128:$src)>; 1041 1042 // 256-bit load/store 1043 def : Pat<(alignedstore256 (v16i16 VR256:$src), addr:$dst), 1044 (VMOVAPSYmr addr:$dst, VR256:$src)>; 1045 def : Pat<(alignedstore256 (v32i8 VR256:$src), addr:$dst), 1046 (VMOVAPSYmr addr:$dst, VR256:$src)>; 1047 def : Pat<(store (v16i16 VR256:$src), addr:$dst), 1048 (VMOVUPSYmr addr:$dst, VR256:$src)>; 1049 def : Pat<(store (v32i8 VR256:$src), addr:$dst), 1050 (VMOVUPSYmr addr:$dst, VR256:$src)>; 1051} 1052 1053// Use movaps / movups for SSE integer load / store (one byte shorter). 1054// The instructions selected below are then converted to MOVDQA/MOVDQU 1055// during the SSE domain pass. 1056let Predicates = [UseSSE1] in { 1057 def : Pat<(alignedloadv2i64 addr:$src), 1058 (MOVAPSrm addr:$src)>; 1059 def : Pat<(loadv2i64 addr:$src), 1060 (MOVUPSrm addr:$src)>; 1061 1062 def : Pat<(alignedstore (v2i64 VR128:$src), addr:$dst), 1063 (MOVAPSmr addr:$dst, VR128:$src)>; 1064 def : Pat<(alignedstore (v4i32 VR128:$src), addr:$dst), 1065 (MOVAPSmr addr:$dst, VR128:$src)>; 1066 def : Pat<(alignedstore (v8i16 VR128:$src), addr:$dst), 1067 (MOVAPSmr addr:$dst, VR128:$src)>; 1068 def : Pat<(alignedstore (v16i8 VR128:$src), addr:$dst), 1069 (MOVAPSmr addr:$dst, VR128:$src)>; 1070 def : Pat<(store (v2i64 VR128:$src), addr:$dst), 1071 (MOVUPSmr addr:$dst, VR128:$src)>; 1072 def : Pat<(store (v4i32 VR128:$src), addr:$dst), 1073 (MOVUPSmr addr:$dst, VR128:$src)>; 1074 def : Pat<(store (v8i16 VR128:$src), addr:$dst), 1075 (MOVUPSmr addr:$dst, VR128:$src)>; 1076 def : Pat<(store (v16i8 VR128:$src), addr:$dst), 1077 (MOVUPSmr addr:$dst, VR128:$src)>; 1078} 1079 1080// Alias instruction to load FR32 or FR64 from f128mem using movaps. Upper 1081// bits are disregarded. FIXME: Set encoding to pseudo! 1082let canFoldAsLoad = 1, isReMaterializable = 1, SchedRW = [WriteLoad] in { 1083let isCodeGenOnly = 1 in { 1084 def FsVMOVAPSrm : VPSI<0x28, MRMSrcMem, (outs FR32:$dst), (ins f128mem:$src), 1085 "movaps\t{$src, $dst|$dst, $src}", 1086 [(set FR32:$dst, (alignedloadfsf32 addr:$src))], 1087 IIC_SSE_MOVA_P_RM>, VEX; 1088 def FsVMOVAPDrm : VPDI<0x28, MRMSrcMem, (outs FR64:$dst), (ins f128mem:$src), 1089 "movapd\t{$src, $dst|$dst, $src}", 1090 [(set FR64:$dst, (alignedloadfsf64 addr:$src))], 1091 IIC_SSE_MOVA_P_RM>, VEX; 1092 def FsMOVAPSrm : PSI<0x28, MRMSrcMem, (outs FR32:$dst), (ins f128mem:$src), 1093 "movaps\t{$src, $dst|$dst, $src}", 1094 [(set FR32:$dst, (alignedloadfsf32 addr:$src))], 1095 IIC_SSE_MOVA_P_RM>; 1096 def FsMOVAPDrm : PDI<0x28, MRMSrcMem, (outs FR64:$dst), (ins f128mem:$src), 1097 "movapd\t{$src, $dst|$dst, $src}", 1098 [(set FR64:$dst, (alignedloadfsf64 addr:$src))], 1099 IIC_SSE_MOVA_P_RM>; 1100} 1101} 1102 1103//===----------------------------------------------------------------------===// 1104// SSE 1 & 2 - Move Low packed FP Instructions 1105//===----------------------------------------------------------------------===// 1106 1107multiclass sse12_mov_hilo_packed_base<bits<8>opc, SDNode psnode, SDNode pdnode, 1108 string base_opc, string asm_opr, 1109 InstrItinClass itin> { 1110 def PSrm : PI<opc, MRMSrcMem, 1111 (outs VR128:$dst), (ins VR128:$src1, f64mem:$src2), 1112 !strconcat(base_opc, "s", asm_opr), 1113 [(set VR128:$dst, 1114 (psnode VR128:$src1, 1115 (bc_v4f32 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))))], 1116 itin, SSEPackedSingle>, PS, 1117 Sched<[WriteFShuffleLd, ReadAfterLd]>; 1118 1119 def PDrm : PI<opc, MRMSrcMem, 1120 (outs VR128:$dst), (ins VR128:$src1, f64mem:$src2), 1121 !strconcat(base_opc, "d", asm_opr), 1122 [(set VR128:$dst, (v2f64 (pdnode VR128:$src1, 1123 (scalar_to_vector (loadf64 addr:$src2)))))], 1124 itin, SSEPackedDouble>, PD, 1125 Sched<[WriteFShuffleLd, ReadAfterLd]>; 1126 1127} 1128 1129multiclass sse12_mov_hilo_packed<bits<8>opc, SDNode psnode, SDNode pdnode, 1130 string base_opc, InstrItinClass itin> { 1131 let Predicates = [UseAVX] in 1132 defm V#NAME : sse12_mov_hilo_packed_base<opc, psnode, pdnode, base_opc, 1133 "\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1134 itin>, VEX_4V; 1135 1136 let Constraints = "$src1 = $dst" in 1137 defm NAME : sse12_mov_hilo_packed_base<opc, psnode, pdnode, base_opc, 1138 "\t{$src2, $dst|$dst, $src2}", 1139 itin>; 1140} 1141 1142let AddedComplexity = 20 in { 1143 defm MOVL : sse12_mov_hilo_packed<0x12, X86Movlps, X86Movlpd, "movlp", 1144 IIC_SSE_MOV_LH>; 1145} 1146 1147let SchedRW = [WriteStore] in { 1148let Predicates = [UseAVX] in { 1149def VMOVLPSmr : VPSI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1150 "movlps\t{$src, $dst|$dst, $src}", 1151 [(store (f64 (extractelt (bc_v2f64 (v4f32 VR128:$src)), 1152 (iPTR 0))), addr:$dst)], 1153 IIC_SSE_MOV_LH>, VEX; 1154def VMOVLPDmr : VPDI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1155 "movlpd\t{$src, $dst|$dst, $src}", 1156 [(store (f64 (extractelt (v2f64 VR128:$src), 1157 (iPTR 0))), addr:$dst)], 1158 IIC_SSE_MOV_LH>, VEX; 1159}// UseAVX 1160def MOVLPSmr : PSI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1161 "movlps\t{$src, $dst|$dst, $src}", 1162 [(store (f64 (extractelt (bc_v2f64 (v4f32 VR128:$src)), 1163 (iPTR 0))), addr:$dst)], 1164 IIC_SSE_MOV_LH>; 1165def MOVLPDmr : PDI<0x13, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1166 "movlpd\t{$src, $dst|$dst, $src}", 1167 [(store (f64 (extractelt (v2f64 VR128:$src), 1168 (iPTR 0))), addr:$dst)], 1169 IIC_SSE_MOV_LH>; 1170} // SchedRW 1171 1172let Predicates = [UseAVX] in { 1173 // Shuffle with VMOVLPS 1174 def : Pat<(v4f32 (X86Movlps VR128:$src1, (load addr:$src2))), 1175 (VMOVLPSrm VR128:$src1, addr:$src2)>; 1176 def : Pat<(v4i32 (X86Movlps VR128:$src1, (load addr:$src2))), 1177 (VMOVLPSrm VR128:$src1, addr:$src2)>; 1178 1179 // Shuffle with VMOVLPD 1180 def : Pat<(v2f64 (X86Movlpd VR128:$src1, (load addr:$src2))), 1181 (VMOVLPDrm VR128:$src1, addr:$src2)>; 1182 def : Pat<(v2i64 (X86Movlpd VR128:$src1, (load addr:$src2))), 1183 (VMOVLPDrm VR128:$src1, addr:$src2)>; 1184 def : Pat<(v2f64 (X86Movsd VR128:$src1, 1185 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))), 1186 (VMOVLPDrm VR128:$src1, addr:$src2)>; 1187 1188 // Store patterns 1189 def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), 1190 addr:$src1), 1191 (VMOVLPSmr addr:$src1, VR128:$src2)>; 1192 def : Pat<(store (v4i32 (X86Movlps 1193 (bc_v4i32 (loadv2i64 addr:$src1)), VR128:$src2)), addr:$src1), 1194 (VMOVLPSmr addr:$src1, VR128:$src2)>; 1195 def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128:$src2)), 1196 addr:$src1), 1197 (VMOVLPDmr addr:$src1, VR128:$src2)>; 1198 def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128:$src2)), 1199 addr:$src1), 1200 (VMOVLPDmr addr:$src1, VR128:$src2)>; 1201} 1202 1203let Predicates = [UseSSE1] in { 1204 // (store (vector_shuffle (load addr), v2, <4, 5, 2, 3>), addr) using MOVLPS 1205 def : Pat<(store (i64 (extractelt (bc_v2i64 (v4f32 VR128:$src2)), 1206 (iPTR 0))), addr:$src1), 1207 (MOVLPSmr addr:$src1, VR128:$src2)>; 1208 1209 // Shuffle with MOVLPS 1210 def : Pat<(v4f32 (X86Movlps VR128:$src1, (load addr:$src2))), 1211 (MOVLPSrm VR128:$src1, addr:$src2)>; 1212 def : Pat<(v4i32 (X86Movlps VR128:$src1, (load addr:$src2))), 1213 (MOVLPSrm VR128:$src1, addr:$src2)>; 1214 def : Pat<(X86Movlps VR128:$src1, 1215 (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))), 1216 (MOVLPSrm VR128:$src1, addr:$src2)>; 1217 1218 // Store patterns 1219 def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128:$src2)), 1220 addr:$src1), 1221 (MOVLPSmr addr:$src1, VR128:$src2)>; 1222 def : Pat<(store (v4i32 (X86Movlps 1223 (bc_v4i32 (loadv2i64 addr:$src1)), VR128:$src2)), 1224 addr:$src1), 1225 (MOVLPSmr addr:$src1, VR128:$src2)>; 1226} 1227 1228let Predicates = [UseSSE2] in { 1229 // Shuffle with MOVLPD 1230 def : Pat<(v2f64 (X86Movlpd VR128:$src1, (load addr:$src2))), 1231 (MOVLPDrm VR128:$src1, addr:$src2)>; 1232 def : Pat<(v2i64 (X86Movlpd VR128:$src1, (load addr:$src2))), 1233 (MOVLPDrm VR128:$src1, addr:$src2)>; 1234 def : Pat<(v2f64 (X86Movsd VR128:$src1, 1235 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))), 1236 (MOVLPDrm VR128:$src1, addr:$src2)>; 1237 1238 // Store patterns 1239 def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128:$src2)), 1240 addr:$src1), 1241 (MOVLPDmr addr:$src1, VR128:$src2)>; 1242 def : Pat<(store (v2i64 (X86Movlpd (load addr:$src1), VR128:$src2)), 1243 addr:$src1), 1244 (MOVLPDmr addr:$src1, VR128:$src2)>; 1245} 1246 1247//===----------------------------------------------------------------------===// 1248// SSE 1 & 2 - Move Hi packed FP Instructions 1249//===----------------------------------------------------------------------===// 1250 1251let AddedComplexity = 20 in { 1252 defm MOVH : sse12_mov_hilo_packed<0x16, X86Movlhps, X86Movlhpd, "movhp", 1253 IIC_SSE_MOV_LH>; 1254} 1255 1256let SchedRW = [WriteStore] in { 1257// v2f64 extract element 1 is always custom lowered to unpack high to low 1258// and extract element 0 so the non-store version isn't too horrible. 1259let Predicates = [UseAVX] in { 1260def VMOVHPSmr : VPSI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1261 "movhps\t{$src, $dst|$dst, $src}", 1262 [(store (f64 (extractelt 1263 (X86Unpckh (bc_v2f64 (v4f32 VR128:$src)), 1264 (bc_v2f64 (v4f32 VR128:$src))), 1265 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>, VEX; 1266def VMOVHPDmr : VPDI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1267 "movhpd\t{$src, $dst|$dst, $src}", 1268 [(store (f64 (extractelt 1269 (v2f64 (X86Unpckh VR128:$src, VR128:$src)), 1270 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>, VEX; 1271} // UseAVX 1272def MOVHPSmr : PSI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1273 "movhps\t{$src, $dst|$dst, $src}", 1274 [(store (f64 (extractelt 1275 (X86Unpckh (bc_v2f64 (v4f32 VR128:$src)), 1276 (bc_v2f64 (v4f32 VR128:$src))), 1277 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>; 1278def MOVHPDmr : PDI<0x17, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 1279 "movhpd\t{$src, $dst|$dst, $src}", 1280 [(store (f64 (extractelt 1281 (v2f64 (X86Unpckh VR128:$src, VR128:$src)), 1282 (iPTR 0))), addr:$dst)], IIC_SSE_MOV_LH>; 1283} // SchedRW 1284 1285let Predicates = [UseAVX] in { 1286 // VMOVHPS patterns 1287 def : Pat<(X86Movlhps VR128:$src1, 1288 (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))), 1289 (VMOVHPSrm VR128:$src1, addr:$src2)>; 1290 def : Pat<(X86Movlhps VR128:$src1, 1291 (bc_v4i32 (v2i64 (X86vzload addr:$src2)))), 1292 (VMOVHPSrm VR128:$src1, addr:$src2)>; 1293 1294 // VMOVHPD patterns 1295 1296 // FIXME: Instead of X86Unpckl, there should be a X86Movlhpd here, the problem 1297 // is during lowering, where it's not possible to recognize the load fold 1298 // cause it has two uses through a bitcast. One use disappears at isel time 1299 // and the fold opportunity reappears. 1300 def : Pat<(v2f64 (X86Unpckl VR128:$src1, 1301 (scalar_to_vector (loadf64 addr:$src2)))), 1302 (VMOVHPDrm VR128:$src1, addr:$src2)>; 1303 // Also handle an i64 load because that may get selected as a faster way to 1304 // load the data. 1305 def : Pat<(v2f64 (X86Unpckl VR128:$src1, 1306 (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src2)))))), 1307 (VMOVHPDrm VR128:$src1, addr:$src2)>; 1308 1309 def : Pat<(store (f64 (extractelt 1310 (v2f64 (X86VPermilpi VR128:$src, (i8 1))), 1311 (iPTR 0))), addr:$dst), 1312 (VMOVHPDmr addr:$dst, VR128:$src)>; 1313} 1314 1315let Predicates = [UseSSE1] in { 1316 // MOVHPS patterns 1317 def : Pat<(X86Movlhps VR128:$src1, 1318 (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))), 1319 (MOVHPSrm VR128:$src1, addr:$src2)>; 1320 def : Pat<(X86Movlhps VR128:$src1, 1321 (bc_v4f32 (v2i64 (X86vzload addr:$src2)))), 1322 (MOVHPSrm VR128:$src1, addr:$src2)>; 1323} 1324 1325let Predicates = [UseSSE2] in { 1326 // MOVHPD patterns 1327 1328 // FIXME: Instead of X86Unpckl, there should be a X86Movlhpd here, the problem 1329 // is during lowering, where it's not possible to recognize the load fold 1330 // cause it has two uses through a bitcast. One use disappears at isel time 1331 // and the fold opportunity reappears. 1332 def : Pat<(v2f64 (X86Unpckl VR128:$src1, 1333 (scalar_to_vector (loadf64 addr:$src2)))), 1334 (MOVHPDrm VR128:$src1, addr:$src2)>; 1335 // Also handle an i64 load because that may get selected as a faster way to 1336 // load the data. 1337 def : Pat<(v2f64 (X86Unpckl VR128:$src1, 1338 (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src2)))))), 1339 (MOVHPDrm VR128:$src1, addr:$src2)>; 1340 1341 def : Pat<(store (f64 (extractelt 1342 (v2f64 (X86Shufp VR128:$src, VR128:$src, (i8 1))), 1343 (iPTR 0))), addr:$dst), 1344 (MOVHPDmr addr:$dst, VR128:$src)>; 1345} 1346 1347//===----------------------------------------------------------------------===// 1348// SSE 1 & 2 - Move Low to High and High to Low packed FP Instructions 1349//===----------------------------------------------------------------------===// 1350 1351let AddedComplexity = 20, Predicates = [UseAVX] in { 1352 def VMOVLHPSrr : VPSI<0x16, MRMSrcReg, (outs VR128:$dst), 1353 (ins VR128:$src1, VR128:$src2), 1354 "movlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1355 [(set VR128:$dst, 1356 (v4f32 (X86Movlhps VR128:$src1, VR128:$src2)))], 1357 IIC_SSE_MOV_LH>, 1358 VEX_4V, Sched<[WriteFShuffle]>; 1359 def VMOVHLPSrr : VPSI<0x12, MRMSrcReg, (outs VR128:$dst), 1360 (ins VR128:$src1, VR128:$src2), 1361 "movhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1362 [(set VR128:$dst, 1363 (v4f32 (X86Movhlps VR128:$src1, VR128:$src2)))], 1364 IIC_SSE_MOV_LH>, 1365 VEX_4V, Sched<[WriteFShuffle]>; 1366} 1367let Constraints = "$src1 = $dst", AddedComplexity = 20 in { 1368 def MOVLHPSrr : PSI<0x16, MRMSrcReg, (outs VR128:$dst), 1369 (ins VR128:$src1, VR128:$src2), 1370 "movlhps\t{$src2, $dst|$dst, $src2}", 1371 [(set VR128:$dst, 1372 (v4f32 (X86Movlhps VR128:$src1, VR128:$src2)))], 1373 IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; 1374 def MOVHLPSrr : PSI<0x12, MRMSrcReg, (outs VR128:$dst), 1375 (ins VR128:$src1, VR128:$src2), 1376 "movhlps\t{$src2, $dst|$dst, $src2}", 1377 [(set VR128:$dst, 1378 (v4f32 (X86Movhlps VR128:$src1, VR128:$src2)))], 1379 IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; 1380} 1381 1382let Predicates = [UseAVX] in { 1383 // MOVLHPS patterns 1384 def : Pat<(v4i32 (X86Movlhps VR128:$src1, VR128:$src2)), 1385 (VMOVLHPSrr VR128:$src1, VR128:$src2)>; 1386 def : Pat<(v2i64 (X86Movlhps VR128:$src1, VR128:$src2)), 1387 (VMOVLHPSrr (v2i64 VR128:$src1), VR128:$src2)>; 1388 1389 // MOVHLPS patterns 1390 def : Pat<(v4i32 (X86Movhlps VR128:$src1, VR128:$src2)), 1391 (VMOVHLPSrr VR128:$src1, VR128:$src2)>; 1392} 1393 1394let Predicates = [UseSSE1] in { 1395 // MOVLHPS patterns 1396 def : Pat<(v4i32 (X86Movlhps VR128:$src1, VR128:$src2)), 1397 (MOVLHPSrr VR128:$src1, VR128:$src2)>; 1398 def : Pat<(v2i64 (X86Movlhps VR128:$src1, VR128:$src2)), 1399 (MOVLHPSrr (v2i64 VR128:$src1), VR128:$src2)>; 1400 1401 // MOVHLPS patterns 1402 def : Pat<(v4i32 (X86Movhlps VR128:$src1, VR128:$src2)), 1403 (MOVHLPSrr VR128:$src1, VR128:$src2)>; 1404} 1405 1406//===----------------------------------------------------------------------===// 1407// SSE 1 & 2 - Conversion Instructions 1408//===----------------------------------------------------------------------===// 1409 1410def SSE_CVT_PD : OpndItins< 1411 IIC_SSE_CVT_PD_RR, IIC_SSE_CVT_PD_RM 1412>; 1413 1414let Sched = WriteCvtI2F in 1415def SSE_CVT_PS : OpndItins< 1416 IIC_SSE_CVT_PS_RR, IIC_SSE_CVT_PS_RM 1417>; 1418 1419let Sched = WriteCvtI2F in 1420def SSE_CVT_Scalar : OpndItins< 1421 IIC_SSE_CVT_Scalar_RR, IIC_SSE_CVT_Scalar_RM 1422>; 1423 1424let Sched = WriteCvtF2I in 1425def SSE_CVT_SS2SI_32 : OpndItins< 1426 IIC_SSE_CVT_SS2SI32_RR, IIC_SSE_CVT_SS2SI32_RM 1427>; 1428 1429let Sched = WriteCvtF2I in 1430def SSE_CVT_SS2SI_64 : OpndItins< 1431 IIC_SSE_CVT_SS2SI64_RR, IIC_SSE_CVT_SS2SI64_RM 1432>; 1433 1434let Sched = WriteCvtF2I in 1435def SSE_CVT_SD2SI : OpndItins< 1436 IIC_SSE_CVT_SD2SI_RR, IIC_SSE_CVT_SD2SI_RM 1437>; 1438 1439// FIXME: We probably want to match the rm form only when optimizing for 1440// size, to avoid false depenendecies (see sse_fp_unop_s for details) 1441multiclass sse12_cvt_s<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 1442 SDNode OpNode, X86MemOperand x86memop, PatFrag ld_frag, 1443 string asm, OpndItins itins> { 1444 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm, 1445 [(set DstRC:$dst, (OpNode SrcRC:$src))], 1446 itins.rr>, Sched<[itins.Sched]>; 1447 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm, 1448 [(set DstRC:$dst, (OpNode (ld_frag addr:$src)))], 1449 itins.rm>, Sched<[itins.Sched.Folded]>; 1450} 1451 1452multiclass sse12_cvt_p<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 1453 X86MemOperand x86memop, string asm, Domain d, 1454 OpndItins itins> { 1455let hasSideEffects = 0 in { 1456 def rr : I<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), asm, 1457 [], itins.rr, d>, Sched<[itins.Sched]>; 1458 let mayLoad = 1 in 1459 def rm : I<opc, MRMSrcMem, (outs DstRC:$dst), (ins x86memop:$src), asm, 1460 [], itins.rm, d>, Sched<[itins.Sched.Folded]>; 1461} 1462} 1463 1464// FIXME: We probably want to match the rm form only when optimizing for 1465// size, to avoid false depenendecies (see sse_fp_unop_s for details) 1466multiclass sse12_vcvt_avx<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 1467 X86MemOperand x86memop, string asm> { 1468let hasSideEffects = 0, Predicates = [UseAVX] in { 1469 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src), 1470 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, 1471 Sched<[WriteCvtI2F]>; 1472 let mayLoad = 1 in 1473 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), 1474 (ins DstRC:$src1, x86memop:$src), 1475 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>, 1476 Sched<[WriteCvtI2FLd, ReadAfterLd]>; 1477} // hasSideEffects = 0 1478} 1479 1480let Predicates = [UseAVX] in { 1481defm VCVTTSS2SI : sse12_cvt_s<0x2C, FR32, GR32, fp_to_sint, f32mem, loadf32, 1482 "cvttss2si\t{$src, $dst|$dst, $src}", 1483 SSE_CVT_SS2SI_32>, 1484 XS, VEX, VEX_LIG; 1485defm VCVTTSS2SI64 : sse12_cvt_s<0x2C, FR32, GR64, fp_to_sint, f32mem, loadf32, 1486 "cvttss2si\t{$src, $dst|$dst, $src}", 1487 SSE_CVT_SS2SI_64>, 1488 XS, VEX, VEX_W, VEX_LIG; 1489defm VCVTTSD2SI : sse12_cvt_s<0x2C, FR64, GR32, fp_to_sint, f64mem, loadf64, 1490 "cvttsd2si\t{$src, $dst|$dst, $src}", 1491 SSE_CVT_SD2SI>, 1492 XD, VEX, VEX_LIG; 1493defm VCVTTSD2SI64 : sse12_cvt_s<0x2C, FR64, GR64, fp_to_sint, f64mem, loadf64, 1494 "cvttsd2si\t{$src, $dst|$dst, $src}", 1495 SSE_CVT_SD2SI>, 1496 XD, VEX, VEX_W, VEX_LIG; 1497 1498def : InstAlias<"vcvttss2si{l}\t{$src, $dst|$dst, $src}", 1499 (VCVTTSS2SIrr GR32:$dst, FR32:$src), 0>; 1500def : InstAlias<"vcvttss2si{l}\t{$src, $dst|$dst, $src}", 1501 (VCVTTSS2SIrm GR32:$dst, f32mem:$src), 0>; 1502def : InstAlias<"vcvttsd2si{l}\t{$src, $dst|$dst, $src}", 1503 (VCVTTSD2SIrr GR32:$dst, FR64:$src), 0>; 1504def : InstAlias<"vcvttsd2si{l}\t{$src, $dst|$dst, $src}", 1505 (VCVTTSD2SIrm GR32:$dst, f64mem:$src), 0>; 1506def : InstAlias<"vcvttss2si{q}\t{$src, $dst|$dst, $src}", 1507 (VCVTTSS2SI64rr GR64:$dst, FR32:$src), 0>; 1508def : InstAlias<"vcvttss2si{q}\t{$src, $dst|$dst, $src}", 1509 (VCVTTSS2SI64rm GR64:$dst, f32mem:$src), 0>; 1510def : InstAlias<"vcvttsd2si{q}\t{$src, $dst|$dst, $src}", 1511 (VCVTTSD2SI64rr GR64:$dst, FR64:$src), 0>; 1512def : InstAlias<"vcvttsd2si{q}\t{$src, $dst|$dst, $src}", 1513 (VCVTTSD2SI64rm GR64:$dst, f64mem:$src), 0>; 1514} 1515// The assembler can recognize rr 64-bit instructions by seeing a rxx 1516// register, but the same isn't true when only using memory operands, 1517// provide other assembly "l" and "q" forms to address this explicitly 1518// where appropriate to do so. 1519defm VCVTSI2SS : sse12_vcvt_avx<0x2A, GR32, FR32, i32mem, "cvtsi2ss{l}">, 1520 XS, VEX_4V, VEX_LIG; 1521defm VCVTSI2SS64 : sse12_vcvt_avx<0x2A, GR64, FR32, i64mem, "cvtsi2ss{q}">, 1522 XS, VEX_4V, VEX_W, VEX_LIG; 1523defm VCVTSI2SD : sse12_vcvt_avx<0x2A, GR32, FR64, i32mem, "cvtsi2sd{l}">, 1524 XD, VEX_4V, VEX_LIG; 1525defm VCVTSI2SD64 : sse12_vcvt_avx<0x2A, GR64, FR64, i64mem, "cvtsi2sd{q}">, 1526 XD, VEX_4V, VEX_W, VEX_LIG; 1527 1528let Predicates = [UseAVX] in { 1529 def : InstAlias<"vcvtsi2ss\t{$src, $src1, $dst|$dst, $src1, $src}", 1530 (VCVTSI2SSrm FR64:$dst, FR64:$src1, i32mem:$src), 0>; 1531 def : InstAlias<"vcvtsi2sd\t{$src, $src1, $dst|$dst, $src1, $src}", 1532 (VCVTSI2SDrm FR64:$dst, FR64:$src1, i32mem:$src), 0>; 1533 1534 def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))), 1535 (VCVTSI2SSrm (f32 (IMPLICIT_DEF)), addr:$src)>; 1536 def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))), 1537 (VCVTSI2SS64rm (f32 (IMPLICIT_DEF)), addr:$src)>; 1538 def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))), 1539 (VCVTSI2SDrm (f64 (IMPLICIT_DEF)), addr:$src)>; 1540 def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))), 1541 (VCVTSI2SD64rm (f64 (IMPLICIT_DEF)), addr:$src)>; 1542 1543 def : Pat<(f32 (sint_to_fp GR32:$src)), 1544 (VCVTSI2SSrr (f32 (IMPLICIT_DEF)), GR32:$src)>; 1545 def : Pat<(f32 (sint_to_fp GR64:$src)), 1546 (VCVTSI2SS64rr (f32 (IMPLICIT_DEF)), GR64:$src)>; 1547 def : Pat<(f64 (sint_to_fp GR32:$src)), 1548 (VCVTSI2SDrr (f64 (IMPLICIT_DEF)), GR32:$src)>; 1549 def : Pat<(f64 (sint_to_fp GR64:$src)), 1550 (VCVTSI2SD64rr (f64 (IMPLICIT_DEF)), GR64:$src)>; 1551} 1552 1553defm CVTTSS2SI : sse12_cvt_s<0x2C, FR32, GR32, fp_to_sint, f32mem, loadf32, 1554 "cvttss2si\t{$src, $dst|$dst, $src}", 1555 SSE_CVT_SS2SI_32>, XS; 1556defm CVTTSS2SI64 : sse12_cvt_s<0x2C, FR32, GR64, fp_to_sint, f32mem, loadf32, 1557 "cvttss2si\t{$src, $dst|$dst, $src}", 1558 SSE_CVT_SS2SI_64>, XS, REX_W; 1559defm CVTTSD2SI : sse12_cvt_s<0x2C, FR64, GR32, fp_to_sint, f64mem, loadf64, 1560 "cvttsd2si\t{$src, $dst|$dst, $src}", 1561 SSE_CVT_SD2SI>, XD; 1562defm CVTTSD2SI64 : sse12_cvt_s<0x2C, FR64, GR64, fp_to_sint, f64mem, loadf64, 1563 "cvttsd2si\t{$src, $dst|$dst, $src}", 1564 SSE_CVT_SD2SI>, XD, REX_W; 1565defm CVTSI2SS : sse12_cvt_s<0x2A, GR32, FR32, sint_to_fp, i32mem, loadi32, 1566 "cvtsi2ss{l}\t{$src, $dst|$dst, $src}", 1567 SSE_CVT_Scalar>, XS; 1568defm CVTSI2SS64 : sse12_cvt_s<0x2A, GR64, FR32, sint_to_fp, i64mem, loadi64, 1569 "cvtsi2ss{q}\t{$src, $dst|$dst, $src}", 1570 SSE_CVT_Scalar>, XS, REX_W; 1571defm CVTSI2SD : sse12_cvt_s<0x2A, GR32, FR64, sint_to_fp, i32mem, loadi32, 1572 "cvtsi2sd{l}\t{$src, $dst|$dst, $src}", 1573 SSE_CVT_Scalar>, XD; 1574defm CVTSI2SD64 : sse12_cvt_s<0x2A, GR64, FR64, sint_to_fp, i64mem, loadi64, 1575 "cvtsi2sd{q}\t{$src, $dst|$dst, $src}", 1576 SSE_CVT_Scalar>, XD, REX_W; 1577 1578def : InstAlias<"cvttss2si{l}\t{$src, $dst|$dst, $src}", 1579 (CVTTSS2SIrr GR32:$dst, FR32:$src), 0>; 1580def : InstAlias<"cvttss2si{l}\t{$src, $dst|$dst, $src}", 1581 (CVTTSS2SIrm GR32:$dst, f32mem:$src), 0>; 1582def : InstAlias<"cvttsd2si{l}\t{$src, $dst|$dst, $src}", 1583 (CVTTSD2SIrr GR32:$dst, FR64:$src), 0>; 1584def : InstAlias<"cvttsd2si{l}\t{$src, $dst|$dst, $src}", 1585 (CVTTSD2SIrm GR32:$dst, f64mem:$src), 0>; 1586def : InstAlias<"cvttss2si{q}\t{$src, $dst|$dst, $src}", 1587 (CVTTSS2SI64rr GR64:$dst, FR32:$src), 0>; 1588def : InstAlias<"cvttss2si{q}\t{$src, $dst|$dst, $src}", 1589 (CVTTSS2SI64rm GR64:$dst, f32mem:$src), 0>; 1590def : InstAlias<"cvttsd2si{q}\t{$src, $dst|$dst, $src}", 1591 (CVTTSD2SI64rr GR64:$dst, FR64:$src), 0>; 1592def : InstAlias<"cvttsd2si{q}\t{$src, $dst|$dst, $src}", 1593 (CVTTSD2SI64rm GR64:$dst, f64mem:$src), 0>; 1594 1595def : InstAlias<"cvtsi2ss\t{$src, $dst|$dst, $src}", 1596 (CVTSI2SSrm FR64:$dst, i32mem:$src), 0>; 1597def : InstAlias<"cvtsi2sd\t{$src, $dst|$dst, $src}", 1598 (CVTSI2SDrm FR64:$dst, i32mem:$src), 0>; 1599 1600// Conversion Instructions Intrinsics - Match intrinsics which expect MM 1601// and/or XMM operand(s). 1602 1603// FIXME: We probably want to match the rm form only when optimizing for 1604// size, to avoid false depenendecies (see sse_fp_unop_s for details) 1605multiclass sse12_cvt_sint<bits<8> opc, RegisterClass SrcRC, RegisterClass DstRC, 1606 Intrinsic Int, Operand memop, ComplexPattern mem_cpat, 1607 string asm, OpndItins itins> { 1608 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins SrcRC:$src), 1609 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 1610 [(set DstRC:$dst, (Int SrcRC:$src))], itins.rr>, 1611 Sched<[itins.Sched]>; 1612 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), (ins memop:$src), 1613 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 1614 [(set DstRC:$dst, (Int mem_cpat:$src))], itins.rm>, 1615 Sched<[itins.Sched.Folded]>; 1616} 1617 1618multiclass sse12_cvt_sint_3addr<bits<8> opc, RegisterClass SrcRC, 1619 RegisterClass DstRC, Intrinsic Int, X86MemOperand x86memop, 1620 PatFrag ld_frag, string asm, OpndItins itins, 1621 bit Is2Addr = 1> { 1622 def rr : SI<opc, MRMSrcReg, (outs DstRC:$dst), (ins DstRC:$src1, SrcRC:$src2), 1623 !if(Is2Addr, 1624 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"), 1625 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 1626 [(set DstRC:$dst, (Int DstRC:$src1, SrcRC:$src2))], 1627 itins.rr>, Sched<[itins.Sched]>; 1628 def rm : SI<opc, MRMSrcMem, (outs DstRC:$dst), 1629 (ins DstRC:$src1, x86memop:$src2), 1630 !if(Is2Addr, 1631 !strconcat(asm, "\t{$src2, $dst|$dst, $src2}"), 1632 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 1633 [(set DstRC:$dst, (Int DstRC:$src1, (ld_frag addr:$src2)))], 1634 itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 1635} 1636 1637let Predicates = [UseAVX] in { 1638defm VCVTSD2SI : sse12_cvt_sint<0x2D, VR128, GR32, 1639 int_x86_sse2_cvtsd2si, sdmem, sse_load_f64, "cvtsd2si", 1640 SSE_CVT_SD2SI>, XD, VEX, VEX_LIG; 1641defm VCVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, 1642 int_x86_sse2_cvtsd2si64, sdmem, sse_load_f64, "cvtsd2si", 1643 SSE_CVT_SD2SI>, XD, VEX, VEX_W, VEX_LIG; 1644} 1645defm CVTSD2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse2_cvtsd2si, 1646 sdmem, sse_load_f64, "cvtsd2si", SSE_CVT_SD2SI>, XD; 1647defm CVTSD2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse2_cvtsd2si64, 1648 sdmem, sse_load_f64, "cvtsd2si", SSE_CVT_SD2SI>, XD, REX_W; 1649 1650 1651let isCodeGenOnly = 1 in { 1652 let Predicates = [UseAVX] in { 1653 defm Int_VCVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128, 1654 int_x86_sse_cvtsi2ss, i32mem, loadi32, "cvtsi2ss{l}", 1655 SSE_CVT_Scalar, 0>, XS, VEX_4V; 1656 defm Int_VCVTSI2SS64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, 1657 int_x86_sse_cvtsi642ss, i64mem, loadi64, "cvtsi2ss{q}", 1658 SSE_CVT_Scalar, 0>, XS, VEX_4V, 1659 VEX_W; 1660 defm Int_VCVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128, 1661 int_x86_sse2_cvtsi2sd, i32mem, loadi32, "cvtsi2sd{l}", 1662 SSE_CVT_Scalar, 0>, XD, VEX_4V; 1663 defm Int_VCVTSI2SD64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, 1664 int_x86_sse2_cvtsi642sd, i64mem, loadi64, "cvtsi2sd{q}", 1665 SSE_CVT_Scalar, 0>, XD, 1666 VEX_4V, VEX_W; 1667 } 1668 let Constraints = "$src1 = $dst" in { 1669 defm Int_CVTSI2SS : sse12_cvt_sint_3addr<0x2A, GR32, VR128, 1670 int_x86_sse_cvtsi2ss, i32mem, loadi32, 1671 "cvtsi2ss{l}", SSE_CVT_Scalar>, XS; 1672 defm Int_CVTSI2SS64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, 1673 int_x86_sse_cvtsi642ss, i64mem, loadi64, 1674 "cvtsi2ss{q}", SSE_CVT_Scalar>, XS, REX_W; 1675 defm Int_CVTSI2SD : sse12_cvt_sint_3addr<0x2A, GR32, VR128, 1676 int_x86_sse2_cvtsi2sd, i32mem, loadi32, 1677 "cvtsi2sd{l}", SSE_CVT_Scalar>, XD; 1678 defm Int_CVTSI2SD64 : sse12_cvt_sint_3addr<0x2A, GR64, VR128, 1679 int_x86_sse2_cvtsi642sd, i64mem, loadi64, 1680 "cvtsi2sd{q}", SSE_CVT_Scalar>, XD, REX_W; 1681 } 1682} // isCodeGenOnly = 1 1683 1684/// SSE 1 Only 1685 1686// Aliases for intrinsics 1687let isCodeGenOnly = 1 in { 1688let Predicates = [UseAVX] in { 1689defm Int_VCVTTSS2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse_cvttss2si, 1690 ssmem, sse_load_f32, "cvttss2si", 1691 SSE_CVT_SS2SI_32>, XS, VEX; 1692defm Int_VCVTTSS2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, 1693 int_x86_sse_cvttss2si64, ssmem, sse_load_f32, 1694 "cvttss2si", SSE_CVT_SS2SI_64>, 1695 XS, VEX, VEX_W; 1696defm Int_VCVTTSD2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse2_cvttsd2si, 1697 sdmem, sse_load_f64, "cvttsd2si", 1698 SSE_CVT_SD2SI>, XD, VEX; 1699defm Int_VCVTTSD2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, 1700 int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64, 1701 "cvttsd2si", SSE_CVT_SD2SI>, 1702 XD, VEX, VEX_W; 1703} 1704defm Int_CVTTSS2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse_cvttss2si, 1705 ssmem, sse_load_f32, "cvttss2si", 1706 SSE_CVT_SS2SI_32>, XS; 1707defm Int_CVTTSS2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, 1708 int_x86_sse_cvttss2si64, ssmem, sse_load_f32, 1709 "cvttss2si", SSE_CVT_SS2SI_64>, XS, REX_W; 1710defm Int_CVTTSD2SI : sse12_cvt_sint<0x2C, VR128, GR32, int_x86_sse2_cvttsd2si, 1711 sdmem, sse_load_f64, "cvttsd2si", 1712 SSE_CVT_SD2SI>, XD; 1713defm Int_CVTTSD2SI64 : sse12_cvt_sint<0x2C, VR128, GR64, 1714 int_x86_sse2_cvttsd2si64, sdmem, sse_load_f64, 1715 "cvttsd2si", SSE_CVT_SD2SI>, XD, REX_W; 1716} // isCodeGenOnly = 1 1717 1718let Predicates = [UseAVX] in { 1719defm VCVTSS2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse_cvtss2si, 1720 ssmem, sse_load_f32, "cvtss2si", 1721 SSE_CVT_SS2SI_32>, XS, VEX, VEX_LIG; 1722defm VCVTSS2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse_cvtss2si64, 1723 ssmem, sse_load_f32, "cvtss2si", 1724 SSE_CVT_SS2SI_64>, XS, VEX, VEX_W, VEX_LIG; 1725} 1726defm CVTSS2SI : sse12_cvt_sint<0x2D, VR128, GR32, int_x86_sse_cvtss2si, 1727 ssmem, sse_load_f32, "cvtss2si", 1728 SSE_CVT_SS2SI_32>, XS; 1729defm CVTSS2SI64 : sse12_cvt_sint<0x2D, VR128, GR64, int_x86_sse_cvtss2si64, 1730 ssmem, sse_load_f32, "cvtss2si", 1731 SSE_CVT_SS2SI_64>, XS, REX_W; 1732 1733defm VCVTDQ2PS : sse12_cvt_p<0x5B, VR128, VR128, i128mem, 1734 "vcvtdq2ps\t{$src, $dst|$dst, $src}", 1735 SSEPackedSingle, SSE_CVT_PS>, 1736 PS, VEX, Requires<[HasAVX]>; 1737defm VCVTDQ2PSY : sse12_cvt_p<0x5B, VR256, VR256, i256mem, 1738 "vcvtdq2ps\t{$src, $dst|$dst, $src}", 1739 SSEPackedSingle, SSE_CVT_PS>, 1740 PS, VEX, VEX_L, Requires<[HasAVX]>; 1741 1742defm CVTDQ2PS : sse12_cvt_p<0x5B, VR128, VR128, i128mem, 1743 "cvtdq2ps\t{$src, $dst|$dst, $src}", 1744 SSEPackedSingle, SSE_CVT_PS>, 1745 PS, Requires<[UseSSE2]>; 1746 1747let Predicates = [UseAVX] in { 1748def : InstAlias<"vcvtss2si{l}\t{$src, $dst|$dst, $src}", 1749 (VCVTSS2SIrr GR32:$dst, VR128:$src), 0>; 1750def : InstAlias<"vcvtss2si{l}\t{$src, $dst|$dst, $src}", 1751 (VCVTSS2SIrm GR32:$dst, ssmem:$src), 0>; 1752def : InstAlias<"vcvtsd2si{l}\t{$src, $dst|$dst, $src}", 1753 (VCVTSD2SIrr GR32:$dst, VR128:$src), 0>; 1754def : InstAlias<"vcvtsd2si{l}\t{$src, $dst|$dst, $src}", 1755 (VCVTSD2SIrm GR32:$dst, sdmem:$src), 0>; 1756def : InstAlias<"vcvtss2si{q}\t{$src, $dst|$dst, $src}", 1757 (VCVTSS2SI64rr GR64:$dst, VR128:$src), 0>; 1758def : InstAlias<"vcvtss2si{q}\t{$src, $dst|$dst, $src}", 1759 (VCVTSS2SI64rm GR64:$dst, ssmem:$src), 0>; 1760def : InstAlias<"vcvtsd2si{q}\t{$src, $dst|$dst, $src}", 1761 (VCVTSD2SI64rr GR64:$dst, VR128:$src), 0>; 1762def : InstAlias<"vcvtsd2si{q}\t{$src, $dst|$dst, $src}", 1763 (VCVTSD2SI64rm GR64:$dst, sdmem:$src), 0>; 1764} 1765 1766def : InstAlias<"cvtss2si{l}\t{$src, $dst|$dst, $src}", 1767 (CVTSS2SIrr GR32:$dst, VR128:$src), 0>; 1768def : InstAlias<"cvtss2si{l}\t{$src, $dst|$dst, $src}", 1769 (CVTSS2SIrm GR32:$dst, ssmem:$src), 0>; 1770def : InstAlias<"cvtsd2si{l}\t{$src, $dst|$dst, $src}", 1771 (CVTSD2SIrr GR32:$dst, VR128:$src), 0>; 1772def : InstAlias<"cvtsd2si{l}\t{$src, $dst|$dst, $src}", 1773 (CVTSD2SIrm GR32:$dst, sdmem:$src), 0>; 1774def : InstAlias<"cvtss2si{q}\t{$src, $dst|$dst, $src}", 1775 (CVTSS2SI64rr GR64:$dst, VR128:$src), 0>; 1776def : InstAlias<"cvtss2si{q}\t{$src, $dst|$dst, $src}", 1777 (CVTSS2SI64rm GR64:$dst, ssmem:$src), 0>; 1778def : InstAlias<"cvtsd2si{q}\t{$src, $dst|$dst, $src}", 1779 (CVTSD2SI64rr GR64:$dst, VR128:$src), 0>; 1780def : InstAlias<"cvtsd2si{q}\t{$src, $dst|$dst, $src}", 1781 (CVTSD2SI64rm GR64:$dst, sdmem:$src), 0>; 1782 1783/// SSE 2 Only 1784 1785// Convert scalar double to scalar single 1786let hasSideEffects = 0, Predicates = [UseAVX] in { 1787def VCVTSD2SSrr : VSDI<0x5A, MRMSrcReg, (outs FR32:$dst), 1788 (ins FR64:$src1, FR64:$src2), 1789 "cvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", [], 1790 IIC_SSE_CVT_Scalar_RR>, VEX_4V, VEX_LIG, 1791 Sched<[WriteCvtF2F]>; 1792let mayLoad = 1 in 1793def VCVTSD2SSrm : I<0x5A, MRMSrcMem, (outs FR32:$dst), 1794 (ins FR64:$src1, f64mem:$src2), 1795 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1796 [], IIC_SSE_CVT_Scalar_RM>, 1797 XD, Requires<[HasAVX, OptForSize]>, VEX_4V, VEX_LIG, 1798 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1799} 1800 1801def : Pat<(f32 (fround FR64:$src)), (VCVTSD2SSrr FR64:$src, FR64:$src)>, 1802 Requires<[UseAVX]>; 1803 1804def CVTSD2SSrr : SDI<0x5A, MRMSrcReg, (outs FR32:$dst), (ins FR64:$src), 1805 "cvtsd2ss\t{$src, $dst|$dst, $src}", 1806 [(set FR32:$dst, (fround FR64:$src))], 1807 IIC_SSE_CVT_Scalar_RR>, Sched<[WriteCvtF2F]>; 1808def CVTSD2SSrm : I<0x5A, MRMSrcMem, (outs FR32:$dst), (ins f64mem:$src), 1809 "cvtsd2ss\t{$src, $dst|$dst, $src}", 1810 [(set FR32:$dst, (fround (loadf64 addr:$src)))], 1811 IIC_SSE_CVT_Scalar_RM>, 1812 XD, 1813 Requires<[UseSSE2, OptForSize]>, Sched<[WriteCvtF2FLd]>; 1814 1815let isCodeGenOnly = 1 in { 1816def Int_VCVTSD2SSrr: I<0x5A, MRMSrcReg, 1817 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 1818 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1819 [(set VR128:$dst, 1820 (int_x86_sse2_cvtsd2ss VR128:$src1, VR128:$src2))], 1821 IIC_SSE_CVT_Scalar_RR>, XD, VEX_4V, Requires<[HasAVX]>, 1822 Sched<[WriteCvtF2F]>; 1823def Int_VCVTSD2SSrm: I<0x5A, MRMSrcReg, 1824 (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2), 1825 "vcvtsd2ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1826 [(set VR128:$dst, (int_x86_sse2_cvtsd2ss 1827 VR128:$src1, sse_load_f64:$src2))], 1828 IIC_SSE_CVT_Scalar_RM>, XD, VEX_4V, Requires<[HasAVX]>, 1829 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1830 1831let Constraints = "$src1 = $dst" in { 1832def Int_CVTSD2SSrr: I<0x5A, MRMSrcReg, 1833 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 1834 "cvtsd2ss\t{$src2, $dst|$dst, $src2}", 1835 [(set VR128:$dst, 1836 (int_x86_sse2_cvtsd2ss VR128:$src1, VR128:$src2))], 1837 IIC_SSE_CVT_Scalar_RR>, XD, Requires<[UseSSE2]>, 1838 Sched<[WriteCvtF2F]>; 1839def Int_CVTSD2SSrm: I<0x5A, MRMSrcReg, 1840 (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2), 1841 "cvtsd2ss\t{$src2, $dst|$dst, $src2}", 1842 [(set VR128:$dst, (int_x86_sse2_cvtsd2ss 1843 VR128:$src1, sse_load_f64:$src2))], 1844 IIC_SSE_CVT_Scalar_RM>, XD, Requires<[UseSSE2]>, 1845 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1846} 1847} // isCodeGenOnly = 1 1848 1849// Convert scalar single to scalar double 1850// SSE2 instructions with XS prefix 1851let hasSideEffects = 0, Predicates = [UseAVX] in { 1852def VCVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst), 1853 (ins FR32:$src1, FR32:$src2), 1854 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1855 [], IIC_SSE_CVT_Scalar_RR>, 1856 XS, Requires<[HasAVX]>, VEX_4V, VEX_LIG, 1857 Sched<[WriteCvtF2F]>; 1858let mayLoad = 1 in 1859def VCVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst), 1860 (ins FR32:$src1, f32mem:$src2), 1861 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1862 [], IIC_SSE_CVT_Scalar_RM>, 1863 XS, VEX_4V, VEX_LIG, Requires<[HasAVX, OptForSize]>, 1864 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1865} 1866 1867def : Pat<(f64 (fextend FR32:$src)), 1868 (VCVTSS2SDrr FR32:$src, FR32:$src)>, Requires<[UseAVX]>; 1869def : Pat<(fextend (loadf32 addr:$src)), 1870 (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>, Requires<[UseAVX]>; 1871 1872def : Pat<(extloadf32 addr:$src), 1873 (VCVTSS2SDrm (f32 (IMPLICIT_DEF)), addr:$src)>, 1874 Requires<[UseAVX, OptForSize]>; 1875def : Pat<(extloadf32 addr:$src), 1876 (VCVTSS2SDrr (f32 (IMPLICIT_DEF)), (VMOVSSrm addr:$src))>, 1877 Requires<[UseAVX, OptForSpeed]>; 1878 1879def CVTSS2SDrr : I<0x5A, MRMSrcReg, (outs FR64:$dst), (ins FR32:$src), 1880 "cvtss2sd\t{$src, $dst|$dst, $src}", 1881 [(set FR64:$dst, (fextend FR32:$src))], 1882 IIC_SSE_CVT_Scalar_RR>, XS, 1883 Requires<[UseSSE2]>, Sched<[WriteCvtF2F]>; 1884def CVTSS2SDrm : I<0x5A, MRMSrcMem, (outs FR64:$dst), (ins f32mem:$src), 1885 "cvtss2sd\t{$src, $dst|$dst, $src}", 1886 [(set FR64:$dst, (extloadf32 addr:$src))], 1887 IIC_SSE_CVT_Scalar_RM>, XS, 1888 Requires<[UseSSE2, OptForSize]>, Sched<[WriteCvtF2FLd]>; 1889 1890// extload f32 -> f64. This matches load+fextend because we have a hack in 1891// the isel (PreprocessForFPConvert) that can introduce loads after dag 1892// combine. 1893// Since these loads aren't folded into the fextend, we have to match it 1894// explicitly here. 1895def : Pat<(fextend (loadf32 addr:$src)), 1896 (CVTSS2SDrm addr:$src)>, Requires<[UseSSE2]>; 1897def : Pat<(extloadf32 addr:$src), 1898 (CVTSS2SDrr (MOVSSrm addr:$src))>, Requires<[UseSSE2, OptForSpeed]>; 1899 1900let isCodeGenOnly = 1 in { 1901def Int_VCVTSS2SDrr: I<0x5A, MRMSrcReg, 1902 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 1903 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1904 [(set VR128:$dst, 1905 (int_x86_sse2_cvtss2sd VR128:$src1, VR128:$src2))], 1906 IIC_SSE_CVT_Scalar_RR>, XS, VEX_4V, Requires<[HasAVX]>, 1907 Sched<[WriteCvtF2F]>; 1908def Int_VCVTSS2SDrm: I<0x5A, MRMSrcMem, 1909 (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2), 1910 "vcvtss2sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 1911 [(set VR128:$dst, 1912 (int_x86_sse2_cvtss2sd VR128:$src1, sse_load_f32:$src2))], 1913 IIC_SSE_CVT_Scalar_RM>, XS, VEX_4V, Requires<[HasAVX]>, 1914 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1915let Constraints = "$src1 = $dst" in { // SSE2 instructions with XS prefix 1916def Int_CVTSS2SDrr: I<0x5A, MRMSrcReg, 1917 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 1918 "cvtss2sd\t{$src2, $dst|$dst, $src2}", 1919 [(set VR128:$dst, 1920 (int_x86_sse2_cvtss2sd VR128:$src1, VR128:$src2))], 1921 IIC_SSE_CVT_Scalar_RR>, XS, Requires<[UseSSE2]>, 1922 Sched<[WriteCvtF2F]>; 1923def Int_CVTSS2SDrm: I<0x5A, MRMSrcMem, 1924 (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2), 1925 "cvtss2sd\t{$src2, $dst|$dst, $src2}", 1926 [(set VR128:$dst, 1927 (int_x86_sse2_cvtss2sd VR128:$src1, sse_load_f32:$src2))], 1928 IIC_SSE_CVT_Scalar_RM>, XS, Requires<[UseSSE2]>, 1929 Sched<[WriteCvtF2FLd, ReadAfterLd]>; 1930} 1931} // isCodeGenOnly = 1 1932 1933// Convert packed single/double fp to doubleword 1934def VCVTPS2DQrr : VPDI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 1935 "cvtps2dq\t{$src, $dst|$dst, $src}", 1936 [(set VR128:$dst, (int_x86_sse2_cvtps2dq VR128:$src))], 1937 IIC_SSE_CVT_PS_RR>, VEX, Sched<[WriteCvtF2I]>; 1938def VCVTPS2DQrm : VPDI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 1939 "cvtps2dq\t{$src, $dst|$dst, $src}", 1940 [(set VR128:$dst, 1941 (int_x86_sse2_cvtps2dq (loadv4f32 addr:$src)))], 1942 IIC_SSE_CVT_PS_RM>, VEX, Sched<[WriteCvtF2ILd]>; 1943def VCVTPS2DQYrr : VPDI<0x5B, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 1944 "cvtps2dq\t{$src, $dst|$dst, $src}", 1945 [(set VR256:$dst, 1946 (int_x86_avx_cvt_ps2dq_256 VR256:$src))], 1947 IIC_SSE_CVT_PS_RR>, VEX, VEX_L, Sched<[WriteCvtF2I]>; 1948def VCVTPS2DQYrm : VPDI<0x5B, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 1949 "cvtps2dq\t{$src, $dst|$dst, $src}", 1950 [(set VR256:$dst, 1951 (int_x86_avx_cvt_ps2dq_256 (loadv8f32 addr:$src)))], 1952 IIC_SSE_CVT_PS_RM>, VEX, VEX_L, Sched<[WriteCvtF2ILd]>; 1953def CVTPS2DQrr : PDI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 1954 "cvtps2dq\t{$src, $dst|$dst, $src}", 1955 [(set VR128:$dst, (int_x86_sse2_cvtps2dq VR128:$src))], 1956 IIC_SSE_CVT_PS_RR>, Sched<[WriteCvtF2I]>; 1957def CVTPS2DQrm : PDI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 1958 "cvtps2dq\t{$src, $dst|$dst, $src}", 1959 [(set VR128:$dst, 1960 (int_x86_sse2_cvtps2dq (memopv4f32 addr:$src)))], 1961 IIC_SSE_CVT_PS_RM>, Sched<[WriteCvtF2ILd]>; 1962 1963 1964// Convert Packed Double FP to Packed DW Integers 1965let Predicates = [HasAVX] in { 1966// The assembler can recognize rr 256-bit instructions by seeing a ymm 1967// register, but the same isn't true when using memory operands instead. 1968// Provide other assembly rr and rm forms to address this explicitly. 1969def VCVTPD2DQrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 1970 "vcvtpd2dq\t{$src, $dst|$dst, $src}", 1971 [(set VR128:$dst, (int_x86_sse2_cvtpd2dq VR128:$src))]>, 1972 VEX, Sched<[WriteCvtF2I]>; 1973 1974// XMM only 1975def : InstAlias<"vcvtpd2dqx\t{$src, $dst|$dst, $src}", 1976 (VCVTPD2DQrr VR128:$dst, VR128:$src), 0>; 1977def VCVTPD2DQXrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 1978 "vcvtpd2dqx\t{$src, $dst|$dst, $src}", 1979 [(set VR128:$dst, 1980 (int_x86_sse2_cvtpd2dq (loadv2f64 addr:$src)))]>, VEX, 1981 Sched<[WriteCvtF2ILd]>; 1982 1983// YMM only 1984def VCVTPD2DQYrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src), 1985 "vcvtpd2dq{y}\t{$src, $dst|$dst, $src}", 1986 [(set VR128:$dst, 1987 (int_x86_avx_cvt_pd2dq_256 VR256:$src))]>, VEX, VEX_L, 1988 Sched<[WriteCvtF2I]>; 1989def VCVTPD2DQYrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src), 1990 "vcvtpd2dq{y}\t{$src, $dst|$dst, $src}", 1991 [(set VR128:$dst, 1992 (int_x86_avx_cvt_pd2dq_256 (loadv4f64 addr:$src)))]>, 1993 VEX, VEX_L, Sched<[WriteCvtF2ILd]>; 1994def : InstAlias<"vcvtpd2dq\t{$src, $dst|$dst, $src}", 1995 (VCVTPD2DQYrr VR128:$dst, VR256:$src), 0>; 1996} 1997 1998def CVTPD2DQrm : SDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 1999 "cvtpd2dq\t{$src, $dst|$dst, $src}", 2000 [(set VR128:$dst, 2001 (int_x86_sse2_cvtpd2dq (memopv2f64 addr:$src)))], 2002 IIC_SSE_CVT_PD_RM>, Sched<[WriteCvtF2ILd]>; 2003def CVTPD2DQrr : SDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2004 "cvtpd2dq\t{$src, $dst|$dst, $src}", 2005 [(set VR128:$dst, (int_x86_sse2_cvtpd2dq VR128:$src))], 2006 IIC_SSE_CVT_PD_RR>, Sched<[WriteCvtF2I]>; 2007 2008// Convert with truncation packed single/double fp to doubleword 2009// SSE2 packed instructions with XS prefix 2010def VCVTTPS2DQrr : VS2SI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2011 "cvttps2dq\t{$src, $dst|$dst, $src}", 2012 [], IIC_SSE_CVT_PS_RR>, VEX, Sched<[WriteCvtF2I]>; 2013def VCVTTPS2DQrm : VS2SI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2014 "cvttps2dq\t{$src, $dst|$dst, $src}", 2015 [], IIC_SSE_CVT_PS_RM>, VEX, Sched<[WriteCvtF2ILd]>; 2016def VCVTTPS2DQYrr : VS2SI<0x5B, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 2017 "cvttps2dq\t{$src, $dst|$dst, $src}", 2018 [], IIC_SSE_CVT_PS_RR>, VEX, VEX_L, Sched<[WriteCvtF2I]>; 2019def VCVTTPS2DQYrm : VS2SI<0x5B, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 2020 "cvttps2dq\t{$src, $dst|$dst, $src}", 2021 [], IIC_SSE_CVT_PS_RM>, VEX, VEX_L, 2022 Sched<[WriteCvtF2ILd]>; 2023 2024def CVTTPS2DQrr : S2SI<0x5B, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2025 "cvttps2dq\t{$src, $dst|$dst, $src}", 2026 [], IIC_SSE_CVT_PS_RR>, Sched<[WriteCvtF2I]>; 2027def CVTTPS2DQrm : S2SI<0x5B, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2028 "cvttps2dq\t{$src, $dst|$dst, $src}", 2029 [], IIC_SSE_CVT_PS_RM>, Sched<[WriteCvtF2ILd]>; 2030 2031let Predicates = [HasAVX] in { 2032 def : Pat<(int_x86_sse2_cvtdq2ps VR128:$src), 2033 (VCVTDQ2PSrr VR128:$src)>; 2034 def : Pat<(int_x86_sse2_cvtdq2ps (bc_v4i32 (loadv2i64 addr:$src))), 2035 (VCVTDQ2PSrm addr:$src)>; 2036} 2037 2038let Predicates = [HasAVX, NoVLX] in { 2039 def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))), 2040 (VCVTDQ2PSrr VR128:$src)>; 2041 def : Pat<(v4f32 (sint_to_fp (bc_v4i32 (loadv2i64 addr:$src)))), 2042 (VCVTDQ2PSrm addr:$src)>; 2043 2044 def : Pat<(v4i32 (fp_to_sint (v4f32 VR128:$src))), 2045 (VCVTTPS2DQrr VR128:$src)>; 2046 def : Pat<(v4i32 (fp_to_sint (loadv4f32 addr:$src))), 2047 (VCVTTPS2DQrm addr:$src)>; 2048 2049 def : Pat<(v8f32 (sint_to_fp (v8i32 VR256:$src))), 2050 (VCVTDQ2PSYrr VR256:$src)>; 2051 def : Pat<(v8f32 (sint_to_fp (bc_v8i32 (loadv4i64 addr:$src)))), 2052 (VCVTDQ2PSYrm addr:$src)>; 2053 2054 def : Pat<(v8i32 (fp_to_sint (v8f32 VR256:$src))), 2055 (VCVTTPS2DQYrr VR256:$src)>; 2056 def : Pat<(v8i32 (fp_to_sint (loadv8f32 addr:$src))), 2057 (VCVTTPS2DQYrm addr:$src)>; 2058} 2059 2060let Predicates = [UseSSE2] in { 2061 def : Pat<(v4f32 (sint_to_fp (v4i32 VR128:$src))), 2062 (CVTDQ2PSrr VR128:$src)>; 2063 def : Pat<(v4f32 (sint_to_fp (bc_v4i32 (memopv2i64 addr:$src)))), 2064 (CVTDQ2PSrm addr:$src)>; 2065 2066 def : Pat<(int_x86_sse2_cvtdq2ps VR128:$src), 2067 (CVTDQ2PSrr VR128:$src)>; 2068 def : Pat<(int_x86_sse2_cvtdq2ps (bc_v4i32 (memopv2i64 addr:$src))), 2069 (CVTDQ2PSrm addr:$src)>; 2070 2071 def : Pat<(v4i32 (fp_to_sint (v4f32 VR128:$src))), 2072 (CVTTPS2DQrr VR128:$src)>; 2073 def : Pat<(v4i32 (fp_to_sint (memopv4f32 addr:$src))), 2074 (CVTTPS2DQrm addr:$src)>; 2075} 2076 2077def VCVTTPD2DQrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2078 "cvttpd2dq\t{$src, $dst|$dst, $src}", 2079 [(set VR128:$dst, 2080 (int_x86_sse2_cvttpd2dq VR128:$src))], 2081 IIC_SSE_CVT_PD_RR>, VEX, Sched<[WriteCvtF2I]>; 2082 2083// The assembler can recognize rr 256-bit instructions by seeing a ymm 2084// register, but the same isn't true when using memory operands instead. 2085// Provide other assembly rr and rm forms to address this explicitly. 2086 2087// XMM only 2088def : InstAlias<"vcvttpd2dqx\t{$src, $dst|$dst, $src}", 2089 (VCVTTPD2DQrr VR128:$dst, VR128:$src), 0>; 2090def VCVTTPD2DQXrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2091 "cvttpd2dqx\t{$src, $dst|$dst, $src}", 2092 [(set VR128:$dst, (int_x86_sse2_cvttpd2dq 2093 (loadv2f64 addr:$src)))], 2094 IIC_SSE_CVT_PD_RM>, VEX, Sched<[WriteCvtF2ILd]>; 2095 2096// YMM only 2097def VCVTTPD2DQYrr : VPDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src), 2098 "cvttpd2dq{y}\t{$src, $dst|$dst, $src}", 2099 [], IIC_SSE_CVT_PD_RR>, VEX, VEX_L, Sched<[WriteCvtF2I]>; 2100def VCVTTPD2DQYrm : VPDI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src), 2101 "cvttpd2dq{y}\t{$src, $dst|$dst, $src}", 2102 [], IIC_SSE_CVT_PD_RM>, VEX, VEX_L, Sched<[WriteCvtF2ILd]>; 2103def : InstAlias<"vcvttpd2dq\t{$src, $dst|$dst, $src}", 2104 (VCVTTPD2DQYrr VR128:$dst, VR256:$src), 0>; 2105 2106let Predicates = [HasAVX, NoVLX] in { 2107 def : Pat<(v4i32 (fp_to_sint (v4f64 VR256:$src))), 2108 (VCVTTPD2DQYrr VR256:$src)>; 2109 def : Pat<(v4i32 (fp_to_sint (loadv4f64 addr:$src))), 2110 (VCVTTPD2DQYrm addr:$src)>; 2111} // Predicates = [HasAVX] 2112 2113def CVTTPD2DQrr : PDI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2114 "cvttpd2dq\t{$src, $dst|$dst, $src}", 2115 [(set VR128:$dst, (int_x86_sse2_cvttpd2dq VR128:$src))], 2116 IIC_SSE_CVT_PD_RR>, Sched<[WriteCvtF2I]>; 2117def CVTTPD2DQrm : PDI<0xE6, MRMSrcMem, (outs VR128:$dst),(ins f128mem:$src), 2118 "cvttpd2dq\t{$src, $dst|$dst, $src}", 2119 [(set VR128:$dst, (int_x86_sse2_cvttpd2dq 2120 (memopv2f64 addr:$src)))], 2121 IIC_SSE_CVT_PD_RM>, 2122 Sched<[WriteCvtF2ILd]>; 2123 2124// Convert packed single to packed double 2125let Predicates = [HasAVX] in { 2126 // SSE2 instructions without OpSize prefix 2127def VCVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2128 "vcvtps2pd\t{$src, $dst|$dst, $src}", 2129 [], IIC_SSE_CVT_PD_RR>, PS, VEX, Sched<[WriteCvtF2F]>; 2130def VCVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src), 2131 "vcvtps2pd\t{$src, $dst|$dst, $src}", 2132 [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))], 2133 IIC_SSE_CVT_PD_RM>, PS, VEX, Sched<[WriteCvtF2FLd]>; 2134def VCVTPS2PDYrr : I<0x5A, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), 2135 "vcvtps2pd\t{$src, $dst|$dst, $src}", 2136 [], IIC_SSE_CVT_PD_RR>, PS, VEX, VEX_L, Sched<[WriteCvtF2F]>; 2137def VCVTPS2PDYrm : I<0x5A, MRMSrcMem, (outs VR256:$dst), (ins f128mem:$src), 2138 "vcvtps2pd\t{$src, $dst|$dst, $src}", 2139 [], IIC_SSE_CVT_PD_RM>, PS, VEX, VEX_L, Sched<[WriteCvtF2FLd]>; 2140} 2141 2142let Predicates = [UseSSE2] in { 2143def CVTPS2PDrr : I<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2144 "cvtps2pd\t{$src, $dst|$dst, $src}", 2145 [], IIC_SSE_CVT_PD_RR>, PS, Sched<[WriteCvtF2F]>; 2146def CVTPS2PDrm : I<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src), 2147 "cvtps2pd\t{$src, $dst|$dst, $src}", 2148 [(set VR128:$dst, (v2f64 (extloadv2f32 addr:$src)))], 2149 IIC_SSE_CVT_PD_RM>, PS, Sched<[WriteCvtF2FLd]>; 2150} 2151 2152// Convert Packed DW Integers to Packed Double FP 2153let Predicates = [HasAVX] in { 2154let hasSideEffects = 0, mayLoad = 1 in 2155def VCVTDQ2PDrm : S2SI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 2156 "vcvtdq2pd\t{$src, $dst|$dst, $src}", 2157 []>, VEX, Sched<[WriteCvtI2FLd]>; 2158def VCVTDQ2PDrr : S2SI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2159 "vcvtdq2pd\t{$src, $dst|$dst, $src}", 2160 []>, VEX, Sched<[WriteCvtI2F]>; 2161def VCVTDQ2PDYrm : S2SI<0xE6, MRMSrcMem, (outs VR256:$dst), (ins i128mem:$src), 2162 "vcvtdq2pd\t{$src, $dst|$dst, $src}", 2163 []>, VEX, VEX_L, Sched<[WriteCvtI2FLd]>; 2164def VCVTDQ2PDYrr : S2SI<0xE6, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), 2165 "vcvtdq2pd\t{$src, $dst|$dst, $src}", 2166 []>, VEX, VEX_L, Sched<[WriteCvtI2F]>; 2167} 2168 2169let hasSideEffects = 0, mayLoad = 1 in 2170def CVTDQ2PDrm : S2SI<0xE6, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 2171 "cvtdq2pd\t{$src, $dst|$dst, $src}", [], 2172 IIC_SSE_CVT_PD_RR>, Sched<[WriteCvtI2FLd]>; 2173def CVTDQ2PDrr : S2SI<0xE6, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2174 "cvtdq2pd\t{$src, $dst|$dst, $src}", [], 2175 IIC_SSE_CVT_PD_RM>, Sched<[WriteCvtI2F]>; 2176 2177// AVX register conversion intrinsics 2178let Predicates = [HasAVX] in { 2179 def : Pat<(v2f64 (X86cvtdq2pd (v4i32 VR128:$src))), 2180 (VCVTDQ2PDrr VR128:$src)>; 2181 def : Pat<(v2f64 (X86cvtdq2pd (bc_v4i32 (loadv2i64 addr:$src)))), 2182 (VCVTDQ2PDrm addr:$src)>; 2183 def : Pat<(v2f64 (X86cvtdq2pd (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 2184 (VCVTDQ2PDrm addr:$src)>; 2185 2186 def : Pat<(v4f64 (sint_to_fp (v4i32 VR128:$src))), 2187 (VCVTDQ2PDYrr VR128:$src)>; 2188 def : Pat<(v4f64 (sint_to_fp (bc_v4i32 (loadv2i64 addr:$src)))), 2189 (VCVTDQ2PDYrm addr:$src)>; 2190} // Predicates = [HasAVX] 2191 2192// SSE2 register conversion intrinsics 2193let Predicates = [HasSSE2] in { 2194 def : Pat<(v2f64 (X86cvtdq2pd (v4i32 VR128:$src))), 2195 (CVTDQ2PDrr VR128:$src)>; 2196 def : Pat<(v2f64 (X86cvtdq2pd (bc_v4i32 (loadv2i64 addr:$src)))), 2197 (CVTDQ2PDrm addr:$src)>; 2198 def : Pat<(v2f64 (X86cvtdq2pd (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 2199 (CVTDQ2PDrm addr:$src)>; 2200} // Predicates = [HasSSE2] 2201 2202// Convert packed double to packed single 2203// The assembler can recognize rr 256-bit instructions by seeing a ymm 2204// register, but the same isn't true when using memory operands instead. 2205// Provide other assembly rr and rm forms to address this explicitly. 2206def VCVTPD2PSrr : VPDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2207 "cvtpd2ps\t{$src, $dst|$dst, $src}", 2208 [(set VR128:$dst, (int_x86_sse2_cvtpd2ps VR128:$src))], 2209 IIC_SSE_CVT_PD_RR>, VEX, Sched<[WriteCvtF2F]>; 2210 2211// XMM only 2212def : InstAlias<"vcvtpd2psx\t{$src, $dst|$dst, $src}", 2213 (VCVTPD2PSrr VR128:$dst, VR128:$src), 0>; 2214def VCVTPD2PSXrm : VPDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2215 "cvtpd2psx\t{$src, $dst|$dst, $src}", 2216 [(set VR128:$dst, 2217 (int_x86_sse2_cvtpd2ps (loadv2f64 addr:$src)))], 2218 IIC_SSE_CVT_PD_RM>, VEX, Sched<[WriteCvtF2FLd]>; 2219 2220// YMM only 2221def VCVTPD2PSYrr : VPDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR256:$src), 2222 "cvtpd2ps{y}\t{$src, $dst|$dst, $src}", 2223 [(set VR128:$dst, 2224 (int_x86_avx_cvt_pd2_ps_256 VR256:$src))], 2225 IIC_SSE_CVT_PD_RR>, VEX, VEX_L, Sched<[WriteCvtF2F]>; 2226def VCVTPD2PSYrm : VPDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f256mem:$src), 2227 "cvtpd2ps{y}\t{$src, $dst|$dst, $src}", 2228 [(set VR128:$dst, 2229 (int_x86_avx_cvt_pd2_ps_256 (loadv4f64 addr:$src)))], 2230 IIC_SSE_CVT_PD_RM>, VEX, VEX_L, Sched<[WriteCvtF2FLd]>; 2231def : InstAlias<"vcvtpd2ps\t{$src, $dst|$dst, $src}", 2232 (VCVTPD2PSYrr VR128:$dst, VR256:$src), 0>; 2233 2234def CVTPD2PSrr : PDI<0x5A, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 2235 "cvtpd2ps\t{$src, $dst|$dst, $src}", 2236 [(set VR128:$dst, (int_x86_sse2_cvtpd2ps VR128:$src))], 2237 IIC_SSE_CVT_PD_RR>, Sched<[WriteCvtF2F]>; 2238def CVTPD2PSrm : PDI<0x5A, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 2239 "cvtpd2ps\t{$src, $dst|$dst, $src}", 2240 [(set VR128:$dst, 2241 (int_x86_sse2_cvtpd2ps (memopv2f64 addr:$src)))], 2242 IIC_SSE_CVT_PD_RM>, Sched<[WriteCvtF2FLd]>; 2243 2244 2245// AVX 256-bit register conversion intrinsics 2246// FIXME: Migrate SSE conversion intrinsics matching to use patterns as below 2247// whenever possible to avoid declaring two versions of each one. 2248let Predicates = [HasAVX] in { 2249 def : Pat<(int_x86_avx_cvtdq2_ps_256 VR256:$src), 2250 (VCVTDQ2PSYrr VR256:$src)>; 2251 def : Pat<(int_x86_avx_cvtdq2_ps_256 (bitconvert (loadv4i64 addr:$src))), 2252 (VCVTDQ2PSYrm addr:$src)>; 2253} 2254 2255let Predicates = [HasAVX, NoVLX] in { 2256 // Match fround and fextend for 128/256-bit conversions 2257 def : Pat<(v4f32 (X86vfpround (v2f64 VR128:$src))), 2258 (VCVTPD2PSrr VR128:$src)>; 2259 def : Pat<(v4f32 (X86vfpround (loadv2f64 addr:$src))), 2260 (VCVTPD2PSXrm addr:$src)>; 2261 def : Pat<(v4f32 (fround (v4f64 VR256:$src))), 2262 (VCVTPD2PSYrr VR256:$src)>; 2263 def : Pat<(v4f32 (fround (loadv4f64 addr:$src))), 2264 (VCVTPD2PSYrm addr:$src)>; 2265 2266 def : Pat<(v2f64 (X86vfpext (v4f32 VR128:$src))), 2267 (VCVTPS2PDrr VR128:$src)>; 2268 def : Pat<(v4f64 (fextend (v4f32 VR128:$src))), 2269 (VCVTPS2PDYrr VR128:$src)>; 2270 def : Pat<(v4f64 (extloadv4f32 addr:$src)), 2271 (VCVTPS2PDYrm addr:$src)>; 2272} 2273 2274let Predicates = [UseSSE2] in { 2275 // Match fround and fextend for 128 conversions 2276 def : Pat<(v4f32 (X86vfpround (v2f64 VR128:$src))), 2277 (CVTPD2PSrr VR128:$src)>; 2278 def : Pat<(v4f32 (X86vfpround (memopv2f64 addr:$src))), 2279 (CVTPD2PSrm addr:$src)>; 2280 2281 def : Pat<(v2f64 (X86vfpext (v4f32 VR128:$src))), 2282 (CVTPS2PDrr VR128:$src)>; 2283} 2284 2285//===----------------------------------------------------------------------===// 2286// SSE 1 & 2 - Compare Instructions 2287//===----------------------------------------------------------------------===// 2288 2289// sse12_cmp_scalar - sse 1 & 2 compare scalar instructions 2290multiclass sse12_cmp_scalar<RegisterClass RC, X86MemOperand x86memop, 2291 Operand CC, SDNode OpNode, ValueType VT, 2292 PatFrag ld_frag, string asm, string asm_alt, 2293 OpndItins itins, ImmLeaf immLeaf> { 2294 def rr : SIi8<0xC2, MRMSrcReg, 2295 (outs RC:$dst), (ins RC:$src1, RC:$src2, CC:$cc), asm, 2296 [(set RC:$dst, (OpNode (VT RC:$src1), RC:$src2, immLeaf:$cc))], 2297 itins.rr>, Sched<[itins.Sched]>; 2298 def rm : SIi8<0xC2, MRMSrcMem, 2299 (outs RC:$dst), (ins RC:$src1, x86memop:$src2, CC:$cc), asm, 2300 [(set RC:$dst, (OpNode (VT RC:$src1), 2301 (ld_frag addr:$src2), immLeaf:$cc))], 2302 itins.rm>, 2303 Sched<[itins.Sched.Folded, ReadAfterLd]>; 2304 2305 // Accept explicit immediate argument form instead of comparison code. 2306 let isAsmParserOnly = 1, hasSideEffects = 0 in { 2307 def rr_alt : SIi8<0xC2, MRMSrcReg, (outs RC:$dst), 2308 (ins RC:$src1, RC:$src2, u8imm:$cc), asm_alt, [], 2309 IIC_SSE_ALU_F32S_RR>, Sched<[itins.Sched]>; 2310 let mayLoad = 1 in 2311 def rm_alt : SIi8<0xC2, MRMSrcMem, (outs RC:$dst), 2312 (ins RC:$src1, x86memop:$src2, u8imm:$cc), asm_alt, [], 2313 IIC_SSE_ALU_F32S_RM>, 2314 Sched<[itins.Sched.Folded, ReadAfterLd]>; 2315 } 2316} 2317 2318defm VCMPSS : sse12_cmp_scalar<FR32, f32mem, AVXCC, X86cmps, f32, loadf32, 2319 "cmp${cc}ss\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2320 "cmpss\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2321 SSE_ALU_F32S, i8immZExt5>, XS, VEX_4V, VEX_LIG; 2322defm VCMPSD : sse12_cmp_scalar<FR64, f64mem, AVXCC, X86cmps, f64, loadf64, 2323 "cmp${cc}sd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2324 "cmpsd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2325 SSE_ALU_F32S, i8immZExt5>, // same latency as 32 bit compare 2326 XD, VEX_4V, VEX_LIG; 2327 2328let Constraints = "$src1 = $dst" in { 2329 defm CMPSS : sse12_cmp_scalar<FR32, f32mem, SSECC, X86cmps, f32, loadf32, 2330 "cmp${cc}ss\t{$src2, $dst|$dst, $src2}", 2331 "cmpss\t{$cc, $src2, $dst|$dst, $src2, $cc}", SSE_ALU_F32S, 2332 i8immZExt3>, XS; 2333 defm CMPSD : sse12_cmp_scalar<FR64, f64mem, SSECC, X86cmps, f64, loadf64, 2334 "cmp${cc}sd\t{$src2, $dst|$dst, $src2}", 2335 "cmpsd\t{$cc, $src2, $dst|$dst, $src2, $cc}", 2336 SSE_ALU_F64S, i8immZExt3>, XD; 2337} 2338 2339multiclass sse12_cmp_scalar_int<X86MemOperand x86memop, Operand CC, 2340 Intrinsic Int, string asm, OpndItins itins, 2341 ImmLeaf immLeaf> { 2342 def rr : SIi8<0xC2, MRMSrcReg, (outs VR128:$dst), 2343 (ins VR128:$src1, VR128:$src, CC:$cc), asm, 2344 [(set VR128:$dst, (Int VR128:$src1, 2345 VR128:$src, immLeaf:$cc))], 2346 itins.rr>, 2347 Sched<[itins.Sched]>; 2348 def rm : SIi8<0xC2, MRMSrcMem, (outs VR128:$dst), 2349 (ins VR128:$src1, x86memop:$src, CC:$cc), asm, 2350 [(set VR128:$dst, (Int VR128:$src1, 2351 (load addr:$src), immLeaf:$cc))], 2352 itins.rm>, 2353 Sched<[itins.Sched.Folded, ReadAfterLd]>; 2354} 2355 2356let isCodeGenOnly = 1 in { 2357 // Aliases to match intrinsics which expect XMM operand(s). 2358 defm Int_VCMPSS : sse12_cmp_scalar_int<f32mem, AVXCC, int_x86_sse_cmp_ss, 2359 "cmp${cc}ss\t{$src, $src1, $dst|$dst, $src1, $src}", 2360 SSE_ALU_F32S, i8immZExt5>, 2361 XS, VEX_4V; 2362 defm Int_VCMPSD : sse12_cmp_scalar_int<f64mem, AVXCC, int_x86_sse2_cmp_sd, 2363 "cmp${cc}sd\t{$src, $src1, $dst|$dst, $src1, $src}", 2364 SSE_ALU_F32S, i8immZExt5>, // same latency as f32 2365 XD, VEX_4V; 2366 let Constraints = "$src1 = $dst" in { 2367 defm Int_CMPSS : sse12_cmp_scalar_int<f32mem, SSECC, int_x86_sse_cmp_ss, 2368 "cmp${cc}ss\t{$src, $dst|$dst, $src}", 2369 SSE_ALU_F32S, i8immZExt3>, XS; 2370 defm Int_CMPSD : sse12_cmp_scalar_int<f64mem, SSECC, int_x86_sse2_cmp_sd, 2371 "cmp${cc}sd\t{$src, $dst|$dst, $src}", 2372 SSE_ALU_F64S, i8immZExt3>, 2373 XD; 2374} 2375} 2376 2377 2378// sse12_ord_cmp - Unordered/Ordered scalar fp compare and set EFLAGS 2379multiclass sse12_ord_cmp<bits<8> opc, RegisterClass RC, SDNode OpNode, 2380 ValueType vt, X86MemOperand x86memop, 2381 PatFrag ld_frag, string OpcodeStr> { 2382 def rr: SI<opc, MRMSrcReg, (outs), (ins RC:$src1, RC:$src2), 2383 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), 2384 [(set EFLAGS, (OpNode (vt RC:$src1), RC:$src2))], 2385 IIC_SSE_COMIS_RR>, 2386 Sched<[WriteFAdd]>; 2387 def rm: SI<opc, MRMSrcMem, (outs), (ins RC:$src1, x86memop:$src2), 2388 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), 2389 [(set EFLAGS, (OpNode (vt RC:$src1), 2390 (ld_frag addr:$src2)))], 2391 IIC_SSE_COMIS_RM>, 2392 Sched<[WriteFAddLd, ReadAfterLd]>; 2393} 2394 2395let Defs = [EFLAGS] in { 2396 defm VUCOMISS : sse12_ord_cmp<0x2E, FR32, X86cmp, f32, f32mem, loadf32, 2397 "ucomiss">, PS, VEX, VEX_LIG; 2398 defm VUCOMISD : sse12_ord_cmp<0x2E, FR64, X86cmp, f64, f64mem, loadf64, 2399 "ucomisd">, PD, VEX, VEX_LIG; 2400 let Pattern = []<dag> in { 2401 defm VCOMISS : sse12_ord_cmp<0x2F, FR32, undef, f32, f32mem, loadf32, 2402 "comiss">, PS, VEX, VEX_LIG; 2403 defm VCOMISD : sse12_ord_cmp<0x2F, FR64, undef, f64, f64mem, loadf64, 2404 "comisd">, PD, VEX, VEX_LIG; 2405 } 2406 2407 let isCodeGenOnly = 1 in { 2408 defm Int_VUCOMISS : sse12_ord_cmp<0x2E, VR128, X86ucomi, v4f32, f128mem, 2409 load, "ucomiss">, PS, VEX; 2410 defm Int_VUCOMISD : sse12_ord_cmp<0x2E, VR128, X86ucomi, v2f64, f128mem, 2411 load, "ucomisd">, PD, VEX; 2412 2413 defm Int_VCOMISS : sse12_ord_cmp<0x2F, VR128, X86comi, v4f32, f128mem, 2414 load, "comiss">, PS, VEX; 2415 defm Int_VCOMISD : sse12_ord_cmp<0x2F, VR128, X86comi, v2f64, f128mem, 2416 load, "comisd">, PD, VEX; 2417 } 2418 defm UCOMISS : sse12_ord_cmp<0x2E, FR32, X86cmp, f32, f32mem, loadf32, 2419 "ucomiss">, PS; 2420 defm UCOMISD : sse12_ord_cmp<0x2E, FR64, X86cmp, f64, f64mem, loadf64, 2421 "ucomisd">, PD; 2422 2423 let Pattern = []<dag> in { 2424 defm COMISS : sse12_ord_cmp<0x2F, FR32, undef, f32, f32mem, loadf32, 2425 "comiss">, PS; 2426 defm COMISD : sse12_ord_cmp<0x2F, FR64, undef, f64, f64mem, loadf64, 2427 "comisd">, PD; 2428 } 2429 2430 let isCodeGenOnly = 1 in { 2431 defm Int_UCOMISS : sse12_ord_cmp<0x2E, VR128, X86ucomi, v4f32, f128mem, 2432 load, "ucomiss">, PS; 2433 defm Int_UCOMISD : sse12_ord_cmp<0x2E, VR128, X86ucomi, v2f64, f128mem, 2434 load, "ucomisd">, PD; 2435 2436 defm Int_COMISS : sse12_ord_cmp<0x2F, VR128, X86comi, v4f32, f128mem, load, 2437 "comiss">, PS; 2438 defm Int_COMISD : sse12_ord_cmp<0x2F, VR128, X86comi, v2f64, f128mem, load, 2439 "comisd">, PD; 2440 } 2441} // Defs = [EFLAGS] 2442 2443// sse12_cmp_packed - sse 1 & 2 compare packed instructions 2444multiclass sse12_cmp_packed<RegisterClass RC, X86MemOperand x86memop, 2445 Operand CC, Intrinsic Int, string asm, 2446 string asm_alt, Domain d, ImmLeaf immLeaf, 2447 PatFrag ld_frag, OpndItins itins = SSE_ALU_F32P> { 2448 let isCommutable = 1 in 2449 def rri : PIi8<0xC2, MRMSrcReg, 2450 (outs RC:$dst), (ins RC:$src1, RC:$src2, CC:$cc), asm, 2451 [(set RC:$dst, (Int RC:$src1, RC:$src2, immLeaf:$cc))], 2452 itins.rr, d>, 2453 Sched<[WriteFAdd]>; 2454 def rmi : PIi8<0xC2, MRMSrcMem, 2455 (outs RC:$dst), (ins RC:$src1, x86memop:$src2, CC:$cc), asm, 2456 [(set RC:$dst, (Int RC:$src1, (ld_frag addr:$src2), immLeaf:$cc))], 2457 itins.rm, d>, 2458 Sched<[WriteFAddLd, ReadAfterLd]>; 2459 2460 // Accept explicit immediate argument form instead of comparison code. 2461 let isAsmParserOnly = 1, hasSideEffects = 0 in { 2462 def rri_alt : PIi8<0xC2, MRMSrcReg, 2463 (outs RC:$dst), (ins RC:$src1, RC:$src2, u8imm:$cc), 2464 asm_alt, [], itins.rr, d>, Sched<[WriteFAdd]>; 2465 let mayLoad = 1 in 2466 def rmi_alt : PIi8<0xC2, MRMSrcMem, 2467 (outs RC:$dst), (ins RC:$src1, x86memop:$src2, u8imm:$cc), 2468 asm_alt, [], itins.rm, d>, 2469 Sched<[WriteFAddLd, ReadAfterLd]>; 2470 } 2471} 2472 2473defm VCMPPS : sse12_cmp_packed<VR128, f128mem, AVXCC, int_x86_sse_cmp_ps, 2474 "cmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2475 "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2476 SSEPackedSingle, i8immZExt5, loadv4f32>, PS, VEX_4V; 2477defm VCMPPD : sse12_cmp_packed<VR128, f128mem, AVXCC, int_x86_sse2_cmp_pd, 2478 "cmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2479 "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2480 SSEPackedDouble, i8immZExt5, loadv2f64>, PD, VEX_4V; 2481defm VCMPPSY : sse12_cmp_packed<VR256, f256mem, AVXCC, int_x86_avx_cmp_ps_256, 2482 "cmp${cc}ps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2483 "cmpps\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2484 SSEPackedSingle, i8immZExt5, loadv8f32>, PS, VEX_4V, VEX_L; 2485defm VCMPPDY : sse12_cmp_packed<VR256, f256mem, AVXCC, int_x86_avx_cmp_pd_256, 2486 "cmp${cc}pd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2487 "cmppd\t{$cc, $src2, $src1, $dst|$dst, $src1, $src2, $cc}", 2488 SSEPackedDouble, i8immZExt5, loadv4f64>, PD, VEX_4V, VEX_L; 2489let Constraints = "$src1 = $dst" in { 2490 defm CMPPS : sse12_cmp_packed<VR128, f128mem, SSECC, int_x86_sse_cmp_ps, 2491 "cmp${cc}ps\t{$src2, $dst|$dst, $src2}", 2492 "cmpps\t{$cc, $src2, $dst|$dst, $src2, $cc}", 2493 SSEPackedSingle, i8immZExt5, memopv4f32, SSE_ALU_F32P>, PS; 2494 defm CMPPD : sse12_cmp_packed<VR128, f128mem, SSECC, int_x86_sse2_cmp_pd, 2495 "cmp${cc}pd\t{$src2, $dst|$dst, $src2}", 2496 "cmppd\t{$cc, $src2, $dst|$dst, $src2, $cc}", 2497 SSEPackedDouble, i8immZExt5, memopv2f64, SSE_ALU_F64P>, PD; 2498} 2499 2500let Predicates = [HasAVX] in { 2501def : Pat<(v4f32 (X86cmpp (v4f32 VR128:$src1), VR128:$src2, imm:$cc)), 2502 (VCMPPSrri (v4f32 VR128:$src1), (v4f32 VR128:$src2), imm:$cc)>; 2503def : Pat<(v4f32 (X86cmpp (v4f32 VR128:$src1), (loadv4f32 addr:$src2), imm:$cc)), 2504 (VCMPPSrmi (v4f32 VR128:$src1), addr:$src2, imm:$cc)>; 2505def : Pat<(v2f64 (X86cmpp (v2f64 VR128:$src1), VR128:$src2, imm:$cc)), 2506 (VCMPPDrri VR128:$src1, VR128:$src2, imm:$cc)>; 2507def : Pat<(v2f64 (X86cmpp (v2f64 VR128:$src1), (loadv2f64 addr:$src2), imm:$cc)), 2508 (VCMPPDrmi VR128:$src1, addr:$src2, imm:$cc)>; 2509 2510def : Pat<(v8f32 (X86cmpp (v8f32 VR256:$src1), VR256:$src2, imm:$cc)), 2511 (VCMPPSYrri (v8f32 VR256:$src1), (v8f32 VR256:$src2), imm:$cc)>; 2512def : Pat<(v8f32 (X86cmpp (v8f32 VR256:$src1), (loadv8f32 addr:$src2), imm:$cc)), 2513 (VCMPPSYrmi (v8f32 VR256:$src1), addr:$src2, imm:$cc)>; 2514def : Pat<(v4f64 (X86cmpp (v4f64 VR256:$src1), VR256:$src2, imm:$cc)), 2515 (VCMPPDYrri VR256:$src1, VR256:$src2, imm:$cc)>; 2516def : Pat<(v4f64 (X86cmpp (v4f64 VR256:$src1), (loadv4f64 addr:$src2), imm:$cc)), 2517 (VCMPPDYrmi VR256:$src1, addr:$src2, imm:$cc)>; 2518} 2519 2520let Predicates = [UseSSE1] in { 2521def : Pat<(v4f32 (X86cmpp (v4f32 VR128:$src1), VR128:$src2, imm:$cc)), 2522 (CMPPSrri (v4f32 VR128:$src1), (v4f32 VR128:$src2), imm:$cc)>; 2523def : Pat<(v4f32 (X86cmpp (v4f32 VR128:$src1), (memopv4f32 addr:$src2), imm:$cc)), 2524 (CMPPSrmi (v4f32 VR128:$src1), addr:$src2, imm:$cc)>; 2525} 2526 2527let Predicates = [UseSSE2] in { 2528def : Pat<(v2f64 (X86cmpp (v2f64 VR128:$src1), VR128:$src2, imm:$cc)), 2529 (CMPPDrri VR128:$src1, VR128:$src2, imm:$cc)>; 2530def : Pat<(v2f64 (X86cmpp (v2f64 VR128:$src1), (memopv2f64 addr:$src2), imm:$cc)), 2531 (CMPPDrmi VR128:$src1, addr:$src2, imm:$cc)>; 2532} 2533 2534//===----------------------------------------------------------------------===// 2535// SSE 1 & 2 - Shuffle Instructions 2536//===----------------------------------------------------------------------===// 2537 2538/// sse12_shuffle - sse 1 & 2 fp shuffle instructions 2539multiclass sse12_shuffle<RegisterClass RC, X86MemOperand x86memop, 2540 ValueType vt, string asm, PatFrag mem_frag, 2541 Domain d> { 2542 def rmi : PIi8<0xC6, MRMSrcMem, (outs RC:$dst), 2543 (ins RC:$src1, x86memop:$src2, u8imm:$src3), asm, 2544 [(set RC:$dst, (vt (X86Shufp RC:$src1, (mem_frag addr:$src2), 2545 (i8 imm:$src3))))], IIC_SSE_SHUFP, d>, 2546 Sched<[WriteFShuffleLd, ReadAfterLd]>; 2547 def rri : PIi8<0xC6, MRMSrcReg, (outs RC:$dst), 2548 (ins RC:$src1, RC:$src2, u8imm:$src3), asm, 2549 [(set RC:$dst, (vt (X86Shufp RC:$src1, RC:$src2, 2550 (i8 imm:$src3))))], IIC_SSE_SHUFP, d>, 2551 Sched<[WriteFShuffle]>; 2552} 2553 2554let Predicates = [HasAVX, NoVLX] in { 2555 defm VSHUFPS : sse12_shuffle<VR128, f128mem, v4f32, 2556 "shufps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 2557 loadv4f32, SSEPackedSingle>, PS, VEX_4V; 2558 defm VSHUFPSY : sse12_shuffle<VR256, f256mem, v8f32, 2559 "shufps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 2560 loadv8f32, SSEPackedSingle>, PS, VEX_4V, VEX_L; 2561 defm VSHUFPD : sse12_shuffle<VR128, f128mem, v2f64, 2562 "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 2563 loadv2f64, SSEPackedDouble>, PD, VEX_4V; 2564 defm VSHUFPDY : sse12_shuffle<VR256, f256mem, v4f64, 2565 "shufpd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 2566 loadv4f64, SSEPackedDouble>, PD, VEX_4V, VEX_L; 2567} 2568let Constraints = "$src1 = $dst" in { 2569 defm SHUFPS : sse12_shuffle<VR128, f128mem, v4f32, 2570 "shufps\t{$src3, $src2, $dst|$dst, $src2, $src3}", 2571 memopv4f32, SSEPackedSingle>, PS; 2572 defm SHUFPD : sse12_shuffle<VR128, f128mem, v2f64, 2573 "shufpd\t{$src3, $src2, $dst|$dst, $src2, $src3}", 2574 memopv2f64, SSEPackedDouble>, PD; 2575} 2576 2577let Predicates = [HasAVX, NoVLX] in { 2578 def : Pat<(v4i32 (X86Shufp VR128:$src1, 2579 (bc_v4i32 (loadv2i64 addr:$src2)), (i8 imm:$imm))), 2580 (VSHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>; 2581 def : Pat<(v4i32 (X86Shufp VR128:$src1, VR128:$src2, (i8 imm:$imm))), 2582 (VSHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>; 2583 2584 def : Pat<(v2i64 (X86Shufp VR128:$src1, 2585 (loadv2i64 addr:$src2), (i8 imm:$imm))), 2586 (VSHUFPDrmi VR128:$src1, addr:$src2, imm:$imm)>; 2587 def : Pat<(v2i64 (X86Shufp VR128:$src1, VR128:$src2, (i8 imm:$imm))), 2588 (VSHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>; 2589 2590 // 256-bit patterns 2591 def : Pat<(v8i32 (X86Shufp VR256:$src1, VR256:$src2, (i8 imm:$imm))), 2592 (VSHUFPSYrri VR256:$src1, VR256:$src2, imm:$imm)>; 2593 def : Pat<(v8i32 (X86Shufp VR256:$src1, 2594 (bc_v8i32 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 2595 (VSHUFPSYrmi VR256:$src1, addr:$src2, imm:$imm)>; 2596 2597 def : Pat<(v4i64 (X86Shufp VR256:$src1, VR256:$src2, (i8 imm:$imm))), 2598 (VSHUFPDYrri VR256:$src1, VR256:$src2, imm:$imm)>; 2599 def : Pat<(v4i64 (X86Shufp VR256:$src1, 2600 (loadv4i64 addr:$src2), (i8 imm:$imm))), 2601 (VSHUFPDYrmi VR256:$src1, addr:$src2, imm:$imm)>; 2602} 2603 2604let Predicates = [UseSSE1] in { 2605 def : Pat<(v4i32 (X86Shufp VR128:$src1, 2606 (bc_v4i32 (memopv2i64 addr:$src2)), (i8 imm:$imm))), 2607 (SHUFPSrmi VR128:$src1, addr:$src2, imm:$imm)>; 2608 def : Pat<(v4i32 (X86Shufp VR128:$src1, VR128:$src2, (i8 imm:$imm))), 2609 (SHUFPSrri VR128:$src1, VR128:$src2, imm:$imm)>; 2610} 2611 2612let Predicates = [UseSSE2] in { 2613 // Generic SHUFPD patterns 2614 def : Pat<(v2i64 (X86Shufp VR128:$src1, 2615 (memopv2i64 addr:$src2), (i8 imm:$imm))), 2616 (SHUFPDrmi VR128:$src1, addr:$src2, imm:$imm)>; 2617 def : Pat<(v2i64 (X86Shufp VR128:$src1, VR128:$src2, (i8 imm:$imm))), 2618 (SHUFPDrri VR128:$src1, VR128:$src2, imm:$imm)>; 2619} 2620 2621//===----------------------------------------------------------------------===// 2622// SSE 1 & 2 - Unpack FP Instructions 2623//===----------------------------------------------------------------------===// 2624 2625/// sse12_unpack_interleave - sse 1 & 2 fp unpack and interleave 2626multiclass sse12_unpack_interleave<bits<8> opc, SDNode OpNode, ValueType vt, 2627 PatFrag mem_frag, RegisterClass RC, 2628 X86MemOperand x86memop, string asm, 2629 Domain d> { 2630 def rr : PI<opc, MRMSrcReg, 2631 (outs RC:$dst), (ins RC:$src1, RC:$src2), 2632 asm, [(set RC:$dst, 2633 (vt (OpNode RC:$src1, RC:$src2)))], 2634 IIC_SSE_UNPCK, d>, Sched<[WriteFShuffle]>; 2635 def rm : PI<opc, MRMSrcMem, 2636 (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 2637 asm, [(set RC:$dst, 2638 (vt (OpNode RC:$src1, 2639 (mem_frag addr:$src2))))], 2640 IIC_SSE_UNPCK, d>, 2641 Sched<[WriteFShuffleLd, ReadAfterLd]>; 2642} 2643 2644let Predicates = [HasAVX, NoVLX] in { 2645defm VUNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, loadv4f32, 2646 VR128, f128mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2647 SSEPackedSingle>, PS, VEX_4V; 2648defm VUNPCKHPD: sse12_unpack_interleave<0x15, X86Unpckh, v2f64, loadv2f64, 2649 VR128, f128mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2650 SSEPackedDouble>, PD, VEX_4V; 2651defm VUNPCKLPS: sse12_unpack_interleave<0x14, X86Unpckl, v4f32, loadv4f32, 2652 VR128, f128mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2653 SSEPackedSingle>, PS, VEX_4V; 2654defm VUNPCKLPD: sse12_unpack_interleave<0x14, X86Unpckl, v2f64, loadv2f64, 2655 VR128, f128mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2656 SSEPackedDouble>, PD, VEX_4V; 2657 2658defm VUNPCKHPSY: sse12_unpack_interleave<0x15, X86Unpckh, v8f32, loadv8f32, 2659 VR256, f256mem, "unpckhps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2660 SSEPackedSingle>, PS, VEX_4V, VEX_L; 2661defm VUNPCKHPDY: sse12_unpack_interleave<0x15, X86Unpckh, v4f64, loadv4f64, 2662 VR256, f256mem, "unpckhpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2663 SSEPackedDouble>, PD, VEX_4V, VEX_L; 2664defm VUNPCKLPSY: sse12_unpack_interleave<0x14, X86Unpckl, v8f32, loadv8f32, 2665 VR256, f256mem, "unpcklps\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2666 SSEPackedSingle>, PS, VEX_4V, VEX_L; 2667defm VUNPCKLPDY: sse12_unpack_interleave<0x14, X86Unpckl, v4f64, loadv4f64, 2668 VR256, f256mem, "unpcklpd\t{$src2, $src1, $dst|$dst, $src1, $src2}", 2669 SSEPackedDouble>, PD, VEX_4V, VEX_L; 2670}// Predicates = [HasAVX, NoVLX] 2671let Constraints = "$src1 = $dst" in { 2672 defm UNPCKHPS: sse12_unpack_interleave<0x15, X86Unpckh, v4f32, memopv4f32, 2673 VR128, f128mem, "unpckhps\t{$src2, $dst|$dst, $src2}", 2674 SSEPackedSingle>, PS; 2675 defm UNPCKHPD: sse12_unpack_interleave<0x15, X86Unpckh, v2f64, memopv2f64, 2676 VR128, f128mem, "unpckhpd\t{$src2, $dst|$dst, $src2}", 2677 SSEPackedDouble>, PD; 2678 defm UNPCKLPS: sse12_unpack_interleave<0x14, X86Unpckl, v4f32, memopv4f32, 2679 VR128, f128mem, "unpcklps\t{$src2, $dst|$dst, $src2}", 2680 SSEPackedSingle>, PS; 2681 defm UNPCKLPD: sse12_unpack_interleave<0x14, X86Unpckl, v2f64, memopv2f64, 2682 VR128, f128mem, "unpcklpd\t{$src2, $dst|$dst, $src2}", 2683 SSEPackedDouble>, PD; 2684} // Constraints = "$src1 = $dst" 2685 2686let Predicates = [HasAVX1Only] in { 2687 def : Pat<(v8i32 (X86Unpckl VR256:$src1, (bc_v8i32 (loadv4i64 addr:$src2)))), 2688 (VUNPCKLPSYrm VR256:$src1, addr:$src2)>; 2689 def : Pat<(v8i32 (X86Unpckl VR256:$src1, VR256:$src2)), 2690 (VUNPCKLPSYrr VR256:$src1, VR256:$src2)>; 2691 def : Pat<(v8i32 (X86Unpckh VR256:$src1, (bc_v8i32 (loadv4i64 addr:$src2)))), 2692 (VUNPCKHPSYrm VR256:$src1, addr:$src2)>; 2693 def : Pat<(v8i32 (X86Unpckh VR256:$src1, VR256:$src2)), 2694 (VUNPCKHPSYrr VR256:$src1, VR256:$src2)>; 2695 2696 def : Pat<(v4i64 (X86Unpckl VR256:$src1, (loadv4i64 addr:$src2))), 2697 (VUNPCKLPDYrm VR256:$src1, addr:$src2)>; 2698 def : Pat<(v4i64 (X86Unpckl VR256:$src1, VR256:$src2)), 2699 (VUNPCKLPDYrr VR256:$src1, VR256:$src2)>; 2700 def : Pat<(v4i64 (X86Unpckh VR256:$src1, (loadv4i64 addr:$src2))), 2701 (VUNPCKHPDYrm VR256:$src1, addr:$src2)>; 2702 def : Pat<(v4i64 (X86Unpckh VR256:$src1, VR256:$src2)), 2703 (VUNPCKHPDYrr VR256:$src1, VR256:$src2)>; 2704} 2705 2706//===----------------------------------------------------------------------===// 2707// SSE 1 & 2 - Extract Floating-Point Sign mask 2708//===----------------------------------------------------------------------===// 2709 2710/// sse12_extr_sign_mask - sse 1 & 2 unpack and interleave 2711multiclass sse12_extr_sign_mask<RegisterClass RC, ValueType vt, 2712 string asm, Domain d> { 2713 def rr : PI<0x50, MRMSrcReg, (outs GR32orGR64:$dst), (ins RC:$src), 2714 !strconcat(asm, "\t{$src, $dst|$dst, $src}"), 2715 [(set GR32orGR64:$dst, (X86movmsk (vt RC:$src)))], IIC_SSE_MOVMSK, d>, 2716 Sched<[WriteVecLogic]>; 2717} 2718 2719let Predicates = [HasAVX] in { 2720 defm VMOVMSKPS : sse12_extr_sign_mask<VR128, v4f32, "movmskps", 2721 SSEPackedSingle>, PS, VEX; 2722 defm VMOVMSKPD : sse12_extr_sign_mask<VR128, v2f64, "movmskpd", 2723 SSEPackedDouble>, PD, VEX; 2724 defm VMOVMSKPSY : sse12_extr_sign_mask<VR256, v8f32, "movmskps", 2725 SSEPackedSingle>, PS, VEX, VEX_L; 2726 defm VMOVMSKPDY : sse12_extr_sign_mask<VR256, v4f64, "movmskpd", 2727 SSEPackedDouble>, PD, VEX, VEX_L; 2728} 2729 2730defm MOVMSKPS : sse12_extr_sign_mask<VR128, v4f32, "movmskps", 2731 SSEPackedSingle>, PS; 2732defm MOVMSKPD : sse12_extr_sign_mask<VR128, v2f64, "movmskpd", 2733 SSEPackedDouble>, PD; 2734 2735//===---------------------------------------------------------------------===// 2736// SSE2 - Packed Integer Logical Instructions 2737//===---------------------------------------------------------------------===// 2738 2739let ExeDomain = SSEPackedInt in { // SSE integer instructions 2740 2741/// PDI_binop_rm - Simple SSE2 binary operator. 2742multiclass PDI_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 2743 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 2744 X86MemOperand x86memop, OpndItins itins, 2745 bit IsCommutable, bit Is2Addr> { 2746 let isCommutable = IsCommutable in 2747 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst), 2748 (ins RC:$src1, RC:$src2), 2749 !if(Is2Addr, 2750 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 2751 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 2752 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))], itins.rr>, 2753 Sched<[itins.Sched]>; 2754 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst), 2755 (ins RC:$src1, x86memop:$src2), 2756 !if(Is2Addr, 2757 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 2758 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 2759 [(set RC:$dst, (OpVT (OpNode RC:$src1, 2760 (bitconvert (memop_frag addr:$src2)))))], 2761 itins.rm>, 2762 Sched<[itins.Sched.Folded, ReadAfterLd]>; 2763} 2764} // ExeDomain = SSEPackedInt 2765 2766multiclass PDI_binop_all<bits<8> opc, string OpcodeStr, SDNode Opcode, 2767 ValueType OpVT128, ValueType OpVT256, 2768 OpndItins itins, bit IsCommutable = 0, Predicate prd> { 2769let Predicates = [HasAVX, prd] in 2770 defm V#NAME : PDI_binop_rm<opc, !strconcat("v", OpcodeStr), Opcode, OpVT128, 2771 VR128, loadv2i64, i128mem, itins, IsCommutable, 0>, VEX_4V; 2772 2773let Constraints = "$src1 = $dst" in 2774 defm NAME : PDI_binop_rm<opc, OpcodeStr, Opcode, OpVT128, VR128, 2775 memopv2i64, i128mem, itins, IsCommutable, 1>; 2776 2777let Predicates = [HasAVX2, prd] in 2778 defm V#NAME#Y : PDI_binop_rm<opc, !strconcat("v", OpcodeStr), Opcode, 2779 OpVT256, VR256, loadv4i64, i256mem, itins, 2780 IsCommutable, 0>, VEX_4V, VEX_L; 2781} 2782 2783// These are ordered here for pattern ordering requirements with the fp versions 2784 2785defm PAND : PDI_binop_all<0xDB, "pand", and, v2i64, v4i64, 2786 SSE_VEC_BIT_ITINS_P, 1, NoVLX>; 2787defm POR : PDI_binop_all<0xEB, "por", or, v2i64, v4i64, 2788 SSE_VEC_BIT_ITINS_P, 1, NoVLX>; 2789defm PXOR : PDI_binop_all<0xEF, "pxor", xor, v2i64, v4i64, 2790 SSE_VEC_BIT_ITINS_P, 1, NoVLX>; 2791defm PANDN : PDI_binop_all<0xDF, "pandn", X86andnp, v2i64, v4i64, 2792 SSE_VEC_BIT_ITINS_P, 0, NoVLX>; 2793 2794//===----------------------------------------------------------------------===// 2795// SSE 1 & 2 - Logical Instructions 2796//===----------------------------------------------------------------------===// 2797 2798// Multiclass for scalars using the X86 logical operation aliases for FP. 2799multiclass sse12_fp_packed_scalar_logical_alias< 2800 bits<8> opc, string OpcodeStr, SDNode OpNode, OpndItins itins> { 2801 defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, 2802 FR32, f32, f128mem, loadf32_128, SSEPackedSingle, itins, 0>, 2803 PS, VEX_4V; 2804 2805 defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, 2806 FR64, f64, f128mem, loadf64_128, SSEPackedDouble, itins, 0>, 2807 PD, VEX_4V; 2808 2809 let Constraints = "$src1 = $dst" in { 2810 defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, FR32, 2811 f32, f128mem, memopfsf32_128, SSEPackedSingle, itins>, PS; 2812 2813 defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, FR64, 2814 f64, f128mem, memopfsf64_128, SSEPackedDouble, itins>, PD; 2815 } 2816} 2817 2818let isCodeGenOnly = 1 in { 2819 defm FsAND : sse12_fp_packed_scalar_logical_alias<0x54, "and", X86fand, 2820 SSE_BIT_ITINS_P>; 2821 defm FsOR : sse12_fp_packed_scalar_logical_alias<0x56, "or", X86for, 2822 SSE_BIT_ITINS_P>; 2823 defm FsXOR : sse12_fp_packed_scalar_logical_alias<0x57, "xor", X86fxor, 2824 SSE_BIT_ITINS_P>; 2825 2826 let isCommutable = 0 in 2827 defm FsANDN : sse12_fp_packed_scalar_logical_alias<0x55, "andn", X86fandn, 2828 SSE_BIT_ITINS_P>; 2829} 2830 2831// Multiclass for vectors using the X86 logical operation aliases for FP. 2832multiclass sse12_fp_packed_vector_logical_alias< 2833 bits<8> opc, string OpcodeStr, SDNode OpNode, OpndItins itins> { 2834 let Predicates = [HasAVX, NoVLX_Or_NoDQI] in { 2835 defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, 2836 VR128, v4f32, f128mem, loadv4f32, SSEPackedSingle, itins, 0>, 2837 PS, VEX_4V; 2838 2839 defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, 2840 VR128, v2f64, f128mem, loadv2f64, SSEPackedDouble, itins, 0>, 2841 PD, VEX_4V; 2842 2843 defm V#NAME#PSY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, 2844 VR256, v8f32, f256mem, loadv8f32, SSEPackedSingle, itins, 0>, 2845 PS, VEX_4V, VEX_L; 2846 2847 defm V#NAME#PDY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, 2848 VR256, v4f64, f256mem, loadv4f64, SSEPackedDouble, itins, 0>, 2849 PD, VEX_4V, VEX_L; 2850 } 2851 2852 let Constraints = "$src1 = $dst" in { 2853 defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR128, 2854 v4f32, f128mem, memopv4f32, SSEPackedSingle, itins>, 2855 PS; 2856 2857 defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR128, 2858 v2f64, f128mem, memopv2f64, SSEPackedDouble, itins>, 2859 PD; 2860 } 2861} 2862 2863let isCodeGenOnly = 1 in { 2864 defm FvAND : sse12_fp_packed_vector_logical_alias<0x54, "and", X86fand, 2865 SSE_BIT_ITINS_P>; 2866 defm FvOR : sse12_fp_packed_vector_logical_alias<0x56, "or", X86for, 2867 SSE_BIT_ITINS_P>; 2868 defm FvXOR : sse12_fp_packed_vector_logical_alias<0x57, "xor", X86fxor, 2869 SSE_BIT_ITINS_P>; 2870 2871 let isCommutable = 0 in 2872 defm FvANDN : sse12_fp_packed_vector_logical_alias<0x55, "andn", X86fandn, 2873 SSE_BIT_ITINS_P>; 2874} 2875 2876/// sse12_fp_packed_logical - SSE 1 & 2 packed FP logical ops 2877/// 2878multiclass sse12_fp_packed_logical<bits<8> opc, string OpcodeStr, 2879 SDNode OpNode> { 2880 let Predicates = [HasAVX, NoVLX] in { 2881 defm V#NAME#PSY : sse12_fp_packed_logical_rm<opc, VR256, SSEPackedSingle, 2882 !strconcat(OpcodeStr, "ps"), f256mem, 2883 [(set VR256:$dst, (v4i64 (OpNode VR256:$src1, VR256:$src2)))], 2884 [(set VR256:$dst, (OpNode (bc_v4i64 (v8f32 VR256:$src1)), 2885 (loadv4i64 addr:$src2)))], 0>, PS, VEX_4V, VEX_L; 2886 2887 defm V#NAME#PDY : sse12_fp_packed_logical_rm<opc, VR256, SSEPackedDouble, 2888 !strconcat(OpcodeStr, "pd"), f256mem, 2889 [(set VR256:$dst, (OpNode (bc_v4i64 (v4f64 VR256:$src1)), 2890 (bc_v4i64 (v4f64 VR256:$src2))))], 2891 [(set VR256:$dst, (OpNode (bc_v4i64 (v4f64 VR256:$src1)), 2892 (loadv4i64 addr:$src2)))], 0>, 2893 PD, VEX_4V, VEX_L; 2894 2895 // In AVX no need to add a pattern for 128-bit logical rr ps, because they 2896 // are all promoted to v2i64, and the patterns are covered by the int 2897 // version. This is needed in SSE only, because v2i64 isn't supported on 2898 // SSE1, but only on SSE2. 2899 defm V#NAME#PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle, 2900 !strconcat(OpcodeStr, "ps"), f128mem, [], 2901 [(set VR128:$dst, (OpNode (bc_v2i64 (v4f32 VR128:$src1)), 2902 (loadv2i64 addr:$src2)))], 0>, PS, VEX_4V; 2903 2904 defm V#NAME#PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble, 2905 !strconcat(OpcodeStr, "pd"), f128mem, 2906 [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), 2907 (bc_v2i64 (v2f64 VR128:$src2))))], 2908 [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), 2909 (loadv2i64 addr:$src2)))], 0>, 2910 PD, VEX_4V; 2911 } 2912 2913 let Constraints = "$src1 = $dst" in { 2914 defm PS : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedSingle, 2915 !strconcat(OpcodeStr, "ps"), f128mem, 2916 [(set VR128:$dst, (v2i64 (OpNode VR128:$src1, VR128:$src2)))], 2917 [(set VR128:$dst, (OpNode (bc_v2i64 (v4f32 VR128:$src1)), 2918 (memopv2i64 addr:$src2)))]>, PS; 2919 2920 defm PD : sse12_fp_packed_logical_rm<opc, VR128, SSEPackedDouble, 2921 !strconcat(OpcodeStr, "pd"), f128mem, 2922 [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), 2923 (bc_v2i64 (v2f64 VR128:$src2))))], 2924 [(set VR128:$dst, (OpNode (bc_v2i64 (v2f64 VR128:$src1)), 2925 (memopv2i64 addr:$src2)))]>, PD; 2926 } 2927} 2928 2929defm AND : sse12_fp_packed_logical<0x54, "and", and>; 2930defm OR : sse12_fp_packed_logical<0x56, "or", or>; 2931defm XOR : sse12_fp_packed_logical<0x57, "xor", xor>; 2932let isCommutable = 0 in 2933 defm ANDN : sse12_fp_packed_logical<0x55, "andn", X86andnp>; 2934 2935// AVX1 requires type coercions in order to fold loads directly into logical 2936// operations. 2937let Predicates = [HasAVX1Only] in { 2938 def : Pat<(bc_v8f32 (and VR256:$src1, (loadv4i64 addr:$src2))), 2939 (VANDPSYrm VR256:$src1, addr:$src2)>; 2940 def : Pat<(bc_v8f32 (or VR256:$src1, (loadv4i64 addr:$src2))), 2941 (VORPSYrm VR256:$src1, addr:$src2)>; 2942 def : Pat<(bc_v8f32 (xor VR256:$src1, (loadv4i64 addr:$src2))), 2943 (VXORPSYrm VR256:$src1, addr:$src2)>; 2944 def : Pat<(bc_v8f32 (X86andnp VR256:$src1, (loadv4i64 addr:$src2))), 2945 (VANDNPSYrm VR256:$src1, addr:$src2)>; 2946} 2947 2948//===----------------------------------------------------------------------===// 2949// SSE 1 & 2 - Arithmetic Instructions 2950//===----------------------------------------------------------------------===// 2951 2952/// basic_sse12_fp_binop_xxx - SSE 1 & 2 binops come in both scalar and 2953/// vector forms. 2954/// 2955/// In addition, we also have a special variant of the scalar form here to 2956/// represent the associated intrinsic operation. This form is unlike the 2957/// plain scalar form, in that it takes an entire vector (instead of a scalar) 2958/// and leaves the top elements unmodified (therefore these cannot be commuted). 2959/// 2960/// These three forms can each be reg+reg or reg+mem. 2961/// 2962 2963/// FIXME: once all 256-bit intrinsics are matched, cleanup and refactor those 2964/// classes below 2965multiclass basic_sse12_fp_binop_p<bits<8> opc, string OpcodeStr, 2966 SDNode OpNode, SizeItins itins> { 2967 let Predicates = [HasAVX, NoVLX] in { 2968 defm V#NAME#PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, 2969 VR128, v4f32, f128mem, loadv4f32, 2970 SSEPackedSingle, itins.s, 0>, PS, VEX_4V; 2971 defm V#NAME#PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, 2972 VR128, v2f64, f128mem, loadv2f64, 2973 SSEPackedDouble, itins.d, 0>, PD, VEX_4V; 2974 2975 defm V#NAME#PSY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), 2976 OpNode, VR256, v8f32, f256mem, loadv8f32, 2977 SSEPackedSingle, itins.s, 0>, PS, VEX_4V, VEX_L; 2978 defm V#NAME#PDY : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), 2979 OpNode, VR256, v4f64, f256mem, loadv4f64, 2980 SSEPackedDouble, itins.d, 0>, PD, VEX_4V, VEX_L; 2981 } 2982 2983 let Constraints = "$src1 = $dst" in { 2984 defm PS : sse12_fp_packed<opc, !strconcat(OpcodeStr, "ps"), OpNode, VR128, 2985 v4f32, f128mem, memopv4f32, SSEPackedSingle, 2986 itins.s>, PS; 2987 defm PD : sse12_fp_packed<opc, !strconcat(OpcodeStr, "pd"), OpNode, VR128, 2988 v2f64, f128mem, memopv2f64, SSEPackedDouble, 2989 itins.d>, PD; 2990 } 2991} 2992 2993multiclass basic_sse12_fp_binop_s<bits<8> opc, string OpcodeStr, SDNode OpNode, 2994 SizeItins itins> { 2995 defm V#NAME#SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"), 2996 OpNode, FR32, f32mem, SSEPackedSingle, itins.s, 0>, 2997 XS, VEX_4V, VEX_LIG; 2998 defm V#NAME#SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"), 2999 OpNode, FR64, f64mem, SSEPackedDouble, itins.d, 0>, 3000 XD, VEX_4V, VEX_LIG; 3001 3002 let Constraints = "$src1 = $dst" in { 3003 defm SS : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "ss"), 3004 OpNode, FR32, f32mem, SSEPackedSingle, 3005 itins.s>, XS; 3006 defm SD : sse12_fp_scalar<opc, !strconcat(OpcodeStr, "sd"), 3007 OpNode, FR64, f64mem, SSEPackedDouble, 3008 itins.d>, XD; 3009 } 3010} 3011 3012multiclass basic_sse12_fp_binop_s_int<bits<8> opc, string OpcodeStr, 3013 SizeItins itins> { 3014 defm V#NAME#SS : sse12_fp_scalar_int<opc, OpcodeStr, VR128, 3015 !strconcat(OpcodeStr, "ss"), "", "_ss", ssmem, sse_load_f32, 3016 SSEPackedSingle, itins.s, 0>, XS, VEX_4V, VEX_LIG; 3017 defm V#NAME#SD : sse12_fp_scalar_int<opc, OpcodeStr, VR128, 3018 !strconcat(OpcodeStr, "sd"), "2", "_sd", sdmem, sse_load_f64, 3019 SSEPackedDouble, itins.d, 0>, XD, VEX_4V, VEX_LIG; 3020 3021 let Constraints = "$src1 = $dst" in { 3022 defm SS : sse12_fp_scalar_int<opc, OpcodeStr, VR128, 3023 !strconcat(OpcodeStr, "ss"), "", "_ss", ssmem, sse_load_f32, 3024 SSEPackedSingle, itins.s>, XS; 3025 defm SD : sse12_fp_scalar_int<opc, OpcodeStr, VR128, 3026 !strconcat(OpcodeStr, "sd"), "2", "_sd", sdmem, sse_load_f64, 3027 SSEPackedDouble, itins.d>, XD; 3028 } 3029} 3030 3031// Binary Arithmetic instructions 3032defm ADD : basic_sse12_fp_binop_p<0x58, "add", fadd, SSE_ALU_ITINS_P>, 3033 basic_sse12_fp_binop_s<0x58, "add", fadd, SSE_ALU_ITINS_S>, 3034 basic_sse12_fp_binop_s_int<0x58, "add", SSE_ALU_ITINS_S>; 3035defm MUL : basic_sse12_fp_binop_p<0x59, "mul", fmul, SSE_MUL_ITINS_P>, 3036 basic_sse12_fp_binop_s<0x59, "mul", fmul, SSE_MUL_ITINS_S>, 3037 basic_sse12_fp_binop_s_int<0x59, "mul", SSE_MUL_ITINS_S>; 3038let isCommutable = 0 in { 3039 defm SUB : basic_sse12_fp_binop_p<0x5C, "sub", fsub, SSE_ALU_ITINS_P>, 3040 basic_sse12_fp_binop_s<0x5C, "sub", fsub, SSE_ALU_ITINS_S>, 3041 basic_sse12_fp_binop_s_int<0x5C, "sub", SSE_ALU_ITINS_S>; 3042 defm DIV : basic_sse12_fp_binop_p<0x5E, "div", fdiv, SSE_DIV_ITINS_P>, 3043 basic_sse12_fp_binop_s<0x5E, "div", fdiv, SSE_DIV_ITINS_S>, 3044 basic_sse12_fp_binop_s_int<0x5E, "div", SSE_DIV_ITINS_S>; 3045 defm MAX : basic_sse12_fp_binop_p<0x5F, "max", X86fmax, SSE_ALU_ITINS_P>, 3046 basic_sse12_fp_binop_s<0x5F, "max", X86fmax, SSE_ALU_ITINS_S>, 3047 basic_sse12_fp_binop_s_int<0x5F, "max", SSE_ALU_ITINS_S>; 3048 defm MIN : basic_sse12_fp_binop_p<0x5D, "min", X86fmin, SSE_ALU_ITINS_P>, 3049 basic_sse12_fp_binop_s<0x5D, "min", X86fmin, SSE_ALU_ITINS_S>, 3050 basic_sse12_fp_binop_s_int<0x5D, "min", SSE_ALU_ITINS_S>; 3051} 3052 3053let isCodeGenOnly = 1 in { 3054 defm MAXC: basic_sse12_fp_binop_p<0x5F, "max", X86fmaxc, SSE_ALU_ITINS_P>, 3055 basic_sse12_fp_binop_s<0x5F, "max", X86fmaxc, SSE_ALU_ITINS_S>; 3056 defm MINC: basic_sse12_fp_binop_p<0x5D, "min", X86fminc, SSE_ALU_ITINS_P>, 3057 basic_sse12_fp_binop_s<0x5D, "min", X86fminc, SSE_ALU_ITINS_S>; 3058} 3059 3060// Patterns used to select SSE scalar fp arithmetic instructions from 3061// either: 3062// 3063// (1) a scalar fp operation followed by a blend 3064// 3065// The effect is that the backend no longer emits unnecessary vector 3066// insert instructions immediately after SSE scalar fp instructions 3067// like addss or mulss. 3068// 3069// For example, given the following code: 3070// __m128 foo(__m128 A, __m128 B) { 3071// A[0] += B[0]; 3072// return A; 3073// } 3074// 3075// Previously we generated: 3076// addss %xmm0, %xmm1 3077// movss %xmm1, %xmm0 3078// 3079// We now generate: 3080// addss %xmm1, %xmm0 3081// 3082// (2) a vector packed single/double fp operation followed by a vector insert 3083// 3084// The effect is that the backend converts the packed fp instruction 3085// followed by a vector insert into a single SSE scalar fp instruction. 3086// 3087// For example, given the following code: 3088// __m128 foo(__m128 A, __m128 B) { 3089// __m128 C = A + B; 3090// return (__m128) {c[0], a[1], a[2], a[3]}; 3091// } 3092// 3093// Previously we generated: 3094// addps %xmm0, %xmm1 3095// movss %xmm1, %xmm0 3096// 3097// We now generate: 3098// addss %xmm1, %xmm0 3099 3100// TODO: Some canonicalization in lowering would simplify the number of 3101// patterns we have to try to match. 3102multiclass scalar_math_f32_patterns<SDNode Op, string OpcPrefix> { 3103 let Predicates = [UseSSE1] in { 3104 // extracted scalar math op with insert via movss 3105 def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), (v4f32 (scalar_to_vector 3106 (Op (f32 (extractelt (v4f32 VR128:$dst), (iPTR 0))), 3107 FR32:$src))))), 3108 (!cast<I>(OpcPrefix#SSrr_Int) v4f32:$dst, 3109 (COPY_TO_REGCLASS FR32:$src, VR128))>; 3110 3111 // vector math op with insert via movss 3112 def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), 3113 (Op (v4f32 VR128:$dst), (v4f32 VR128:$src)))), 3114 (!cast<I>(OpcPrefix#SSrr_Int) v4f32:$dst, v4f32:$src)>; 3115 } 3116 3117 // With SSE 4.1, blendi is preferred to movsd, so match that too. 3118 let Predicates = [UseSSE41] in { 3119 // extracted scalar math op with insert via blend 3120 def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), (v4f32 (scalar_to_vector 3121 (Op (f32 (extractelt (v4f32 VR128:$dst), (iPTR 0))), 3122 FR32:$src))), (i8 1))), 3123 (!cast<I>(OpcPrefix#SSrr_Int) v4f32:$dst, 3124 (COPY_TO_REGCLASS FR32:$src, VR128))>; 3125 3126 // vector math op with insert via blend 3127 def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), 3128 (Op (v4f32 VR128:$dst), (v4f32 VR128:$src)), (i8 1))), 3129 (!cast<I>(OpcPrefix#SSrr_Int)v4f32:$dst, v4f32:$src)>; 3130 3131 } 3132 3133 // Repeat everything for AVX, except for the movss + scalar combo... 3134 // because that one shouldn't occur with AVX codegen? 3135 let Predicates = [HasAVX] in { 3136 // extracted scalar math op with insert via blend 3137 def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), (v4f32 (scalar_to_vector 3138 (Op (f32 (extractelt (v4f32 VR128:$dst), (iPTR 0))), 3139 FR32:$src))), (i8 1))), 3140 (!cast<I>("V"#OpcPrefix#SSrr_Int) v4f32:$dst, 3141 (COPY_TO_REGCLASS FR32:$src, VR128))>; 3142 3143 // vector math op with insert via movss 3144 def : Pat<(v4f32 (X86Movss (v4f32 VR128:$dst), 3145 (Op (v4f32 VR128:$dst), (v4f32 VR128:$src)))), 3146 (!cast<I>("V"#OpcPrefix#SSrr_Int) v4f32:$dst, v4f32:$src)>; 3147 3148 // vector math op with insert via blend 3149 def : Pat<(v4f32 (X86Blendi (v4f32 VR128:$dst), 3150 (Op (v4f32 VR128:$dst), (v4f32 VR128:$src)), (i8 1))), 3151 (!cast<I>("V"#OpcPrefix#SSrr_Int) v4f32:$dst, v4f32:$src)>; 3152 } 3153} 3154 3155defm : scalar_math_f32_patterns<fadd, "ADD">; 3156defm : scalar_math_f32_patterns<fsub, "SUB">; 3157defm : scalar_math_f32_patterns<fmul, "MUL">; 3158defm : scalar_math_f32_patterns<fdiv, "DIV">; 3159 3160multiclass scalar_math_f64_patterns<SDNode Op, string OpcPrefix> { 3161 let Predicates = [UseSSE2] in { 3162 // extracted scalar math op with insert via movsd 3163 def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector 3164 (Op (f64 (extractelt (v2f64 VR128:$dst), (iPTR 0))), 3165 FR64:$src))))), 3166 (!cast<I>(OpcPrefix#SDrr_Int) v2f64:$dst, 3167 (COPY_TO_REGCLASS FR64:$src, VR128))>; 3168 3169 // vector math op with insert via movsd 3170 def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), 3171 (Op (v2f64 VR128:$dst), (v2f64 VR128:$src)))), 3172 (!cast<I>(OpcPrefix#SDrr_Int) v2f64:$dst, v2f64:$src)>; 3173 } 3174 3175 // With SSE 4.1, blendi is preferred to movsd, so match those too. 3176 let Predicates = [UseSSE41] in { 3177 // extracted scalar math op with insert via blend 3178 def : Pat<(v2f64 (X86Blendi (v2f64 VR128:$dst), (v2f64 (scalar_to_vector 3179 (Op (f64 (extractelt (v2f64 VR128:$dst), (iPTR 0))), 3180 FR64:$src))), (i8 1))), 3181 (!cast<I>(OpcPrefix#SDrr_Int) v2f64:$dst, 3182 (COPY_TO_REGCLASS FR64:$src, VR128))>; 3183 3184 // vector math op with insert via blend 3185 def : Pat<(v2f64 (X86Blendi (v2f64 VR128:$dst), 3186 (Op (v2f64 VR128:$dst), (v2f64 VR128:$src)), (i8 1))), 3187 (!cast<I>(OpcPrefix#SDrr_Int) v2f64:$dst, v2f64:$src)>; 3188 } 3189 3190 // Repeat everything for AVX. 3191 let Predicates = [HasAVX] in { 3192 // extracted scalar math op with insert via movsd 3193 def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), (v2f64 (scalar_to_vector 3194 (Op (f64 (extractelt (v2f64 VR128:$dst), (iPTR 0))), 3195 FR64:$src))))), 3196 (!cast<I>("V"#OpcPrefix#SDrr_Int) v2f64:$dst, 3197 (COPY_TO_REGCLASS FR64:$src, VR128))>; 3198 3199 // extracted scalar math op with insert via blend 3200 def : Pat<(v2f64 (X86Blendi (v2f64 VR128:$dst), (v2f64 (scalar_to_vector 3201 (Op (f64 (extractelt (v2f64 VR128:$dst), (iPTR 0))), 3202 FR64:$src))), (i8 1))), 3203 (!cast<I>("V"#OpcPrefix#SDrr_Int) v2f64:$dst, 3204 (COPY_TO_REGCLASS FR64:$src, VR128))>; 3205 3206 // vector math op with insert via movsd 3207 def : Pat<(v2f64 (X86Movsd (v2f64 VR128:$dst), 3208 (Op (v2f64 VR128:$dst), (v2f64 VR128:$src)))), 3209 (!cast<I>("V"#OpcPrefix#SDrr_Int) v2f64:$dst, v2f64:$src)>; 3210 3211 // vector math op with insert via blend 3212 def : Pat<(v2f64 (X86Blendi (v2f64 VR128:$dst), 3213 (Op (v2f64 VR128:$dst), (v2f64 VR128:$src)), (i8 1))), 3214 (!cast<I>("V"#OpcPrefix#SDrr_Int) v2f64:$dst, v2f64:$src)>; 3215 } 3216} 3217 3218defm : scalar_math_f64_patterns<fadd, "ADD">; 3219defm : scalar_math_f64_patterns<fsub, "SUB">; 3220defm : scalar_math_f64_patterns<fmul, "MUL">; 3221defm : scalar_math_f64_patterns<fdiv, "DIV">; 3222 3223 3224/// Unop Arithmetic 3225/// In addition, we also have a special variant of the scalar form here to 3226/// represent the associated intrinsic operation. This form is unlike the 3227/// plain scalar form, in that it takes an entire vector (instead of a 3228/// scalar) and leaves the top elements undefined. 3229/// 3230/// And, we have a special variant form for a full-vector intrinsic form. 3231 3232let Sched = WriteFSqrt in { 3233def SSE_SQRTPS : OpndItins< 3234 IIC_SSE_SQRTPS_RR, IIC_SSE_SQRTPS_RM 3235>; 3236 3237def SSE_SQRTSS : OpndItins< 3238 IIC_SSE_SQRTSS_RR, IIC_SSE_SQRTSS_RM 3239>; 3240 3241def SSE_SQRTPD : OpndItins< 3242 IIC_SSE_SQRTPD_RR, IIC_SSE_SQRTPD_RM 3243>; 3244 3245def SSE_SQRTSD : OpndItins< 3246 IIC_SSE_SQRTSD_RR, IIC_SSE_SQRTSD_RM 3247>; 3248} 3249 3250let Sched = WriteFRsqrt in { 3251def SSE_RSQRTPS : OpndItins< 3252 IIC_SSE_RSQRTPS_RR, IIC_SSE_RSQRTPS_RM 3253>; 3254 3255def SSE_RSQRTSS : OpndItins< 3256 IIC_SSE_RSQRTSS_RR, IIC_SSE_RSQRTSS_RM 3257>; 3258} 3259 3260let Sched = WriteFRcp in { 3261def SSE_RCPP : OpndItins< 3262 IIC_SSE_RCPP_RR, IIC_SSE_RCPP_RM 3263>; 3264 3265def SSE_RCPS : OpndItins< 3266 IIC_SSE_RCPS_RR, IIC_SSE_RCPS_RM 3267>; 3268} 3269 3270/// sse_fp_unop_s - SSE1 unops in scalar form 3271/// For the non-AVX defs, we need $src1 to be tied to $dst because 3272/// the HW instructions are 2 operand / destructive. 3273multiclass sse_fp_unop_s<bits<8> opc, string OpcodeStr, RegisterClass RC, 3274 ValueType vt, ValueType ScalarVT, 3275 X86MemOperand x86memop, Operand vec_memop, 3276 ComplexPattern mem_cpat, Intrinsic Intr, 3277 SDNode OpNode, Domain d, OpndItins itins, 3278 Predicate target, string Suffix> { 3279 let hasSideEffects = 0 in { 3280 def r : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1), 3281 !strconcat(OpcodeStr, "\t{$src1, $dst|$dst, $src1}"), 3282 [(set RC:$dst, (OpNode RC:$src1))], itins.rr, d>, Sched<[itins.Sched]>, 3283 Requires<[target]>; 3284 let mayLoad = 1 in 3285 def m : I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src1), 3286 !strconcat(OpcodeStr, "\t{$src1, $dst|$dst, $src1}"), 3287 [(set RC:$dst, (OpNode (load addr:$src1)))], itins.rm, d>, 3288 Sched<[itins.Sched.Folded, ReadAfterLd]>, 3289 Requires<[target, OptForSize]>; 3290 3291 let isCodeGenOnly = 1, Constraints = "$src1 = $dst" in { 3292 def r_Int : I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 3293 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3294 []>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3295 let mayLoad = 1 in 3296 def m_Int : I<opc, MRMSrcMem, (outs VR128:$dst), (ins VR128:$src1, vec_memop:$src2), 3297 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3298 []>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3299 } 3300 } 3301 3302 let Predicates = [target] in { 3303 def : Pat<(vt (OpNode mem_cpat:$src)), 3304 (vt (COPY_TO_REGCLASS (vt (!cast<Instruction>(NAME#Suffix##m_Int) 3305 (vt (IMPLICIT_DEF)), mem_cpat:$src)), RC))>; 3306 // These are unary operations, but they are modeled as having 2 source operands 3307 // because the high elements of the destination are unchanged in SSE. 3308 def : Pat<(Intr VR128:$src), 3309 (!cast<Instruction>(NAME#Suffix##r_Int) VR128:$src, VR128:$src)>; 3310 def : Pat<(Intr (load addr:$src)), 3311 (vt (COPY_TO_REGCLASS(!cast<Instruction>(NAME#Suffix##m) 3312 addr:$src), VR128))>; 3313 } 3314 // We don't want to fold scalar loads into these instructions unless 3315 // optimizing for size. This is because the folded instruction will have a 3316 // partial register update, while the unfolded sequence will not, e.g. 3317 // movss mem, %xmm0 3318 // rcpss %xmm0, %xmm0 3319 // which has a clobber before the rcp, vs. 3320 // rcpss mem, %xmm0 3321 let Predicates = [target, OptForSize] in { 3322 def : Pat<(Intr mem_cpat:$src), 3323 (!cast<Instruction>(NAME#Suffix##m_Int) 3324 (vt (IMPLICIT_DEF)), mem_cpat:$src)>; 3325 } 3326} 3327 3328multiclass avx_fp_unop_s<bits<8> opc, string OpcodeStr, RegisterClass RC, 3329 ValueType vt, ValueType ScalarVT, 3330 X86MemOperand x86memop, Operand vec_memop, 3331 ComplexPattern mem_cpat, 3332 Intrinsic Intr, SDNode OpNode, Domain d, 3333 OpndItins itins, string Suffix> { 3334 let hasSideEffects = 0 in { 3335 def r : I<opc, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 3336 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3337 [], itins.rr, d>, Sched<[itins.Sched]>; 3338 let mayLoad = 1 in 3339 def m : I<opc, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 3340 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3341 [], itins.rm, d>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3342 let isCodeGenOnly = 1 in { 3343 def r_Int : I<opc, MRMSrcReg, (outs VR128:$dst), 3344 (ins VR128:$src1, VR128:$src2), 3345 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3346 []>, Sched<[itins.Sched.Folded]>; 3347 let mayLoad = 1 in 3348 def m_Int : I<opc, MRMSrcMem, (outs VR128:$dst), 3349 (ins VR128:$src1, vec_memop:$src2), 3350 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 3351 []>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3352 } 3353 } 3354 3355 // We don't want to fold scalar loads into these instructions unless 3356 // optimizing for size. This is because the folded instruction will have a 3357 // partial register update, while the unfolded sequence will not, e.g. 3358 // vmovss mem, %xmm0 3359 // vrcpss %xmm0, %xmm0, %xmm0 3360 // which has a clobber before the rcp, vs. 3361 // vrcpss mem, %xmm0, %xmm0 3362 // TODO: In theory, we could fold the load, and avoid the stall caused by 3363 // the partial register store, either in ExeDepFix or with smarter RA. 3364 let Predicates = [UseAVX] in { 3365 def : Pat<(OpNode RC:$src), (!cast<Instruction>("V"#NAME#Suffix##r) 3366 (ScalarVT (IMPLICIT_DEF)), RC:$src)>; 3367 } 3368 let Predicates = [HasAVX] in { 3369 def : Pat<(Intr VR128:$src), 3370 (!cast<Instruction>("V"#NAME#Suffix##r_Int) (vt (IMPLICIT_DEF)), 3371 VR128:$src)>; 3372 } 3373 let Predicates = [HasAVX, OptForSize] in { 3374 def : Pat<(Intr mem_cpat:$src), 3375 (!cast<Instruction>("V"#NAME#Suffix##m_Int) 3376 (vt (IMPLICIT_DEF)), mem_cpat:$src)>; 3377 } 3378 let Predicates = [UseAVX, OptForSize] in { 3379 def : Pat<(ScalarVT (OpNode (load addr:$src))), 3380 (!cast<Instruction>("V"#NAME#Suffix##m) (ScalarVT (IMPLICIT_DEF)), 3381 addr:$src)>; 3382 def : Pat<(vt (OpNode mem_cpat:$src)), 3383 (!cast<Instruction>("V"#NAME#Suffix##m_Int) (vt (IMPLICIT_DEF)), 3384 mem_cpat:$src)>; 3385 } 3386} 3387 3388/// sse1_fp_unop_p - SSE1 unops in packed form. 3389multiclass sse1_fp_unop_p<bits<8> opc, string OpcodeStr, SDNode OpNode, 3390 OpndItins itins, list<Predicate> prds> { 3391let Predicates = prds in { 3392 def V#NAME#PSr : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3393 !strconcat("v", OpcodeStr, 3394 "ps\t{$src, $dst|$dst, $src}"), 3395 [(set VR128:$dst, (v4f32 (OpNode VR128:$src)))], 3396 itins.rr>, VEX, Sched<[itins.Sched]>; 3397 def V#NAME#PSm : PSI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 3398 !strconcat("v", OpcodeStr, 3399 "ps\t{$src, $dst|$dst, $src}"), 3400 [(set VR128:$dst, (OpNode (loadv4f32 addr:$src)))], 3401 itins.rm>, VEX, Sched<[itins.Sched.Folded]>; 3402 def V#NAME#PSYr : PSI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 3403 !strconcat("v", OpcodeStr, 3404 "ps\t{$src, $dst|$dst, $src}"), 3405 [(set VR256:$dst, (v8f32 (OpNode VR256:$src)))], 3406 itins.rr>, VEX, VEX_L, Sched<[itins.Sched]>; 3407 def V#NAME#PSYm : PSI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 3408 !strconcat("v", OpcodeStr, 3409 "ps\t{$src, $dst|$dst, $src}"), 3410 [(set VR256:$dst, (OpNode (loadv8f32 addr:$src)))], 3411 itins.rm>, VEX, VEX_L, Sched<[itins.Sched.Folded]>; 3412} 3413 3414 def PSr : PSI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3415 !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"), 3416 [(set VR128:$dst, (v4f32 (OpNode VR128:$src)))], itins.rr>, 3417 Sched<[itins.Sched]>; 3418 def PSm : PSI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 3419 !strconcat(OpcodeStr, "ps\t{$src, $dst|$dst, $src}"), 3420 [(set VR128:$dst, (OpNode (memopv4f32 addr:$src)))], itins.rm>, 3421 Sched<[itins.Sched.Folded]>; 3422} 3423 3424/// sse2_fp_unop_p - SSE2 unops in vector forms. 3425multiclass sse2_fp_unop_p<bits<8> opc, string OpcodeStr, 3426 SDNode OpNode, OpndItins itins> { 3427let Predicates = [HasAVX] in { 3428 def V#NAME#PDr : PDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3429 !strconcat("v", OpcodeStr, 3430 "pd\t{$src, $dst|$dst, $src}"), 3431 [(set VR128:$dst, (v2f64 (OpNode VR128:$src)))], 3432 itins.rr>, VEX, Sched<[itins.Sched]>; 3433 def V#NAME#PDm : PDI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 3434 !strconcat("v", OpcodeStr, 3435 "pd\t{$src, $dst|$dst, $src}"), 3436 [(set VR128:$dst, (OpNode (loadv2f64 addr:$src)))], 3437 itins.rm>, VEX, Sched<[itins.Sched.Folded]>; 3438 def V#NAME#PDYr : PDI<opc, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 3439 !strconcat("v", OpcodeStr, 3440 "pd\t{$src, $dst|$dst, $src}"), 3441 [(set VR256:$dst, (v4f64 (OpNode VR256:$src)))], 3442 itins.rr>, VEX, VEX_L, Sched<[itins.Sched]>; 3443 def V#NAME#PDYm : PDI<opc, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 3444 !strconcat("v", OpcodeStr, 3445 "pd\t{$src, $dst|$dst, $src}"), 3446 [(set VR256:$dst, (OpNode (loadv4f64 addr:$src)))], 3447 itins.rm>, VEX, VEX_L, Sched<[itins.Sched.Folded]>; 3448} 3449 3450 def PDr : PDI<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3451 !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"), 3452 [(set VR128:$dst, (v2f64 (OpNode VR128:$src)))], itins.rr>, 3453 Sched<[itins.Sched]>; 3454 def PDm : PDI<opc, MRMSrcMem, (outs VR128:$dst), (ins f128mem:$src), 3455 !strconcat(OpcodeStr, "pd\t{$src, $dst|$dst, $src}"), 3456 [(set VR128:$dst, (OpNode (memopv2f64 addr:$src)))], itins.rm>, 3457 Sched<[itins.Sched.Folded]>; 3458} 3459 3460multiclass sse1_fp_unop_s<bits<8> opc, string OpcodeStr, SDNode OpNode, 3461 OpndItins itins> { 3462 defm SS : sse_fp_unop_s<opc, OpcodeStr##ss, FR32, v4f32, f32, f32mem, 3463 ssmem, sse_load_f32, 3464 !cast<Intrinsic>("int_x86_sse_"##OpcodeStr##_ss), OpNode, 3465 SSEPackedSingle, itins, UseSSE1, "SS">, XS; 3466 defm V#NAME#SS : avx_fp_unop_s<opc, "v"#OpcodeStr##ss, FR32, v4f32, f32, 3467 f32mem, ssmem, sse_load_f32, 3468 !cast<Intrinsic>("int_x86_sse_"##OpcodeStr##_ss), OpNode, 3469 SSEPackedSingle, itins, "SS">, XS, VEX_4V, VEX_LIG; 3470} 3471 3472multiclass sse2_fp_unop_s<bits<8> opc, string OpcodeStr, SDNode OpNode, 3473 OpndItins itins> { 3474 defm SD : sse_fp_unop_s<opc, OpcodeStr##sd, FR64, v2f64, f64, f64mem, 3475 sdmem, sse_load_f64, 3476 !cast<Intrinsic>("int_x86_sse2_"##OpcodeStr##_sd), 3477 OpNode, SSEPackedDouble, itins, UseSSE2, "SD">, XD; 3478 defm V#NAME#SD : avx_fp_unop_s<opc, "v"#OpcodeStr##sd, FR64, v2f64, f64, 3479 f64mem, sdmem, sse_load_f64, 3480 !cast<Intrinsic>("int_x86_sse2_"##OpcodeStr##_sd), 3481 OpNode, SSEPackedDouble, itins, "SD">, 3482 XD, VEX_4V, VEX_LIG; 3483} 3484 3485// Square root. 3486defm SQRT : sse1_fp_unop_s<0x51, "sqrt", fsqrt, SSE_SQRTSS>, 3487 sse1_fp_unop_p<0x51, "sqrt", fsqrt, SSE_SQRTPS, [HasAVX]>, 3488 sse2_fp_unop_s<0x51, "sqrt", fsqrt, SSE_SQRTSD>, 3489 sse2_fp_unop_p<0x51, "sqrt", fsqrt, SSE_SQRTPD>; 3490 3491// Reciprocal approximations. Note that these typically require refinement 3492// in order to obtain suitable precision. 3493defm RSQRT : sse1_fp_unop_s<0x52, "rsqrt", X86frsqrt, SSE_RSQRTSS>, 3494 sse1_fp_unop_p<0x52, "rsqrt", X86frsqrt, SSE_RSQRTPS, [HasAVX, NoVLX] >; 3495defm RCP : sse1_fp_unop_s<0x53, "rcp", X86frcp, SSE_RCPS>, 3496 sse1_fp_unop_p<0x53, "rcp", X86frcp, SSE_RCPP, [HasAVX, NoVLX]>; 3497 3498// There is no f64 version of the reciprocal approximation instructions. 3499 3500// TODO: We should add *scalar* op patterns for these just like we have for 3501// the binops above. If the binop and unop patterns could all be unified 3502// that would be even better. 3503 3504multiclass scalar_unary_math_patterns<Intrinsic Intr, string OpcPrefix, 3505 SDNode Move, ValueType VT, 3506 Predicate BasePredicate> { 3507 let Predicates = [BasePredicate] in { 3508 def : Pat<(VT (Move VT:$dst, (Intr VT:$src))), 3509 (!cast<I>(OpcPrefix#r_Int) VT:$dst, VT:$src)>; 3510 } 3511 3512 // With SSE 4.1, blendi is preferred to movs*, so match that too. 3513 let Predicates = [UseSSE41] in { 3514 def : Pat<(VT (X86Blendi VT:$dst, (Intr VT:$src), (i8 1))), 3515 (!cast<I>(OpcPrefix#r_Int) VT:$dst, VT:$src)>; 3516 } 3517 3518 // Repeat for AVX versions of the instructions. 3519 let Predicates = [HasAVX] in { 3520 def : Pat<(VT (Move VT:$dst, (Intr VT:$src))), 3521 (!cast<I>("V"#OpcPrefix#r_Int) VT:$dst, VT:$src)>; 3522 3523 def : Pat<(VT (X86Blendi VT:$dst, (Intr VT:$src), (i8 1))), 3524 (!cast<I>("V"#OpcPrefix#r_Int) VT:$dst, VT:$src)>; 3525 } 3526} 3527 3528defm : scalar_unary_math_patterns<int_x86_sse_rcp_ss, "RCPSS", X86Movss, 3529 v4f32, UseSSE1>; 3530defm : scalar_unary_math_patterns<int_x86_sse_rsqrt_ss, "RSQRTSS", X86Movss, 3531 v4f32, UseSSE1>; 3532defm : scalar_unary_math_patterns<int_x86_sse_sqrt_ss, "SQRTSS", X86Movss, 3533 v4f32, UseSSE1>; 3534defm : scalar_unary_math_patterns<int_x86_sse2_sqrt_sd, "SQRTSD", X86Movsd, 3535 v2f64, UseSSE2>; 3536 3537 3538//===----------------------------------------------------------------------===// 3539// SSE 1 & 2 - Non-temporal stores 3540//===----------------------------------------------------------------------===// 3541 3542let AddedComplexity = 400 in { // Prefer non-temporal versions 3543let SchedRW = [WriteStore] in { 3544let Predicates = [HasAVX, NoVLX] in { 3545def VMOVNTPSmr : VPSI<0x2B, MRMDestMem, (outs), 3546 (ins f128mem:$dst, VR128:$src), 3547 "movntps\t{$src, $dst|$dst, $src}", 3548 [(alignednontemporalstore (v4f32 VR128:$src), 3549 addr:$dst)], 3550 IIC_SSE_MOVNT>, VEX; 3551def VMOVNTPDmr : VPDI<0x2B, MRMDestMem, (outs), 3552 (ins f128mem:$dst, VR128:$src), 3553 "movntpd\t{$src, $dst|$dst, $src}", 3554 [(alignednontemporalstore (v2f64 VR128:$src), 3555 addr:$dst)], 3556 IIC_SSE_MOVNT>, VEX; 3557 3558let ExeDomain = SSEPackedInt in 3559def VMOVNTDQmr : VPDI<0xE7, MRMDestMem, (outs), 3560 (ins f128mem:$dst, VR128:$src), 3561 "movntdq\t{$src, $dst|$dst, $src}", 3562 [(alignednontemporalstore (v2i64 VR128:$src), 3563 addr:$dst)], 3564 IIC_SSE_MOVNT>, VEX; 3565 3566def VMOVNTPSYmr : VPSI<0x2B, MRMDestMem, (outs), 3567 (ins f256mem:$dst, VR256:$src), 3568 "movntps\t{$src, $dst|$dst, $src}", 3569 [(alignednontemporalstore (v8f32 VR256:$src), 3570 addr:$dst)], 3571 IIC_SSE_MOVNT>, VEX, VEX_L; 3572def VMOVNTPDYmr : VPDI<0x2B, MRMDestMem, (outs), 3573 (ins f256mem:$dst, VR256:$src), 3574 "movntpd\t{$src, $dst|$dst, $src}", 3575 [(alignednontemporalstore (v4f64 VR256:$src), 3576 addr:$dst)], 3577 IIC_SSE_MOVNT>, VEX, VEX_L; 3578let ExeDomain = SSEPackedInt in 3579def VMOVNTDQYmr : VPDI<0xE7, MRMDestMem, (outs), 3580 (ins f256mem:$dst, VR256:$src), 3581 "movntdq\t{$src, $dst|$dst, $src}", 3582 [(alignednontemporalstore (v4i64 VR256:$src), 3583 addr:$dst)], 3584 IIC_SSE_MOVNT>, VEX, VEX_L; 3585} 3586 3587def MOVNTPSmr : PSI<0x2B, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 3588 "movntps\t{$src, $dst|$dst, $src}", 3589 [(alignednontemporalstore (v4f32 VR128:$src), addr:$dst)], 3590 IIC_SSE_MOVNT>; 3591def MOVNTPDmr : PDI<0x2B, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 3592 "movntpd\t{$src, $dst|$dst, $src}", 3593 [(alignednontemporalstore(v2f64 VR128:$src), addr:$dst)], 3594 IIC_SSE_MOVNT>; 3595 3596let ExeDomain = SSEPackedInt in 3597def MOVNTDQmr : PDI<0xE7, MRMDestMem, (outs), (ins f128mem:$dst, VR128:$src), 3598 "movntdq\t{$src, $dst|$dst, $src}", 3599 [(alignednontemporalstore (v2i64 VR128:$src), addr:$dst)], 3600 IIC_SSE_MOVNT>; 3601 3602// There is no AVX form for instructions below this point 3603def MOVNTImr : I<0xC3, MRMDestMem, (outs), (ins i32mem:$dst, GR32:$src), 3604 "movnti{l}\t{$src, $dst|$dst, $src}", 3605 [(nontemporalstore (i32 GR32:$src), addr:$dst)], 3606 IIC_SSE_MOVNT>, 3607 PS, Requires<[HasSSE2]>; 3608def MOVNTI_64mr : RI<0xC3, MRMDestMem, (outs), (ins i64mem:$dst, GR64:$src), 3609 "movnti{q}\t{$src, $dst|$dst, $src}", 3610 [(nontemporalstore (i64 GR64:$src), addr:$dst)], 3611 IIC_SSE_MOVNT>, 3612 PS, Requires<[HasSSE2]>; 3613} // SchedRW = [WriteStore] 3614 3615let Predicates = [HasAVX, NoVLX] in { 3616 def : Pat<(alignednontemporalstore (v8i32 VR256:$src), addr:$dst), 3617 (VMOVNTDQYmr addr:$dst, VR256:$src)>; 3618 def : Pat<(alignednontemporalstore (v16i16 VR256:$src), addr:$dst), 3619 (VMOVNTDQYmr addr:$dst, VR256:$src)>; 3620 def : Pat<(alignednontemporalstore (v32i8 VR256:$src), addr:$dst), 3621 (VMOVNTDQYmr addr:$dst, VR256:$src)>; 3622 3623 def : Pat<(alignednontemporalstore (v4i32 VR128:$src), addr:$dst), 3624 (VMOVNTDQmr addr:$dst, VR128:$src)>; 3625 def : Pat<(alignednontemporalstore (v8i16 VR128:$src), addr:$dst), 3626 (VMOVNTDQmr addr:$dst, VR128:$src)>; 3627 def : Pat<(alignednontemporalstore (v16i8 VR128:$src), addr:$dst), 3628 (VMOVNTDQmr addr:$dst, VR128:$src)>; 3629} 3630 3631let Predicates = [UseSSE2] in { 3632 def : Pat<(alignednontemporalstore (v4i32 VR128:$src), addr:$dst), 3633 (MOVNTDQmr addr:$dst, VR128:$src)>; 3634 def : Pat<(alignednontemporalstore (v8i16 VR128:$src), addr:$dst), 3635 (MOVNTDQmr addr:$dst, VR128:$src)>; 3636 def : Pat<(alignednontemporalstore (v16i8 VR128:$src), addr:$dst), 3637 (MOVNTDQmr addr:$dst, VR128:$src)>; 3638} 3639 3640} // AddedComplexity 3641 3642//===----------------------------------------------------------------------===// 3643// SSE 1 & 2 - Prefetch and memory fence 3644//===----------------------------------------------------------------------===// 3645 3646// Prefetch intrinsic. 3647let Predicates = [HasSSE1], SchedRW = [WriteLoad] in { 3648def PREFETCHT0 : I<0x18, MRM1m, (outs), (ins i8mem:$src), 3649 "prefetcht0\t$src", [(prefetch addr:$src, imm, (i32 3), (i32 1))], 3650 IIC_SSE_PREFETCH>, TB; 3651def PREFETCHT1 : I<0x18, MRM2m, (outs), (ins i8mem:$src), 3652 "prefetcht1\t$src", [(prefetch addr:$src, imm, (i32 2), (i32 1))], 3653 IIC_SSE_PREFETCH>, TB; 3654def PREFETCHT2 : I<0x18, MRM3m, (outs), (ins i8mem:$src), 3655 "prefetcht2\t$src", [(prefetch addr:$src, imm, (i32 1), (i32 1))], 3656 IIC_SSE_PREFETCH>, TB; 3657def PREFETCHNTA : I<0x18, MRM0m, (outs), (ins i8mem:$src), 3658 "prefetchnta\t$src", [(prefetch addr:$src, imm, (i32 0), (i32 1))], 3659 IIC_SSE_PREFETCH>, TB; 3660} 3661 3662// FIXME: How should flush instruction be modeled? 3663let SchedRW = [WriteLoad] in { 3664// Flush cache 3665def CLFLUSH : I<0xAE, MRM7m, (outs), (ins i8mem:$src), 3666 "clflush\t$src", [(int_x86_sse2_clflush addr:$src)], 3667 IIC_SSE_PREFETCH>, PS, Requires<[HasSSE2]>; 3668} 3669 3670let SchedRW = [WriteNop] in { 3671// Pause. This "instruction" is encoded as "rep; nop", so even though it 3672// was introduced with SSE2, it's backward compatible. 3673def PAUSE : I<0x90, RawFrm, (outs), (ins), 3674 "pause", [(int_x86_sse2_pause)], IIC_SSE_PAUSE>, 3675 OBXS, Requires<[HasSSE2]>; 3676} 3677 3678let SchedRW = [WriteFence] in { 3679// Load, store, and memory fence 3680// TODO: As with mfence, we may want to ease the availablity of sfence/lfence 3681// to include any 64-bit target. 3682def SFENCE : I<0xAE, MRM_F8, (outs), (ins), 3683 "sfence", [(int_x86_sse_sfence)], IIC_SSE_SFENCE>, 3684 PS, Requires<[HasSSE1]>; 3685def LFENCE : I<0xAE, MRM_E8, (outs), (ins), 3686 "lfence", [(int_x86_sse2_lfence)], IIC_SSE_LFENCE>, 3687 TB, Requires<[HasSSE2]>; 3688def MFENCE : I<0xAE, MRM_F0, (outs), (ins), 3689 "mfence", [(int_x86_sse2_mfence)], IIC_SSE_MFENCE>, 3690 TB, Requires<[HasMFence]>; 3691} // SchedRW 3692 3693def : Pat<(X86MFence), (MFENCE)>; 3694 3695//===----------------------------------------------------------------------===// 3696// SSE 1 & 2 - Load/Store XCSR register 3697//===----------------------------------------------------------------------===// 3698 3699def VLDMXCSR : VPSI<0xAE, MRM2m, (outs), (ins i32mem:$src), 3700 "ldmxcsr\t$src", [(int_x86_sse_ldmxcsr addr:$src)], 3701 IIC_SSE_LDMXCSR>, VEX, Sched<[WriteLoad]>; 3702def VSTMXCSR : VPSI<0xAE, MRM3m, (outs), (ins i32mem:$dst), 3703 "stmxcsr\t$dst", [(int_x86_sse_stmxcsr addr:$dst)], 3704 IIC_SSE_STMXCSR>, VEX, Sched<[WriteStore]>; 3705 3706let Predicates = [UseSSE1] in { 3707def LDMXCSR : I<0xAE, MRM2m, (outs), (ins i32mem:$src), 3708 "ldmxcsr\t$src", [(int_x86_sse_ldmxcsr addr:$src)], 3709 IIC_SSE_LDMXCSR>, TB, Sched<[WriteLoad]>; 3710def STMXCSR : I<0xAE, MRM3m, (outs), (ins i32mem:$dst), 3711 "stmxcsr\t$dst", [(int_x86_sse_stmxcsr addr:$dst)], 3712 IIC_SSE_STMXCSR>, TB, Sched<[WriteStore]>; 3713} 3714 3715//===---------------------------------------------------------------------===// 3716// SSE2 - Move Aligned/Unaligned Packed Integer Instructions 3717//===---------------------------------------------------------------------===// 3718 3719let ExeDomain = SSEPackedInt in { // SSE integer instructions 3720 3721let hasSideEffects = 0, SchedRW = [WriteMove] in { 3722def VMOVDQArr : VPDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3723 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>, 3724 VEX; 3725def VMOVDQAYrr : VPDI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 3726 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>, 3727 VEX, VEX_L; 3728def VMOVDQUrr : VSSI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3729 "movdqu\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVU_P_RR>, 3730 VEX; 3731def VMOVDQUYrr : VSSI<0x6F, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 3732 "movdqu\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVU_P_RR>, 3733 VEX, VEX_L; 3734} 3735 3736// For Disassembler 3737let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, 3738 SchedRW = [WriteMove] in { 3739def VMOVDQArr_REV : VPDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 3740 "movdqa\t{$src, $dst|$dst, $src}", [], 3741 IIC_SSE_MOVA_P_RR>, 3742 VEX; 3743def VMOVDQAYrr_REV : VPDI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src), 3744 "movdqa\t{$src, $dst|$dst, $src}", [], 3745 IIC_SSE_MOVA_P_RR>, VEX, VEX_L; 3746def VMOVDQUrr_REV : VSSI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 3747 "movdqu\t{$src, $dst|$dst, $src}", [], 3748 IIC_SSE_MOVU_P_RR>, 3749 VEX; 3750def VMOVDQUYrr_REV : VSSI<0x7F, MRMDestReg, (outs VR256:$dst), (ins VR256:$src), 3751 "movdqu\t{$src, $dst|$dst, $src}", [], 3752 IIC_SSE_MOVU_P_RR>, VEX, VEX_L; 3753} 3754 3755let canFoldAsLoad = 1, mayLoad = 1, isReMaterializable = 1, 3756 hasSideEffects = 0, SchedRW = [WriteLoad] in { 3757def VMOVDQArm : VPDI<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 3758 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RM>, 3759 VEX; 3760def VMOVDQAYrm : VPDI<0x6F, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), 3761 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RM>, 3762 VEX, VEX_L; 3763let Predicates = [HasAVX] in { 3764 def VMOVDQUrm : I<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 3765 "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_RM>, 3766 XS, VEX; 3767 def VMOVDQUYrm : I<0x6F, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), 3768 "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_RM>, 3769 XS, VEX, VEX_L; 3770} 3771} 3772 3773let mayStore = 1, hasSideEffects = 0, SchedRW = [WriteStore] in { 3774def VMOVDQAmr : VPDI<0x7F, MRMDestMem, (outs), 3775 (ins i128mem:$dst, VR128:$src), 3776 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_MR>, 3777 VEX; 3778def VMOVDQAYmr : VPDI<0x7F, MRMDestMem, (outs), 3779 (ins i256mem:$dst, VR256:$src), 3780 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_MR>, 3781 VEX, VEX_L; 3782let Predicates = [HasAVX] in { 3783def VMOVDQUmr : I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src), 3784 "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_MR>, 3785 XS, VEX; 3786def VMOVDQUYmr : I<0x7F, MRMDestMem, (outs), (ins i256mem:$dst, VR256:$src), 3787 "vmovdqu\t{$src, $dst|$dst, $src}",[], IIC_SSE_MOVU_P_MR>, 3788 XS, VEX, VEX_L; 3789} 3790} 3791 3792let SchedRW = [WriteMove] in { 3793let hasSideEffects = 0 in 3794def MOVDQArr : PDI<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3795 "movdqa\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVA_P_RR>; 3796 3797def MOVDQUrr : I<0x6F, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 3798 "movdqu\t{$src, $dst|$dst, $src}", 3799 [], IIC_SSE_MOVU_P_RR>, XS, Requires<[UseSSE2]>; 3800 3801// For Disassembler 3802let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in { 3803def MOVDQArr_REV : PDI<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 3804 "movdqa\t{$src, $dst|$dst, $src}", [], 3805 IIC_SSE_MOVA_P_RR>; 3806 3807def MOVDQUrr_REV : I<0x7F, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 3808 "movdqu\t{$src, $dst|$dst, $src}", 3809 [], IIC_SSE_MOVU_P_RR>, XS, Requires<[UseSSE2]>; 3810} 3811} // SchedRW 3812 3813let canFoldAsLoad = 1, mayLoad = 1, isReMaterializable = 1, 3814 hasSideEffects = 0, SchedRW = [WriteLoad] in { 3815def MOVDQArm : PDI<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 3816 "movdqa\t{$src, $dst|$dst, $src}", 3817 [/*(set VR128:$dst, (alignedloadv2i64 addr:$src))*/], 3818 IIC_SSE_MOVA_P_RM>; 3819def MOVDQUrm : I<0x6F, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 3820 "movdqu\t{$src, $dst|$dst, $src}", 3821 [/*(set VR128:$dst, (loadv2i64 addr:$src))*/], 3822 IIC_SSE_MOVU_P_RM>, 3823 XS, Requires<[UseSSE2]>; 3824} 3825 3826let mayStore = 1, hasSideEffects = 0, SchedRW = [WriteStore] in { 3827def MOVDQAmr : PDI<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src), 3828 "movdqa\t{$src, $dst|$dst, $src}", 3829 [/*(alignedstore (v2i64 VR128:$src), addr:$dst)*/], 3830 IIC_SSE_MOVA_P_MR>; 3831def MOVDQUmr : I<0x7F, MRMDestMem, (outs), (ins i128mem:$dst, VR128:$src), 3832 "movdqu\t{$src, $dst|$dst, $src}", 3833 [/*(store (v2i64 VR128:$src), addr:$dst)*/], 3834 IIC_SSE_MOVU_P_MR>, 3835 XS, Requires<[UseSSE2]>; 3836} 3837 3838} // ExeDomain = SSEPackedInt 3839 3840// Aliases to help the assembler pick two byte VEX encodings by swapping the 3841// operands relative to the normal instructions to use VEX.R instead of VEX.B. 3842def : InstAlias<"vmovdqa\t{$src, $dst|$dst, $src}", 3843 (VMOVDQArr_REV VR128L:$dst, VR128H:$src), 0>; 3844def : InstAlias<"vmovdqa\t{$src, $dst|$dst, $src}", 3845 (VMOVDQAYrr_REV VR256L:$dst, VR256H:$src), 0>; 3846def : InstAlias<"vmovdqu\t{$src, $dst|$dst, $src}", 3847 (VMOVDQUrr_REV VR128L:$dst, VR128H:$src), 0>; 3848def : InstAlias<"vmovdqu\t{$src, $dst|$dst, $src}", 3849 (VMOVDQUYrr_REV VR256L:$dst, VR256H:$src), 0>; 3850 3851//===---------------------------------------------------------------------===// 3852// SSE2 - Packed Integer Arithmetic Instructions 3853//===---------------------------------------------------------------------===// 3854 3855let Sched = WriteVecIMul in 3856def SSE_PMADD : OpndItins< 3857 IIC_SSE_PMADD, IIC_SSE_PMADD 3858>; 3859 3860let ExeDomain = SSEPackedInt in { // SSE integer instructions 3861 3862multiclass PDI_binop_rm_int<bits<8> opc, string OpcodeStr, Intrinsic IntId, 3863 RegisterClass RC, PatFrag memop_frag, 3864 X86MemOperand x86memop, 3865 OpndItins itins, 3866 bit IsCommutable = 0, 3867 bit Is2Addr = 1> { 3868 let isCommutable = IsCommutable in 3869 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst), 3870 (ins RC:$src1, RC:$src2), 3871 !if(Is2Addr, 3872 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3873 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3874 [(set RC:$dst, (IntId RC:$src1, RC:$src2))], itins.rr>, 3875 Sched<[itins.Sched]>; 3876 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst), 3877 (ins RC:$src1, x86memop:$src2), 3878 !if(Is2Addr, 3879 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3880 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3881 [(set RC:$dst, (IntId RC:$src1, (bitconvert (memop_frag addr:$src2))))], 3882 itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 3883} 3884 3885multiclass PDI_binop_all_int<bits<8> opc, string OpcodeStr, Intrinsic IntId128, 3886 Intrinsic IntId256, OpndItins itins, 3887 bit IsCommutable = 0> { 3888let Predicates = [HasAVX] in 3889 defm V#NAME : PDI_binop_rm_int<opc, !strconcat("v", OpcodeStr), IntId128, 3890 VR128, loadv2i64, i128mem, itins, 3891 IsCommutable, 0>, VEX_4V; 3892 3893let Constraints = "$src1 = $dst" in 3894 defm NAME : PDI_binop_rm_int<opc, OpcodeStr, IntId128, VR128, memopv2i64, 3895 i128mem, itins, IsCommutable, 1>; 3896 3897let Predicates = [HasAVX2] in 3898 defm V#NAME#Y : PDI_binop_rm_int<opc, !strconcat("v", OpcodeStr), IntId256, 3899 VR256, loadv4i64, i256mem, itins, 3900 IsCommutable, 0>, VEX_4V, VEX_L; 3901} 3902 3903multiclass PDI_binop_rmi<bits<8> opc, bits<8> opc2, Format ImmForm, 3904 string OpcodeStr, SDNode OpNode, 3905 SDNode OpNode2, RegisterClass RC, 3906 ValueType DstVT, ValueType SrcVT, 3907 PatFrag ld_frag, ShiftOpndItins itins, 3908 bit Is2Addr = 1> { 3909 // src2 is always 128-bit 3910 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst), 3911 (ins RC:$src1, VR128:$src2), 3912 !if(Is2Addr, 3913 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3914 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3915 [(set RC:$dst, (DstVT (OpNode RC:$src1, (SrcVT VR128:$src2))))], 3916 itins.rr>, Sched<[WriteVecShift]>; 3917 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst), 3918 (ins RC:$src1, i128mem:$src2), 3919 !if(Is2Addr, 3920 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3921 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3922 [(set RC:$dst, (DstVT (OpNode RC:$src1, 3923 (SrcVT (bitconvert (ld_frag addr:$src2))))))], itins.rm>, 3924 Sched<[WriteVecShiftLd, ReadAfterLd]>; 3925 def ri : PDIi8<opc2, ImmForm, (outs RC:$dst), 3926 (ins RC:$src1, u8imm:$src2), 3927 !if(Is2Addr, 3928 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3929 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3930 [(set RC:$dst, (DstVT (OpNode2 RC:$src1, (i8 imm:$src2))))], itins.ri>, 3931 Sched<[WriteVecShift]>; 3932} 3933 3934/// PDI_binop_rm2 - Simple SSE2 binary operator with different src and dst types 3935multiclass PDI_binop_rm2<bits<8> opc, string OpcodeStr, SDNode OpNode, 3936 ValueType DstVT, ValueType SrcVT, RegisterClass RC, 3937 PatFrag memop_frag, X86MemOperand x86memop, 3938 OpndItins itins, 3939 bit IsCommutable = 0, bit Is2Addr = 1> { 3940 let isCommutable = IsCommutable in 3941 def rr : PDI<opc, MRMSrcReg, (outs RC:$dst), 3942 (ins RC:$src1, RC:$src2), 3943 !if(Is2Addr, 3944 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3945 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3946 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1), RC:$src2)))]>, 3947 Sched<[itins.Sched]>; 3948 def rm : PDI<opc, MRMSrcMem, (outs RC:$dst), 3949 (ins RC:$src1, x86memop:$src2), 3950 !if(Is2Addr, 3951 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 3952 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 3953 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1), 3954 (bitconvert (memop_frag addr:$src2)))))]>, 3955 Sched<[itins.Sched.Folded, ReadAfterLd]>; 3956} 3957} // ExeDomain = SSEPackedInt 3958 3959defm PADDB : PDI_binop_all<0xFC, "paddb", add, v16i8, v32i8, 3960 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 3961defm PADDW : PDI_binop_all<0xFD, "paddw", add, v8i16, v16i16, 3962 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 3963defm PADDD : PDI_binop_all<0xFE, "paddd", add, v4i32, v8i32, 3964 SSE_INTALU_ITINS_P, 1, NoVLX>; 3965defm PADDQ : PDI_binop_all<0xD4, "paddq", add, v2i64, v4i64, 3966 SSE_INTALUQ_ITINS_P, 1, NoVLX>; 3967defm PADDSB : PDI_binop_all<0xEC, "paddsb", X86adds, v16i8, v32i8, 3968 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 3969defm PADDSW : PDI_binop_all<0xED, "paddsw", X86adds, v8i16, v16i16, 3970 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 3971defm PADDUSB : PDI_binop_all<0xDC, "paddusb", X86addus, v16i8, v32i8, 3972 SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>; 3973defm PADDUSW : PDI_binop_all<0xDD, "paddusw", X86addus, v8i16, v16i16, 3974 SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>; 3975defm PMULLW : PDI_binop_all<0xD5, "pmullw", mul, v8i16, v16i16, 3976 SSE_INTMUL_ITINS_P, 1, NoVLX_Or_NoBWI>; 3977defm PMULHUW : PDI_binop_all<0xE4, "pmulhuw", mulhu, v8i16, v16i16, 3978 SSE_INTMUL_ITINS_P, 1, NoVLX_Or_NoBWI>; 3979defm PMULHW : PDI_binop_all<0xE5, "pmulhw", mulhs, v8i16, v16i16, 3980 SSE_INTMUL_ITINS_P, 1, NoVLX_Or_NoBWI>; 3981defm PSUBB : PDI_binop_all<0xF8, "psubb", sub, v16i8, v32i8, 3982 SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>; 3983defm PSUBW : PDI_binop_all<0xF9, "psubw", sub, v8i16, v16i16, 3984 SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>; 3985defm PSUBD : PDI_binop_all<0xFA, "psubd", sub, v4i32, v8i32, 3986 SSE_INTALU_ITINS_P, 0, NoVLX>; 3987defm PSUBQ : PDI_binop_all<0xFB, "psubq", sub, v2i64, v4i64, 3988 SSE_INTALUQ_ITINS_P, 0, NoVLX>; 3989defm PSUBSB : PDI_binop_all<0xE8, "psubsb", X86subs, v16i8, v32i8, 3990 SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>; 3991defm PSUBSW : PDI_binop_all<0xE9, "psubsw", X86subs, v8i16, v16i16, 3992 SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>; 3993defm PSUBUSB : PDI_binop_all<0xD8, "psubusb", X86subus, v16i8, v32i8, 3994 SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>; 3995defm PSUBUSW : PDI_binop_all<0xD9, "psubusw", X86subus, v8i16, v16i16, 3996 SSE_INTALU_ITINS_P, 0, NoVLX_Or_NoBWI>; 3997defm PMINUB : PDI_binop_all<0xDA, "pminub", umin, v16i8, v32i8, 3998 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 3999defm PMINSW : PDI_binop_all<0xEA, "pminsw", smin, v8i16, v16i16, 4000 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 4001defm PMAXUB : PDI_binop_all<0xDE, "pmaxub", umax, v16i8, v32i8, 4002 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 4003defm PMAXSW : PDI_binop_all<0xEE, "pmaxsw", smax, v8i16, v16i16, 4004 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 4005defm PAVGB : PDI_binop_all<0xE0, "pavgb", X86avg, v16i8, v32i8, 4006 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 4007defm PAVGW : PDI_binop_all<0xE3, "pavgw", X86avg, v8i16, v16i16, 4008 SSE_INTALU_ITINS_P, 1, NoVLX_Or_NoBWI>; 4009 4010// Intrinsic forms 4011defm PMADDWD : PDI_binop_all_int<0xF5, "pmaddwd", int_x86_sse2_pmadd_wd, 4012 int_x86_avx2_pmadd_wd, SSE_PMADD, 1>; 4013 4014let Predicates = [HasAVX, NoVLX_Or_NoBWI] in 4015defm VPSADBW : PDI_binop_rm2<0xF6, "vpsadbw", X86psadbw, v2i64, v16i8, VR128, 4016 loadv2i64, i128mem, SSE_INTMUL_ITINS_P, 1, 0>, 4017 VEX_4V; 4018let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in 4019defm VPSADBWY : PDI_binop_rm2<0xF6, "vpsadbw", X86psadbw, v4i64, v32i8, VR256, 4020 loadv4i64, i256mem, SSE_INTMUL_ITINS_P, 1, 0>, 4021 VEX_4V, VEX_L; 4022let Constraints = "$src1 = $dst" in 4023defm PSADBW : PDI_binop_rm2<0xF6, "psadbw", X86psadbw, v2i64, v16i8, VR128, 4024 memopv2i64, i128mem, SSE_INTALU_ITINS_P, 1>; 4025 4026let Predicates = [HasAVX, NoVLX] in 4027defm VPMULUDQ : PDI_binop_rm2<0xF4, "vpmuludq", X86pmuludq, v2i64, v4i32, VR128, 4028 loadv2i64, i128mem, SSE_INTMUL_ITINS_P, 1, 0>, 4029 VEX_4V; 4030let Predicates = [HasAVX2, NoVLX] in 4031defm VPMULUDQY : PDI_binop_rm2<0xF4, "vpmuludq", X86pmuludq, v4i64, v8i32, 4032 VR256, loadv4i64, i256mem, 4033 SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V, VEX_L; 4034let Constraints = "$src1 = $dst" in 4035defm PMULUDQ : PDI_binop_rm2<0xF4, "pmuludq", X86pmuludq, v2i64, v4i32, VR128, 4036 memopv2i64, i128mem, SSE_INTMUL_ITINS_P, 1>; 4037 4038//===---------------------------------------------------------------------===// 4039// SSE2 - Packed Integer Logical Instructions 4040//===---------------------------------------------------------------------===// 4041 4042let Predicates = [HasAVX, NoVLX] in { 4043defm VPSLLD : PDI_binop_rmi<0xF2, 0x72, MRM6r, "vpslld", X86vshl, X86vshli, 4044 VR128, v4i32, v4i32, loadv2i64, 4045 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4046defm VPSLLQ : PDI_binop_rmi<0xF3, 0x73, MRM6r, "vpsllq", X86vshl, X86vshli, 4047 VR128, v2i64, v2i64, loadv2i64, 4048 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4049 4050defm VPSRLD : PDI_binop_rmi<0xD2, 0x72, MRM2r, "vpsrld", X86vsrl, X86vsrli, 4051 VR128, v4i32, v4i32, loadv2i64, 4052 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4053defm VPSRLQ : PDI_binop_rmi<0xD3, 0x73, MRM2r, "vpsrlq", X86vsrl, X86vsrli, 4054 VR128, v2i64, v2i64, loadv2i64, 4055 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4056 4057defm VPSRAD : PDI_binop_rmi<0xE2, 0x72, MRM4r, "vpsrad", X86vsra, X86vsrai, 4058 VR128, v4i32, v4i32, loadv2i64, 4059 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4060} // Predicates = [HasAVX, NoVLX] 4061 4062let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 4063defm VPSLLW : PDI_binop_rmi<0xF1, 0x71, MRM6r, "vpsllw", X86vshl, X86vshli, 4064 VR128, v8i16, v8i16, loadv2i64, 4065 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4066defm VPSRLW : PDI_binop_rmi<0xD1, 0x71, MRM2r, "vpsrlw", X86vsrl, X86vsrli, 4067 VR128, v8i16, v8i16, loadv2i64, 4068 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4069defm VPSRAW : PDI_binop_rmi<0xE1, 0x71, MRM4r, "vpsraw", X86vsra, X86vsrai, 4070 VR128, v8i16, v8i16, loadv2i64, 4071 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V; 4072} // Predicates = [HasAVX, NoVLX_Or_NoBWI] 4073 4074 4075let ExeDomain = SSEPackedInt, SchedRW = [WriteVecShift] , 4076 Predicates = [HasAVX, NoVLX_Or_NoBWI]in { 4077 // 128-bit logical shifts. 4078 def VPSLLDQri : PDIi8<0x73, MRM7r, 4079 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4080 "vpslldq\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4081 [(set VR128:$dst, 4082 (v16i8 (X86vshldq VR128:$src1, (i8 imm:$src2))))]>, 4083 VEX_4V; 4084 def VPSRLDQri : PDIi8<0x73, MRM3r, 4085 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4086 "vpsrldq\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4087 [(set VR128:$dst, 4088 (v16i8 (X86vshrdq VR128:$src1, (i8 imm:$src2))))]>, 4089 VEX_4V; 4090 // PSRADQri doesn't exist in SSE[1-3]. 4091} // Predicates = [HasAVX, NoVLX_Or_NoBWI] 4092 4093let Predicates = [HasAVX2, NoVLX] in { 4094defm VPSLLDY : PDI_binop_rmi<0xF2, 0x72, MRM6r, "vpslld", X86vshl, X86vshli, 4095 VR256, v8i32, v4i32, loadv2i64, 4096 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4097defm VPSLLQY : PDI_binop_rmi<0xF3, 0x73, MRM6r, "vpsllq", X86vshl, X86vshli, 4098 VR256, v4i64, v2i64, loadv2i64, 4099 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4100 4101defm VPSRLDY : PDI_binop_rmi<0xD2, 0x72, MRM2r, "vpsrld", X86vsrl, X86vsrli, 4102 VR256, v8i32, v4i32, loadv2i64, 4103 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4104defm VPSRLQY : PDI_binop_rmi<0xD3, 0x73, MRM2r, "vpsrlq", X86vsrl, X86vsrli, 4105 VR256, v4i64, v2i64, loadv2i64, 4106 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4107 4108defm VPSRADY : PDI_binop_rmi<0xE2, 0x72, MRM4r, "vpsrad", X86vsra, X86vsrai, 4109 VR256, v8i32, v4i32, loadv2i64, 4110 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4111}// Predicates = [HasAVX2, NoVLX] 4112 4113let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in { 4114defm VPSLLWY : PDI_binop_rmi<0xF1, 0x71, MRM6r, "vpsllw", X86vshl, X86vshli, 4115 VR256, v16i16, v8i16, loadv2i64, 4116 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4117defm VPSRLWY : PDI_binop_rmi<0xD1, 0x71, MRM2r, "vpsrlw", X86vsrl, X86vsrli, 4118 VR256, v16i16, v8i16, loadv2i64, 4119 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4120defm VPSRAWY : PDI_binop_rmi<0xE1, 0x71, MRM4r, "vpsraw", X86vsra, X86vsrai, 4121 VR256, v16i16, v8i16, loadv2i64, 4122 SSE_INTSHIFT_ITINS_P, 0>, VEX_4V, VEX_L; 4123}// Predicates = [HasAVX2, NoVLX_Or_NoBWI] 4124 4125let ExeDomain = SSEPackedInt, SchedRW = [WriteVecShift], hasSideEffects = 0 , 4126 Predicates = [HasAVX2, NoVLX_Or_NoBWI] in { 4127 // 256-bit logical shifts. 4128 def VPSLLDQYri : PDIi8<0x73, MRM7r, 4129 (outs VR256:$dst), (ins VR256:$src1, u8imm:$src2), 4130 "vpslldq\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4131 [(set VR256:$dst, 4132 (v32i8 (X86vshldq VR256:$src1, (i8 imm:$src2))))]>, 4133 VEX_4V, VEX_L; 4134 def VPSRLDQYri : PDIi8<0x73, MRM3r, 4135 (outs VR256:$dst), (ins VR256:$src1, u8imm:$src2), 4136 "vpsrldq\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4137 [(set VR256:$dst, 4138 (v32i8 (X86vshrdq VR256:$src1, (i8 imm:$src2))))]>, 4139 VEX_4V, VEX_L; 4140 // PSRADQYri doesn't exist in SSE[1-3]. 4141} // Predicates = [HasAVX2, NoVLX_Or_NoBWI] 4142 4143let Constraints = "$src1 = $dst" in { 4144defm PSLLW : PDI_binop_rmi<0xF1, 0x71, MRM6r, "psllw", X86vshl, X86vshli, 4145 VR128, v8i16, v8i16, memopv2i64, 4146 SSE_INTSHIFT_ITINS_P>; 4147defm PSLLD : PDI_binop_rmi<0xF2, 0x72, MRM6r, "pslld", X86vshl, X86vshli, 4148 VR128, v4i32, v4i32, memopv2i64, 4149 SSE_INTSHIFT_ITINS_P>; 4150defm PSLLQ : PDI_binop_rmi<0xF3, 0x73, MRM6r, "psllq", X86vshl, X86vshli, 4151 VR128, v2i64, v2i64, memopv2i64, 4152 SSE_INTSHIFT_ITINS_P>; 4153 4154defm PSRLW : PDI_binop_rmi<0xD1, 0x71, MRM2r, "psrlw", X86vsrl, X86vsrli, 4155 VR128, v8i16, v8i16, memopv2i64, 4156 SSE_INTSHIFT_ITINS_P>; 4157defm PSRLD : PDI_binop_rmi<0xD2, 0x72, MRM2r, "psrld", X86vsrl, X86vsrli, 4158 VR128, v4i32, v4i32, memopv2i64, 4159 SSE_INTSHIFT_ITINS_P>; 4160defm PSRLQ : PDI_binop_rmi<0xD3, 0x73, MRM2r, "psrlq", X86vsrl, X86vsrli, 4161 VR128, v2i64, v2i64, memopv2i64, 4162 SSE_INTSHIFT_ITINS_P>; 4163 4164defm PSRAW : PDI_binop_rmi<0xE1, 0x71, MRM4r, "psraw", X86vsra, X86vsrai, 4165 VR128, v8i16, v8i16, memopv2i64, 4166 SSE_INTSHIFT_ITINS_P>; 4167defm PSRAD : PDI_binop_rmi<0xE2, 0x72, MRM4r, "psrad", X86vsra, X86vsrai, 4168 VR128, v4i32, v4i32, memopv2i64, 4169 SSE_INTSHIFT_ITINS_P>; 4170 4171let ExeDomain = SSEPackedInt, SchedRW = [WriteVecShift], hasSideEffects = 0 in { 4172 // 128-bit logical shifts. 4173 def PSLLDQri : PDIi8<0x73, MRM7r, 4174 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4175 "pslldq\t{$src2, $dst|$dst, $src2}", 4176 [(set VR128:$dst, 4177 (v16i8 (X86vshldq VR128:$src1, (i8 imm:$src2))))], 4178 IIC_SSE_INTSHDQ_P_RI>; 4179 def PSRLDQri : PDIi8<0x73, MRM3r, 4180 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4181 "psrldq\t{$src2, $dst|$dst, $src2}", 4182 [(set VR128:$dst, 4183 (v16i8 (X86vshrdq VR128:$src1, (i8 imm:$src2))))], 4184 IIC_SSE_INTSHDQ_P_RI>; 4185 // PSRADQri doesn't exist in SSE[1-3]. 4186} 4187} // Constraints = "$src1 = $dst" 4188 4189//===---------------------------------------------------------------------===// 4190// SSE2 - Packed Integer Comparison Instructions 4191//===---------------------------------------------------------------------===// 4192 4193defm PCMPEQB : PDI_binop_all<0x74, "pcmpeqb", X86pcmpeq, v16i8, v32i8, 4194 SSE_INTALU_ITINS_P, 1, TruePredicate>; 4195defm PCMPEQW : PDI_binop_all<0x75, "pcmpeqw", X86pcmpeq, v8i16, v16i16, 4196 SSE_INTALU_ITINS_P, 1, TruePredicate>; 4197defm PCMPEQD : PDI_binop_all<0x76, "pcmpeqd", X86pcmpeq, v4i32, v8i32, 4198 SSE_INTALU_ITINS_P, 1, TruePredicate>; 4199defm PCMPGTB : PDI_binop_all<0x64, "pcmpgtb", X86pcmpgt, v16i8, v32i8, 4200 SSE_INTALU_ITINS_P, 0, TruePredicate>; 4201defm PCMPGTW : PDI_binop_all<0x65, "pcmpgtw", X86pcmpgt, v8i16, v16i16, 4202 SSE_INTALU_ITINS_P, 0, TruePredicate>; 4203defm PCMPGTD : PDI_binop_all<0x66, "pcmpgtd", X86pcmpgt, v4i32, v8i32, 4204 SSE_INTALU_ITINS_P, 0, TruePredicate>; 4205 4206//===---------------------------------------------------------------------===// 4207// SSE2 - Packed Integer Shuffle Instructions 4208//===---------------------------------------------------------------------===// 4209 4210let ExeDomain = SSEPackedInt in { 4211multiclass sse2_pshuffle<string OpcodeStr, ValueType vt128, ValueType vt256, 4212 SDNode OpNode, Predicate prd> { 4213let Predicates = [HasAVX, prd] in { 4214 def V#NAME#ri : Ii8<0x70, MRMSrcReg, (outs VR128:$dst), 4215 (ins VR128:$src1, u8imm:$src2), 4216 !strconcat("v", OpcodeStr, 4217 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4218 [(set VR128:$dst, 4219 (vt128 (OpNode VR128:$src1, (i8 imm:$src2))))], 4220 IIC_SSE_PSHUF_RI>, VEX, Sched<[WriteShuffle]>; 4221 def V#NAME#mi : Ii8<0x70, MRMSrcMem, (outs VR128:$dst), 4222 (ins i128mem:$src1, u8imm:$src2), 4223 !strconcat("v", OpcodeStr, 4224 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4225 [(set VR128:$dst, 4226 (vt128 (OpNode (bitconvert (loadv2i64 addr:$src1)), 4227 (i8 imm:$src2))))], IIC_SSE_PSHUF_MI>, VEX, 4228 Sched<[WriteShuffleLd]>; 4229} 4230 4231let Predicates = [HasAVX2, prd] in { 4232 def V#NAME#Yri : Ii8<0x70, MRMSrcReg, (outs VR256:$dst), 4233 (ins VR256:$src1, u8imm:$src2), 4234 !strconcat("v", OpcodeStr, 4235 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4236 [(set VR256:$dst, 4237 (vt256 (OpNode VR256:$src1, (i8 imm:$src2))))], 4238 IIC_SSE_PSHUF_RI>, VEX, VEX_L, Sched<[WriteShuffle]>; 4239 def V#NAME#Ymi : Ii8<0x70, MRMSrcMem, (outs VR256:$dst), 4240 (ins i256mem:$src1, u8imm:$src2), 4241 !strconcat("v", OpcodeStr, 4242 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4243 [(set VR256:$dst, 4244 (vt256 (OpNode (bitconvert (loadv4i64 addr:$src1)), 4245 (i8 imm:$src2))))], IIC_SSE_PSHUF_MI>, VEX, VEX_L, 4246 Sched<[WriteShuffleLd]>; 4247} 4248 4249let Predicates = [UseSSE2] in { 4250 def ri : Ii8<0x70, MRMSrcReg, 4251 (outs VR128:$dst), (ins VR128:$src1, u8imm:$src2), 4252 !strconcat(OpcodeStr, 4253 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4254 [(set VR128:$dst, 4255 (vt128 (OpNode VR128:$src1, (i8 imm:$src2))))], 4256 IIC_SSE_PSHUF_RI>, Sched<[WriteShuffle]>; 4257 def mi : Ii8<0x70, MRMSrcMem, 4258 (outs VR128:$dst), (ins i128mem:$src1, u8imm:$src2), 4259 !strconcat(OpcodeStr, 4260 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4261 [(set VR128:$dst, 4262 (vt128 (OpNode (bitconvert (memopv2i64 addr:$src1)), 4263 (i8 imm:$src2))))], IIC_SSE_PSHUF_MI>, 4264 Sched<[WriteShuffleLd, ReadAfterLd]>; 4265} 4266} 4267} // ExeDomain = SSEPackedInt 4268 4269defm PSHUFD : sse2_pshuffle<"pshufd", v4i32, v8i32, X86PShufd, NoVLX>, PD; 4270defm PSHUFHW : sse2_pshuffle<"pshufhw", v8i16, v16i16, X86PShufhw, 4271 NoVLX_Or_NoBWI>, XS; 4272defm PSHUFLW : sse2_pshuffle<"pshuflw", v8i16, v16i16, X86PShuflw, 4273 NoVLX_Or_NoBWI>, XD; 4274 4275let Predicates = [HasAVX] in { 4276 def : Pat<(v4f32 (X86PShufd (loadv4f32 addr:$src1), (i8 imm:$imm))), 4277 (VPSHUFDmi addr:$src1, imm:$imm)>; 4278 def : Pat<(v4f32 (X86PShufd VR128:$src1, (i8 imm:$imm))), 4279 (VPSHUFDri VR128:$src1, imm:$imm)>; 4280} 4281 4282let Predicates = [UseSSE2] in { 4283 def : Pat<(v4f32 (X86PShufd (memopv4f32 addr:$src1), (i8 imm:$imm))), 4284 (PSHUFDmi addr:$src1, imm:$imm)>; 4285 def : Pat<(v4f32 (X86PShufd VR128:$src1, (i8 imm:$imm))), 4286 (PSHUFDri VR128:$src1, imm:$imm)>; 4287} 4288 4289//===---------------------------------------------------------------------===// 4290// Packed Integer Pack Instructions (SSE & AVX) 4291//===---------------------------------------------------------------------===// 4292 4293let ExeDomain = SSEPackedInt in { 4294multiclass sse2_pack<bits<8> opc, string OpcodeStr, ValueType OutVT, 4295 ValueType ArgVT, SDNode OpNode, PatFrag ld_frag, 4296 bit Is2Addr = 1> { 4297 def rr : PDI<opc, MRMSrcReg, 4298 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 4299 !if(Is2Addr, 4300 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4301 !strconcat(OpcodeStr, 4302 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4303 [(set VR128:$dst, 4304 (OutVT (OpNode (ArgVT VR128:$src1), VR128:$src2)))]>, 4305 Sched<[WriteShuffle]>; 4306 def rm : PDI<opc, MRMSrcMem, 4307 (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), 4308 !if(Is2Addr, 4309 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4310 !strconcat(OpcodeStr, 4311 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4312 [(set VR128:$dst, 4313 (OutVT (OpNode (ArgVT VR128:$src1), 4314 (bitconvert (ld_frag addr:$src2)))))]>, 4315 Sched<[WriteShuffleLd, ReadAfterLd]>; 4316} 4317 4318multiclass sse2_pack_y<bits<8> opc, string OpcodeStr, ValueType OutVT, 4319 ValueType ArgVT, SDNode OpNode> { 4320 def Yrr : PDI<opc, MRMSrcReg, 4321 (outs VR256:$dst), (ins VR256:$src1, VR256:$src2), 4322 !strconcat(OpcodeStr, 4323 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4324 [(set VR256:$dst, 4325 (OutVT (OpNode (ArgVT VR256:$src1), VR256:$src2)))]>, 4326 Sched<[WriteShuffle]>; 4327 def Yrm : PDI<opc, MRMSrcMem, 4328 (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), 4329 !strconcat(OpcodeStr, 4330 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4331 [(set VR256:$dst, 4332 (OutVT (OpNode (ArgVT VR256:$src1), 4333 (bitconvert (loadv4i64 addr:$src2)))))]>, 4334 Sched<[WriteShuffleLd, ReadAfterLd]>; 4335} 4336 4337multiclass sse4_pack<bits<8> opc, string OpcodeStr, ValueType OutVT, 4338 ValueType ArgVT, SDNode OpNode, PatFrag ld_frag, 4339 bit Is2Addr = 1> { 4340 def rr : SS48I<opc, MRMSrcReg, 4341 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 4342 !if(Is2Addr, 4343 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4344 !strconcat(OpcodeStr, 4345 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4346 [(set VR128:$dst, 4347 (OutVT (OpNode (ArgVT VR128:$src1), VR128:$src2)))]>, 4348 Sched<[WriteShuffle]>; 4349 def rm : SS48I<opc, MRMSrcMem, 4350 (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), 4351 !if(Is2Addr, 4352 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 4353 !strconcat(OpcodeStr, 4354 "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4355 [(set VR128:$dst, 4356 (OutVT (OpNode (ArgVT VR128:$src1), 4357 (bitconvert (ld_frag addr:$src2)))))]>, 4358 Sched<[WriteShuffleLd, ReadAfterLd]>; 4359} 4360 4361multiclass sse4_pack_y<bits<8> opc, string OpcodeStr, ValueType OutVT, 4362 ValueType ArgVT, SDNode OpNode> { 4363 def Yrr : SS48I<opc, MRMSrcReg, 4364 (outs VR256:$dst), (ins VR256:$src1, VR256:$src2), 4365 !strconcat(OpcodeStr, 4366 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4367 [(set VR256:$dst, 4368 (OutVT (OpNode (ArgVT VR256:$src1), VR256:$src2)))]>, 4369 Sched<[WriteShuffle]>; 4370 def Yrm : SS48I<opc, MRMSrcMem, 4371 (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), 4372 !strconcat(OpcodeStr, 4373 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4374 [(set VR256:$dst, 4375 (OutVT (OpNode (ArgVT VR256:$src1), 4376 (bitconvert (loadv4i64 addr:$src2)))))]>, 4377 Sched<[WriteShuffleLd, ReadAfterLd]>; 4378} 4379 4380let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 4381 defm VPACKSSWB : sse2_pack<0x63, "vpacksswb", v16i8, v8i16, X86Packss, 4382 loadv2i64, 0>, VEX_4V; 4383 defm VPACKSSDW : sse2_pack<0x6B, "vpackssdw", v8i16, v4i32, X86Packss, 4384 loadv2i64, 0>, VEX_4V; 4385 4386 defm VPACKUSWB : sse2_pack<0x67, "vpackuswb", v16i8, v8i16, X86Packus, 4387 loadv2i64, 0>, VEX_4V; 4388 defm VPACKUSDW : sse4_pack<0x2B, "vpackusdw", v8i16, v4i32, X86Packus, 4389 loadv2i64, 0>, VEX_4V; 4390} 4391 4392let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in { 4393 defm VPACKSSWB : sse2_pack_y<0x63, "vpacksswb", v32i8, v16i16, X86Packss>, 4394 VEX_4V, VEX_L; 4395 defm VPACKSSDW : sse2_pack_y<0x6B, "vpackssdw", v16i16, v8i32, X86Packss>, 4396 VEX_4V, VEX_L; 4397 4398 defm VPACKUSWB : sse2_pack_y<0x67, "vpackuswb", v32i8, v16i16, X86Packus>, 4399 VEX_4V, VEX_L; 4400 defm VPACKUSDW : sse4_pack_y<0x2B, "vpackusdw", v16i16, v8i32, X86Packus>, 4401 VEX_4V, VEX_L; 4402} 4403 4404let Constraints = "$src1 = $dst" in { 4405 defm PACKSSWB : sse2_pack<0x63, "packsswb", v16i8, v8i16, X86Packss, 4406 memopv2i64>; 4407 defm PACKSSDW : sse2_pack<0x6B, "packssdw", v8i16, v4i32, X86Packss, 4408 memopv2i64>; 4409 4410 defm PACKUSWB : sse2_pack<0x67, "packuswb", v16i8, v8i16, X86Packus, 4411 memopv2i64>; 4412 4413 defm PACKUSDW : sse4_pack<0x2B, "packusdw", v8i16, v4i32, X86Packus, 4414 memopv2i64>; 4415} 4416} // ExeDomain = SSEPackedInt 4417 4418//===---------------------------------------------------------------------===// 4419// SSE2 - Packed Integer Unpack Instructions 4420//===---------------------------------------------------------------------===// 4421 4422let ExeDomain = SSEPackedInt in { 4423multiclass sse2_unpack<bits<8> opc, string OpcodeStr, ValueType vt, 4424 SDNode OpNode, PatFrag ld_frag, bit Is2Addr = 1> { 4425 def rr : PDI<opc, MRMSrcReg, 4426 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2), 4427 !if(Is2Addr, 4428 !strconcat(OpcodeStr,"\t{$src2, $dst|$dst, $src2}"), 4429 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4430 [(set VR128:$dst, (vt (OpNode VR128:$src1, VR128:$src2)))], 4431 IIC_SSE_UNPCK>, Sched<[WriteShuffle]>; 4432 def rm : PDI<opc, MRMSrcMem, 4433 (outs VR128:$dst), (ins VR128:$src1, i128mem:$src2), 4434 !if(Is2Addr, 4435 !strconcat(OpcodeStr,"\t{$src2, $dst|$dst, $src2}"), 4436 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 4437 [(set VR128:$dst, (vt (OpNode VR128:$src1, 4438 (bitconvert (ld_frag addr:$src2)))))], 4439 IIC_SSE_UNPCK>, 4440 Sched<[WriteShuffleLd, ReadAfterLd]>; 4441} 4442 4443multiclass sse2_unpack_y<bits<8> opc, string OpcodeStr, ValueType vt, 4444 SDNode OpNode> { 4445 def Yrr : PDI<opc, MRMSrcReg, 4446 (outs VR256:$dst), (ins VR256:$src1, VR256:$src2), 4447 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4448 [(set VR256:$dst, (vt (OpNode VR256:$src1, VR256:$src2)))]>, 4449 Sched<[WriteShuffle]>; 4450 def Yrm : PDI<opc, MRMSrcMem, 4451 (outs VR256:$dst), (ins VR256:$src1, i256mem:$src2), 4452 !strconcat(OpcodeStr,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 4453 [(set VR256:$dst, (vt (OpNode VR256:$src1, 4454 (bitconvert (loadv4i64 addr:$src2)))))]>, 4455 Sched<[WriteShuffleLd, ReadAfterLd]>; 4456} 4457 4458 4459let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 4460 defm VPUNPCKLBW : sse2_unpack<0x60, "vpunpcklbw", v16i8, X86Unpckl, 4461 loadv2i64, 0>, VEX_4V; 4462 defm VPUNPCKLWD : sse2_unpack<0x61, "vpunpcklwd", v8i16, X86Unpckl, 4463 loadv2i64, 0>, VEX_4V; 4464 defm VPUNPCKHBW : sse2_unpack<0x68, "vpunpckhbw", v16i8, X86Unpckh, 4465 loadv2i64, 0>, VEX_4V; 4466 defm VPUNPCKHWD : sse2_unpack<0x69, "vpunpckhwd", v8i16, X86Unpckh, 4467 loadv2i64, 0>, VEX_4V; 4468} 4469let Predicates = [HasAVX, NoVLX] in { 4470 defm VPUNPCKLDQ : sse2_unpack<0x62, "vpunpckldq", v4i32, X86Unpckl, 4471 loadv2i64, 0>, VEX_4V; 4472 defm VPUNPCKLQDQ : sse2_unpack<0x6C, "vpunpcklqdq", v2i64, X86Unpckl, 4473 loadv2i64, 0>, VEX_4V; 4474 defm VPUNPCKHDQ : sse2_unpack<0x6A, "vpunpckhdq", v4i32, X86Unpckh, 4475 loadv2i64, 0>, VEX_4V; 4476 defm VPUNPCKHQDQ : sse2_unpack<0x6D, "vpunpckhqdq", v2i64, X86Unpckh, 4477 loadv2i64, 0>, VEX_4V; 4478} 4479 4480let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in { 4481 defm VPUNPCKLBW : sse2_unpack_y<0x60, "vpunpcklbw", v32i8, X86Unpckl>, 4482 VEX_4V, VEX_L; 4483 defm VPUNPCKLWD : sse2_unpack_y<0x61, "vpunpcklwd", v16i16, X86Unpckl>, 4484 VEX_4V, VEX_L; 4485 defm VPUNPCKHBW : sse2_unpack_y<0x68, "vpunpckhbw", v32i8, X86Unpckh>, 4486 VEX_4V, VEX_L; 4487 defm VPUNPCKHWD : sse2_unpack_y<0x69, "vpunpckhwd", v16i16, X86Unpckh>, 4488 VEX_4V, VEX_L; 4489} 4490let Predicates = [HasAVX2, NoVLX] in { 4491 defm VPUNPCKLDQ : sse2_unpack_y<0x62, "vpunpckldq", v8i32, X86Unpckl>, 4492 VEX_4V, VEX_L; 4493 defm VPUNPCKLQDQ : sse2_unpack_y<0x6C, "vpunpcklqdq", v4i64, X86Unpckl>, 4494 VEX_4V, VEX_L; 4495 defm VPUNPCKHDQ : sse2_unpack_y<0x6A, "vpunpckhdq", v8i32, X86Unpckh>, 4496 VEX_4V, VEX_L; 4497 defm VPUNPCKHQDQ : sse2_unpack_y<0x6D, "vpunpckhqdq", v4i64, X86Unpckh>, 4498 VEX_4V, VEX_L; 4499} 4500 4501let Constraints = "$src1 = $dst" in { 4502 defm PUNPCKLBW : sse2_unpack<0x60, "punpcklbw", v16i8, X86Unpckl, 4503 memopv2i64>; 4504 defm PUNPCKLWD : sse2_unpack<0x61, "punpcklwd", v8i16, X86Unpckl, 4505 memopv2i64>; 4506 defm PUNPCKLDQ : sse2_unpack<0x62, "punpckldq", v4i32, X86Unpckl, 4507 memopv2i64>; 4508 defm PUNPCKLQDQ : sse2_unpack<0x6C, "punpcklqdq", v2i64, X86Unpckl, 4509 memopv2i64>; 4510 4511 defm PUNPCKHBW : sse2_unpack<0x68, "punpckhbw", v16i8, X86Unpckh, 4512 memopv2i64>; 4513 defm PUNPCKHWD : sse2_unpack<0x69, "punpckhwd", v8i16, X86Unpckh, 4514 memopv2i64>; 4515 defm PUNPCKHDQ : sse2_unpack<0x6A, "punpckhdq", v4i32, X86Unpckh, 4516 memopv2i64>; 4517 defm PUNPCKHQDQ : sse2_unpack<0x6D, "punpckhqdq", v2i64, X86Unpckh, 4518 memopv2i64>; 4519} 4520} // ExeDomain = SSEPackedInt 4521 4522//===---------------------------------------------------------------------===// 4523// SSE2 - Packed Integer Extract and Insert 4524//===---------------------------------------------------------------------===// 4525 4526let ExeDomain = SSEPackedInt in { 4527multiclass sse2_pinsrw<bit Is2Addr = 1> { 4528 def rri : Ii8<0xC4, MRMSrcReg, 4529 (outs VR128:$dst), (ins VR128:$src1, 4530 GR32orGR64:$src2, u8imm:$src3), 4531 !if(Is2Addr, 4532 "pinsrw\t{$src3, $src2, $dst|$dst, $src2, $src3}", 4533 "vpinsrw\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 4534 [(set VR128:$dst, 4535 (X86pinsrw VR128:$src1, GR32orGR64:$src2, imm:$src3))], 4536 IIC_SSE_PINSRW>, Sched<[WriteShuffle]>; 4537 def rmi : Ii8<0xC4, MRMSrcMem, 4538 (outs VR128:$dst), (ins VR128:$src1, 4539 i16mem:$src2, u8imm:$src3), 4540 !if(Is2Addr, 4541 "pinsrw\t{$src3, $src2, $dst|$dst, $src2, $src3}", 4542 "vpinsrw\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 4543 [(set VR128:$dst, 4544 (X86pinsrw VR128:$src1, (extloadi16 addr:$src2), 4545 imm:$src3))], IIC_SSE_PINSRW>, 4546 Sched<[WriteShuffleLd, ReadAfterLd]>; 4547} 4548 4549// Extract 4550let Predicates = [HasAVX, NoBWI] in 4551def VPEXTRWri : Ii8<0xC5, MRMSrcReg, 4552 (outs GR32orGR64:$dst), (ins VR128:$src1, u8imm:$src2), 4553 "vpextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4554 [(set GR32orGR64:$dst, (X86pextrw (v8i16 VR128:$src1), 4555 imm:$src2))]>, PD, VEX, 4556 Sched<[WriteShuffle]>; 4557def PEXTRWri : PDIi8<0xC5, MRMSrcReg, 4558 (outs GR32orGR64:$dst), (ins VR128:$src1, u8imm:$src2), 4559 "pextrw\t{$src2, $src1, $dst|$dst, $src1, $src2}", 4560 [(set GR32orGR64:$dst, (X86pextrw (v8i16 VR128:$src1), 4561 imm:$src2))], IIC_SSE_PEXTRW>, 4562 Sched<[WriteShuffleLd, ReadAfterLd]>; 4563 4564// Insert 4565let Predicates = [HasAVX, NoBWI] in 4566defm VPINSRW : sse2_pinsrw<0>, PD, VEX_4V; 4567 4568let Predicates = [UseSSE2], Constraints = "$src1 = $dst" in 4569defm PINSRW : sse2_pinsrw, PD; 4570 4571} // ExeDomain = SSEPackedInt 4572 4573//===---------------------------------------------------------------------===// 4574// SSE2 - Packed Mask Creation 4575//===---------------------------------------------------------------------===// 4576 4577let ExeDomain = SSEPackedInt, SchedRW = [WriteVecLogic] in { 4578 4579def VPMOVMSKBrr : VPDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst), 4580 (ins VR128:$src), 4581 "pmovmskb\t{$src, $dst|$dst, $src}", 4582 [(set GR32orGR64:$dst, (X86movmsk (v16i8 VR128:$src)))], 4583 IIC_SSE_MOVMSK>, VEX; 4584 4585let Predicates = [HasAVX2] in { 4586def VPMOVMSKBYrr : VPDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst), 4587 (ins VR256:$src), 4588 "pmovmskb\t{$src, $dst|$dst, $src}", 4589 [(set GR32orGR64:$dst, (X86movmsk (v32i8 VR256:$src)))]>, 4590 VEX, VEX_L; 4591} 4592 4593def PMOVMSKBrr : PDI<0xD7, MRMSrcReg, (outs GR32orGR64:$dst), (ins VR128:$src), 4594 "pmovmskb\t{$src, $dst|$dst, $src}", 4595 [(set GR32orGR64:$dst, (X86movmsk (v16i8 VR128:$src)))], 4596 IIC_SSE_MOVMSK>; 4597 4598} // ExeDomain = SSEPackedInt 4599 4600//===---------------------------------------------------------------------===// 4601// SSE2 - Conditional Store 4602//===---------------------------------------------------------------------===// 4603 4604let ExeDomain = SSEPackedInt, SchedRW = [WriteStore] in { 4605 4606let Uses = [EDI], Predicates = [HasAVX,Not64BitMode] in 4607def VMASKMOVDQU : VPDI<0xF7, MRMSrcReg, (outs), 4608 (ins VR128:$src, VR128:$mask), 4609 "maskmovdqu\t{$mask, $src|$src, $mask}", 4610 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, EDI)], 4611 IIC_SSE_MASKMOV>, VEX; 4612let Uses = [RDI], Predicates = [HasAVX,In64BitMode] in 4613def VMASKMOVDQU64 : VPDI<0xF7, MRMSrcReg, (outs), 4614 (ins VR128:$src, VR128:$mask), 4615 "maskmovdqu\t{$mask, $src|$src, $mask}", 4616 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, RDI)], 4617 IIC_SSE_MASKMOV>, VEX; 4618 4619let Uses = [EDI], Predicates = [UseSSE2,Not64BitMode] in 4620def MASKMOVDQU : PDI<0xF7, MRMSrcReg, (outs), (ins VR128:$src, VR128:$mask), 4621 "maskmovdqu\t{$mask, $src|$src, $mask}", 4622 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, EDI)], 4623 IIC_SSE_MASKMOV>; 4624let Uses = [RDI], Predicates = [UseSSE2,In64BitMode] in 4625def MASKMOVDQU64 : PDI<0xF7, MRMSrcReg, (outs), (ins VR128:$src, VR128:$mask), 4626 "maskmovdqu\t{$mask, $src|$src, $mask}", 4627 [(int_x86_sse2_maskmov_dqu VR128:$src, VR128:$mask, RDI)], 4628 IIC_SSE_MASKMOV>; 4629 4630} // ExeDomain = SSEPackedInt 4631 4632//===---------------------------------------------------------------------===// 4633// SSE2 - Move Doubleword/Quadword 4634//===---------------------------------------------------------------------===// 4635 4636//===---------------------------------------------------------------------===// 4637// Move Int Doubleword to Packed Double Int 4638// 4639def VMOVDI2PDIrr : VS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR32:$src), 4640 "movd\t{$src, $dst|$dst, $src}", 4641 [(set VR128:$dst, 4642 (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>, 4643 VEX, Sched<[WriteMove]>; 4644def VMOVDI2PDIrm : VS2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i32mem:$src), 4645 "movd\t{$src, $dst|$dst, $src}", 4646 [(set VR128:$dst, 4647 (v4i32 (scalar_to_vector (loadi32 addr:$src))))], 4648 IIC_SSE_MOVDQ>, 4649 VEX, Sched<[WriteLoad]>; 4650def VMOV64toPQIrr : VRS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR64:$src), 4651 "movq\t{$src, $dst|$dst, $src}", 4652 [(set VR128:$dst, 4653 (v2i64 (scalar_to_vector GR64:$src)))], 4654 IIC_SSE_MOVDQ>, VEX, Sched<[WriteMove]>; 4655let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in 4656def VMOV64toPQIrm : VRS2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4657 "movq\t{$src, $dst|$dst, $src}", 4658 [], IIC_SSE_MOVDQ>, VEX, Sched<[WriteLoad]>; 4659let isCodeGenOnly = 1 in 4660def VMOV64toSDrr : VRS2I<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src), 4661 "movq\t{$src, $dst|$dst, $src}", 4662 [(set FR64:$dst, (bitconvert GR64:$src))], 4663 IIC_SSE_MOVDQ>, VEX, Sched<[WriteMove]>; 4664 4665def MOVDI2PDIrr : S2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR32:$src), 4666 "movd\t{$src, $dst|$dst, $src}", 4667 [(set VR128:$dst, 4668 (v4i32 (scalar_to_vector GR32:$src)))], IIC_SSE_MOVDQ>, 4669 Sched<[WriteMove]>; 4670def MOVDI2PDIrm : S2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i32mem:$src), 4671 "movd\t{$src, $dst|$dst, $src}", 4672 [(set VR128:$dst, 4673 (v4i32 (scalar_to_vector (loadi32 addr:$src))))], 4674 IIC_SSE_MOVDQ>, Sched<[WriteLoad]>; 4675def MOV64toPQIrr : RS2I<0x6E, MRMSrcReg, (outs VR128:$dst), (ins GR64:$src), 4676 "mov{d|q}\t{$src, $dst|$dst, $src}", 4677 [(set VR128:$dst, 4678 (v2i64 (scalar_to_vector GR64:$src)))], 4679 IIC_SSE_MOVDQ>, Sched<[WriteMove]>; 4680let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in 4681def MOV64toPQIrm : RS2I<0x6E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4682 "mov{d|q}\t{$src, $dst|$dst, $src}", 4683 [], IIC_SSE_MOVDQ>, Sched<[WriteLoad]>; 4684let isCodeGenOnly = 1 in 4685def MOV64toSDrr : RS2I<0x6E, MRMSrcReg, (outs FR64:$dst), (ins GR64:$src), 4686 "mov{d|q}\t{$src, $dst|$dst, $src}", 4687 [(set FR64:$dst, (bitconvert GR64:$src))], 4688 IIC_SSE_MOVDQ>, Sched<[WriteMove]>; 4689 4690//===---------------------------------------------------------------------===// 4691// Move Int Doubleword to Single Scalar 4692// 4693let isCodeGenOnly = 1 in { 4694 def VMOVDI2SSrr : VS2I<0x6E, MRMSrcReg, (outs FR32:$dst), (ins GR32:$src), 4695 "movd\t{$src, $dst|$dst, $src}", 4696 [(set FR32:$dst, (bitconvert GR32:$src))], 4697 IIC_SSE_MOVDQ>, VEX, Sched<[WriteMove]>; 4698 4699 def VMOVDI2SSrm : VS2I<0x6E, MRMSrcMem, (outs FR32:$dst), (ins i32mem:$src), 4700 "movd\t{$src, $dst|$dst, $src}", 4701 [(set FR32:$dst, (bitconvert (loadi32 addr:$src)))], 4702 IIC_SSE_MOVDQ>, 4703 VEX, Sched<[WriteLoad]>; 4704 def MOVDI2SSrr : S2I<0x6E, MRMSrcReg, (outs FR32:$dst), (ins GR32:$src), 4705 "movd\t{$src, $dst|$dst, $src}", 4706 [(set FR32:$dst, (bitconvert GR32:$src))], 4707 IIC_SSE_MOVDQ>, Sched<[WriteMove]>; 4708 4709 def MOVDI2SSrm : S2I<0x6E, MRMSrcMem, (outs FR32:$dst), (ins i32mem:$src), 4710 "movd\t{$src, $dst|$dst, $src}", 4711 [(set FR32:$dst, (bitconvert (loadi32 addr:$src)))], 4712 IIC_SSE_MOVDQ>, Sched<[WriteLoad]>; 4713} 4714 4715//===---------------------------------------------------------------------===// 4716// Move Packed Doubleword Int to Packed Double Int 4717// 4718def VMOVPDI2DIrr : VS2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src), 4719 "movd\t{$src, $dst|$dst, $src}", 4720 [(set GR32:$dst, (extractelt (v4i32 VR128:$src), 4721 (iPTR 0)))], IIC_SSE_MOVD_ToGP>, VEX, 4722 Sched<[WriteMove]>; 4723def VMOVPDI2DImr : VS2I<0x7E, MRMDestMem, (outs), 4724 (ins i32mem:$dst, VR128:$src), 4725 "movd\t{$src, $dst|$dst, $src}", 4726 [(store (i32 (extractelt (v4i32 VR128:$src), 4727 (iPTR 0))), addr:$dst)], IIC_SSE_MOVDQ>, 4728 VEX, Sched<[WriteStore]>; 4729def MOVPDI2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128:$src), 4730 "movd\t{$src, $dst|$dst, $src}", 4731 [(set GR32:$dst, (extractelt (v4i32 VR128:$src), 4732 (iPTR 0)))], IIC_SSE_MOVD_ToGP>, 4733 Sched<[WriteMove]>; 4734def MOVPDI2DImr : S2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, VR128:$src), 4735 "movd\t{$src, $dst|$dst, $src}", 4736 [(store (i32 (extractelt (v4i32 VR128:$src), 4737 (iPTR 0))), addr:$dst)], 4738 IIC_SSE_MOVDQ>, Sched<[WriteStore]>; 4739 4740def : Pat<(v8i32 (X86Vinsert (v8i32 immAllZerosV), GR32:$src2, (iPTR 0))), 4741 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIrr GR32:$src2), sub_xmm)>; 4742 4743def : Pat<(v4i64 (X86Vinsert (bc_v4i64 (v8i32 immAllZerosV)), GR64:$src2, (iPTR 0))), 4744 (SUBREG_TO_REG (i32 0), (VMOV64toPQIrr GR64:$src2), sub_xmm)>; 4745 4746def : Pat<(v8i32 (X86Vinsert undef, GR32:$src2, (iPTR 0))), 4747 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIrr GR32:$src2), sub_xmm)>; 4748 4749def : Pat<(v4i64 (X86Vinsert undef, GR64:$src2, (iPTR 0))), 4750 (SUBREG_TO_REG (i32 0), (VMOV64toPQIrr GR64:$src2), sub_xmm)>; 4751 4752//===---------------------------------------------------------------------===// 4753// Move Packed Doubleword Int first element to Doubleword Int 4754// 4755let SchedRW = [WriteMove] in { 4756def VMOVPQIto64rr : VRS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src), 4757 "movq\t{$src, $dst|$dst, $src}", 4758 [(set GR64:$dst, (extractelt (v2i64 VR128:$src), 4759 (iPTR 0)))], 4760 IIC_SSE_MOVD_ToGP>, 4761 VEX; 4762 4763def MOVPQIto64rr : RS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128:$src), 4764 "mov{d|q}\t{$src, $dst|$dst, $src}", 4765 [(set GR64:$dst, (extractelt (v2i64 VR128:$src), 4766 (iPTR 0)))], 4767 IIC_SSE_MOVD_ToGP>; 4768} //SchedRW 4769 4770let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in 4771def VMOVPQIto64rm : VRS2I<0x7E, MRMDestMem, (outs), 4772 (ins i64mem:$dst, VR128:$src), 4773 "movq\t{$src, $dst|$dst, $src}", 4774 [], IIC_SSE_MOVDQ>, VEX, Sched<[WriteStore]>; 4775let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in 4776def MOVPQIto64rm : RS2I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), 4777 "mov{d|q}\t{$src, $dst|$dst, $src}", 4778 [], IIC_SSE_MOVDQ>, Sched<[WriteStore]>; 4779 4780//===---------------------------------------------------------------------===// 4781// Bitcast FR64 <-> GR64 4782// 4783let isCodeGenOnly = 1 in { 4784 let Predicates = [UseAVX] in 4785 def VMOV64toSDrm : VS2SI<0x7E, MRMSrcMem, (outs FR64:$dst), (ins i64mem:$src), 4786 "movq\t{$src, $dst|$dst, $src}", 4787 [(set FR64:$dst, (bitconvert (loadi64 addr:$src)))]>, 4788 VEX, Sched<[WriteLoad]>; 4789 def VMOVSDto64rr : VRS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src), 4790 "movq\t{$src, $dst|$dst, $src}", 4791 [(set GR64:$dst, (bitconvert FR64:$src))], 4792 IIC_SSE_MOVDQ>, VEX, Sched<[WriteMove]>; 4793 def VMOVSDto64mr : VRS2I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src), 4794 "movq\t{$src, $dst|$dst, $src}", 4795 [(store (i64 (bitconvert FR64:$src)), addr:$dst)], 4796 IIC_SSE_MOVDQ>, VEX, Sched<[WriteStore]>; 4797 4798 def MOV64toSDrm : S2SI<0x7E, MRMSrcMem, (outs FR64:$dst), (ins i64mem:$src), 4799 "movq\t{$src, $dst|$dst, $src}", 4800 [(set FR64:$dst, (bitconvert (loadi64 addr:$src)))], 4801 IIC_SSE_MOVDQ>, Sched<[WriteLoad]>; 4802 def MOVSDto64rr : RS2I<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64:$src), 4803 "mov{d|q}\t{$src, $dst|$dst, $src}", 4804 [(set GR64:$dst, (bitconvert FR64:$src))], 4805 IIC_SSE_MOVD_ToGP>, Sched<[WriteMove]>; 4806 def MOVSDto64mr : RS2I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64:$src), 4807 "movq\t{$src, $dst|$dst, $src}", 4808 [(store (i64 (bitconvert FR64:$src)), addr:$dst)], 4809 IIC_SSE_MOVDQ>, Sched<[WriteStore]>; 4810} 4811 4812//===---------------------------------------------------------------------===// 4813// Move Scalar Single to Double Int 4814// 4815let isCodeGenOnly = 1 in { 4816 def VMOVSS2DIrr : VS2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins FR32:$src), 4817 "movd\t{$src, $dst|$dst, $src}", 4818 [(set GR32:$dst, (bitconvert FR32:$src))], 4819 IIC_SSE_MOVD_ToGP>, VEX, Sched<[WriteMove]>; 4820 def VMOVSS2DImr : VS2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, FR32:$src), 4821 "movd\t{$src, $dst|$dst, $src}", 4822 [(store (i32 (bitconvert FR32:$src)), addr:$dst)], 4823 IIC_SSE_MOVDQ>, VEX, Sched<[WriteStore]>; 4824 def MOVSS2DIrr : S2I<0x7E, MRMDestReg, (outs GR32:$dst), (ins FR32:$src), 4825 "movd\t{$src, $dst|$dst, $src}", 4826 [(set GR32:$dst, (bitconvert FR32:$src))], 4827 IIC_SSE_MOVD_ToGP>, Sched<[WriteMove]>; 4828 def MOVSS2DImr : S2I<0x7E, MRMDestMem, (outs), (ins i32mem:$dst, FR32:$src), 4829 "movd\t{$src, $dst|$dst, $src}", 4830 [(store (i32 (bitconvert FR32:$src)), addr:$dst)], 4831 IIC_SSE_MOVDQ>, Sched<[WriteStore]>; 4832} 4833 4834let Predicates = [UseAVX] in { 4835 let AddedComplexity = 15 in { 4836 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))), 4837 (VMOVDI2PDIrr GR32:$src)>; 4838 4839 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))), 4840 (VMOV64toPQIrr GR64:$src)>; 4841 4842 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, 4843 (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))), 4844 (SUBREG_TO_REG (i64 0), (VMOV64toPQIrr GR64:$src), sub_xmm)>; 4845 } 4846 // AVX 128-bit movd/movq instructions write zeros in the high 128-bit part. 4847 // These instructions also write zeros in the high part of a 256-bit register. 4848 let AddedComplexity = 20 in { 4849 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))), 4850 (VMOVDI2PDIrm addr:$src)>; 4851 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))), 4852 (VMOVDI2PDIrm addr:$src)>; 4853 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))), 4854 (VMOVDI2PDIrm addr:$src)>; 4855 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, 4856 (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))), 4857 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIrm addr:$src), sub_xmm)>; 4858 } 4859 // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext. 4860 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef, 4861 (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))), 4862 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIrr GR32:$src), sub_xmm)>; 4863} 4864 4865let Predicates = [UseSSE2] in { 4866 let AddedComplexity = 15 in { 4867 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))), 4868 (MOVDI2PDIrr GR32:$src)>; 4869 4870 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))), 4871 (MOV64toPQIrr GR64:$src)>; 4872 } 4873 let AddedComplexity = 20 in { 4874 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))), 4875 (MOVDI2PDIrm addr:$src)>; 4876 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))), 4877 (MOVDI2PDIrm addr:$src)>; 4878 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))), 4879 (MOVDI2PDIrm addr:$src)>; 4880 } 4881} 4882 4883// These are the correct encodings of the instructions so that we know how to 4884// read correct assembly, even though we continue to emit the wrong ones for 4885// compatibility with Darwin's buggy assembler. 4886def : InstAlias<"movq\t{$src, $dst|$dst, $src}", 4887 (MOV64toPQIrr VR128:$dst, GR64:$src), 0>; 4888def : InstAlias<"movq\t{$src, $dst|$dst, $src}", 4889 (MOVPQIto64rr GR64:$dst, VR128:$src), 0>; 4890// Allow "vmovd" but print "vmovq" since we don't need compatibility for AVX. 4891def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}", 4892 (VMOV64toPQIrr VR128:$dst, GR64:$src), 0>; 4893def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}", 4894 (VMOVPQIto64rr GR64:$dst, VR128:$src), 0>; 4895 4896//===---------------------------------------------------------------------===// 4897// SSE2 - Move Quadword 4898//===---------------------------------------------------------------------===// 4899 4900//===---------------------------------------------------------------------===// 4901// Move Quadword Int to Packed Quadword Int 4902// 4903 4904let ExeDomain = SSEPackedInt, SchedRW = [WriteLoad] in { 4905def VMOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4906 "vmovq\t{$src, $dst|$dst, $src}", 4907 [(set VR128:$dst, 4908 (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>, XS, 4909 VEX, Requires<[UseAVX]>; 4910def MOVQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4911 "movq\t{$src, $dst|$dst, $src}", 4912 [(set VR128:$dst, 4913 (v2i64 (scalar_to_vector (loadi64 addr:$src))))], 4914 IIC_SSE_MOVDQ>, XS, 4915 Requires<[UseSSE2]>; // SSE2 instruction with XS Prefix 4916} // ExeDomain, SchedRW 4917 4918//===---------------------------------------------------------------------===// 4919// Move Packed Quadword Int to Quadword Int 4920// 4921let ExeDomain = SSEPackedInt, SchedRW = [WriteStore] in { 4922def VMOVPQI2QImr : VS2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), 4923 "movq\t{$src, $dst|$dst, $src}", 4924 [(store (i64 (extractelt (v2i64 VR128:$src), 4925 (iPTR 0))), addr:$dst)], 4926 IIC_SSE_MOVDQ>, VEX; 4927def MOVPQI2QImr : S2I<0xD6, MRMDestMem, (outs), (ins i64mem:$dst, VR128:$src), 4928 "movq\t{$src, $dst|$dst, $src}", 4929 [(store (i64 (extractelt (v2i64 VR128:$src), 4930 (iPTR 0))), addr:$dst)], 4931 IIC_SSE_MOVDQ>; 4932} // ExeDomain, SchedRW 4933 4934// For disassembler only 4935let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, 4936 SchedRW = [WriteVecLogic] in { 4937def VMOVPQI2QIrr : VS2I<0xD6, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 4938 "movq\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVQ_RR>, VEX; 4939def MOVPQI2QIrr : S2I<0xD6, MRMDestReg, (outs VR128:$dst), (ins VR128:$src), 4940 "movq\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVQ_RR>; 4941} 4942 4943// Aliases to help the assembler pick two byte VEX encodings by swapping the 4944// operands relative to the normal instructions to use VEX.R instead of VEX.B. 4945def : InstAlias<"vmovq\t{$src, $dst|$dst, $src}", 4946 (VMOVPQI2QIrr VR128L:$dst, VR128H:$src), 0>; 4947 4948//===---------------------------------------------------------------------===// 4949// Store / copy lower 64-bits of a XMM register. 4950// 4951let ExeDomain = SSEPackedInt, isCodeGenOnly = 1, AddedComplexity = 20 in { 4952def VMOVZQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4953 "vmovq\t{$src, $dst|$dst, $src}", 4954 [(set VR128:$dst, 4955 (v2i64 (X86vzmovl (v2i64 (scalar_to_vector 4956 (loadi64 addr:$src))))))], 4957 IIC_SSE_MOVDQ>, 4958 XS, VEX, Requires<[UseAVX]>, Sched<[WriteLoad]>; 4959 4960def MOVZQI2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i64mem:$src), 4961 "movq\t{$src, $dst|$dst, $src}", 4962 [(set VR128:$dst, 4963 (v2i64 (X86vzmovl (v2i64 (scalar_to_vector 4964 (loadi64 addr:$src))))))], 4965 IIC_SSE_MOVDQ>, 4966 XS, Requires<[UseSSE2]>, Sched<[WriteLoad]>; 4967} // ExeDomain, isCodeGenOnly, AddedComplexity 4968 4969let Predicates = [UseAVX], AddedComplexity = 20 in { 4970 def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4f32 addr:$src)))), 4971 (VMOVZQI2PQIrm addr:$src)>; 4972 def : Pat<(v2i64 (X86vzload addr:$src)), 4973 (VMOVZQI2PQIrm addr:$src)>; 4974 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef, 4975 (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))), 4976 (SUBREG_TO_REG (i64 0), (VMOVZQI2PQIrm addr:$src), sub_xmm)>; 4977 def : Pat<(v4i64 (X86vzload addr:$src)), 4978 (SUBREG_TO_REG (i64 0), (VMOVZQI2PQIrm addr:$src), sub_xmm)>; 4979} 4980 4981let Predicates = [UseSSE2], AddedComplexity = 20 in { 4982 def : Pat<(v2i64 (X86vzmovl (bc_v2i64 (loadv4f32 addr:$src)))), 4983 (MOVZQI2PQIrm addr:$src)>; 4984 def : Pat<(v2i64 (X86vzload addr:$src)), (MOVZQI2PQIrm addr:$src)>; 4985} 4986 4987//===---------------------------------------------------------------------===// 4988// Moving from XMM to XMM and clear upper 64 bits. Note, there is a bug in 4989// IA32 document. movq xmm1, xmm2 does clear the high bits. 4990// 4991let ExeDomain = SSEPackedInt, SchedRW = [WriteVecLogic] in { 4992let AddedComplexity = 15 in 4993def VMOVZPQILo2PQIrr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 4994 "vmovq\t{$src, $dst|$dst, $src}", 4995 [(set VR128:$dst, (v2i64 (X86vzmovl (v2i64 VR128:$src))))], 4996 IIC_SSE_MOVQ_RR>, 4997 XS, VEX, Requires<[UseAVX]>; 4998let AddedComplexity = 15 in 4999def MOVZPQILo2PQIrr : I<0x7E, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 5000 "movq\t{$src, $dst|$dst, $src}", 5001 [(set VR128:$dst, (v2i64 (X86vzmovl (v2i64 VR128:$src))))], 5002 IIC_SSE_MOVQ_RR>, 5003 XS, Requires<[UseSSE2]>; 5004} // ExeDomain, SchedRW 5005 5006let ExeDomain = SSEPackedInt, isCodeGenOnly = 1, SchedRW = [WriteVecLogicLd] in { 5007let AddedComplexity = 20 in 5008def VMOVZPQILo2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 5009 "vmovq\t{$src, $dst|$dst, $src}", 5010 [(set VR128:$dst, (v2i64 (X86vzmovl 5011 (loadv2i64 addr:$src))))], 5012 IIC_SSE_MOVDQ>, 5013 XS, VEX, Requires<[UseAVX]>; 5014let AddedComplexity = 20 in { 5015def MOVZPQILo2PQIrm : I<0x7E, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 5016 "movq\t{$src, $dst|$dst, $src}", 5017 [(set VR128:$dst, (v2i64 (X86vzmovl 5018 (loadv2i64 addr:$src))))], 5019 IIC_SSE_MOVDQ>, 5020 XS, Requires<[UseSSE2]>; 5021} 5022} // ExeDomain, isCodeGenOnly, SchedRW 5023 5024let AddedComplexity = 20 in { 5025 let Predicates = [UseAVX] in { 5026 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))), 5027 (VMOVZPQILo2PQIrr VR128:$src)>; 5028 } 5029 let Predicates = [UseSSE2] in { 5030 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128:$src))), 5031 (MOVZPQILo2PQIrr VR128:$src)>; 5032 } 5033} 5034 5035//===---------------------------------------------------------------------===// 5036// SSE3 - Replicate Single FP - MOVSHDUP and MOVSLDUP 5037//===---------------------------------------------------------------------===// 5038multiclass sse3_replicate_sfp<bits<8> op, SDNode OpNode, string OpcodeStr, 5039 ValueType vt, RegisterClass RC, PatFrag mem_frag, 5040 X86MemOperand x86memop> { 5041def rr : S3SI<op, MRMSrcReg, (outs RC:$dst), (ins RC:$src), 5042 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5043 [(set RC:$dst, (vt (OpNode RC:$src)))], 5044 IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; 5045def rm : S3SI<op, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 5046 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5047 [(set RC:$dst, (OpNode (mem_frag addr:$src)))], 5048 IIC_SSE_MOV_LH>, Sched<[WriteLoad]>; 5049} 5050 5051let Predicates = [HasAVX, NoVLX] in { 5052 defm VMOVSHDUP : sse3_replicate_sfp<0x16, X86Movshdup, "vmovshdup", 5053 v4f32, VR128, loadv4f32, f128mem>, VEX; 5054 defm VMOVSLDUP : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup", 5055 v4f32, VR128, loadv4f32, f128mem>, VEX; 5056 defm VMOVSHDUPY : sse3_replicate_sfp<0x16, X86Movshdup, "vmovshdup", 5057 v8f32, VR256, loadv8f32, f256mem>, VEX, VEX_L; 5058 defm VMOVSLDUPY : sse3_replicate_sfp<0x12, X86Movsldup, "vmovsldup", 5059 v8f32, VR256, loadv8f32, f256mem>, VEX, VEX_L; 5060} 5061defm MOVSHDUP : sse3_replicate_sfp<0x16, X86Movshdup, "movshdup", v4f32, VR128, 5062 memopv4f32, f128mem>; 5063defm MOVSLDUP : sse3_replicate_sfp<0x12, X86Movsldup, "movsldup", v4f32, VR128, 5064 memopv4f32, f128mem>; 5065 5066let Predicates = [HasAVX, NoVLX] in { 5067 def : Pat<(v4i32 (X86Movshdup VR128:$src)), 5068 (VMOVSHDUPrr VR128:$src)>; 5069 def : Pat<(v4i32 (X86Movshdup (bc_v4i32 (loadv2i64 addr:$src)))), 5070 (VMOVSHDUPrm addr:$src)>; 5071 def : Pat<(v4i32 (X86Movsldup VR128:$src)), 5072 (VMOVSLDUPrr VR128:$src)>; 5073 def : Pat<(v4i32 (X86Movsldup (bc_v4i32 (loadv2i64 addr:$src)))), 5074 (VMOVSLDUPrm addr:$src)>; 5075 def : Pat<(v8i32 (X86Movshdup VR256:$src)), 5076 (VMOVSHDUPYrr VR256:$src)>; 5077 def : Pat<(v8i32 (X86Movshdup (bc_v8i32 (loadv4i64 addr:$src)))), 5078 (VMOVSHDUPYrm addr:$src)>; 5079 def : Pat<(v8i32 (X86Movsldup VR256:$src)), 5080 (VMOVSLDUPYrr VR256:$src)>; 5081 def : Pat<(v8i32 (X86Movsldup (bc_v8i32 (loadv4i64 addr:$src)))), 5082 (VMOVSLDUPYrm addr:$src)>; 5083} 5084 5085let Predicates = [UseSSE3] in { 5086 def : Pat<(v4i32 (X86Movshdup VR128:$src)), 5087 (MOVSHDUPrr VR128:$src)>; 5088 def : Pat<(v4i32 (X86Movshdup (bc_v4i32 (memopv2i64 addr:$src)))), 5089 (MOVSHDUPrm addr:$src)>; 5090 def : Pat<(v4i32 (X86Movsldup VR128:$src)), 5091 (MOVSLDUPrr VR128:$src)>; 5092 def : Pat<(v4i32 (X86Movsldup (bc_v4i32 (memopv2i64 addr:$src)))), 5093 (MOVSLDUPrm addr:$src)>; 5094} 5095 5096//===---------------------------------------------------------------------===// 5097// SSE3 - Replicate Double FP - MOVDDUP 5098//===---------------------------------------------------------------------===// 5099 5100multiclass sse3_replicate_dfp<string OpcodeStr> { 5101def rr : S3DI<0x12, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 5102 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5103 [(set VR128:$dst, (v2f64 (X86Movddup VR128:$src)))], 5104 IIC_SSE_MOV_LH>, Sched<[WriteFShuffle]>; 5105def rm : S3DI<0x12, MRMSrcMem, (outs VR128:$dst), (ins f64mem:$src), 5106 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5107 [(set VR128:$dst, 5108 (v2f64 (X86Movddup 5109 (scalar_to_vector (loadf64 addr:$src)))))], 5110 IIC_SSE_MOV_LH>, Sched<[WriteLoad]>; 5111} 5112 5113// FIXME: Merge with above classe when there're patterns for the ymm version 5114multiclass sse3_replicate_dfp_y<string OpcodeStr> { 5115def rr : S3DI<0x12, MRMSrcReg, (outs VR256:$dst), (ins VR256:$src), 5116 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5117 [(set VR256:$dst, (v4f64 (X86Movddup VR256:$src)))]>, 5118 Sched<[WriteFShuffle]>; 5119def rm : S3DI<0x12, MRMSrcMem, (outs VR256:$dst), (ins f256mem:$src), 5120 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5121 [(set VR256:$dst, 5122 (v4f64 (X86Movddup (loadv4f64 addr:$src))))]>, 5123 Sched<[WriteLoad]>; 5124} 5125 5126let Predicates = [HasAVX, NoVLX] in { 5127 defm VMOVDDUP : sse3_replicate_dfp<"vmovddup">, VEX; 5128 defm VMOVDDUPY : sse3_replicate_dfp_y<"vmovddup">, VEX, VEX_L; 5129} 5130 5131defm MOVDDUP : sse3_replicate_dfp<"movddup">; 5132 5133 5134let Predicates = [HasAVX, NoVLX] in { 5135 def : Pat<(X86Movddup (loadv2f64 addr:$src)), 5136 (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>; 5137 5138 // 256-bit version 5139 def : Pat<(X86Movddup (loadv4i64 addr:$src)), 5140 (VMOVDDUPYrm addr:$src)>; 5141 def : Pat<(X86Movddup (v4i64 VR256:$src)), 5142 (VMOVDDUPYrr VR256:$src)>; 5143} 5144 5145let Predicates = [HasAVX] in { 5146 def : Pat<(X86Movddup (bc_v2f64 (loadv4f32 addr:$src))), 5147 (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>; 5148 def : Pat<(X86Movddup (bc_v2f64 (loadv2i64 addr:$src))), 5149 (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>; 5150 def : Pat<(X86Movddup (bc_v2f64 5151 (v2i64 (scalar_to_vector (loadi64 addr:$src))))), 5152 (VMOVDDUPrm addr:$src)>, Requires<[HasAVX]>; 5153} 5154 5155let Predicates = [UseAVX, OptForSize] in { 5156 def : Pat<(v2f64 (X86VBroadcast (loadf64 addr:$src))), 5157 (VMOVDDUPrm addr:$src)>; 5158 def : Pat<(v2i64 (X86VBroadcast (loadi64 addr:$src))), 5159 (VMOVDDUPrm addr:$src)>; 5160} 5161 5162let Predicates = [UseSSE3] in { 5163 def : Pat<(X86Movddup (memopv2f64 addr:$src)), 5164 (MOVDDUPrm addr:$src)>; 5165 def : Pat<(X86Movddup (bc_v2f64 (memopv4f32 addr:$src))), 5166 (MOVDDUPrm addr:$src)>; 5167 def : Pat<(X86Movddup (bc_v2f64 (memopv2i64 addr:$src))), 5168 (MOVDDUPrm addr:$src)>; 5169 def : Pat<(X86Movddup (bc_v2f64 5170 (v2i64 (scalar_to_vector (loadi64 addr:$src))))), 5171 (MOVDDUPrm addr:$src)>; 5172} 5173 5174//===---------------------------------------------------------------------===// 5175// SSE3 - Move Unaligned Integer 5176//===---------------------------------------------------------------------===// 5177 5178let SchedRW = [WriteLoad] in { 5179let Predicates = [HasAVX] in { 5180 def VLDDQUrm : S3DI<0xF0, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 5181 "vlddqu\t{$src, $dst|$dst, $src}", 5182 [(set VR128:$dst, (int_x86_sse3_ldu_dq addr:$src))]>, VEX; 5183 def VLDDQUYrm : S3DI<0xF0, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), 5184 "vlddqu\t{$src, $dst|$dst, $src}", 5185 [(set VR256:$dst, (int_x86_avx_ldu_dq_256 addr:$src))]>, 5186 VEX, VEX_L; 5187} 5188def LDDQUrm : S3DI<0xF0, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 5189 "lddqu\t{$src, $dst|$dst, $src}", 5190 [(set VR128:$dst, (int_x86_sse3_ldu_dq addr:$src))], 5191 IIC_SSE_LDDQU>; 5192} 5193 5194//===---------------------------------------------------------------------===// 5195// SSE3 - Arithmetic 5196//===---------------------------------------------------------------------===// 5197 5198multiclass sse3_addsub<Intrinsic Int, string OpcodeStr, RegisterClass RC, 5199 X86MemOperand x86memop, OpndItins itins, 5200 PatFrag ld_frag, bit Is2Addr = 1> { 5201 def rr : I<0xD0, MRMSrcReg, 5202 (outs RC:$dst), (ins RC:$src1, RC:$src2), 5203 !if(Is2Addr, 5204 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5205 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5206 [(set RC:$dst, (Int RC:$src1, RC:$src2))], itins.rr>, 5207 Sched<[itins.Sched]>; 5208 def rm : I<0xD0, MRMSrcMem, 5209 (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 5210 !if(Is2Addr, 5211 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5212 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5213 [(set RC:$dst, (Int RC:$src1, (ld_frag addr:$src2)))], itins.rr>, 5214 Sched<[itins.Sched.Folded, ReadAfterLd]>; 5215} 5216 5217let Predicates = [HasAVX] in { 5218 let ExeDomain = SSEPackedSingle in { 5219 defm VADDSUBPS : sse3_addsub<int_x86_sse3_addsub_ps, "vaddsubps", VR128, 5220 f128mem, SSE_ALU_F32P, loadv4f32, 0>, XD, VEX_4V; 5221 defm VADDSUBPSY : sse3_addsub<int_x86_avx_addsub_ps_256, "vaddsubps", VR256, 5222 f256mem, SSE_ALU_F32P, loadv8f32, 0>, XD, VEX_4V, VEX_L; 5223 } 5224 let ExeDomain = SSEPackedDouble in { 5225 defm VADDSUBPD : sse3_addsub<int_x86_sse3_addsub_pd, "vaddsubpd", VR128, 5226 f128mem, SSE_ALU_F64P, loadv2f64, 0>, PD, VEX_4V; 5227 defm VADDSUBPDY : sse3_addsub<int_x86_avx_addsub_pd_256, "vaddsubpd", VR256, 5228 f256mem, SSE_ALU_F64P, loadv4f64, 0>, PD, VEX_4V, VEX_L; 5229 } 5230} 5231let Constraints = "$src1 = $dst", Predicates = [UseSSE3] in { 5232 let ExeDomain = SSEPackedSingle in 5233 defm ADDSUBPS : sse3_addsub<int_x86_sse3_addsub_ps, "addsubps", VR128, 5234 f128mem, SSE_ALU_F32P, memopv4f32>, XD; 5235 let ExeDomain = SSEPackedDouble in 5236 defm ADDSUBPD : sse3_addsub<int_x86_sse3_addsub_pd, "addsubpd", VR128, 5237 f128mem, SSE_ALU_F64P, memopv2f64>, PD; 5238} 5239 5240// Patterns used to select 'addsub' instructions. 5241let Predicates = [HasAVX] in { 5242 def : Pat<(v4f32 (X86Addsub (v4f32 VR128:$lhs), (v4f32 VR128:$rhs))), 5243 (VADDSUBPSrr VR128:$lhs, VR128:$rhs)>; 5244 def : Pat<(v4f32 (X86Addsub (v4f32 VR128:$lhs), (loadv4f32 addr:$rhs))), 5245 (VADDSUBPSrm VR128:$lhs, f128mem:$rhs)>; 5246 def : Pat<(v2f64 (X86Addsub (v2f64 VR128:$lhs), (v2f64 VR128:$rhs))), 5247 (VADDSUBPDrr VR128:$lhs, VR128:$rhs)>; 5248 def : Pat<(v2f64 (X86Addsub (v2f64 VR128:$lhs), (loadv2f64 addr:$rhs))), 5249 (VADDSUBPDrm VR128:$lhs, f128mem:$rhs)>; 5250 5251 def : Pat<(v8f32 (X86Addsub (v8f32 VR256:$lhs), (v8f32 VR256:$rhs))), 5252 (VADDSUBPSYrr VR256:$lhs, VR256:$rhs)>; 5253 def : Pat<(v8f32 (X86Addsub (v8f32 VR256:$lhs), (loadv8f32 addr:$rhs))), 5254 (VADDSUBPSYrm VR256:$lhs, f256mem:$rhs)>; 5255 def : Pat<(v4f64 (X86Addsub (v4f64 VR256:$lhs), (v4f64 VR256:$rhs))), 5256 (VADDSUBPDYrr VR256:$lhs, VR256:$rhs)>; 5257 def : Pat<(v4f64 (X86Addsub (v4f64 VR256:$lhs), (loadv4f64 addr:$rhs))), 5258 (VADDSUBPDYrm VR256:$lhs, f256mem:$rhs)>; 5259} 5260 5261let Predicates = [UseSSE3] in { 5262 def : Pat<(v4f32 (X86Addsub (v4f32 VR128:$lhs), (v4f32 VR128:$rhs))), 5263 (ADDSUBPSrr VR128:$lhs, VR128:$rhs)>; 5264 def : Pat<(v4f32 (X86Addsub (v4f32 VR128:$lhs), (memopv4f32 addr:$rhs))), 5265 (ADDSUBPSrm VR128:$lhs, f128mem:$rhs)>; 5266 def : Pat<(v2f64 (X86Addsub (v2f64 VR128:$lhs), (v2f64 VR128:$rhs))), 5267 (ADDSUBPDrr VR128:$lhs, VR128:$rhs)>; 5268 def : Pat<(v2f64 (X86Addsub (v2f64 VR128:$lhs), (memopv2f64 addr:$rhs))), 5269 (ADDSUBPDrm VR128:$lhs, f128mem:$rhs)>; 5270} 5271 5272//===---------------------------------------------------------------------===// 5273// SSE3 Instructions 5274//===---------------------------------------------------------------------===// 5275 5276// Horizontal ops 5277multiclass S3D_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC, 5278 X86MemOperand x86memop, SDNode OpNode, PatFrag ld_frag, 5279 bit Is2Addr = 1> { 5280 def rr : S3DI<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 5281 !if(Is2Addr, 5282 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5283 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5284 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], IIC_SSE_HADDSUB_RR>, 5285 Sched<[WriteFAdd]>; 5286 5287 def rm : S3DI<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 5288 !if(Is2Addr, 5289 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5290 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5291 [(set RC:$dst, (vt (OpNode RC:$src1, (ld_frag addr:$src2))))], 5292 IIC_SSE_HADDSUB_RM>, Sched<[WriteFAddLd, ReadAfterLd]>; 5293} 5294multiclass S3_Int<bits<8> o, string OpcodeStr, ValueType vt, RegisterClass RC, 5295 X86MemOperand x86memop, SDNode OpNode, PatFrag ld_frag, 5296 bit Is2Addr = 1> { 5297 def rr : S3I<o, MRMSrcReg, (outs RC:$dst), (ins RC:$src1, RC:$src2), 5298 !if(Is2Addr, 5299 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5300 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5301 [(set RC:$dst, (vt (OpNode RC:$src1, RC:$src2)))], IIC_SSE_HADDSUB_RR>, 5302 Sched<[WriteFAdd]>; 5303 5304 def rm : S3I<o, MRMSrcMem, (outs RC:$dst), (ins RC:$src1, x86memop:$src2), 5305 !if(Is2Addr, 5306 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5307 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5308 [(set RC:$dst, (vt (OpNode RC:$src1, (ld_frag addr:$src2))))], 5309 IIC_SSE_HADDSUB_RM>, Sched<[WriteFAddLd, ReadAfterLd]>; 5310} 5311 5312let Predicates = [HasAVX] in { 5313 let ExeDomain = SSEPackedSingle in { 5314 defm VHADDPS : S3D_Int<0x7C, "vhaddps", v4f32, VR128, f128mem, 5315 X86fhadd, loadv4f32, 0>, VEX_4V; 5316 defm VHSUBPS : S3D_Int<0x7D, "vhsubps", v4f32, VR128, f128mem, 5317 X86fhsub, loadv4f32, 0>, VEX_4V; 5318 defm VHADDPSY : S3D_Int<0x7C, "vhaddps", v8f32, VR256, f256mem, 5319 X86fhadd, loadv8f32, 0>, VEX_4V, VEX_L; 5320 defm VHSUBPSY : S3D_Int<0x7D, "vhsubps", v8f32, VR256, f256mem, 5321 X86fhsub, loadv8f32, 0>, VEX_4V, VEX_L; 5322 } 5323 let ExeDomain = SSEPackedDouble in { 5324 defm VHADDPD : S3_Int <0x7C, "vhaddpd", v2f64, VR128, f128mem, 5325 X86fhadd, loadv2f64, 0>, VEX_4V; 5326 defm VHSUBPD : S3_Int <0x7D, "vhsubpd", v2f64, VR128, f128mem, 5327 X86fhsub, loadv2f64, 0>, VEX_4V; 5328 defm VHADDPDY : S3_Int <0x7C, "vhaddpd", v4f64, VR256, f256mem, 5329 X86fhadd, loadv4f64, 0>, VEX_4V, VEX_L; 5330 defm VHSUBPDY : S3_Int <0x7D, "vhsubpd", v4f64, VR256, f256mem, 5331 X86fhsub, loadv4f64, 0>, VEX_4V, VEX_L; 5332 } 5333} 5334 5335let Constraints = "$src1 = $dst" in { 5336 let ExeDomain = SSEPackedSingle in { 5337 defm HADDPS : S3D_Int<0x7C, "haddps", v4f32, VR128, f128mem, X86fhadd, 5338 memopv4f32>; 5339 defm HSUBPS : S3D_Int<0x7D, "hsubps", v4f32, VR128, f128mem, X86fhsub, 5340 memopv4f32>; 5341 } 5342 let ExeDomain = SSEPackedDouble in { 5343 defm HADDPD : S3_Int<0x7C, "haddpd", v2f64, VR128, f128mem, X86fhadd, 5344 memopv2f64>; 5345 defm HSUBPD : S3_Int<0x7D, "hsubpd", v2f64, VR128, f128mem, X86fhsub, 5346 memopv2f64>; 5347 } 5348} 5349 5350//===---------------------------------------------------------------------===// 5351// SSSE3 - Packed Absolute Instructions 5352//===---------------------------------------------------------------------===// 5353 5354 5355/// SS3I_unop_rm_int - Simple SSSE3 unary op whose type can be v*{i8,i16,i32}. 5356multiclass SS3I_unop_rm<bits<8> opc, string OpcodeStr, ValueType vt, 5357 SDNode OpNode, PatFrag ld_frag> { 5358 def rr128 : SS38I<opc, MRMSrcReg, (outs VR128:$dst), 5359 (ins VR128:$src), 5360 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5361 [(set VR128:$dst, (vt (OpNode VR128:$src)))], 5362 IIC_SSE_PABS_RR>, Sched<[WriteVecALU]>; 5363 5364 def rm128 : SS38I<opc, MRMSrcMem, (outs VR128:$dst), 5365 (ins i128mem:$src), 5366 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5367 [(set VR128:$dst, 5368 (vt (OpNode (bitconvert (ld_frag addr:$src)))))], 5369 IIC_SSE_PABS_RM>, Sched<[WriteVecALULd]>; 5370} 5371 5372/// SS3I_unop_rm_int_y - Simple SSSE3 unary op whose type can be v*{i8,i16,i32}. 5373multiclass SS3I_unop_rm_y<bits<8> opc, string OpcodeStr, ValueType vt, 5374 SDNode OpNode> { 5375 def rr256 : SS38I<opc, MRMSrcReg, (outs VR256:$dst), 5376 (ins VR256:$src), 5377 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5378 [(set VR256:$dst, (vt (OpNode VR256:$src)))]>, 5379 Sched<[WriteVecALU]>; 5380 5381 def rm256 : SS38I<opc, MRMSrcMem, (outs VR256:$dst), 5382 (ins i256mem:$src), 5383 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5384 [(set VR256:$dst, 5385 (vt (OpNode (bitconvert (loadv4i64 addr:$src)))))]>, 5386 Sched<[WriteVecALULd]>; 5387} 5388 5389// Helper fragments to match sext vXi1 to vXiY. 5390def v16i1sextv16i8 : PatLeaf<(v16i8 (X86pcmpgt (bc_v16i8 (v4i32 immAllZerosV)), 5391 VR128:$src))>; 5392def v8i1sextv8i16 : PatLeaf<(v8i16 (X86vsrai VR128:$src, (i8 15)))>; 5393def v4i1sextv4i32 : PatLeaf<(v4i32 (X86vsrai VR128:$src, (i8 31)))>; 5394def v32i1sextv32i8 : PatLeaf<(v32i8 (X86pcmpgt (bc_v32i8 (v8i32 immAllZerosV)), 5395 VR256:$src))>; 5396def v16i1sextv16i16: PatLeaf<(v16i16 (X86vsrai VR256:$src, (i8 15)))>; 5397def v8i1sextv8i32 : PatLeaf<(v8i32 (X86vsrai VR256:$src, (i8 31)))>; 5398 5399let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 5400 defm VPABSB : SS3I_unop_rm<0x1C, "vpabsb", v16i8, X86Abs, loadv2i64>, VEX; 5401 defm VPABSW : SS3I_unop_rm<0x1D, "vpabsw", v8i16, X86Abs, loadv2i64>, VEX; 5402} 5403let Predicates = [HasAVX, NoVLX] in { 5404 defm VPABSD : SS3I_unop_rm<0x1E, "vpabsd", v4i32, X86Abs, loadv2i64>, VEX; 5405} 5406 5407let Predicates = [HasAVX] in { 5408 def : Pat<(xor 5409 (bc_v2i64 (v16i1sextv16i8)), 5410 (bc_v2i64 (add (v16i8 VR128:$src), (v16i1sextv16i8)))), 5411 (VPABSBrr128 VR128:$src)>; 5412 def : Pat<(xor 5413 (bc_v2i64 (v8i1sextv8i16)), 5414 (bc_v2i64 (add (v8i16 VR128:$src), (v8i1sextv8i16)))), 5415 (VPABSWrr128 VR128:$src)>; 5416 def : Pat<(xor 5417 (bc_v2i64 (v4i1sextv4i32)), 5418 (bc_v2i64 (add (v4i32 VR128:$src), (v4i1sextv4i32)))), 5419 (VPABSDrr128 VR128:$src)>; 5420} 5421 5422let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in { 5423 defm VPABSB : SS3I_unop_rm_y<0x1C, "vpabsb", v32i8, X86Abs>, VEX, VEX_L; 5424 defm VPABSW : SS3I_unop_rm_y<0x1D, "vpabsw", v16i16, X86Abs>, VEX, VEX_L; 5425} 5426let Predicates = [HasAVX2, NoVLX] in { 5427 defm VPABSD : SS3I_unop_rm_y<0x1E, "vpabsd", v8i32, X86Abs>, VEX, VEX_L; 5428} 5429 5430let Predicates = [HasAVX2] in { 5431 def : Pat<(xor 5432 (bc_v4i64 (v32i1sextv32i8)), 5433 (bc_v4i64 (add (v32i8 VR256:$src), (v32i1sextv32i8)))), 5434 (VPABSBrr256 VR256:$src)>; 5435 def : Pat<(xor 5436 (bc_v4i64 (v16i1sextv16i16)), 5437 (bc_v4i64 (add (v16i16 VR256:$src), (v16i1sextv16i16)))), 5438 (VPABSWrr256 VR256:$src)>; 5439 def : Pat<(xor 5440 (bc_v4i64 (v8i1sextv8i32)), 5441 (bc_v4i64 (add (v8i32 VR256:$src), (v8i1sextv8i32)))), 5442 (VPABSDrr256 VR256:$src)>; 5443} 5444 5445defm PABSB : SS3I_unop_rm<0x1C, "pabsb", v16i8, X86Abs, memopv2i64>; 5446defm PABSW : SS3I_unop_rm<0x1D, "pabsw", v8i16, X86Abs, memopv2i64>; 5447defm PABSD : SS3I_unop_rm<0x1E, "pabsd", v4i32, X86Abs, memopv2i64>; 5448 5449let Predicates = [UseSSSE3] in { 5450 def : Pat<(xor 5451 (bc_v2i64 (v16i1sextv16i8)), 5452 (bc_v2i64 (add (v16i8 VR128:$src), (v16i1sextv16i8)))), 5453 (PABSBrr128 VR128:$src)>; 5454 def : Pat<(xor 5455 (bc_v2i64 (v8i1sextv8i16)), 5456 (bc_v2i64 (add (v8i16 VR128:$src), (v8i1sextv8i16)))), 5457 (PABSWrr128 VR128:$src)>; 5458 def : Pat<(xor 5459 (bc_v2i64 (v4i1sextv4i32)), 5460 (bc_v2i64 (add (v4i32 VR128:$src), (v4i1sextv4i32)))), 5461 (PABSDrr128 VR128:$src)>; 5462} 5463 5464//===---------------------------------------------------------------------===// 5465// SSSE3 - Packed Binary Operator Instructions 5466//===---------------------------------------------------------------------===// 5467 5468let Sched = WriteVecALU in { 5469def SSE_PHADDSUBD : OpndItins< 5470 IIC_SSE_PHADDSUBD_RR, IIC_SSE_PHADDSUBD_RM 5471>; 5472def SSE_PHADDSUBSW : OpndItins< 5473 IIC_SSE_PHADDSUBSW_RR, IIC_SSE_PHADDSUBSW_RM 5474>; 5475def SSE_PHADDSUBW : OpndItins< 5476 IIC_SSE_PHADDSUBW_RR, IIC_SSE_PHADDSUBW_RM 5477>; 5478} 5479let Sched = WriteShuffle in 5480def SSE_PSHUFB : OpndItins< 5481 IIC_SSE_PSHUFB_RR, IIC_SSE_PSHUFB_RM 5482>; 5483let Sched = WriteVecALU in 5484def SSE_PSIGN : OpndItins< 5485 IIC_SSE_PSIGN_RR, IIC_SSE_PSIGN_RM 5486>; 5487let Sched = WriteVecIMul in 5488def SSE_PMULHRSW : OpndItins< 5489 IIC_SSE_PMULHRSW, IIC_SSE_PMULHRSW 5490>; 5491 5492/// SS3I_binop_rm - Simple SSSE3 bin op 5493multiclass SS3I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 5494 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 5495 X86MemOperand x86memop, OpndItins itins, 5496 bit Is2Addr = 1> { 5497 let isCommutable = 1 in 5498 def rr : SS38I<opc, MRMSrcReg, (outs RC:$dst), 5499 (ins RC:$src1, RC:$src2), 5500 !if(Is2Addr, 5501 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5502 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5503 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))], itins.rr>, 5504 Sched<[itins.Sched]>; 5505 def rm : SS38I<opc, MRMSrcMem, (outs RC:$dst), 5506 (ins RC:$src1, x86memop:$src2), 5507 !if(Is2Addr, 5508 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5509 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5510 [(set RC:$dst, 5511 (OpVT (OpNode RC:$src1, 5512 (bitconvert (memop_frag addr:$src2)))))], itins.rm>, 5513 Sched<[itins.Sched.Folded, ReadAfterLd]>; 5514} 5515 5516/// SS3I_binop_rm_int - Simple SSSE3 bin op whose type can be v*{i8,i16,i32}. 5517multiclass SS3I_binop_rm_int<bits<8> opc, string OpcodeStr, 5518 Intrinsic IntId128, OpndItins itins, 5519 PatFrag ld_frag, bit Is2Addr = 1> { 5520 let isCommutable = 1 in 5521 def rr128 : SS38I<opc, MRMSrcReg, (outs VR128:$dst), 5522 (ins VR128:$src1, VR128:$src2), 5523 !if(Is2Addr, 5524 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5525 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5526 [(set VR128:$dst, (IntId128 VR128:$src1, VR128:$src2))]>, 5527 Sched<[itins.Sched]>; 5528 def rm128 : SS38I<opc, MRMSrcMem, (outs VR128:$dst), 5529 (ins VR128:$src1, i128mem:$src2), 5530 !if(Is2Addr, 5531 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 5532 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 5533 [(set VR128:$dst, 5534 (IntId128 VR128:$src1, 5535 (bitconvert (ld_frag addr:$src2))))]>, 5536 Sched<[itins.Sched.Folded, ReadAfterLd]>; 5537} 5538 5539multiclass SS3I_binop_rm_int_y<bits<8> opc, string OpcodeStr, 5540 Intrinsic IntId256, 5541 X86FoldableSchedWrite Sched> { 5542 let isCommutable = 1 in 5543 def rr256 : SS38I<opc, MRMSrcReg, (outs VR256:$dst), 5544 (ins VR256:$src1, VR256:$src2), 5545 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5546 [(set VR256:$dst, (IntId256 VR256:$src1, VR256:$src2))]>, 5547 Sched<[Sched]>; 5548 def rm256 : SS38I<opc, MRMSrcMem, (outs VR256:$dst), 5549 (ins VR256:$src1, i256mem:$src2), 5550 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 5551 [(set VR256:$dst, 5552 (IntId256 VR256:$src1, (bitconvert (loadv4i64 addr:$src2))))]>, 5553 Sched<[Sched.Folded, ReadAfterLd]>; 5554} 5555 5556let ImmT = NoImm, Predicates = [HasAVX] in { 5557let isCommutable = 0 in { 5558 defm VPHADDW : SS3I_binop_rm<0x01, "vphaddw", X86hadd, v8i16, VR128, 5559 loadv2i64, i128mem, 5560 SSE_PHADDSUBW, 0>, VEX_4V; 5561 defm VPHADDD : SS3I_binop_rm<0x02, "vphaddd", X86hadd, v4i32, VR128, 5562 loadv2i64, i128mem, 5563 SSE_PHADDSUBD, 0>, VEX_4V; 5564 defm VPHSUBW : SS3I_binop_rm<0x05, "vphsubw", X86hsub, v8i16, VR128, 5565 loadv2i64, i128mem, 5566 SSE_PHADDSUBW, 0>, VEX_4V; 5567 defm VPHSUBD : SS3I_binop_rm<0x06, "vphsubd", X86hsub, v4i32, VR128, 5568 loadv2i64, i128mem, 5569 SSE_PHADDSUBD, 0>, VEX_4V; 5570 defm VPSIGNB : SS3I_binop_rm_int<0x08, "vpsignb", 5571 int_x86_ssse3_psign_b_128, 5572 SSE_PSIGN, loadv2i64, 0>, VEX_4V; 5573 defm VPSIGNW : SS3I_binop_rm_int<0x09, "vpsignw", 5574 int_x86_ssse3_psign_w_128, 5575 SSE_PSIGN, loadv2i64, 0>, VEX_4V; 5576 defm VPSIGND : SS3I_binop_rm_int<0x0A, "vpsignd", 5577 int_x86_ssse3_psign_d_128, 5578 SSE_PSIGN, loadv2i64, 0>, VEX_4V; 5579 defm VPSHUFB : SS3I_binop_rm<0x00, "vpshufb", X86pshufb, v16i8, VR128, 5580 loadv2i64, i128mem, 5581 SSE_PSHUFB, 0>, VEX_4V; 5582 defm VPHADDSW : SS3I_binop_rm_int<0x03, "vphaddsw", 5583 int_x86_ssse3_phadd_sw_128, 5584 SSE_PHADDSUBSW, loadv2i64, 0>, VEX_4V; 5585 defm VPHSUBSW : SS3I_binop_rm_int<0x07, "vphsubsw", 5586 int_x86_ssse3_phsub_sw_128, 5587 SSE_PHADDSUBSW, loadv2i64, 0>, VEX_4V; 5588 defm VPMADDUBSW : SS3I_binop_rm_int<0x04, "vpmaddubsw", 5589 int_x86_ssse3_pmadd_ub_sw_128, 5590 SSE_PMADD, loadv2i64, 0>, VEX_4V; 5591} 5592defm VPMULHRSW : SS3I_binop_rm_int<0x0B, "vpmulhrsw", 5593 int_x86_ssse3_pmul_hr_sw_128, 5594 SSE_PMULHRSW, loadv2i64, 0>, VEX_4V; 5595} 5596 5597let ImmT = NoImm, Predicates = [HasAVX2] in { 5598let isCommutable = 0 in { 5599 defm VPHADDWY : SS3I_binop_rm<0x01, "vphaddw", X86hadd, v16i16, VR256, 5600 loadv4i64, i256mem, 5601 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5602 defm VPHADDDY : SS3I_binop_rm<0x02, "vphaddd", X86hadd, v8i32, VR256, 5603 loadv4i64, i256mem, 5604 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5605 defm VPHSUBWY : SS3I_binop_rm<0x05, "vphsubw", X86hsub, v16i16, VR256, 5606 loadv4i64, i256mem, 5607 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5608 defm VPHSUBDY : SS3I_binop_rm<0x06, "vphsubd", X86hsub, v8i32, VR256, 5609 loadv4i64, i256mem, 5610 SSE_PHADDSUBW, 0>, VEX_4V, VEX_L; 5611 defm VPSIGNBY : SS3I_binop_rm_int_y<0x08, "vpsignb", int_x86_avx2_psign_b, 5612 WriteVecALU>, VEX_4V, VEX_L; 5613 defm VPSIGNWY : SS3I_binop_rm_int_y<0x09, "vpsignw", int_x86_avx2_psign_w, 5614 WriteVecALU>, VEX_4V, VEX_L; 5615 defm VPSIGNDY : SS3I_binop_rm_int_y<0x0A, "vpsignd", int_x86_avx2_psign_d, 5616 WriteVecALU>, VEX_4V, VEX_L; 5617 defm VPSHUFBY : SS3I_binop_rm<0x00, "vpshufb", X86pshufb, v32i8, VR256, 5618 loadv4i64, i256mem, 5619 SSE_PSHUFB, 0>, VEX_4V, VEX_L; 5620 defm VPHADDSW : SS3I_binop_rm_int_y<0x03, "vphaddsw", 5621 int_x86_avx2_phadd_sw, 5622 WriteVecALU>, VEX_4V, VEX_L; 5623 defm VPHSUBSW : SS3I_binop_rm_int_y<0x07, "vphsubsw", 5624 int_x86_avx2_phsub_sw, 5625 WriteVecALU>, VEX_4V, VEX_L; 5626 defm VPMADDUBSW : SS3I_binop_rm_int_y<0x04, "vpmaddubsw", 5627 int_x86_avx2_pmadd_ub_sw, 5628 WriteVecIMul>, VEX_4V, VEX_L; 5629} 5630defm VPMULHRSW : SS3I_binop_rm_int_y<0x0B, "vpmulhrsw", 5631 int_x86_avx2_pmul_hr_sw, 5632 WriteVecIMul>, VEX_4V, VEX_L; 5633} 5634 5635// None of these have i8 immediate fields. 5636let ImmT = NoImm, Constraints = "$src1 = $dst" in { 5637let isCommutable = 0 in { 5638 defm PHADDW : SS3I_binop_rm<0x01, "phaddw", X86hadd, v8i16, VR128, 5639 memopv2i64, i128mem, SSE_PHADDSUBW>; 5640 defm PHADDD : SS3I_binop_rm<0x02, "phaddd", X86hadd, v4i32, VR128, 5641 memopv2i64, i128mem, SSE_PHADDSUBD>; 5642 defm PHSUBW : SS3I_binop_rm<0x05, "phsubw", X86hsub, v8i16, VR128, 5643 memopv2i64, i128mem, SSE_PHADDSUBW>; 5644 defm PHSUBD : SS3I_binop_rm<0x06, "phsubd", X86hsub, v4i32, VR128, 5645 memopv2i64, i128mem, SSE_PHADDSUBD>; 5646 defm PSIGNB : SS3I_binop_rm_int<0x08, "psignb", int_x86_ssse3_psign_b_128, 5647 SSE_PSIGN, memopv2i64>; 5648 defm PSIGNW : SS3I_binop_rm_int<0x09, "psignw", int_x86_ssse3_psign_w_128, 5649 SSE_PSIGN, memopv2i64>; 5650 defm PSIGND : SS3I_binop_rm_int<0x0A, "psignd", int_x86_ssse3_psign_d_128, 5651 SSE_PSIGN, memopv2i64>; 5652 defm PSHUFB : SS3I_binop_rm<0x00, "pshufb", X86pshufb, v16i8, VR128, 5653 memopv2i64, i128mem, SSE_PSHUFB>; 5654 defm PHADDSW : SS3I_binop_rm_int<0x03, "phaddsw", 5655 int_x86_ssse3_phadd_sw_128, 5656 SSE_PHADDSUBSW, memopv2i64>; 5657 defm PHSUBSW : SS3I_binop_rm_int<0x07, "phsubsw", 5658 int_x86_ssse3_phsub_sw_128, 5659 SSE_PHADDSUBSW, memopv2i64>; 5660 defm PMADDUBSW : SS3I_binop_rm_int<0x04, "pmaddubsw", 5661 int_x86_ssse3_pmadd_ub_sw_128, 5662 SSE_PMADD, memopv2i64>; 5663} 5664defm PMULHRSW : SS3I_binop_rm_int<0x0B, "pmulhrsw", 5665 int_x86_ssse3_pmul_hr_sw_128, 5666 SSE_PMULHRSW, memopv2i64>; 5667} 5668 5669//===---------------------------------------------------------------------===// 5670// SSSE3 - Packed Align Instruction Patterns 5671//===---------------------------------------------------------------------===// 5672 5673multiclass ssse3_palignr<string asm, bit Is2Addr = 1> { 5674 let hasSideEffects = 0 in { 5675 def rri : SS3AI<0x0F, MRMSrcReg, (outs VR128:$dst), 5676 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 5677 !if(Is2Addr, 5678 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 5679 !strconcat(asm, 5680 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 5681 [], IIC_SSE_PALIGNRR>, Sched<[WriteShuffle]>; 5682 let mayLoad = 1 in 5683 def rmi : SS3AI<0x0F, MRMSrcMem, (outs VR128:$dst), 5684 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 5685 !if(Is2Addr, 5686 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 5687 !strconcat(asm, 5688 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 5689 [], IIC_SSE_PALIGNRM>, Sched<[WriteShuffleLd, ReadAfterLd]>; 5690 } 5691} 5692 5693multiclass ssse3_palignr_y<string asm, bit Is2Addr = 1> { 5694 let hasSideEffects = 0 in { 5695 def Yrri : SS3AI<0x0F, MRMSrcReg, (outs VR256:$dst), 5696 (ins VR256:$src1, VR256:$src2, u8imm:$src3), 5697 !strconcat(asm, 5698 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 5699 []>, Sched<[WriteShuffle]>; 5700 let mayLoad = 1 in 5701 def Yrmi : SS3AI<0x0F, MRMSrcMem, (outs VR256:$dst), 5702 (ins VR256:$src1, i256mem:$src2, u8imm:$src3), 5703 !strconcat(asm, 5704 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 5705 []>, Sched<[WriteShuffleLd, ReadAfterLd]>; 5706 } 5707} 5708 5709let Predicates = [HasAVX] in 5710 defm VPALIGNR : ssse3_palignr<"vpalignr", 0>, VEX_4V; 5711let Predicates = [HasAVX2] in 5712 defm VPALIGNR : ssse3_palignr_y<"vpalignr", 0>, VEX_4V, VEX_L; 5713let Constraints = "$src1 = $dst", Predicates = [UseSSSE3] in 5714 defm PALIGNR : ssse3_palignr<"palignr">; 5715 5716let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in { 5717def : Pat<(v8i32 (X86PAlignr VR256:$src1, VR256:$src2, (i8 imm:$imm))), 5718 (VPALIGNRYrri VR256:$src1, VR256:$src2, imm:$imm)>; 5719def : Pat<(v8f32 (X86PAlignr VR256:$src1, VR256:$src2, (i8 imm:$imm))), 5720 (VPALIGNRYrri VR256:$src1, VR256:$src2, imm:$imm)>; 5721def : Pat<(v16i16 (X86PAlignr VR256:$src1, VR256:$src2, (i8 imm:$imm))), 5722 (VPALIGNRYrri VR256:$src1, VR256:$src2, imm:$imm)>; 5723def : Pat<(v32i8 (X86PAlignr VR256:$src1, VR256:$src2, (i8 imm:$imm))), 5724 (VPALIGNRYrri VR256:$src1, VR256:$src2, imm:$imm)>; 5725} 5726 5727let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 5728def : Pat<(v4i32 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5729 (VPALIGNRrri VR128:$src1, VR128:$src2, imm:$imm)>; 5730def : Pat<(v4f32 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5731 (VPALIGNRrri VR128:$src1, VR128:$src2, imm:$imm)>; 5732def : Pat<(v8i16 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5733 (VPALIGNRrri VR128:$src1, VR128:$src2, imm:$imm)>; 5734def : Pat<(v16i8 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5735 (VPALIGNRrri VR128:$src1, VR128:$src2, imm:$imm)>; 5736} 5737 5738let Predicates = [UseSSSE3] in { 5739def : Pat<(v4i32 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5740 (PALIGNRrri VR128:$src1, VR128:$src2, imm:$imm)>; 5741def : Pat<(v4f32 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5742 (PALIGNRrri VR128:$src1, VR128:$src2, imm:$imm)>; 5743def : Pat<(v8i16 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5744 (PALIGNRrri VR128:$src1, VR128:$src2, imm:$imm)>; 5745def : Pat<(v16i8 (X86PAlignr VR128:$src1, VR128:$src2, (i8 imm:$imm))), 5746 (PALIGNRrri VR128:$src1, VR128:$src2, imm:$imm)>; 5747} 5748 5749//===---------------------------------------------------------------------===// 5750// SSSE3 - Thread synchronization 5751//===---------------------------------------------------------------------===// 5752 5753let SchedRW = [WriteSystem] in { 5754let usesCustomInserter = 1 in { 5755def MONITOR : PseudoI<(outs), (ins i32mem:$src1, GR32:$src2, GR32:$src3), 5756 [(int_x86_sse3_monitor addr:$src1, GR32:$src2, GR32:$src3)]>, 5757 Requires<[HasSSE3]>; 5758} 5759 5760let Uses = [EAX, ECX, EDX] in 5761def MONITORrrr : I<0x01, MRM_C8, (outs), (ins), "monitor", [], IIC_SSE_MONITOR>, 5762 TB, Requires<[HasSSE3]>; 5763 5764let Uses = [ECX, EAX] in 5765def MWAITrr : I<0x01, MRM_C9, (outs), (ins), "mwait", 5766 [(int_x86_sse3_mwait ECX, EAX)], IIC_SSE_MWAIT>, 5767 TB, Requires<[HasSSE3]>; 5768} // SchedRW 5769 5770def : InstAlias<"mwait\t{%eax, %ecx|ecx, eax}", (MWAITrr)>, Requires<[Not64BitMode]>; 5771def : InstAlias<"mwait\t{%rax, %rcx|rcx, rax}", (MWAITrr)>, Requires<[In64BitMode]>; 5772 5773def : InstAlias<"monitor\t{%eax, %ecx, %edx|edx, ecx, eax}", (MONITORrrr)>, 5774 Requires<[Not64BitMode]>; 5775def : InstAlias<"monitor\t{%rax, %rcx, %rdx|rdx, rcx, rax}", (MONITORrrr)>, 5776 Requires<[In64BitMode]>; 5777 5778//===----------------------------------------------------------------------===// 5779// SSE4.1 - Packed Move with Sign/Zero Extend 5780//===----------------------------------------------------------------------===// 5781 5782multiclass SS41I_pmovx_rrrm<bits<8> opc, string OpcodeStr, X86MemOperand MemOp, 5783 RegisterClass OutRC, RegisterClass InRC, 5784 OpndItins itins> { 5785 def rr : SS48I<opc, MRMSrcReg, (outs OutRC:$dst), (ins InRC:$src), 5786 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5787 [], itins.rr>, 5788 Sched<[itins.Sched]>; 5789 5790 def rm : SS48I<opc, MRMSrcMem, (outs OutRC:$dst), (ins MemOp:$src), 5791 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 5792 [], 5793 itins.rm>, Sched<[itins.Sched.Folded]>; 5794} 5795 5796multiclass SS41I_pmovx_rm_all<bits<8> opc, string OpcodeStr, 5797 X86MemOperand MemOp, X86MemOperand MemYOp, 5798 OpndItins SSEItins, OpndItins AVXItins, 5799 OpndItins AVX2Itins, Predicate prd> { 5800 defm NAME : SS41I_pmovx_rrrm<opc, OpcodeStr, MemOp, VR128, VR128, SSEItins>; 5801 let Predicates = [HasAVX, prd] in 5802 defm V#NAME : SS41I_pmovx_rrrm<opc, !strconcat("v", OpcodeStr), MemOp, 5803 VR128, VR128, AVXItins>, VEX; 5804 let Predicates = [HasAVX2, prd] in 5805 defm V#NAME#Y : SS41I_pmovx_rrrm<opc, !strconcat("v", OpcodeStr), MemYOp, 5806 VR256, VR128, AVX2Itins>, VEX, VEX_L; 5807} 5808 5809multiclass SS41I_pmovx_rm<bits<8> opc, string OpcodeStr, X86MemOperand MemOp, 5810 X86MemOperand MemYOp, Predicate prd> { 5811 defm PMOVSX#NAME : SS41I_pmovx_rm_all<opc, !strconcat("pmovsx", OpcodeStr), 5812 MemOp, MemYOp, 5813 SSE_INTALU_ITINS_SHUFF_P, 5814 DEFAULT_ITINS_SHUFFLESCHED, 5815 DEFAULT_ITINS_SHUFFLESCHED, prd>; 5816 defm PMOVZX#NAME : SS41I_pmovx_rm_all<!add(opc, 0x10), 5817 !strconcat("pmovzx", OpcodeStr), 5818 MemOp, MemYOp, 5819 SSE_INTALU_ITINS_SHUFF_P, 5820 DEFAULT_ITINS_SHUFFLESCHED, 5821 DEFAULT_ITINS_SHUFFLESCHED, prd>; 5822} 5823 5824defm BW : SS41I_pmovx_rm<0x20, "bw", i64mem, i128mem, NoVLX_Or_NoBWI>; 5825defm WD : SS41I_pmovx_rm<0x23, "wd", i64mem, i128mem, NoVLX>; 5826defm DQ : SS41I_pmovx_rm<0x25, "dq", i64mem, i128mem, NoVLX>; 5827 5828defm BD : SS41I_pmovx_rm<0x21, "bd", i32mem, i64mem, NoVLX>; 5829defm WQ : SS41I_pmovx_rm<0x24, "wq", i32mem, i64mem, NoVLX>; 5830 5831defm BQ : SS41I_pmovx_rm<0x22, "bq", i16mem, i32mem, NoVLX>; 5832 5833// AVX2 Patterns 5834multiclass SS41I_pmovx_avx2_patterns<string OpcPrefix, string ExtTy, SDNode ExtOp> { 5835 // Register-Register patterns 5836 let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 5837 def : Pat<(v16i16 (ExtOp (v16i8 VR128:$src))), 5838 (!cast<I>(OpcPrefix#BWYrr) VR128:$src)>; 5839 } 5840 let Predicates = [HasAVX, NoVLX] in { 5841 def : Pat<(v8i32 (ExtOp (v16i8 VR128:$src))), 5842 (!cast<I>(OpcPrefix#BDYrr) VR128:$src)>; 5843 def : Pat<(v4i64 (ExtOp (v16i8 VR128:$src))), 5844 (!cast<I>(OpcPrefix#BQYrr) VR128:$src)>; 5845 5846 def : Pat<(v8i32 (ExtOp (v8i16 VR128:$src))), 5847 (!cast<I>(OpcPrefix#WDYrr) VR128:$src)>; 5848 def : Pat<(v4i64 (ExtOp (v8i16 VR128:$src))), 5849 (!cast<I>(OpcPrefix#WQYrr) VR128:$src)>; 5850 5851 def : Pat<(v4i64 (ExtOp (v4i32 VR128:$src))), 5852 (!cast<I>(OpcPrefix#DQYrr) VR128:$src)>; 5853 } 5854 5855 // Simple Register-Memory patterns 5856 let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 5857 def : Pat<(v16i16 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 5858 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5859 } 5860 let Predicates = [HasAVX, NoVLX] in { 5861 def : Pat<(v8i32 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 5862 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5863 def : Pat<(v4i64 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 5864 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5865 5866 def : Pat<(v8i32 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)), 5867 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5868 def : Pat<(v4i64 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)), 5869 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 5870 5871 def : Pat<(v4i64 (!cast<PatFrag>(ExtTy#"extloadvi32") addr:$src)), 5872 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 5873 } 5874 5875 // AVX2 Register-Memory patterns 5876 let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 5877 def : Pat<(v16i16 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5878 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5879 def : Pat<(v16i16 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), 5880 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5881 def : Pat<(v16i16 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 5882 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5883 def : Pat<(v16i16 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5884 (!cast<I>(OpcPrefix#BWYrm) addr:$src)>; 5885 } 5886 let Predicates = [HasAVX, NoVLX] in { 5887 def : Pat<(v8i32 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 5888 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5889 def : Pat<(v8i32 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), 5890 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5891 def : Pat<(v8i32 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 5892 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5893 def : Pat<(v8i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5894 (!cast<I>(OpcPrefix#BDYrm) addr:$src)>; 5895 5896 def : Pat<(v4i64 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), 5897 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5898 def : Pat<(v4i64 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), 5899 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5900 def : Pat<(v4i64 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 5901 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5902 def : Pat<(v4i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5903 (!cast<I>(OpcPrefix#BQYrm) addr:$src)>; 5904 5905 def : Pat<(v8i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 5906 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5907 def : Pat<(v8i32 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), 5908 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5909 def : Pat<(v8i32 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), 5910 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5911 def : Pat<(v8i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 5912 (!cast<I>(OpcPrefix#WDYrm) addr:$src)>; 5913 5914 def : Pat<(v4i64 (ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 5915 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 5916 def : Pat<(v4i64 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), 5917 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 5918 def : Pat<(v4i64 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), 5919 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 5920 def : Pat<(v4i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 5921 (!cast<I>(OpcPrefix#WQYrm) addr:$src)>; 5922 5923 def : Pat<(v4i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))), 5924 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 5925 def : Pat<(v4i64 (ExtOp (v4i32 (vzmovl_v2i64 addr:$src)))), 5926 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 5927 def : Pat<(v4i64 (ExtOp (v4i32 (vzload_v2i64 addr:$src)))), 5928 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 5929 def : Pat<(v4i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))), 5930 (!cast<I>(OpcPrefix#DQYrm) addr:$src)>; 5931 } 5932} 5933 5934defm : SS41I_pmovx_avx2_patterns<"VPMOVSX", "s", X86vsext>; 5935defm : SS41I_pmovx_avx2_patterns<"VPMOVZX", "z", X86vzext>; 5936 5937// SSE4.1/AVX patterns. 5938multiclass SS41I_pmovx_patterns<string OpcPrefix, string ExtTy, 5939 SDNode ExtOp, PatFrag ExtLoad16> { 5940 let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 5941 def : Pat<(v8i16 (ExtOp (v16i8 VR128:$src))), 5942 (!cast<I>(OpcPrefix#BWrr) VR128:$src)>; 5943 } 5944 let Predicates = [HasAVX, NoVLX] in { 5945 def : Pat<(v4i32 (ExtOp (v16i8 VR128:$src))), 5946 (!cast<I>(OpcPrefix#BDrr) VR128:$src)>; 5947 def : Pat<(v2i64 (ExtOp (v16i8 VR128:$src))), 5948 (!cast<I>(OpcPrefix#BQrr) VR128:$src)>; 5949 5950 def : Pat<(v4i32 (ExtOp (v8i16 VR128:$src))), 5951 (!cast<I>(OpcPrefix#WDrr) VR128:$src)>; 5952 def : Pat<(v2i64 (ExtOp (v8i16 VR128:$src))), 5953 (!cast<I>(OpcPrefix#WQrr) VR128:$src)>; 5954 5955 def : Pat<(v2i64 (ExtOp (v4i32 VR128:$src))), 5956 (!cast<I>(OpcPrefix#DQrr) VR128:$src)>; 5957 } 5958 let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 5959 def : Pat<(v8i16 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 5960 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 5961 } 5962 let Predicates = [HasAVX, NoVLX] in { 5963 def : Pat<(v4i32 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 5964 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 5965 def : Pat<(v2i64 (!cast<PatFrag>(ExtTy#"extloadvi8") addr:$src)), 5966 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 5967 5968 def : Pat<(v4i32 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)), 5969 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 5970 def : Pat<(v2i64 (!cast<PatFrag>(ExtTy#"extloadvi16") addr:$src)), 5971 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 5972 5973 def : Pat<(v2i64 (!cast<PatFrag>(ExtTy#"extloadvi32") addr:$src)), 5974 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 5975 } 5976 let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 5977 def : Pat<(v8i16 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 5978 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 5979 def : Pat<(v8i16 (ExtOp (bc_v16i8 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))), 5980 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 5981 def : Pat<(v8i16 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))), 5982 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 5983 def : Pat<(v8i16 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 5984 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 5985 def : Pat<(v8i16 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5986 (!cast<I>(OpcPrefix#BWrm) addr:$src)>; 5987 } 5988 let Predicates = [HasAVX, NoVLX] in { 5989 def : Pat<(v4i32 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), 5990 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 5991 def : Pat<(v4i32 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), 5992 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 5993 def : Pat<(v4i32 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 5994 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 5995 def : Pat<(v4i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 5996 (!cast<I>(OpcPrefix#BDrm) addr:$src)>; 5997 5998 def : Pat<(v2i64 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (ExtLoad16 addr:$src)))))), 5999 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 6000 def : Pat<(v2i64 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))), 6001 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 6002 def : Pat<(v2i64 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))), 6003 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 6004 def : Pat<(v2i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))), 6005 (!cast<I>(OpcPrefix#BQrm) addr:$src)>; 6006 6007 def : Pat<(v4i32 (ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 6008 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6009 def : Pat<(v4i32 (ExtOp (bc_v8i16 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))), 6010 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6011 def : Pat<(v4i32 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))), 6012 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6013 def : Pat<(v4i32 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), 6014 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6015 def : Pat<(v4i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 6016 (!cast<I>(OpcPrefix#WDrm) addr:$src)>; 6017 6018 def : Pat<(v2i64 (ExtOp (bc_v8i16 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))), 6019 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 6020 def : Pat<(v2i64 (ExtOp (v8i16 (vzmovl_v4i32 addr:$src)))), 6021 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 6022 def : Pat<(v2i64 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))), 6023 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 6024 def : Pat<(v2i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))), 6025 (!cast<I>(OpcPrefix#WQrm) addr:$src)>; 6026 6027 def : Pat<(v2i64 (ExtOp (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))), 6028 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6029 def : Pat<(v2i64 (ExtOp (bc_v4i32 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))), 6030 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6031 def : Pat<(v2i64 (ExtOp (v4i32 (vzmovl_v2i64 addr:$src)))), 6032 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6033 def : Pat<(v2i64 (ExtOp (v4i32 (vzload_v2i64 addr:$src)))), 6034 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6035 def : Pat<(v2i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))), 6036 (!cast<I>(OpcPrefix#DQrm) addr:$src)>; 6037 } 6038} 6039 6040defm : SS41I_pmovx_patterns<"VPMOVSX", "s", X86vsext, extloadi32i16>; 6041defm : SS41I_pmovx_patterns<"VPMOVZX", "z", X86vzext, loadi16_anyext>; 6042 6043let Predicates = [UseSSE41] in { 6044 defm : SS41I_pmovx_patterns<"PMOVSX", "s", X86vsext, extloadi32i16>; 6045 defm : SS41I_pmovx_patterns<"PMOVZX", "z", X86vzext, loadi16_anyext>; 6046} 6047 6048//===----------------------------------------------------------------------===// 6049// SSE4.1 - Extract Instructions 6050//===----------------------------------------------------------------------===// 6051 6052/// SS41I_binop_ext8 - SSE 4.1 extract 8 bits to 32 bit reg or 8 bit mem 6053multiclass SS41I_extract8<bits<8> opc, string OpcodeStr> { 6054 def rr : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst), 6055 (ins VR128:$src1, u8imm:$src2), 6056 !strconcat(OpcodeStr, 6057 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6058 [(set GR32orGR64:$dst, (X86pextrb (v16i8 VR128:$src1), 6059 imm:$src2))]>, 6060 Sched<[WriteShuffle]>; 6061 let hasSideEffects = 0, mayStore = 1, 6062 SchedRW = [WriteShuffleLd, WriteRMW] in 6063 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6064 (ins i8mem:$dst, VR128:$src1, u8imm:$src2), 6065 !strconcat(OpcodeStr, 6066 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6067 [(store (i8 (trunc (assertzext (X86pextrb (v16i8 VR128:$src1), 6068 imm:$src2)))), addr:$dst)]>; 6069} 6070 6071let Predicates = [HasAVX, NoBWI] in 6072 defm VPEXTRB : SS41I_extract8<0x14, "vpextrb">, VEX; 6073 6074defm PEXTRB : SS41I_extract8<0x14, "pextrb">; 6075 6076 6077/// SS41I_extract16 - SSE 4.1 extract 16 bits to memory destination 6078multiclass SS41I_extract16<bits<8> opc, string OpcodeStr> { 6079 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0 in 6080 def rr_REV : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst), 6081 (ins VR128:$src1, u8imm:$src2), 6082 !strconcat(OpcodeStr, 6083 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6084 []>, Sched<[WriteShuffle]>; 6085 6086 let hasSideEffects = 0, mayStore = 1, 6087 SchedRW = [WriteShuffleLd, WriteRMW] in 6088 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6089 (ins i16mem:$dst, VR128:$src1, u8imm:$src2), 6090 !strconcat(OpcodeStr, 6091 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6092 [(store (i16 (trunc (assertzext (X86pextrw (v8i16 VR128:$src1), 6093 imm:$src2)))), addr:$dst)]>; 6094} 6095 6096let Predicates = [HasAVX, NoBWI] in 6097 defm VPEXTRW : SS41I_extract16<0x15, "vpextrw">, VEX; 6098 6099defm PEXTRW : SS41I_extract16<0x15, "pextrw">; 6100 6101 6102/// SS41I_extract32 - SSE 4.1 extract 32 bits to int reg or memory destination 6103multiclass SS41I_extract32<bits<8> opc, string OpcodeStr> { 6104 def rr : SS4AIi8<opc, MRMDestReg, (outs GR32:$dst), 6105 (ins VR128:$src1, u8imm:$src2), 6106 !strconcat(OpcodeStr, 6107 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6108 [(set GR32:$dst, 6109 (extractelt (v4i32 VR128:$src1), imm:$src2))]>, 6110 Sched<[WriteShuffle]>; 6111 let SchedRW = [WriteShuffleLd, WriteRMW] in 6112 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6113 (ins i32mem:$dst, VR128:$src1, u8imm:$src2), 6114 !strconcat(OpcodeStr, 6115 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6116 [(store (extractelt (v4i32 VR128:$src1), imm:$src2), 6117 addr:$dst)]>; 6118} 6119 6120let Predicates = [HasAVX, NoDQI] in 6121 defm VPEXTRD : SS41I_extract32<0x16, "vpextrd">, VEX; 6122 6123defm PEXTRD : SS41I_extract32<0x16, "pextrd">; 6124 6125/// SS41I_extract32 - SSE 4.1 extract 32 bits to int reg or memory destination 6126multiclass SS41I_extract64<bits<8> opc, string OpcodeStr> { 6127 def rr : SS4AIi8<opc, MRMDestReg, (outs GR64:$dst), 6128 (ins VR128:$src1, u8imm:$src2), 6129 !strconcat(OpcodeStr, 6130 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6131 [(set GR64:$dst, 6132 (extractelt (v2i64 VR128:$src1), imm:$src2))]>, 6133 Sched<[WriteShuffle]>, REX_W; 6134 let SchedRW = [WriteShuffleLd, WriteRMW] in 6135 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6136 (ins i64mem:$dst, VR128:$src1, u8imm:$src2), 6137 !strconcat(OpcodeStr, 6138 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6139 [(store (extractelt (v2i64 VR128:$src1), imm:$src2), 6140 addr:$dst)]>, REX_W; 6141} 6142 6143let Predicates = [HasAVX, NoDQI] in 6144 defm VPEXTRQ : SS41I_extract64<0x16, "vpextrq">, VEX, VEX_W; 6145 6146defm PEXTRQ : SS41I_extract64<0x16, "pextrq">; 6147 6148/// SS41I_extractf32 - SSE 4.1 extract 32 bits fp value to int reg or memory 6149/// destination 6150multiclass SS41I_extractf32<bits<8> opc, string OpcodeStr, 6151 OpndItins itins = DEFAULT_ITINS> { 6152 def rr : SS4AIi8<opc, MRMDestReg, (outs GR32orGR64:$dst), 6153 (ins VR128:$src1, u8imm:$src2), 6154 !strconcat(OpcodeStr, 6155 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6156 [(set GR32orGR64:$dst, 6157 (extractelt (bc_v4i32 (v4f32 VR128:$src1)), imm:$src2))], 6158 itins.rr>, Sched<[WriteFBlend]>; 6159 let SchedRW = [WriteFBlendLd, WriteRMW] in 6160 def mr : SS4AIi8<opc, MRMDestMem, (outs), 6161 (ins f32mem:$dst, VR128:$src1, u8imm:$src2), 6162 !strconcat(OpcodeStr, 6163 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6164 [(store (extractelt (bc_v4i32 (v4f32 VR128:$src1)), imm:$src2), 6165 addr:$dst)], itins.rm>; 6166} 6167 6168let ExeDomain = SSEPackedSingle in { 6169 let Predicates = [UseAVX] in 6170 defm VEXTRACTPS : SS41I_extractf32<0x17, "vextractps">, VEX; 6171 defm EXTRACTPS : SS41I_extractf32<0x17, "extractps", SSE_EXTRACT_ITINS>; 6172} 6173 6174// Also match an EXTRACTPS store when the store is done as f32 instead of i32. 6175def : Pat<(store (f32 (bitconvert (extractelt (bc_v4i32 (v4f32 VR128:$src1)), 6176 imm:$src2))), 6177 addr:$dst), 6178 (VEXTRACTPSmr addr:$dst, VR128:$src1, imm:$src2)>, 6179 Requires<[HasAVX]>; 6180def : Pat<(store (f32 (bitconvert (extractelt (bc_v4i32 (v4f32 VR128:$src1)), 6181 imm:$src2))), 6182 addr:$dst), 6183 (EXTRACTPSmr addr:$dst, VR128:$src1, imm:$src2)>, 6184 Requires<[UseSSE41]>; 6185 6186//===----------------------------------------------------------------------===// 6187// SSE4.1 - Insert Instructions 6188//===----------------------------------------------------------------------===// 6189 6190multiclass SS41I_insert8<bits<8> opc, string asm, bit Is2Addr = 1> { 6191 def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst), 6192 (ins VR128:$src1, GR32orGR64:$src2, u8imm:$src3), 6193 !if(Is2Addr, 6194 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6195 !strconcat(asm, 6196 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6197 [(set VR128:$dst, 6198 (X86pinsrb VR128:$src1, GR32orGR64:$src2, imm:$src3))]>, 6199 Sched<[WriteShuffle]>; 6200 def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), 6201 (ins VR128:$src1, i8mem:$src2, u8imm:$src3), 6202 !if(Is2Addr, 6203 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6204 !strconcat(asm, 6205 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6206 [(set VR128:$dst, 6207 (X86pinsrb VR128:$src1, (extloadi8 addr:$src2), 6208 imm:$src3))]>, Sched<[WriteShuffleLd, ReadAfterLd]>; 6209} 6210 6211let Predicates = [HasAVX, NoBWI] in 6212 defm VPINSRB : SS41I_insert8<0x20, "vpinsrb", 0>, VEX_4V; 6213let Constraints = "$src1 = $dst" in 6214 defm PINSRB : SS41I_insert8<0x20, "pinsrb">; 6215 6216multiclass SS41I_insert32<bits<8> opc, string asm, bit Is2Addr = 1> { 6217 def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst), 6218 (ins VR128:$src1, GR32:$src2, u8imm:$src3), 6219 !if(Is2Addr, 6220 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6221 !strconcat(asm, 6222 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6223 [(set VR128:$dst, 6224 (v4i32 (insertelt VR128:$src1, GR32:$src2, imm:$src3)))]>, 6225 Sched<[WriteShuffle]>; 6226 def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), 6227 (ins VR128:$src1, i32mem:$src2, u8imm:$src3), 6228 !if(Is2Addr, 6229 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6230 !strconcat(asm, 6231 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6232 [(set VR128:$dst, 6233 (v4i32 (insertelt VR128:$src1, (loadi32 addr:$src2), 6234 imm:$src3)))]>, Sched<[WriteShuffleLd, ReadAfterLd]>; 6235} 6236 6237let Predicates = [HasAVX, NoDQI] in 6238 defm VPINSRD : SS41I_insert32<0x22, "vpinsrd", 0>, VEX_4V; 6239let Constraints = "$src1 = $dst" in 6240 defm PINSRD : SS41I_insert32<0x22, "pinsrd">; 6241 6242multiclass SS41I_insert64<bits<8> opc, string asm, bit Is2Addr = 1> { 6243 def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst), 6244 (ins VR128:$src1, GR64:$src2, u8imm:$src3), 6245 !if(Is2Addr, 6246 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6247 !strconcat(asm, 6248 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6249 [(set VR128:$dst, 6250 (v2i64 (insertelt VR128:$src1, GR64:$src2, imm:$src3)))]>, 6251 Sched<[WriteShuffle]>; 6252 def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), 6253 (ins VR128:$src1, i64mem:$src2, u8imm:$src3), 6254 !if(Is2Addr, 6255 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6256 !strconcat(asm, 6257 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6258 [(set VR128:$dst, 6259 (v2i64 (insertelt VR128:$src1, (loadi64 addr:$src2), 6260 imm:$src3)))]>, Sched<[WriteShuffleLd, ReadAfterLd]>; 6261} 6262 6263let Predicates = [HasAVX, NoDQI] in 6264 defm VPINSRQ : SS41I_insert64<0x22, "vpinsrq", 0>, VEX_4V, VEX_W; 6265let Constraints = "$src1 = $dst" in 6266 defm PINSRQ : SS41I_insert64<0x22, "pinsrq">, REX_W; 6267 6268// insertps has a few different modes, there's the first two here below which 6269// are optimized inserts that won't zero arbitrary elements in the destination 6270// vector. The next one matches the intrinsic and could zero arbitrary elements 6271// in the target vector. 6272multiclass SS41I_insertf32<bits<8> opc, string asm, bit Is2Addr = 1, 6273 OpndItins itins = DEFAULT_ITINS> { 6274 def rr : SS4AIi8<opc, MRMSrcReg, (outs VR128:$dst), 6275 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 6276 !if(Is2Addr, 6277 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6278 !strconcat(asm, 6279 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6280 [(set VR128:$dst, 6281 (X86insertps VR128:$src1, VR128:$src2, imm:$src3))], itins.rr>, 6282 Sched<[WriteFShuffle]>; 6283 def rm : SS4AIi8<opc, MRMSrcMem, (outs VR128:$dst), 6284 (ins VR128:$src1, f32mem:$src2, u8imm:$src3), 6285 !if(Is2Addr, 6286 !strconcat(asm, "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6287 !strconcat(asm, 6288 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6289 [(set VR128:$dst, 6290 (X86insertps VR128:$src1, 6291 (v4f32 (scalar_to_vector (loadf32 addr:$src2))), 6292 imm:$src3))], itins.rm>, 6293 Sched<[WriteFShuffleLd, ReadAfterLd]>; 6294} 6295 6296let ExeDomain = SSEPackedSingle in { 6297 let Predicates = [UseAVX] in 6298 defm VINSERTPS : SS41I_insertf32<0x21, "vinsertps", 0>, VEX_4V; 6299 let Constraints = "$src1 = $dst" in 6300 defm INSERTPS : SS41I_insertf32<0x21, "insertps", 1, SSE_INSERT_ITINS>; 6301} 6302 6303let Predicates = [UseSSE41] in { 6304 // If we're inserting an element from a load or a null pshuf of a load, 6305 // fold the load into the insertps instruction. 6306 def : Pat<(v4f32 (X86insertps (v4f32 VR128:$src1), (X86PShufd (v4f32 6307 (scalar_to_vector (loadf32 addr:$src2))), (i8 0)), 6308 imm:$src3)), 6309 (INSERTPSrm VR128:$src1, addr:$src2, imm:$src3)>; 6310 def : Pat<(v4f32 (X86insertps (v4f32 VR128:$src1), (X86PShufd 6311 (loadv4f32 addr:$src2), (i8 0)), imm:$src3)), 6312 (INSERTPSrm VR128:$src1, addr:$src2, imm:$src3)>; 6313} 6314 6315let Predicates = [UseAVX] in { 6316 // If we're inserting an element from a vbroadcast of a load, fold the 6317 // load into the X86insertps instruction. 6318 def : Pat<(v4f32 (X86insertps (v4f32 VR128:$src1), 6319 (X86VBroadcast (loadf32 addr:$src2)), imm:$src3)), 6320 (VINSERTPSrm VR128:$src1, addr:$src2, imm:$src3)>; 6321 def : Pat<(v4f32 (X86insertps (v4f32 VR128:$src1), 6322 (X86VBroadcast (loadv4f32 addr:$src2)), imm:$src3)), 6323 (VINSERTPSrm VR128:$src1, addr:$src2, imm:$src3)>; 6324} 6325 6326//===----------------------------------------------------------------------===// 6327// SSE4.1 - Round Instructions 6328//===----------------------------------------------------------------------===// 6329 6330multiclass sse41_fp_unop_rm<bits<8> opcps, bits<8> opcpd, string OpcodeStr, 6331 X86MemOperand x86memop, RegisterClass RC, 6332 PatFrag mem_frag32, PatFrag mem_frag64, 6333 Intrinsic V4F32Int, Intrinsic V2F64Int> { 6334let ExeDomain = SSEPackedSingle in { 6335 // Intrinsic operation, reg. 6336 // Vector intrinsic operation, reg 6337 def PSr : SS4AIi8<opcps, MRMSrcReg, 6338 (outs RC:$dst), (ins RC:$src1, i32u8imm:$src2), 6339 !strconcat(OpcodeStr, 6340 "ps\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6341 [(set RC:$dst, (V4F32Int RC:$src1, imm:$src2))], 6342 IIC_SSE_ROUNDPS_REG>, Sched<[WriteFAdd]>; 6343 6344 // Vector intrinsic operation, mem 6345 def PSm : SS4AIi8<opcps, MRMSrcMem, 6346 (outs RC:$dst), (ins x86memop:$src1, i32u8imm:$src2), 6347 !strconcat(OpcodeStr, 6348 "ps\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6349 [(set RC:$dst, 6350 (V4F32Int (mem_frag32 addr:$src1),imm:$src2))], 6351 IIC_SSE_ROUNDPS_MEM>, Sched<[WriteFAddLd]>; 6352} // ExeDomain = SSEPackedSingle 6353 6354let ExeDomain = SSEPackedDouble in { 6355 // Vector intrinsic operation, reg 6356 def PDr : SS4AIi8<opcpd, MRMSrcReg, 6357 (outs RC:$dst), (ins RC:$src1, i32u8imm:$src2), 6358 !strconcat(OpcodeStr, 6359 "pd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6360 [(set RC:$dst, (V2F64Int RC:$src1, imm:$src2))], 6361 IIC_SSE_ROUNDPS_REG>, Sched<[WriteFAdd]>; 6362 6363 // Vector intrinsic operation, mem 6364 def PDm : SS4AIi8<opcpd, MRMSrcMem, 6365 (outs RC:$dst), (ins x86memop:$src1, i32u8imm:$src2), 6366 !strconcat(OpcodeStr, 6367 "pd\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 6368 [(set RC:$dst, 6369 (V2F64Int (mem_frag64 addr:$src1),imm:$src2))], 6370 IIC_SSE_ROUNDPS_REG>, Sched<[WriteFAddLd]>; 6371} // ExeDomain = SSEPackedDouble 6372} 6373 6374multiclass sse41_fp_binop_rm<bits<8> opcss, bits<8> opcsd, 6375 string OpcodeStr, 6376 Intrinsic F32Int, 6377 Intrinsic F64Int, bit Is2Addr = 1> { 6378let ExeDomain = GenericDomain in { 6379 // Operation, reg. 6380 let hasSideEffects = 0 in 6381 def SSr : SS4AIi8<opcss, MRMSrcReg, 6382 (outs FR32:$dst), (ins FR32:$src1, FR32:$src2, i32u8imm:$src3), 6383 !if(Is2Addr, 6384 !strconcat(OpcodeStr, 6385 "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6386 !strconcat(OpcodeStr, 6387 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6388 []>, Sched<[WriteFAdd]>; 6389 6390 // Intrinsic operation, reg. 6391 let isCodeGenOnly = 1 in 6392 def SSr_Int : SS4AIi8<opcss, MRMSrcReg, 6393 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32u8imm:$src3), 6394 !if(Is2Addr, 6395 !strconcat(OpcodeStr, 6396 "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6397 !strconcat(OpcodeStr, 6398 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6399 [(set VR128:$dst, (F32Int VR128:$src1, VR128:$src2, imm:$src3))]>, 6400 Sched<[WriteFAdd]>; 6401 6402 // Intrinsic operation, mem. 6403 def SSm : SS4AIi8<opcss, MRMSrcMem, 6404 (outs VR128:$dst), (ins VR128:$src1, ssmem:$src2, i32u8imm:$src3), 6405 !if(Is2Addr, 6406 !strconcat(OpcodeStr, 6407 "ss\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6408 !strconcat(OpcodeStr, 6409 "ss\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6410 [(set VR128:$dst, 6411 (F32Int VR128:$src1, sse_load_f32:$src2, imm:$src3))]>, 6412 Sched<[WriteFAddLd, ReadAfterLd]>; 6413 6414 // Operation, reg. 6415 let hasSideEffects = 0 in 6416 def SDr : SS4AIi8<opcsd, MRMSrcReg, 6417 (outs FR64:$dst), (ins FR64:$src1, FR64:$src2, i32u8imm:$src3), 6418 !if(Is2Addr, 6419 !strconcat(OpcodeStr, 6420 "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6421 !strconcat(OpcodeStr, 6422 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6423 []>, Sched<[WriteFAdd]>; 6424 6425 // Intrinsic operation, reg. 6426 let isCodeGenOnly = 1 in 6427 def SDr_Int : SS4AIi8<opcsd, MRMSrcReg, 6428 (outs VR128:$dst), (ins VR128:$src1, VR128:$src2, i32u8imm:$src3), 6429 !if(Is2Addr, 6430 !strconcat(OpcodeStr, 6431 "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6432 !strconcat(OpcodeStr, 6433 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6434 [(set VR128:$dst, (F64Int VR128:$src1, VR128:$src2, imm:$src3))]>, 6435 Sched<[WriteFAdd]>; 6436 6437 // Intrinsic operation, mem. 6438 def SDm : SS4AIi8<opcsd, MRMSrcMem, 6439 (outs VR128:$dst), (ins VR128:$src1, sdmem:$src2, i32u8imm:$src3), 6440 !if(Is2Addr, 6441 !strconcat(OpcodeStr, 6442 "sd\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6443 !strconcat(OpcodeStr, 6444 "sd\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6445 [(set VR128:$dst, 6446 (F64Int VR128:$src1, sse_load_f64:$src2, imm:$src3))]>, 6447 Sched<[WriteFAddLd, ReadAfterLd]>; 6448} // ExeDomain = GenericDomain 6449} 6450 6451// FP round - roundss, roundps, roundsd, roundpd 6452let Predicates = [HasAVX] in { 6453 // Intrinsic form 6454 defm VROUND : sse41_fp_unop_rm<0x08, 0x09, "vround", f128mem, VR128, 6455 loadv4f32, loadv2f64, 6456 int_x86_sse41_round_ps, 6457 int_x86_sse41_round_pd>, VEX; 6458 defm VROUNDY : sse41_fp_unop_rm<0x08, 0x09, "vround", f256mem, VR256, 6459 loadv8f32, loadv4f64, 6460 int_x86_avx_round_ps_256, 6461 int_x86_avx_round_pd_256>, VEX, VEX_L; 6462 defm VROUND : sse41_fp_binop_rm<0x0A, 0x0B, "vround", 6463 int_x86_sse41_round_ss, 6464 int_x86_sse41_round_sd, 0>, VEX_4V, VEX_LIG; 6465} 6466 6467let Predicates = [UseAVX] in { 6468 def : Pat<(ffloor FR32:$src), 6469 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x9))>; 6470 def : Pat<(f64 (ffloor FR64:$src)), 6471 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x9))>; 6472 def : Pat<(f32 (fnearbyint FR32:$src)), 6473 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xC))>; 6474 def : Pat<(f64 (fnearbyint FR64:$src)), 6475 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xC))>; 6476 def : Pat<(f32 (fceil FR32:$src)), 6477 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xA))>; 6478 def : Pat<(f64 (fceil FR64:$src)), 6479 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xA))>; 6480 def : Pat<(f32 (frint FR32:$src)), 6481 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x4))>; 6482 def : Pat<(f64 (frint FR64:$src)), 6483 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x4))>; 6484 def : Pat<(f32 (ftrunc FR32:$src)), 6485 (VROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xB))>; 6486 def : Pat<(f64 (ftrunc FR64:$src)), 6487 (VROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xB))>; 6488} 6489 6490let Predicates = [HasAVX] in { 6491 def : Pat<(v4f32 (ffloor VR128:$src)), 6492 (VROUNDPSr VR128:$src, (i32 0x9))>; 6493 def : Pat<(v4f32 (fnearbyint VR128:$src)), 6494 (VROUNDPSr VR128:$src, (i32 0xC))>; 6495 def : Pat<(v4f32 (fceil VR128:$src)), 6496 (VROUNDPSr VR128:$src, (i32 0xA))>; 6497 def : Pat<(v4f32 (frint VR128:$src)), 6498 (VROUNDPSr VR128:$src, (i32 0x4))>; 6499 def : Pat<(v4f32 (ftrunc VR128:$src)), 6500 (VROUNDPSr VR128:$src, (i32 0xB))>; 6501 6502 def : Pat<(v2f64 (ffloor VR128:$src)), 6503 (VROUNDPDr VR128:$src, (i32 0x9))>; 6504 def : Pat<(v2f64 (fnearbyint VR128:$src)), 6505 (VROUNDPDr VR128:$src, (i32 0xC))>; 6506 def : Pat<(v2f64 (fceil VR128:$src)), 6507 (VROUNDPDr VR128:$src, (i32 0xA))>; 6508 def : Pat<(v2f64 (frint VR128:$src)), 6509 (VROUNDPDr VR128:$src, (i32 0x4))>; 6510 def : Pat<(v2f64 (ftrunc VR128:$src)), 6511 (VROUNDPDr VR128:$src, (i32 0xB))>; 6512 6513 def : Pat<(v8f32 (ffloor VR256:$src)), 6514 (VROUNDYPSr VR256:$src, (i32 0x9))>; 6515 def : Pat<(v8f32 (fnearbyint VR256:$src)), 6516 (VROUNDYPSr VR256:$src, (i32 0xC))>; 6517 def : Pat<(v8f32 (fceil VR256:$src)), 6518 (VROUNDYPSr VR256:$src, (i32 0xA))>; 6519 def : Pat<(v8f32 (frint VR256:$src)), 6520 (VROUNDYPSr VR256:$src, (i32 0x4))>; 6521 def : Pat<(v8f32 (ftrunc VR256:$src)), 6522 (VROUNDYPSr VR256:$src, (i32 0xB))>; 6523 6524 def : Pat<(v4f64 (ffloor VR256:$src)), 6525 (VROUNDYPDr VR256:$src, (i32 0x9))>; 6526 def : Pat<(v4f64 (fnearbyint VR256:$src)), 6527 (VROUNDYPDr VR256:$src, (i32 0xC))>; 6528 def : Pat<(v4f64 (fceil VR256:$src)), 6529 (VROUNDYPDr VR256:$src, (i32 0xA))>; 6530 def : Pat<(v4f64 (frint VR256:$src)), 6531 (VROUNDYPDr VR256:$src, (i32 0x4))>; 6532 def : Pat<(v4f64 (ftrunc VR256:$src)), 6533 (VROUNDYPDr VR256:$src, (i32 0xB))>; 6534} 6535 6536defm ROUND : sse41_fp_unop_rm<0x08, 0x09, "round", f128mem, VR128, 6537 memopv4f32, memopv2f64, 6538 int_x86_sse41_round_ps, int_x86_sse41_round_pd>; 6539let Constraints = "$src1 = $dst" in 6540defm ROUND : sse41_fp_binop_rm<0x0A, 0x0B, "round", 6541 int_x86_sse41_round_ss, int_x86_sse41_round_sd>; 6542 6543let Predicates = [UseSSE41] in { 6544 def : Pat<(ffloor FR32:$src), 6545 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x9))>; 6546 def : Pat<(f64 (ffloor FR64:$src)), 6547 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x9))>; 6548 def : Pat<(f32 (fnearbyint FR32:$src)), 6549 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xC))>; 6550 def : Pat<(f64 (fnearbyint FR64:$src)), 6551 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xC))>; 6552 def : Pat<(f32 (fceil FR32:$src)), 6553 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xA))>; 6554 def : Pat<(f64 (fceil FR64:$src)), 6555 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xA))>; 6556 def : Pat<(f32 (frint FR32:$src)), 6557 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0x4))>; 6558 def : Pat<(f64 (frint FR64:$src)), 6559 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0x4))>; 6560 def : Pat<(f32 (ftrunc FR32:$src)), 6561 (ROUNDSSr (f32 (IMPLICIT_DEF)), FR32:$src, (i32 0xB))>; 6562 def : Pat<(f64 (ftrunc FR64:$src)), 6563 (ROUNDSDr (f64 (IMPLICIT_DEF)), FR64:$src, (i32 0xB))>; 6564 6565 def : Pat<(v4f32 (ffloor VR128:$src)), 6566 (ROUNDPSr VR128:$src, (i32 0x9))>; 6567 def : Pat<(v4f32 (fnearbyint VR128:$src)), 6568 (ROUNDPSr VR128:$src, (i32 0xC))>; 6569 def : Pat<(v4f32 (fceil VR128:$src)), 6570 (ROUNDPSr VR128:$src, (i32 0xA))>; 6571 def : Pat<(v4f32 (frint VR128:$src)), 6572 (ROUNDPSr VR128:$src, (i32 0x4))>; 6573 def : Pat<(v4f32 (ftrunc VR128:$src)), 6574 (ROUNDPSr VR128:$src, (i32 0xB))>; 6575 6576 def : Pat<(v2f64 (ffloor VR128:$src)), 6577 (ROUNDPDr VR128:$src, (i32 0x9))>; 6578 def : Pat<(v2f64 (fnearbyint VR128:$src)), 6579 (ROUNDPDr VR128:$src, (i32 0xC))>; 6580 def : Pat<(v2f64 (fceil VR128:$src)), 6581 (ROUNDPDr VR128:$src, (i32 0xA))>; 6582 def : Pat<(v2f64 (frint VR128:$src)), 6583 (ROUNDPDr VR128:$src, (i32 0x4))>; 6584 def : Pat<(v2f64 (ftrunc VR128:$src)), 6585 (ROUNDPDr VR128:$src, (i32 0xB))>; 6586} 6587 6588//===----------------------------------------------------------------------===// 6589// SSE4.1 - Packed Bit Test 6590//===----------------------------------------------------------------------===// 6591 6592// ptest instruction we'll lower to this in X86ISelLowering primarily from 6593// the intel intrinsic that corresponds to this. 6594let Defs = [EFLAGS], Predicates = [HasAVX] in { 6595def VPTESTrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2), 6596 "vptest\t{$src2, $src1|$src1, $src2}", 6597 [(set EFLAGS, (X86ptest VR128:$src1, (v2i64 VR128:$src2)))]>, 6598 Sched<[WriteVecLogic]>, VEX; 6599def VPTESTrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR128:$src1, f128mem:$src2), 6600 "vptest\t{$src2, $src1|$src1, $src2}", 6601 [(set EFLAGS,(X86ptest VR128:$src1, (loadv2i64 addr:$src2)))]>, 6602 Sched<[WriteVecLogicLd, ReadAfterLd]>, VEX; 6603 6604def VPTESTYrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR256:$src1, VR256:$src2), 6605 "vptest\t{$src2, $src1|$src1, $src2}", 6606 [(set EFLAGS, (X86ptest VR256:$src1, (v4i64 VR256:$src2)))]>, 6607 Sched<[WriteVecLogic]>, VEX, VEX_L; 6608def VPTESTYrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR256:$src1, i256mem:$src2), 6609 "vptest\t{$src2, $src1|$src1, $src2}", 6610 [(set EFLAGS,(X86ptest VR256:$src1, (loadv4i64 addr:$src2)))]>, 6611 Sched<[WriteVecLogicLd, ReadAfterLd]>, VEX, VEX_L; 6612} 6613 6614let Defs = [EFLAGS] in { 6615def PTESTrr : SS48I<0x17, MRMSrcReg, (outs), (ins VR128:$src1, VR128:$src2), 6616 "ptest\t{$src2, $src1|$src1, $src2}", 6617 [(set EFLAGS, (X86ptest VR128:$src1, (v2i64 VR128:$src2)))]>, 6618 Sched<[WriteVecLogic]>; 6619def PTESTrm : SS48I<0x17, MRMSrcMem, (outs), (ins VR128:$src1, f128mem:$src2), 6620 "ptest\t{$src2, $src1|$src1, $src2}", 6621 [(set EFLAGS, (X86ptest VR128:$src1, (memopv2i64 addr:$src2)))]>, 6622 Sched<[WriteVecLogicLd, ReadAfterLd]>; 6623} 6624 6625// The bit test instructions below are AVX only 6626multiclass avx_bittest<bits<8> opc, string OpcodeStr, RegisterClass RC, 6627 X86MemOperand x86memop, PatFrag mem_frag, ValueType vt> { 6628 def rr : SS48I<opc, MRMSrcReg, (outs), (ins RC:$src1, RC:$src2), 6629 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), 6630 [(set EFLAGS, (X86testp RC:$src1, (vt RC:$src2)))]>, 6631 Sched<[WriteVecLogic]>, VEX; 6632 def rm : SS48I<opc, MRMSrcMem, (outs), (ins RC:$src1, x86memop:$src2), 6633 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"), 6634 [(set EFLAGS, (X86testp RC:$src1, (mem_frag addr:$src2)))]>, 6635 Sched<[WriteVecLogicLd, ReadAfterLd]>, VEX; 6636} 6637 6638let Defs = [EFLAGS], Predicates = [HasAVX] in { 6639let ExeDomain = SSEPackedSingle in { 6640defm VTESTPS : avx_bittest<0x0E, "vtestps", VR128, f128mem, loadv4f32, v4f32>; 6641defm VTESTPSY : avx_bittest<0x0E, "vtestps", VR256, f256mem, loadv8f32, v8f32>, 6642 VEX_L; 6643} 6644let ExeDomain = SSEPackedDouble in { 6645defm VTESTPD : avx_bittest<0x0F, "vtestpd", VR128, f128mem, loadv2f64, v2f64>; 6646defm VTESTPDY : avx_bittest<0x0F, "vtestpd", VR256, f256mem, loadv4f64, v4f64>, 6647 VEX_L; 6648} 6649} 6650 6651//===----------------------------------------------------------------------===// 6652// SSE4.1 - Misc Instructions 6653//===----------------------------------------------------------------------===// 6654 6655let Defs = [EFLAGS], Predicates = [HasPOPCNT] in { 6656 def POPCNT16rr : I<0xB8, MRMSrcReg, (outs GR16:$dst), (ins GR16:$src), 6657 "popcnt{w}\t{$src, $dst|$dst, $src}", 6658 [(set GR16:$dst, (ctpop GR16:$src)), (implicit EFLAGS)], 6659 IIC_SSE_POPCNT_RR>, Sched<[WriteFAdd]>, 6660 OpSize16, XS; 6661 def POPCNT16rm : I<0xB8, MRMSrcMem, (outs GR16:$dst), (ins i16mem:$src), 6662 "popcnt{w}\t{$src, $dst|$dst, $src}", 6663 [(set GR16:$dst, (ctpop (loadi16 addr:$src))), 6664 (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, 6665 Sched<[WriteFAddLd]>, OpSize16, XS; 6666 6667 def POPCNT32rr : I<0xB8, MRMSrcReg, (outs GR32:$dst), (ins GR32:$src), 6668 "popcnt{l}\t{$src, $dst|$dst, $src}", 6669 [(set GR32:$dst, (ctpop GR32:$src)), (implicit EFLAGS)], 6670 IIC_SSE_POPCNT_RR>, Sched<[WriteFAdd]>, 6671 OpSize32, XS; 6672 6673 def POPCNT32rm : I<0xB8, MRMSrcMem, (outs GR32:$dst), (ins i32mem:$src), 6674 "popcnt{l}\t{$src, $dst|$dst, $src}", 6675 [(set GR32:$dst, (ctpop (loadi32 addr:$src))), 6676 (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, 6677 Sched<[WriteFAddLd]>, OpSize32, XS; 6678 6679 def POPCNT64rr : RI<0xB8, MRMSrcReg, (outs GR64:$dst), (ins GR64:$src), 6680 "popcnt{q}\t{$src, $dst|$dst, $src}", 6681 [(set GR64:$dst, (ctpop GR64:$src)), (implicit EFLAGS)], 6682 IIC_SSE_POPCNT_RR>, Sched<[WriteFAdd]>, XS; 6683 def POPCNT64rm : RI<0xB8, MRMSrcMem, (outs GR64:$dst), (ins i64mem:$src), 6684 "popcnt{q}\t{$src, $dst|$dst, $src}", 6685 [(set GR64:$dst, (ctpop (loadi64 addr:$src))), 6686 (implicit EFLAGS)], IIC_SSE_POPCNT_RM>, 6687 Sched<[WriteFAddLd]>, XS; 6688} 6689 6690 6691 6692// SS41I_unop_rm_int_v16 - SSE 4.1 unary operator whose type is v8i16. 6693multiclass SS41I_unop_rm_int_v16<bits<8> opc, string OpcodeStr, 6694 Intrinsic IntId128, PatFrag ld_frag, 6695 X86FoldableSchedWrite Sched> { 6696 def rr128 : SS48I<opc, MRMSrcReg, (outs VR128:$dst), 6697 (ins VR128:$src), 6698 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 6699 [(set VR128:$dst, (IntId128 VR128:$src))]>, 6700 Sched<[Sched]>; 6701 def rm128 : SS48I<opc, MRMSrcMem, (outs VR128:$dst), 6702 (ins i128mem:$src), 6703 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 6704 [(set VR128:$dst, 6705 (IntId128 (bitconvert (ld_frag addr:$src))))]>, 6706 Sched<[Sched.Folded]>; 6707} 6708 6709// PHMIN has the same profile as PSAD, thus we use the same scheduling 6710// model, although the naming is misleading. 6711let Predicates = [HasAVX] in 6712defm VPHMINPOSUW : SS41I_unop_rm_int_v16 <0x41, "vphminposuw", 6713 int_x86_sse41_phminposuw, loadv2i64, 6714 WriteVecIMul>, VEX; 6715defm PHMINPOSUW : SS41I_unop_rm_int_v16 <0x41, "phminposuw", 6716 int_x86_sse41_phminposuw, memopv2i64, 6717 WriteVecIMul>; 6718 6719/// SS48I_binop_rm - Simple SSE41 binary operator. 6720multiclass SS48I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 6721 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 6722 X86MemOperand x86memop, bit Is2Addr = 1, 6723 OpndItins itins = SSE_INTALU_ITINS_P> { 6724 let isCommutable = 1 in 6725 def rr : SS48I<opc, MRMSrcReg, (outs RC:$dst), 6726 (ins RC:$src1, RC:$src2), 6727 !if(Is2Addr, 6728 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 6729 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 6730 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>, 6731 Sched<[itins.Sched]>; 6732 def rm : SS48I<opc, MRMSrcMem, (outs RC:$dst), 6733 (ins RC:$src1, x86memop:$src2), 6734 !if(Is2Addr, 6735 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 6736 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 6737 [(set RC:$dst, 6738 (OpVT (OpNode RC:$src1, (bitconvert (memop_frag addr:$src2)))))]>, 6739 Sched<[itins.Sched.Folded, ReadAfterLd]>; 6740} 6741 6742/// SS48I_binop_rm2 - Simple SSE41 binary operator with different src and dst 6743/// types. 6744multiclass SS48I_binop_rm2<bits<8> opc, string OpcodeStr, SDNode OpNode, 6745 ValueType DstVT, ValueType SrcVT, RegisterClass RC, 6746 PatFrag memop_frag, X86MemOperand x86memop, 6747 OpndItins itins, 6748 bit IsCommutable = 0, bit Is2Addr = 1> { 6749 let isCommutable = IsCommutable in 6750 def rr : SS48I<opc, MRMSrcReg, (outs RC:$dst), 6751 (ins RC:$src1, RC:$src2), 6752 !if(Is2Addr, 6753 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 6754 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 6755 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1), RC:$src2)))]>, 6756 Sched<[itins.Sched]>; 6757 def rm : SS48I<opc, MRMSrcMem, (outs RC:$dst), 6758 (ins RC:$src1, x86memop:$src2), 6759 !if(Is2Addr, 6760 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 6761 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 6762 [(set RC:$dst, (DstVT (OpNode (SrcVT RC:$src1), 6763 (bitconvert (memop_frag addr:$src2)))))]>, 6764 Sched<[itins.Sched.Folded, ReadAfterLd]>; 6765} 6766 6767let Predicates = [HasAVX, NoVLX] in { 6768 defm VPMINSD : SS48I_binop_rm<0x39, "vpminsd", smin, v4i32, VR128, 6769 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6770 VEX_4V; 6771 defm VPMINUD : SS48I_binop_rm<0x3B, "vpminud", umin, v4i32, VR128, 6772 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6773 VEX_4V; 6774 defm VPMAXSD : SS48I_binop_rm<0x3D, "vpmaxsd", smax, v4i32, VR128, 6775 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6776 VEX_4V; 6777 defm VPMAXUD : SS48I_binop_rm<0x3F, "vpmaxud", umax, v4i32, VR128, 6778 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6779 VEX_4V; 6780 defm VPMULDQ : SS48I_binop_rm2<0x28, "vpmuldq", X86pmuldq, v2i64, v4i32, 6781 VR128, loadv2i64, i128mem, 6782 SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V; 6783} 6784let Predicates = [HasAVX, NoVLX_Or_NoBWI] in { 6785 defm VPMINSB : SS48I_binop_rm<0x38, "vpminsb", smin, v16i8, VR128, 6786 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6787 VEX_4V; 6788 defm VPMINUW : SS48I_binop_rm<0x3A, "vpminuw", umin, v8i16, VR128, 6789 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6790 VEX_4V; 6791 defm VPMAXSB : SS48I_binop_rm<0x3C, "vpmaxsb", smax, v16i8, VR128, 6792 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6793 VEX_4V; 6794 defm VPMAXUW : SS48I_binop_rm<0x3E, "vpmaxuw", umax, v8i16, VR128, 6795 loadv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6796 VEX_4V; 6797} 6798 6799let Predicates = [HasAVX2, NoVLX] in { 6800 defm VPMINSDY : SS48I_binop_rm<0x39, "vpminsd", smin, v8i32, VR256, 6801 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6802 VEX_4V, VEX_L; 6803 defm VPMINUDY : SS48I_binop_rm<0x3B, "vpminud", umin, v8i32, VR256, 6804 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6805 VEX_4V, VEX_L; 6806 defm VPMAXSDY : SS48I_binop_rm<0x3D, "vpmaxsd", smax, v8i32, VR256, 6807 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6808 VEX_4V, VEX_L; 6809 defm VPMAXUDY : SS48I_binop_rm<0x3F, "vpmaxud", umax, v8i32, VR256, 6810 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6811 VEX_4V, VEX_L; 6812 defm VPMULDQY : SS48I_binop_rm2<0x28, "vpmuldq", X86pmuldq, v4i64, v8i32, 6813 VR256, loadv4i64, i256mem, 6814 SSE_INTMUL_ITINS_P, 1, 0>, VEX_4V, VEX_L; 6815} 6816let Predicates = [HasAVX2, NoVLX_Or_NoBWI] in { 6817 defm VPMINSBY : SS48I_binop_rm<0x38, "vpminsb", smin, v32i8, VR256, 6818 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6819 VEX_4V, VEX_L; 6820 defm VPMINUWY : SS48I_binop_rm<0x3A, "vpminuw", umin, v16i16, VR256, 6821 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6822 VEX_4V, VEX_L; 6823 defm VPMAXSBY : SS48I_binop_rm<0x3C, "vpmaxsb", smax, v32i8, VR256, 6824 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6825 VEX_4V, VEX_L; 6826 defm VPMAXUWY : SS48I_binop_rm<0x3E, "vpmaxuw", umax, v16i16, VR256, 6827 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6828 VEX_4V, VEX_L; 6829} 6830 6831let Constraints = "$src1 = $dst" in { 6832 defm PMINSB : SS48I_binop_rm<0x38, "pminsb", smin, v16i8, VR128, 6833 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6834 defm PMINSD : SS48I_binop_rm<0x39, "pminsd", smin, v4i32, VR128, 6835 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6836 defm PMINUD : SS48I_binop_rm<0x3B, "pminud", umin, v4i32, VR128, 6837 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6838 defm PMINUW : SS48I_binop_rm<0x3A, "pminuw", umin, v8i16, VR128, 6839 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6840 defm PMAXSB : SS48I_binop_rm<0x3C, "pmaxsb", smax, v16i8, VR128, 6841 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6842 defm PMAXSD : SS48I_binop_rm<0x3D, "pmaxsd", smax, v4i32, VR128, 6843 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6844 defm PMAXUD : SS48I_binop_rm<0x3F, "pmaxud", umax, v4i32, VR128, 6845 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6846 defm PMAXUW : SS48I_binop_rm<0x3E, "pmaxuw", umax, v8i16, VR128, 6847 memopv2i64, i128mem, 1, SSE_INTALU_ITINS_P>; 6848 defm PMULDQ : SS48I_binop_rm2<0x28, "pmuldq", X86pmuldq, v2i64, v4i32, 6849 VR128, memopv2i64, i128mem, 6850 SSE_INTMUL_ITINS_P, 1>; 6851} 6852 6853let Predicates = [HasAVX, NoVLX] in { 6854 defm VPMULLD : SS48I_binop_rm<0x40, "vpmulld", mul, v4i32, VR128, 6855 memopv2i64, i128mem, 0, SSE_PMULLD_ITINS>, 6856 VEX_4V; 6857 defm VPCMPEQQ : SS48I_binop_rm<0x29, "vpcmpeqq", X86pcmpeq, v2i64, VR128, 6858 memopv2i64, i128mem, 0, SSE_INTALU_ITINS_P>, 6859 VEX_4V; 6860} 6861let Predicates = [HasAVX2] in { 6862 defm VPMULLDY : SS48I_binop_rm<0x40, "vpmulld", mul, v8i32, VR256, 6863 loadv4i64, i256mem, 0, SSE_PMULLD_ITINS>, 6864 VEX_4V, VEX_L; 6865 defm VPCMPEQQY : SS48I_binop_rm<0x29, "vpcmpeqq", X86pcmpeq, v4i64, VR256, 6866 loadv4i64, i256mem, 0, SSE_INTALU_ITINS_P>, 6867 VEX_4V, VEX_L; 6868} 6869 6870let Constraints = "$src1 = $dst" in { 6871 defm PMULLD : SS48I_binop_rm<0x40, "pmulld", mul, v4i32, VR128, 6872 memopv2i64, i128mem, 1, SSE_PMULLD_ITINS>; 6873 defm PCMPEQQ : SS48I_binop_rm<0x29, "pcmpeqq", X86pcmpeq, v2i64, VR128, 6874 memopv2i64, i128mem, 1, SSE_INTALUQ_ITINS_P>; 6875} 6876 6877/// SS41I_binop_rmi_int - SSE 4.1 binary operator with 8-bit immediate 6878multiclass SS41I_binop_rmi_int<bits<8> opc, string OpcodeStr, 6879 Intrinsic IntId, RegisterClass RC, PatFrag memop_frag, 6880 X86MemOperand x86memop, bit Is2Addr = 1, 6881 OpndItins itins = DEFAULT_ITINS> { 6882 let isCommutable = 1 in 6883 def rri : SS4AIi8<opc, MRMSrcReg, (outs RC:$dst), 6884 (ins RC:$src1, RC:$src2, u8imm:$src3), 6885 !if(Is2Addr, 6886 !strconcat(OpcodeStr, 6887 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6888 !strconcat(OpcodeStr, 6889 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6890 [(set RC:$dst, (IntId RC:$src1, RC:$src2, imm:$src3))], itins.rr>, 6891 Sched<[itins.Sched]>; 6892 def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst), 6893 (ins RC:$src1, x86memop:$src2, u8imm:$src3), 6894 !if(Is2Addr, 6895 !strconcat(OpcodeStr, 6896 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6897 !strconcat(OpcodeStr, 6898 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6899 [(set RC:$dst, 6900 (IntId RC:$src1, 6901 (bitconvert (memop_frag addr:$src2)), imm:$src3))], itins.rm>, 6902 Sched<[itins.Sched.Folded, ReadAfterLd]>; 6903} 6904 6905/// SS41I_binop_rmi - SSE 4.1 binary operator with 8-bit immediate 6906multiclass SS41I_binop_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode, 6907 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 6908 X86MemOperand x86memop, bit Is2Addr = 1, 6909 OpndItins itins = DEFAULT_ITINS> { 6910 let isCommutable = 1 in 6911 def rri : SS4AIi8<opc, MRMSrcReg, (outs RC:$dst), 6912 (ins RC:$src1, RC:$src2, u8imm:$src3), 6913 !if(Is2Addr, 6914 !strconcat(OpcodeStr, 6915 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6916 !strconcat(OpcodeStr, 6917 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6918 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, imm:$src3)))], 6919 itins.rr>, Sched<[itins.Sched]>; 6920 def rmi : SS4AIi8<opc, MRMSrcMem, (outs RC:$dst), 6921 (ins RC:$src1, x86memop:$src2, u8imm:$src3), 6922 !if(Is2Addr, 6923 !strconcat(OpcodeStr, 6924 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"), 6925 !strconcat(OpcodeStr, 6926 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}")), 6927 [(set RC:$dst, 6928 (OpVT (OpNode RC:$src1, 6929 (bitconvert (memop_frag addr:$src2)), imm:$src3)))], itins.rm>, 6930 Sched<[itins.Sched.Folded, ReadAfterLd]>; 6931} 6932 6933let Predicates = [HasAVX] in { 6934 let isCommutable = 0 in { 6935 defm VMPSADBW : SS41I_binop_rmi_int<0x42, "vmpsadbw", int_x86_sse41_mpsadbw, 6936 VR128, loadv2i64, i128mem, 0, 6937 DEFAULT_ITINS_MPSADSCHED>, VEX_4V; 6938 } 6939 6940 let ExeDomain = SSEPackedSingle in { 6941 defm VBLENDPS : SS41I_binop_rmi<0x0C, "vblendps", X86Blendi, v4f32, 6942 VR128, loadv4f32, f128mem, 0, 6943 DEFAULT_ITINS_FBLENDSCHED>, VEX_4V; 6944 defm VBLENDPSY : SS41I_binop_rmi<0x0C, "vblendps", X86Blendi, v8f32, 6945 VR256, loadv8f32, f256mem, 0, 6946 DEFAULT_ITINS_FBLENDSCHED>, VEX_4V, VEX_L; 6947 } 6948 let ExeDomain = SSEPackedDouble in { 6949 defm VBLENDPD : SS41I_binop_rmi<0x0D, "vblendpd", X86Blendi, v2f64, 6950 VR128, loadv2f64, f128mem, 0, 6951 DEFAULT_ITINS_FBLENDSCHED>, VEX_4V; 6952 defm VBLENDPDY : SS41I_binop_rmi<0x0D, "vblendpd", X86Blendi, v4f64, 6953 VR256, loadv4f64, f256mem, 0, 6954 DEFAULT_ITINS_FBLENDSCHED>, VEX_4V, VEX_L; 6955 } 6956 defm VPBLENDW : SS41I_binop_rmi<0x0E, "vpblendw", X86Blendi, v8i16, 6957 VR128, loadv2i64, i128mem, 0, 6958 DEFAULT_ITINS_BLENDSCHED>, VEX_4V; 6959 6960 let ExeDomain = SSEPackedSingle in 6961 defm VDPPS : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_sse41_dpps, 6962 VR128, loadv4f32, f128mem, 0, 6963 SSE_DPPS_ITINS>, VEX_4V; 6964 let ExeDomain = SSEPackedDouble in 6965 defm VDPPD : SS41I_binop_rmi_int<0x41, "vdppd", int_x86_sse41_dppd, 6966 VR128, loadv2f64, f128mem, 0, 6967 SSE_DPPS_ITINS>, VEX_4V; 6968 let ExeDomain = SSEPackedSingle in 6969 defm VDPPSY : SS41I_binop_rmi_int<0x40, "vdpps", int_x86_avx_dp_ps_256, 6970 VR256, loadv8f32, i256mem, 0, 6971 SSE_DPPS_ITINS>, VEX_4V, VEX_L; 6972} 6973 6974let Predicates = [HasAVX2] in { 6975 let isCommutable = 0 in { 6976 defm VMPSADBWY : SS41I_binop_rmi_int<0x42, "vmpsadbw", int_x86_avx2_mpsadbw, 6977 VR256, loadv4i64, i256mem, 0, 6978 DEFAULT_ITINS_MPSADSCHED>, VEX_4V, VEX_L; 6979 } 6980 defm VPBLENDWY : SS41I_binop_rmi<0x0E, "vpblendw", X86Blendi, v16i16, 6981 VR256, loadv4i64, i256mem, 0, 6982 DEFAULT_ITINS_BLENDSCHED>, VEX_4V, VEX_L; 6983} 6984 6985let Constraints = "$src1 = $dst" in { 6986 let isCommutable = 0 in { 6987 defm MPSADBW : SS41I_binop_rmi_int<0x42, "mpsadbw", int_x86_sse41_mpsadbw, 6988 VR128, memopv2i64, i128mem, 6989 1, SSE_MPSADBW_ITINS>; 6990 } 6991 let ExeDomain = SSEPackedSingle in 6992 defm BLENDPS : SS41I_binop_rmi<0x0C, "blendps", X86Blendi, v4f32, 6993 VR128, memopv4f32, f128mem, 6994 1, SSE_INTALU_ITINS_FBLEND_P>; 6995 let ExeDomain = SSEPackedDouble in 6996 defm BLENDPD : SS41I_binop_rmi<0x0D, "blendpd", X86Blendi, v2f64, 6997 VR128, memopv2f64, f128mem, 6998 1, SSE_INTALU_ITINS_FBLEND_P>; 6999 defm PBLENDW : SS41I_binop_rmi<0x0E, "pblendw", X86Blendi, v8i16, 7000 VR128, memopv2i64, i128mem, 7001 1, SSE_INTALU_ITINS_BLEND_P>; 7002 let ExeDomain = SSEPackedSingle in 7003 defm DPPS : SS41I_binop_rmi_int<0x40, "dpps", int_x86_sse41_dpps, 7004 VR128, memopv4f32, f128mem, 1, 7005 SSE_DPPS_ITINS>; 7006 let ExeDomain = SSEPackedDouble in 7007 defm DPPD : SS41I_binop_rmi_int<0x41, "dppd", int_x86_sse41_dppd, 7008 VR128, memopv2f64, f128mem, 1, 7009 SSE_DPPD_ITINS>; 7010} 7011 7012/// SS41I_quaternary_int_avx - AVX SSE 4.1 with 4 operators 7013multiclass SS41I_quaternary_int_avx<bits<8> opc, string OpcodeStr, 7014 RegisterClass RC, X86MemOperand x86memop, 7015 PatFrag mem_frag, Intrinsic IntId, 7016 X86FoldableSchedWrite Sched> { 7017 def rr : Ii8<opc, MRMSrcReg, (outs RC:$dst), 7018 (ins RC:$src1, RC:$src2, RC:$src3), 7019 !strconcat(OpcodeStr, 7020 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 7021 [(set RC:$dst, (IntId RC:$src1, RC:$src2, RC:$src3))], 7022 NoItinerary, SSEPackedInt>, TAPD, VEX_4V, VEX_I8IMM, 7023 Sched<[Sched]>; 7024 7025 def rm : Ii8<opc, MRMSrcMem, (outs RC:$dst), 7026 (ins RC:$src1, x86memop:$src2, RC:$src3), 7027 !strconcat(OpcodeStr, 7028 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 7029 [(set RC:$dst, 7030 (IntId RC:$src1, (bitconvert (mem_frag addr:$src2)), 7031 RC:$src3))], 7032 NoItinerary, SSEPackedInt>, TAPD, VEX_4V, VEX_I8IMM, 7033 Sched<[Sched.Folded, ReadAfterLd]>; 7034} 7035 7036let Predicates = [HasAVX] in { 7037let ExeDomain = SSEPackedDouble in { 7038defm VBLENDVPD : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR128, f128mem, 7039 loadv2f64, int_x86_sse41_blendvpd, 7040 WriteFVarBlend>; 7041defm VBLENDVPDY : SS41I_quaternary_int_avx<0x4B, "vblendvpd", VR256, f256mem, 7042 loadv4f64, int_x86_avx_blendv_pd_256, 7043 WriteFVarBlend>, VEX_L; 7044} // ExeDomain = SSEPackedDouble 7045let ExeDomain = SSEPackedSingle in { 7046defm VBLENDVPS : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR128, f128mem, 7047 loadv4f32, int_x86_sse41_blendvps, 7048 WriteFVarBlend>; 7049defm VBLENDVPSY : SS41I_quaternary_int_avx<0x4A, "vblendvps", VR256, f256mem, 7050 loadv8f32, int_x86_avx_blendv_ps_256, 7051 WriteFVarBlend>, VEX_L; 7052} // ExeDomain = SSEPackedSingle 7053defm VPBLENDVB : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR128, i128mem, 7054 loadv2i64, int_x86_sse41_pblendvb, 7055 WriteVarBlend>; 7056} 7057 7058let Predicates = [HasAVX2] in { 7059defm VPBLENDVBY : SS41I_quaternary_int_avx<0x4C, "vpblendvb", VR256, i256mem, 7060 loadv4i64, int_x86_avx2_pblendvb, 7061 WriteVarBlend>, VEX_L; 7062} 7063 7064let Predicates = [HasAVX] in { 7065 def : Pat<(v16i8 (vselect (v16i8 VR128:$mask), (v16i8 VR128:$src1), 7066 (v16i8 VR128:$src2))), 7067 (VPBLENDVBrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7068 def : Pat<(v4i32 (vselect (v4i32 VR128:$mask), (v4i32 VR128:$src1), 7069 (v4i32 VR128:$src2))), 7070 (VBLENDVPSrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7071 def : Pat<(v4f32 (vselect (v4i32 VR128:$mask), (v4f32 VR128:$src1), 7072 (v4f32 VR128:$src2))), 7073 (VBLENDVPSrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7074 def : Pat<(v2i64 (vselect (v2i64 VR128:$mask), (v2i64 VR128:$src1), 7075 (v2i64 VR128:$src2))), 7076 (VBLENDVPDrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7077 def : Pat<(v2f64 (vselect (v2i64 VR128:$mask), (v2f64 VR128:$src1), 7078 (v2f64 VR128:$src2))), 7079 (VBLENDVPDrr VR128:$src2, VR128:$src1, VR128:$mask)>; 7080 def : Pat<(v8i32 (vselect (v8i32 VR256:$mask), (v8i32 VR256:$src1), 7081 (v8i32 VR256:$src2))), 7082 (VBLENDVPSYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7083 def : Pat<(v8f32 (vselect (v8i32 VR256:$mask), (v8f32 VR256:$src1), 7084 (v8f32 VR256:$src2))), 7085 (VBLENDVPSYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7086 def : Pat<(v4i64 (vselect (v4i64 VR256:$mask), (v4i64 VR256:$src1), 7087 (v4i64 VR256:$src2))), 7088 (VBLENDVPDYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7089 def : Pat<(v4f64 (vselect (v4i64 VR256:$mask), (v4f64 VR256:$src1), 7090 (v4f64 VR256:$src2))), 7091 (VBLENDVPDYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7092} 7093 7094let Predicates = [HasAVX2] in { 7095 def : Pat<(v32i8 (vselect (v32i8 VR256:$mask), (v32i8 VR256:$src1), 7096 (v32i8 VR256:$src2))), 7097 (VPBLENDVBYrr VR256:$src2, VR256:$src1, VR256:$mask)>; 7098} 7099 7100// Patterns 7101// FIXME: Prefer a movss or movsd over a blendps when optimizing for size or 7102// on targets where they have equal performance. These were changed to use 7103// blends because blends have better throughput on SandyBridge and Haswell, but 7104// movs[s/d] are 1-2 byte shorter instructions. 7105let Predicates = [UseAVX] in { 7106 let AddedComplexity = 15 in { 7107 // Move scalar to XMM zero-extended, zeroing a VR128 then do a 7108 // MOVS{S,D} to the lower bits. 7109 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector FR32:$src)))), 7110 (VMOVSSrr (v4f32 (V_SET0)), FR32:$src)>; 7111 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))), 7112 (VBLENDPSrri (v4f32 (V_SET0)), VR128:$src, (i8 1))>; 7113 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))), 7114 (VPBLENDWrri (v4i32 (V_SET0)), VR128:$src, (i8 3))>; 7115 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64:$src)))), 7116 (VMOVSDrr (v2f64 (V_SET0)), FR64:$src)>; 7117 7118 // Move low f32 and clear high bits. 7119 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256:$src))), 7120 (VBLENDPSYrri (v8f32 (AVX_SET0)), VR256:$src, (i8 1))>; 7121 7122 // Move low f64 and clear high bits. 7123 def : Pat<(v4f64 (X86vzmovl (v4f64 VR256:$src))), 7124 (VBLENDPDYrri (v4f64 (AVX_SET0)), VR256:$src, (i8 1))>; 7125 } 7126 7127 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef, 7128 (v4f32 (scalar_to_vector FR32:$src)), (iPTR 0)))), 7129 (SUBREG_TO_REG (i32 0), 7130 (v4f32 (VMOVSSrr (v4f32 (V_SET0)), FR32:$src)), 7131 sub_xmm)>; 7132 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef, 7133 (v2f64 (scalar_to_vector FR64:$src)), (iPTR 0)))), 7134 (SUBREG_TO_REG (i64 0), 7135 (v2f64 (VMOVSDrr (v2f64 (V_SET0)), FR64:$src)), 7136 sub_xmm)>; 7137 7138 // These will incur an FP/int domain crossing penalty, but it may be the only 7139 // way without AVX2. Do not add any complexity because we may be able to match 7140 // more optimal patterns defined earlier in this file. 7141 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256:$src))), 7142 (VBLENDPSYrri (v8i32 (AVX_SET0)), VR256:$src, (i8 1))>; 7143 def : Pat<(v4i64 (X86vzmovl (v4i64 VR256:$src))), 7144 (VBLENDPDYrri (v4i64 (AVX_SET0)), VR256:$src, (i8 1))>; 7145} 7146 7147// FIXME: Prefer a movss or movsd over a blendps when optimizing for size or 7148// on targets where they have equal performance. These were changed to use 7149// blends because blends have better throughput on SandyBridge and Haswell, but 7150// movs[s/d] are 1-2 byte shorter instructions. 7151let Predicates = [UseSSE41], AddedComplexity = 15 in { 7152 // With SSE41 we can use blends for these patterns. 7153 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128:$src))), 7154 (BLENDPSrri (v4f32 (V_SET0)), VR128:$src, (i8 1))>; 7155 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128:$src))), 7156 (PBLENDWrri (v4i32 (V_SET0)), VR128:$src, (i8 3))>; 7157} 7158 7159 7160/// SS41I_ternary_int - SSE 4.1 ternary operator 7161let Uses = [XMM0], Constraints = "$src1 = $dst" in { 7162 multiclass SS41I_ternary_int<bits<8> opc, string OpcodeStr, PatFrag mem_frag, 7163 X86MemOperand x86memop, Intrinsic IntId, 7164 OpndItins itins = DEFAULT_ITINS> { 7165 def rr0 : SS48I<opc, MRMSrcReg, (outs VR128:$dst), 7166 (ins VR128:$src1, VR128:$src2), 7167 !strconcat(OpcodeStr, 7168 "\t{$src2, $dst|$dst, $src2}"), 7169 [(set VR128:$dst, (IntId VR128:$src1, VR128:$src2, XMM0))], 7170 itins.rr>, Sched<[itins.Sched]>; 7171 7172 def rm0 : SS48I<opc, MRMSrcMem, (outs VR128:$dst), 7173 (ins VR128:$src1, x86memop:$src2), 7174 !strconcat(OpcodeStr, 7175 "\t{$src2, $dst|$dst, $src2}"), 7176 [(set VR128:$dst, 7177 (IntId VR128:$src1, 7178 (bitconvert (mem_frag addr:$src2)), XMM0))], 7179 itins.rm>, Sched<[itins.Sched.Folded, ReadAfterLd]>; 7180 } 7181} 7182 7183let ExeDomain = SSEPackedDouble in 7184defm BLENDVPD : SS41I_ternary_int<0x15, "blendvpd", memopv2f64, f128mem, 7185 int_x86_sse41_blendvpd, 7186 DEFAULT_ITINS_FBLENDSCHED>; 7187let ExeDomain = SSEPackedSingle in 7188defm BLENDVPS : SS41I_ternary_int<0x14, "blendvps", memopv4f32, f128mem, 7189 int_x86_sse41_blendvps, 7190 DEFAULT_ITINS_FBLENDSCHED>; 7191defm PBLENDVB : SS41I_ternary_int<0x10, "pblendvb", memopv2i64, i128mem, 7192 int_x86_sse41_pblendvb, 7193 DEFAULT_ITINS_VARBLENDSCHED>; 7194 7195// Aliases with the implicit xmm0 argument 7196def : InstAlias<"blendvpd\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7197 (BLENDVPDrr0 VR128:$dst, VR128:$src2)>; 7198def : InstAlias<"blendvpd\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7199 (BLENDVPDrm0 VR128:$dst, f128mem:$src2)>; 7200def : InstAlias<"blendvps\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7201 (BLENDVPSrr0 VR128:$dst, VR128:$src2)>; 7202def : InstAlias<"blendvps\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7203 (BLENDVPSrm0 VR128:$dst, f128mem:$src2)>; 7204def : InstAlias<"pblendvb\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7205 (PBLENDVBrr0 VR128:$dst, VR128:$src2)>; 7206def : InstAlias<"pblendvb\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7207 (PBLENDVBrm0 VR128:$dst, i128mem:$src2)>; 7208 7209let Predicates = [UseSSE41] in { 7210 def : Pat<(v16i8 (vselect (v16i8 XMM0), (v16i8 VR128:$src1), 7211 (v16i8 VR128:$src2))), 7212 (PBLENDVBrr0 VR128:$src2, VR128:$src1)>; 7213 def : Pat<(v4i32 (vselect (v4i32 XMM0), (v4i32 VR128:$src1), 7214 (v4i32 VR128:$src2))), 7215 (BLENDVPSrr0 VR128:$src2, VR128:$src1)>; 7216 def : Pat<(v4f32 (vselect (v4i32 XMM0), (v4f32 VR128:$src1), 7217 (v4f32 VR128:$src2))), 7218 (BLENDVPSrr0 VR128:$src2, VR128:$src1)>; 7219 def : Pat<(v2i64 (vselect (v2i64 XMM0), (v2i64 VR128:$src1), 7220 (v2i64 VR128:$src2))), 7221 (BLENDVPDrr0 VR128:$src2, VR128:$src1)>; 7222 def : Pat<(v2f64 (vselect (v2i64 XMM0), (v2f64 VR128:$src1), 7223 (v2f64 VR128:$src2))), 7224 (BLENDVPDrr0 VR128:$src2, VR128:$src1)>; 7225} 7226 7227let AddedComplexity = 400 in { // Prefer non-temporal versions 7228let SchedRW = [WriteLoad] in { 7229let Predicates = [HasAVX, NoVLX] in 7230def VMOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 7231 "vmovntdqa\t{$src, $dst|$dst, $src}", 7232 [(set VR128:$dst, (int_x86_sse41_movntdqa addr:$src))]>, 7233 VEX; 7234let Predicates = [HasAVX2, NoVLX] in 7235def VMOVNTDQAYrm : SS48I<0x2A, MRMSrcMem, (outs VR256:$dst), (ins i256mem:$src), 7236 "vmovntdqa\t{$src, $dst|$dst, $src}", 7237 [(set VR256:$dst, (int_x86_avx2_movntdqa addr:$src))]>, 7238 VEX, VEX_L; 7239def MOVNTDQArm : SS48I<0x2A, MRMSrcMem, (outs VR128:$dst), (ins i128mem:$src), 7240 "movntdqa\t{$src, $dst|$dst, $src}", 7241 [(set VR128:$dst, (int_x86_sse41_movntdqa addr:$src))]>; 7242} // SchedRW 7243 7244let Predicates = [HasAVX2, NoVLX] in { 7245 def : Pat<(v8f32 (alignednontemporalload addr:$src)), 7246 (VMOVNTDQAYrm addr:$src)>; 7247 def : Pat<(v4f64 (alignednontemporalload addr:$src)), 7248 (VMOVNTDQAYrm addr:$src)>; 7249 def : Pat<(v4i64 (alignednontemporalload addr:$src)), 7250 (VMOVNTDQAYrm addr:$src)>; 7251} 7252 7253let Predicates = [HasAVX, NoVLX] in { 7254 def : Pat<(v4f32 (alignednontemporalload addr:$src)), 7255 (VMOVNTDQArm addr:$src)>; 7256 def : Pat<(v2f64 (alignednontemporalload addr:$src)), 7257 (VMOVNTDQArm addr:$src)>; 7258 def : Pat<(v2i64 (alignednontemporalload addr:$src)), 7259 (VMOVNTDQArm addr:$src)>; 7260} 7261 7262let Predicates = [UseSSE41] in { 7263 def : Pat<(v4f32 (alignednontemporalload addr:$src)), 7264 (MOVNTDQArm addr:$src)>; 7265 def : Pat<(v2f64 (alignednontemporalload addr:$src)), 7266 (MOVNTDQArm addr:$src)>; 7267 def : Pat<(v2i64 (alignednontemporalload addr:$src)), 7268 (MOVNTDQArm addr:$src)>; 7269} 7270 7271} // AddedComplexity 7272 7273//===----------------------------------------------------------------------===// 7274// SSE4.2 - Compare Instructions 7275//===----------------------------------------------------------------------===// 7276 7277/// SS42I_binop_rm - Simple SSE 4.2 binary operator 7278multiclass SS42I_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode, 7279 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 7280 X86MemOperand x86memop, bit Is2Addr = 1> { 7281 def rr : SS428I<opc, MRMSrcReg, (outs RC:$dst), 7282 (ins RC:$src1, RC:$src2), 7283 !if(Is2Addr, 7284 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7285 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 7286 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2)))]>; 7287 def rm : SS428I<opc, MRMSrcMem, (outs RC:$dst), 7288 (ins RC:$src1, x86memop:$src2), 7289 !if(Is2Addr, 7290 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7291 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 7292 [(set RC:$dst, 7293 (OpVT (OpNode RC:$src1, (memop_frag addr:$src2))))]>; 7294} 7295 7296let Predicates = [HasAVX] in 7297 defm VPCMPGTQ : SS42I_binop_rm<0x37, "vpcmpgtq", X86pcmpgt, v2i64, VR128, 7298 loadv2i64, i128mem, 0>, VEX_4V; 7299 7300let Predicates = [HasAVX2] in 7301 defm VPCMPGTQY : SS42I_binop_rm<0x37, "vpcmpgtq", X86pcmpgt, v4i64, VR256, 7302 loadv4i64, i256mem, 0>, VEX_4V, VEX_L; 7303 7304let Constraints = "$src1 = $dst" in 7305 defm PCMPGTQ : SS42I_binop_rm<0x37, "pcmpgtq", X86pcmpgt, v2i64, VR128, 7306 memopv2i64, i128mem>; 7307 7308//===----------------------------------------------------------------------===// 7309// SSE4.2 - String/text Processing Instructions 7310//===----------------------------------------------------------------------===// 7311 7312// Packed Compare Implicit Length Strings, Return Mask 7313multiclass pseudo_pcmpistrm<string asm, PatFrag ld_frag> { 7314 def REG : PseudoI<(outs VR128:$dst), 7315 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7316 [(set VR128:$dst, (int_x86_sse42_pcmpistrm128 VR128:$src1, VR128:$src2, 7317 imm:$src3))]>; 7318 def MEM : PseudoI<(outs VR128:$dst), 7319 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7320 [(set VR128:$dst, (int_x86_sse42_pcmpistrm128 VR128:$src1, 7321 (bc_v16i8 (ld_frag addr:$src2)), imm:$src3))]>; 7322} 7323 7324let Defs = [EFLAGS], usesCustomInserter = 1 in { 7325 defm VPCMPISTRM128 : pseudo_pcmpistrm<"#VPCMPISTRM128", loadv2i64>, 7326 Requires<[HasAVX]>; 7327 defm PCMPISTRM128 : pseudo_pcmpistrm<"#PCMPISTRM128", memopv2i64>, 7328 Requires<[UseSSE42]>; 7329} 7330 7331multiclass pcmpistrm_SS42AI<string asm> { 7332 def rr : SS42AI<0x62, MRMSrcReg, (outs), 7333 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7334 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), 7335 []>, Sched<[WritePCmpIStrM]>; 7336 let mayLoad = 1 in 7337 def rm :SS42AI<0x62, MRMSrcMem, (outs), 7338 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7339 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), 7340 []>, Sched<[WritePCmpIStrMLd, ReadAfterLd]>; 7341} 7342 7343let Defs = [XMM0, EFLAGS], hasSideEffects = 0 in { 7344 let Predicates = [HasAVX] in 7345 defm VPCMPISTRM128 : pcmpistrm_SS42AI<"vpcmpistrm">, VEX; 7346 defm PCMPISTRM128 : pcmpistrm_SS42AI<"pcmpistrm"> ; 7347} 7348 7349// Packed Compare Explicit Length Strings, Return Mask 7350multiclass pseudo_pcmpestrm<string asm, PatFrag ld_frag> { 7351 def REG : PseudoI<(outs VR128:$dst), 7352 (ins VR128:$src1, VR128:$src3, u8imm:$src5), 7353 [(set VR128:$dst, (int_x86_sse42_pcmpestrm128 7354 VR128:$src1, EAX, VR128:$src3, EDX, imm:$src5))]>; 7355 def MEM : PseudoI<(outs VR128:$dst), 7356 (ins VR128:$src1, i128mem:$src3, u8imm:$src5), 7357 [(set VR128:$dst, (int_x86_sse42_pcmpestrm128 VR128:$src1, EAX, 7358 (bc_v16i8 (ld_frag addr:$src3)), EDX, imm:$src5))]>; 7359} 7360 7361let Defs = [EFLAGS], Uses = [EAX, EDX], usesCustomInserter = 1 in { 7362 defm VPCMPESTRM128 : pseudo_pcmpestrm<"#VPCMPESTRM128", loadv2i64>, 7363 Requires<[HasAVX]>; 7364 defm PCMPESTRM128 : pseudo_pcmpestrm<"#PCMPESTRM128", memopv2i64>, 7365 Requires<[UseSSE42]>; 7366} 7367 7368multiclass SS42AI_pcmpestrm<string asm> { 7369 def rr : SS42AI<0x60, MRMSrcReg, (outs), 7370 (ins VR128:$src1, VR128:$src3, u8imm:$src5), 7371 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), 7372 []>, Sched<[WritePCmpEStrM]>; 7373 let mayLoad = 1 in 7374 def rm : SS42AI<0x60, MRMSrcMem, (outs), 7375 (ins VR128:$src1, i128mem:$src3, u8imm:$src5), 7376 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), 7377 []>, Sched<[WritePCmpEStrMLd, ReadAfterLd]>; 7378} 7379 7380let Defs = [XMM0, EFLAGS], Uses = [EAX, EDX], hasSideEffects = 0 in { 7381 let Predicates = [HasAVX] in 7382 defm VPCMPESTRM128 : SS42AI_pcmpestrm<"vpcmpestrm">, VEX; 7383 defm PCMPESTRM128 : SS42AI_pcmpestrm<"pcmpestrm">; 7384} 7385 7386// Packed Compare Implicit Length Strings, Return Index 7387multiclass pseudo_pcmpistri<string asm, PatFrag ld_frag> { 7388 def REG : PseudoI<(outs GR32:$dst), 7389 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7390 [(set GR32:$dst, EFLAGS, 7391 (X86pcmpistri VR128:$src1, VR128:$src2, imm:$src3))]>; 7392 def MEM : PseudoI<(outs GR32:$dst), 7393 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7394 [(set GR32:$dst, EFLAGS, (X86pcmpistri VR128:$src1, 7395 (bc_v16i8 (ld_frag addr:$src2)), imm:$src3))]>; 7396} 7397 7398let Defs = [EFLAGS], usesCustomInserter = 1 in { 7399 defm VPCMPISTRI : pseudo_pcmpistri<"#VPCMPISTRI", loadv2i64>, 7400 Requires<[HasAVX]>; 7401 defm PCMPISTRI : pseudo_pcmpistri<"#PCMPISTRI", memopv2i64>, 7402 Requires<[UseSSE42]>; 7403} 7404 7405multiclass SS42AI_pcmpistri<string asm> { 7406 def rr : SS42AI<0x63, MRMSrcReg, (outs), 7407 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7408 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), 7409 []>, Sched<[WritePCmpIStrI]>; 7410 let mayLoad = 1 in 7411 def rm : SS42AI<0x63, MRMSrcMem, (outs), 7412 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7413 !strconcat(asm, "\t{$src3, $src2, $src1|$src1, $src2, $src3}"), 7414 []>, Sched<[WritePCmpIStrILd, ReadAfterLd]>; 7415} 7416 7417let Defs = [ECX, EFLAGS], hasSideEffects = 0 in { 7418 let Predicates = [HasAVX] in 7419 defm VPCMPISTRI : SS42AI_pcmpistri<"vpcmpistri">, VEX; 7420 defm PCMPISTRI : SS42AI_pcmpistri<"pcmpistri">; 7421} 7422 7423// Packed Compare Explicit Length Strings, Return Index 7424multiclass pseudo_pcmpestri<string asm, PatFrag ld_frag> { 7425 def REG : PseudoI<(outs GR32:$dst), 7426 (ins VR128:$src1, VR128:$src3, u8imm:$src5), 7427 [(set GR32:$dst, EFLAGS, 7428 (X86pcmpestri VR128:$src1, EAX, VR128:$src3, EDX, imm:$src5))]>; 7429 def MEM : PseudoI<(outs GR32:$dst), 7430 (ins VR128:$src1, i128mem:$src3, u8imm:$src5), 7431 [(set GR32:$dst, EFLAGS, 7432 (X86pcmpestri VR128:$src1, EAX, (bc_v16i8 (ld_frag addr:$src3)), EDX, 7433 imm:$src5))]>; 7434} 7435 7436let Defs = [EFLAGS], Uses = [EAX, EDX], usesCustomInserter = 1 in { 7437 defm VPCMPESTRI : pseudo_pcmpestri<"#VPCMPESTRI", loadv2i64>, 7438 Requires<[HasAVX]>; 7439 defm PCMPESTRI : pseudo_pcmpestri<"#PCMPESTRI", memopv2i64>, 7440 Requires<[UseSSE42]>; 7441} 7442 7443multiclass SS42AI_pcmpestri<string asm> { 7444 def rr : SS42AI<0x61, MRMSrcReg, (outs), 7445 (ins VR128:$src1, VR128:$src3, u8imm:$src5), 7446 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), 7447 []>, Sched<[WritePCmpEStrI]>; 7448 let mayLoad = 1 in 7449 def rm : SS42AI<0x61, MRMSrcMem, (outs), 7450 (ins VR128:$src1, i128mem:$src3, u8imm:$src5), 7451 !strconcat(asm, "\t{$src5, $src3, $src1|$src1, $src3, $src5}"), 7452 []>, Sched<[WritePCmpEStrILd, ReadAfterLd]>; 7453} 7454 7455let Defs = [ECX, EFLAGS], Uses = [EAX, EDX], hasSideEffects = 0 in { 7456 let Predicates = [HasAVX] in 7457 defm VPCMPESTRI : SS42AI_pcmpestri<"vpcmpestri">, VEX; 7458 defm PCMPESTRI : SS42AI_pcmpestri<"pcmpestri">; 7459} 7460 7461//===----------------------------------------------------------------------===// 7462// SSE4.2 - CRC Instructions 7463//===----------------------------------------------------------------------===// 7464 7465// No CRC instructions have AVX equivalents 7466 7467// crc intrinsic instruction 7468// This set of instructions are only rm, the only difference is the size 7469// of r and m. 7470class SS42I_crc32r<bits<8> opc, string asm, RegisterClass RCOut, 7471 RegisterClass RCIn, SDPatternOperator Int> : 7472 SS42FI<opc, MRMSrcReg, (outs RCOut:$dst), (ins RCOut:$src1, RCIn:$src2), 7473 !strconcat(asm, "\t{$src2, $src1|$src1, $src2}"), 7474 [(set RCOut:$dst, (Int RCOut:$src1, RCIn:$src2))], IIC_CRC32_REG>, 7475 Sched<[WriteFAdd]>; 7476 7477class SS42I_crc32m<bits<8> opc, string asm, RegisterClass RCOut, 7478 X86MemOperand x86memop, SDPatternOperator Int> : 7479 SS42FI<opc, MRMSrcMem, (outs RCOut:$dst), (ins RCOut:$src1, x86memop:$src2), 7480 !strconcat(asm, "\t{$src2, $src1|$src1, $src2}"), 7481 [(set RCOut:$dst, (Int RCOut:$src1, (load addr:$src2)))], 7482 IIC_CRC32_MEM>, Sched<[WriteFAddLd, ReadAfterLd]>; 7483 7484let Constraints = "$src1 = $dst" in { 7485 def CRC32r32m8 : SS42I_crc32m<0xF0, "crc32{b}", GR32, i8mem, 7486 int_x86_sse42_crc32_32_8>; 7487 def CRC32r32r8 : SS42I_crc32r<0xF0, "crc32{b}", GR32, GR8, 7488 int_x86_sse42_crc32_32_8>; 7489 def CRC32r32m16 : SS42I_crc32m<0xF1, "crc32{w}", GR32, i16mem, 7490 int_x86_sse42_crc32_32_16>, OpSize16; 7491 def CRC32r32r16 : SS42I_crc32r<0xF1, "crc32{w}", GR32, GR16, 7492 int_x86_sse42_crc32_32_16>, OpSize16; 7493 def CRC32r32m32 : SS42I_crc32m<0xF1, "crc32{l}", GR32, i32mem, 7494 int_x86_sse42_crc32_32_32>, OpSize32; 7495 def CRC32r32r32 : SS42I_crc32r<0xF1, "crc32{l}", GR32, GR32, 7496 int_x86_sse42_crc32_32_32>, OpSize32; 7497 def CRC32r64m64 : SS42I_crc32m<0xF1, "crc32{q}", GR64, i64mem, 7498 int_x86_sse42_crc32_64_64>, REX_W; 7499 def CRC32r64r64 : SS42I_crc32r<0xF1, "crc32{q}", GR64, GR64, 7500 int_x86_sse42_crc32_64_64>, REX_W; 7501 let hasSideEffects = 0 in { 7502 let mayLoad = 1 in 7503 def CRC32r64m8 : SS42I_crc32m<0xF0, "crc32{b}", GR64, i8mem, 7504 null_frag>, REX_W; 7505 def CRC32r64r8 : SS42I_crc32r<0xF0, "crc32{b}", GR64, GR8, 7506 null_frag>, REX_W; 7507 } 7508} 7509 7510//===----------------------------------------------------------------------===// 7511// SHA-NI Instructions 7512//===----------------------------------------------------------------------===// 7513 7514multiclass SHAI_binop<bits<8> Opc, string OpcodeStr, Intrinsic IntId, 7515 bit UsesXMM0 = 0> { 7516 def rr : I<Opc, MRMSrcReg, (outs VR128:$dst), 7517 (ins VR128:$src1, VR128:$src2), 7518 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7519 [!if(UsesXMM0, 7520 (set VR128:$dst, (IntId VR128:$src1, VR128:$src2, XMM0)), 7521 (set VR128:$dst, (IntId VR128:$src1, VR128:$src2)))]>, T8; 7522 7523 def rm : I<Opc, MRMSrcMem, (outs VR128:$dst), 7524 (ins VR128:$src1, i128mem:$src2), 7525 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7526 [!if(UsesXMM0, 7527 (set VR128:$dst, (IntId VR128:$src1, 7528 (bc_v4i32 (memopv2i64 addr:$src2)), XMM0)), 7529 (set VR128:$dst, (IntId VR128:$src1, 7530 (bc_v4i32 (memopv2i64 addr:$src2)))))]>, T8; 7531} 7532 7533let Constraints = "$src1 = $dst", Predicates = [HasSHA] in { 7534 def SHA1RNDS4rri : Ii8<0xCC, MRMSrcReg, (outs VR128:$dst), 7535 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7536 "sha1rnds4\t{$src3, $src2, $dst|$dst, $src2, $src3}", 7537 [(set VR128:$dst, 7538 (int_x86_sha1rnds4 VR128:$src1, VR128:$src2, 7539 (i8 imm:$src3)))]>, TA; 7540 def SHA1RNDS4rmi : Ii8<0xCC, MRMSrcMem, (outs VR128:$dst), 7541 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7542 "sha1rnds4\t{$src3, $src2, $dst|$dst, $src2, $src3}", 7543 [(set VR128:$dst, 7544 (int_x86_sha1rnds4 VR128:$src1, 7545 (bc_v4i32 (memopv2i64 addr:$src2)), 7546 (i8 imm:$src3)))]>, TA; 7547 7548 defm SHA1NEXTE : SHAI_binop<0xC8, "sha1nexte", int_x86_sha1nexte>; 7549 defm SHA1MSG1 : SHAI_binop<0xC9, "sha1msg1", int_x86_sha1msg1>; 7550 defm SHA1MSG2 : SHAI_binop<0xCA, "sha1msg2", int_x86_sha1msg2>; 7551 7552 let Uses=[XMM0] in 7553 defm SHA256RNDS2 : SHAI_binop<0xCB, "sha256rnds2", int_x86_sha256rnds2, 1>; 7554 7555 defm SHA256MSG1 : SHAI_binop<0xCC, "sha256msg1", int_x86_sha256msg1>; 7556 defm SHA256MSG2 : SHAI_binop<0xCD, "sha256msg2", int_x86_sha256msg2>; 7557} 7558 7559// Aliases with explicit %xmm0 7560def : InstAlias<"sha256rnds2\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7561 (SHA256RNDS2rr VR128:$dst, VR128:$src2)>; 7562def : InstAlias<"sha256rnds2\t{%xmm0, $src2, $dst|$dst, $src2, xmm0}", 7563 (SHA256RNDS2rm VR128:$dst, i128mem:$src2)>; 7564 7565//===----------------------------------------------------------------------===// 7566// AES-NI Instructions 7567//===----------------------------------------------------------------------===// 7568 7569multiclass AESI_binop_rm_int<bits<8> opc, string OpcodeStr, Intrinsic IntId128, 7570 PatFrag ld_frag, bit Is2Addr = 1> { 7571 def rr : AES8I<opc, MRMSrcReg, (outs VR128:$dst), 7572 (ins VR128:$src1, VR128:$src2), 7573 !if(Is2Addr, 7574 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7575 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 7576 [(set VR128:$dst, (IntId128 VR128:$src1, VR128:$src2))]>, 7577 Sched<[WriteAESDecEnc]>; 7578 def rm : AES8I<opc, MRMSrcMem, (outs VR128:$dst), 7579 (ins VR128:$src1, i128mem:$src2), 7580 !if(Is2Addr, 7581 !strconcat(OpcodeStr, "\t{$src2, $dst|$dst, $src2}"), 7582 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}")), 7583 [(set VR128:$dst, 7584 (IntId128 VR128:$src1, (ld_frag addr:$src2)))]>, 7585 Sched<[WriteAESDecEncLd, ReadAfterLd]>; 7586} 7587 7588// Perform One Round of an AES Encryption/Decryption Flow 7589let Predicates = [HasAVX, HasAES] in { 7590 defm VAESENC : AESI_binop_rm_int<0xDC, "vaesenc", 7591 int_x86_aesni_aesenc, loadv2i64, 0>, VEX_4V; 7592 defm VAESENCLAST : AESI_binop_rm_int<0xDD, "vaesenclast", 7593 int_x86_aesni_aesenclast, loadv2i64, 0>, VEX_4V; 7594 defm VAESDEC : AESI_binop_rm_int<0xDE, "vaesdec", 7595 int_x86_aesni_aesdec, loadv2i64, 0>, VEX_4V; 7596 defm VAESDECLAST : AESI_binop_rm_int<0xDF, "vaesdeclast", 7597 int_x86_aesni_aesdeclast, loadv2i64, 0>, VEX_4V; 7598} 7599 7600let Constraints = "$src1 = $dst" in { 7601 defm AESENC : AESI_binop_rm_int<0xDC, "aesenc", 7602 int_x86_aesni_aesenc, memopv2i64>; 7603 defm AESENCLAST : AESI_binop_rm_int<0xDD, "aesenclast", 7604 int_x86_aesni_aesenclast, memopv2i64>; 7605 defm AESDEC : AESI_binop_rm_int<0xDE, "aesdec", 7606 int_x86_aesni_aesdec, memopv2i64>; 7607 defm AESDECLAST : AESI_binop_rm_int<0xDF, "aesdeclast", 7608 int_x86_aesni_aesdeclast, memopv2i64>; 7609} 7610 7611// Perform the AES InvMixColumn Transformation 7612let Predicates = [HasAVX, HasAES] in { 7613 def VAESIMCrr : AES8I<0xDB, MRMSrcReg, (outs VR128:$dst), 7614 (ins VR128:$src1), 7615 "vaesimc\t{$src1, $dst|$dst, $src1}", 7616 [(set VR128:$dst, 7617 (int_x86_aesni_aesimc VR128:$src1))]>, Sched<[WriteAESIMC]>, 7618 VEX; 7619 def VAESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst), 7620 (ins i128mem:$src1), 7621 "vaesimc\t{$src1, $dst|$dst, $src1}", 7622 [(set VR128:$dst, (int_x86_aesni_aesimc (loadv2i64 addr:$src1)))]>, 7623 Sched<[WriteAESIMCLd]>, VEX; 7624} 7625def AESIMCrr : AES8I<0xDB, MRMSrcReg, (outs VR128:$dst), 7626 (ins VR128:$src1), 7627 "aesimc\t{$src1, $dst|$dst, $src1}", 7628 [(set VR128:$dst, 7629 (int_x86_aesni_aesimc VR128:$src1))]>, Sched<[WriteAESIMC]>; 7630def AESIMCrm : AES8I<0xDB, MRMSrcMem, (outs VR128:$dst), 7631 (ins i128mem:$src1), 7632 "aesimc\t{$src1, $dst|$dst, $src1}", 7633 [(set VR128:$dst, (int_x86_aesni_aesimc (memopv2i64 addr:$src1)))]>, 7634 Sched<[WriteAESIMCLd]>; 7635 7636// AES Round Key Generation Assist 7637let Predicates = [HasAVX, HasAES] in { 7638 def VAESKEYGENASSIST128rr : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst), 7639 (ins VR128:$src1, u8imm:$src2), 7640 "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7641 [(set VR128:$dst, 7642 (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>, 7643 Sched<[WriteAESKeyGen]>, VEX; 7644 def VAESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), 7645 (ins i128mem:$src1, u8imm:$src2), 7646 "vaeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7647 [(set VR128:$dst, 7648 (int_x86_aesni_aeskeygenassist (loadv2i64 addr:$src1), imm:$src2))]>, 7649 Sched<[WriteAESKeyGenLd]>, VEX; 7650} 7651def AESKEYGENASSIST128rr : AESAI<0xDF, MRMSrcReg, (outs VR128:$dst), 7652 (ins VR128:$src1, u8imm:$src2), 7653 "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7654 [(set VR128:$dst, 7655 (int_x86_aesni_aeskeygenassist VR128:$src1, imm:$src2))]>, 7656 Sched<[WriteAESKeyGen]>; 7657def AESKEYGENASSIST128rm : AESAI<0xDF, MRMSrcMem, (outs VR128:$dst), 7658 (ins i128mem:$src1, u8imm:$src2), 7659 "aeskeygenassist\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7660 [(set VR128:$dst, 7661 (int_x86_aesni_aeskeygenassist (memopv2i64 addr:$src1), imm:$src2))]>, 7662 Sched<[WriteAESKeyGenLd]>; 7663 7664//===----------------------------------------------------------------------===// 7665// PCLMUL Instructions 7666//===----------------------------------------------------------------------===// 7667 7668// AVX carry-less Multiplication instructions 7669let isCommutable = 1 in 7670def VPCLMULQDQrr : AVXPCLMULIi8<0x44, MRMSrcReg, (outs VR128:$dst), 7671 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7672 "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 7673 [(set VR128:$dst, 7674 (int_x86_pclmulqdq VR128:$src1, VR128:$src2, imm:$src3))]>, 7675 Sched<[WriteCLMul]>; 7676 7677def VPCLMULQDQrm : AVXPCLMULIi8<0x44, MRMSrcMem, (outs VR128:$dst), 7678 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7679 "vpclmulqdq\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 7680 [(set VR128:$dst, (int_x86_pclmulqdq VR128:$src1, 7681 (loadv2i64 addr:$src2), imm:$src3))]>, 7682 Sched<[WriteCLMulLd, ReadAfterLd]>; 7683 7684// Carry-less Multiplication instructions 7685let Constraints = "$src1 = $dst" in { 7686let isCommutable = 1 in 7687def PCLMULQDQrr : PCLMULIi8<0x44, MRMSrcReg, (outs VR128:$dst), 7688 (ins VR128:$src1, VR128:$src2, u8imm:$src3), 7689 "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}", 7690 [(set VR128:$dst, 7691 (int_x86_pclmulqdq VR128:$src1, VR128:$src2, imm:$src3))], 7692 IIC_SSE_PCLMULQDQ_RR>, Sched<[WriteCLMul]>; 7693 7694def PCLMULQDQrm : PCLMULIi8<0x44, MRMSrcMem, (outs VR128:$dst), 7695 (ins VR128:$src1, i128mem:$src2, u8imm:$src3), 7696 "pclmulqdq\t{$src3, $src2, $dst|$dst, $src2, $src3}", 7697 [(set VR128:$dst, (int_x86_pclmulqdq VR128:$src1, 7698 (memopv2i64 addr:$src2), imm:$src3))], 7699 IIC_SSE_PCLMULQDQ_RM>, 7700 Sched<[WriteCLMulLd, ReadAfterLd]>; 7701} // Constraints = "$src1 = $dst" 7702 7703 7704multiclass pclmul_alias<string asm, int immop> { 7705 def : InstAlias<!strconcat("pclmul", asm, "dq {$src, $dst|$dst, $src}"), 7706 (PCLMULQDQrr VR128:$dst, VR128:$src, immop), 0>; 7707 7708 def : InstAlias<!strconcat("pclmul", asm, "dq {$src, $dst|$dst, $src}"), 7709 (PCLMULQDQrm VR128:$dst, i128mem:$src, immop), 0>; 7710 7711 def : InstAlias<!strconcat("vpclmul", asm, 7712 "dq {$src2, $src1, $dst|$dst, $src1, $src2}"), 7713 (VPCLMULQDQrr VR128:$dst, VR128:$src1, VR128:$src2, immop), 7714 0>; 7715 7716 def : InstAlias<!strconcat("vpclmul", asm, 7717 "dq {$src2, $src1, $dst|$dst, $src1, $src2}"), 7718 (VPCLMULQDQrm VR128:$dst, VR128:$src1, i128mem:$src2, immop), 7719 0>; 7720} 7721defm : pclmul_alias<"hqhq", 0x11>; 7722defm : pclmul_alias<"hqlq", 0x01>; 7723defm : pclmul_alias<"lqhq", 0x10>; 7724defm : pclmul_alias<"lqlq", 0x00>; 7725 7726//===----------------------------------------------------------------------===// 7727// SSE4A Instructions 7728//===----------------------------------------------------------------------===// 7729 7730let Predicates = [HasSSE4A] in { 7731 7732let Constraints = "$src = $dst" in { 7733def EXTRQI : Ii8<0x78, MRMXr, (outs VR128:$dst), 7734 (ins VR128:$src, u8imm:$len, u8imm:$idx), 7735 "extrq\t{$idx, $len, $src|$src, $len, $idx}", 7736 [(set VR128:$dst, (X86extrqi VR128:$src, imm:$len, 7737 imm:$idx))]>, PD; 7738def EXTRQ : I<0x79, MRMSrcReg, (outs VR128:$dst), 7739 (ins VR128:$src, VR128:$mask), 7740 "extrq\t{$mask, $src|$src, $mask}", 7741 [(set VR128:$dst, (int_x86_sse4a_extrq VR128:$src, 7742 VR128:$mask))]>, PD; 7743 7744def INSERTQI : Ii8<0x78, MRMSrcReg, (outs VR128:$dst), 7745 (ins VR128:$src, VR128:$src2, u8imm:$len, u8imm:$idx), 7746 "insertq\t{$idx, $len, $src2, $src|$src, $src2, $len, $idx}", 7747 [(set VR128:$dst, (X86insertqi VR128:$src, VR128:$src2, 7748 imm:$len, imm:$idx))]>, XD; 7749def INSERTQ : I<0x79, MRMSrcReg, (outs VR128:$dst), 7750 (ins VR128:$src, VR128:$mask), 7751 "insertq\t{$mask, $src|$src, $mask}", 7752 [(set VR128:$dst, (int_x86_sse4a_insertq VR128:$src, 7753 VR128:$mask))]>, XD; 7754} 7755 7756// Non-temporal (unaligned) scalar stores. 7757let AddedComplexity = 400 in { // Prefer non-temporal versions 7758let mayStore = 1, SchedRW = [WriteStore] in { 7759def MOVNTSS : I<0x2B, MRMDestMem, (outs), (ins f32mem:$dst, VR128:$src), 7760 "movntss\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVNT>, XS; 7761 7762def MOVNTSD : I<0x2B, MRMDestMem, (outs), (ins f64mem:$dst, VR128:$src), 7763 "movntsd\t{$src, $dst|$dst, $src}", [], IIC_SSE_MOVNT>, XD; 7764} // SchedRW 7765 7766def : Pat<(nontemporalstore FR32:$src, addr:$dst), 7767 (MOVNTSS addr:$dst, (COPY_TO_REGCLASS FR32:$src, VR128))>; 7768 7769def : Pat<(nontemporalstore FR64:$src, addr:$dst), 7770 (MOVNTSD addr:$dst, (COPY_TO_REGCLASS FR64:$src, VR128))>; 7771 7772} // AddedComplexity 7773} // HasSSE4A 7774 7775//===----------------------------------------------------------------------===// 7776// AVX Instructions 7777//===----------------------------------------------------------------------===// 7778 7779//===----------------------------------------------------------------------===// 7780// VBROADCAST - Load from memory and broadcast to all elements of the 7781// destination operand 7782// 7783class avx_broadcast_rm<bits<8> opc, string OpcodeStr, RegisterClass RC, 7784 X86MemOperand x86memop, ValueType VT, 7785 PatFrag ld_frag, SchedWrite Sched> : 7786 AVX8I<opc, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 7787 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 7788 [(set RC:$dst, (VT (X86VBroadcast (ld_frag addr:$src))))]>, 7789 Sched<[Sched]>, VEX; 7790 7791// AVX2 adds register forms 7792class avx2_broadcast_rr<bits<8> opc, string OpcodeStr, RegisterClass RC, 7793 ValueType ResVT, ValueType OpVT, SchedWrite Sched> : 7794 AVX28I<opc, MRMSrcReg, (outs RC:$dst), (ins VR128:$src), 7795 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 7796 [(set RC:$dst, (ResVT (X86VBroadcast (OpVT VR128:$src))))]>, 7797 Sched<[Sched]>, VEX; 7798 7799let ExeDomain = SSEPackedSingle, Predicates = [HasAVX, NoVLX] in { 7800 def VBROADCASTSSrm : avx_broadcast_rm<0x18, "vbroadcastss", VR128, 7801 f32mem, v4f32, loadf32, WriteLoad>; 7802 def VBROADCASTSSYrm : avx_broadcast_rm<0x18, "vbroadcastss", VR256, 7803 f32mem, v8f32, loadf32, 7804 WriteFShuffleLd>, VEX_L; 7805} 7806let ExeDomain = SSEPackedDouble, Predicates = [HasAVX, NoVLX] in 7807def VBROADCASTSDYrm : avx_broadcast_rm<0x19, "vbroadcastsd", VR256, f64mem, 7808 v4f64, loadf64, WriteFShuffleLd>, VEX_L; 7809 7810let ExeDomain = SSEPackedSingle, Predicates = [HasAVX2, NoVLX] in { 7811 def VBROADCASTSSrr : avx2_broadcast_rr<0x18, "vbroadcastss", VR128, 7812 v4f32, v4f32, WriteFShuffle>; 7813 def VBROADCASTSSYrr : avx2_broadcast_rr<0x18, "vbroadcastss", VR256, 7814 v8f32, v4f32, WriteFShuffle256>, VEX_L; 7815} 7816let ExeDomain = SSEPackedDouble, Predicates = [HasAVX2, NoVLX] in 7817def VBROADCASTSDYrr : avx2_broadcast_rr<0x19, "vbroadcastsd", VR256, 7818 v4f64, v2f64, WriteFShuffle256>, VEX_L; 7819 7820let mayLoad = 1, hasSideEffects = 0, Predicates = [HasAVX2] in 7821def VBROADCASTI128 : AVX8I<0x5A, MRMSrcMem, (outs VR256:$dst), 7822 (ins i128mem:$src), 7823 "vbroadcasti128\t{$src, $dst|$dst, $src}", []>, 7824 Sched<[WriteLoad]>, VEX, VEX_L; 7825 7826def VBROADCASTF128 : AVX8I<0x1A, MRMSrcMem, (outs VR256:$dst), 7827 (ins f128mem:$src), 7828 "vbroadcastf128\t{$src, $dst|$dst, $src}", 7829 [(set VR256:$dst, 7830 (int_x86_avx_vbroadcastf128_pd_256 addr:$src))]>, 7831 Sched<[WriteFShuffleLd]>, VEX, VEX_L; 7832 7833let Predicates = [HasAVX] in 7834def : Pat<(int_x86_avx_vbroadcastf128_ps_256 addr:$src), 7835 (VBROADCASTF128 addr:$src)>; 7836 7837 7838//===----------------------------------------------------------------------===// 7839// VINSERTF128 - Insert packed floating-point values 7840// 7841let hasSideEffects = 0, ExeDomain = SSEPackedSingle in { 7842def VINSERTF128rr : AVXAIi8<0x18, MRMSrcReg, (outs VR256:$dst), 7843 (ins VR256:$src1, VR128:$src2, u8imm:$src3), 7844 "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 7845 []>, Sched<[WriteFShuffle]>, VEX_4V, VEX_L; 7846let mayLoad = 1 in 7847def VINSERTF128rm : AVXAIi8<0x18, MRMSrcMem, (outs VR256:$dst), 7848 (ins VR256:$src1, f128mem:$src2, u8imm:$src3), 7849 "vinsertf128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 7850 []>, Sched<[WriteFShuffleLd, ReadAfterLd]>, VEX_4V, VEX_L; 7851} 7852 7853let Predicates = [HasAVX, NoVLX] in { 7854def : Pat<(vinsert128_insert:$ins (v8f32 VR256:$src1), (v4f32 VR128:$src2), 7855 (iPTR imm)), 7856 (VINSERTF128rr VR256:$src1, VR128:$src2, 7857 (INSERT_get_vinsert128_imm VR256:$ins))>; 7858def : Pat<(vinsert128_insert:$ins (v4f64 VR256:$src1), (v2f64 VR128:$src2), 7859 (iPTR imm)), 7860 (VINSERTF128rr VR256:$src1, VR128:$src2, 7861 (INSERT_get_vinsert128_imm VR256:$ins))>; 7862 7863def : Pat<(vinsert128_insert:$ins (v8f32 VR256:$src1), (loadv4f32 addr:$src2), 7864 (iPTR imm)), 7865 (VINSERTF128rm VR256:$src1, addr:$src2, 7866 (INSERT_get_vinsert128_imm VR256:$ins))>; 7867def : Pat<(vinsert128_insert:$ins (v4f64 VR256:$src1), (loadv2f64 addr:$src2), 7868 (iPTR imm)), 7869 (VINSERTF128rm VR256:$src1, addr:$src2, 7870 (INSERT_get_vinsert128_imm VR256:$ins))>; 7871} 7872 7873let Predicates = [HasAVX1Only] in { 7874def : Pat<(vinsert128_insert:$ins (v4i64 VR256:$src1), (v2i64 VR128:$src2), 7875 (iPTR imm)), 7876 (VINSERTF128rr VR256:$src1, VR128:$src2, 7877 (INSERT_get_vinsert128_imm VR256:$ins))>; 7878def : Pat<(vinsert128_insert:$ins (v8i32 VR256:$src1), (v4i32 VR128:$src2), 7879 (iPTR imm)), 7880 (VINSERTF128rr VR256:$src1, VR128:$src2, 7881 (INSERT_get_vinsert128_imm VR256:$ins))>; 7882def : Pat<(vinsert128_insert:$ins (v32i8 VR256:$src1), (v16i8 VR128:$src2), 7883 (iPTR imm)), 7884 (VINSERTF128rr VR256:$src1, VR128:$src2, 7885 (INSERT_get_vinsert128_imm VR256:$ins))>; 7886def : Pat<(vinsert128_insert:$ins (v16i16 VR256:$src1), (v8i16 VR128:$src2), 7887 (iPTR imm)), 7888 (VINSERTF128rr VR256:$src1, VR128:$src2, 7889 (INSERT_get_vinsert128_imm VR256:$ins))>; 7890 7891def : Pat<(vinsert128_insert:$ins (v4i64 VR256:$src1), (loadv2i64 addr:$src2), 7892 (iPTR imm)), 7893 (VINSERTF128rm VR256:$src1, addr:$src2, 7894 (INSERT_get_vinsert128_imm VR256:$ins))>; 7895def : Pat<(vinsert128_insert:$ins (v8i32 VR256:$src1), 7896 (bc_v4i32 (loadv2i64 addr:$src2)), 7897 (iPTR imm)), 7898 (VINSERTF128rm VR256:$src1, addr:$src2, 7899 (INSERT_get_vinsert128_imm VR256:$ins))>; 7900def : Pat<(vinsert128_insert:$ins (v32i8 VR256:$src1), 7901 (bc_v16i8 (loadv2i64 addr:$src2)), 7902 (iPTR imm)), 7903 (VINSERTF128rm VR256:$src1, addr:$src2, 7904 (INSERT_get_vinsert128_imm VR256:$ins))>; 7905def : Pat<(vinsert128_insert:$ins (v16i16 VR256:$src1), 7906 (bc_v8i16 (loadv2i64 addr:$src2)), 7907 (iPTR imm)), 7908 (VINSERTF128rm VR256:$src1, addr:$src2, 7909 (INSERT_get_vinsert128_imm VR256:$ins))>; 7910} 7911 7912//===----------------------------------------------------------------------===// 7913// VEXTRACTF128 - Extract packed floating-point values 7914// 7915let hasSideEffects = 0, ExeDomain = SSEPackedSingle in { 7916def VEXTRACTF128rr : AVXAIi8<0x19, MRMDestReg, (outs VR128:$dst), 7917 (ins VR256:$src1, u8imm:$src2), 7918 "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7919 []>, Sched<[WriteFShuffle]>, VEX, VEX_L; 7920let mayStore = 1 in 7921def VEXTRACTF128mr : AVXAIi8<0x19, MRMDestMem, (outs), 7922 (ins f128mem:$dst, VR256:$src1, u8imm:$src2), 7923 "vextractf128\t{$src2, $src1, $dst|$dst, $src1, $src2}", 7924 []>, Sched<[WriteStore]>, VEX, VEX_L; 7925} 7926 7927// AVX1 patterns 7928let Predicates = [HasAVX, NoVLX] in { 7929def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7930 (v4f32 (VEXTRACTF128rr 7931 (v8f32 VR256:$src1), 7932 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7933def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7934 (v2f64 (VEXTRACTF128rr 7935 (v4f64 VR256:$src1), 7936 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7937 7938def : Pat<(store (v4f32 (vextract128_extract:$ext (v8f32 VR256:$src1), 7939 (iPTR imm))), addr:$dst), 7940 (VEXTRACTF128mr addr:$dst, VR256:$src1, 7941 (EXTRACT_get_vextract128_imm VR128:$ext))>; 7942def : Pat<(store (v2f64 (vextract128_extract:$ext (v4f64 VR256:$src1), 7943 (iPTR imm))), addr:$dst), 7944 (VEXTRACTF128mr addr:$dst, VR256:$src1, 7945 (EXTRACT_get_vextract128_imm VR128:$ext))>; 7946} 7947 7948let Predicates = [HasAVX1Only] in { 7949def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7950 (v2i64 (VEXTRACTF128rr 7951 (v4i64 VR256:$src1), 7952 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7953def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7954 (v4i32 (VEXTRACTF128rr 7955 (v8i32 VR256:$src1), 7956 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7957def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7958 (v8i16 (VEXTRACTF128rr 7959 (v16i16 VR256:$src1), 7960 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7961def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 7962 (v16i8 (VEXTRACTF128rr 7963 (v32i8 VR256:$src1), 7964 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 7965 7966def : Pat<(store (v2i64 (vextract128_extract:$ext (v4i64 VR256:$src1), 7967 (iPTR imm))), addr:$dst), 7968 (VEXTRACTF128mr addr:$dst, VR256:$src1, 7969 (EXTRACT_get_vextract128_imm VR128:$ext))>; 7970def : Pat<(store (v4i32 (vextract128_extract:$ext (v8i32 VR256:$src1), 7971 (iPTR imm))), addr:$dst), 7972 (VEXTRACTF128mr addr:$dst, VR256:$src1, 7973 (EXTRACT_get_vextract128_imm VR128:$ext))>; 7974def : Pat<(store (v8i16 (vextract128_extract:$ext (v16i16 VR256:$src1), 7975 (iPTR imm))), addr:$dst), 7976 (VEXTRACTF128mr addr:$dst, VR256:$src1, 7977 (EXTRACT_get_vextract128_imm VR128:$ext))>; 7978def : Pat<(store (v16i8 (vextract128_extract:$ext (v32i8 VR256:$src1), 7979 (iPTR imm))), addr:$dst), 7980 (VEXTRACTF128mr addr:$dst, VR256:$src1, 7981 (EXTRACT_get_vextract128_imm VR128:$ext))>; 7982} 7983 7984//===----------------------------------------------------------------------===// 7985// VMASKMOV - Conditional SIMD Packed Loads and Stores 7986// 7987multiclass avx_movmask_rm<bits<8> opc_rm, bits<8> opc_mr, string OpcodeStr, 7988 Intrinsic IntLd, Intrinsic IntLd256, 7989 Intrinsic IntSt, Intrinsic IntSt256> { 7990 def rm : AVX8I<opc_rm, MRMSrcMem, (outs VR128:$dst), 7991 (ins VR128:$src1, f128mem:$src2), 7992 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 7993 [(set VR128:$dst, (IntLd addr:$src2, VR128:$src1))]>, 7994 VEX_4V; 7995 def Yrm : AVX8I<opc_rm, MRMSrcMem, (outs VR256:$dst), 7996 (ins VR256:$src1, f256mem:$src2), 7997 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 7998 [(set VR256:$dst, (IntLd256 addr:$src2, VR256:$src1))]>, 7999 VEX_4V, VEX_L; 8000 def mr : AVX8I<opc_mr, MRMDestMem, (outs), 8001 (ins f128mem:$dst, VR128:$src1, VR128:$src2), 8002 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8003 [(IntSt addr:$dst, VR128:$src1, VR128:$src2)]>, VEX_4V; 8004 def Ymr : AVX8I<opc_mr, MRMDestMem, (outs), 8005 (ins f256mem:$dst, VR256:$src1, VR256:$src2), 8006 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8007 [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>, VEX_4V, VEX_L; 8008} 8009 8010let ExeDomain = SSEPackedSingle in 8011defm VMASKMOVPS : avx_movmask_rm<0x2C, 0x2E, "vmaskmovps", 8012 int_x86_avx_maskload_ps, 8013 int_x86_avx_maskload_ps_256, 8014 int_x86_avx_maskstore_ps, 8015 int_x86_avx_maskstore_ps_256>; 8016let ExeDomain = SSEPackedDouble in 8017defm VMASKMOVPD : avx_movmask_rm<0x2D, 0x2F, "vmaskmovpd", 8018 int_x86_avx_maskload_pd, 8019 int_x86_avx_maskload_pd_256, 8020 int_x86_avx_maskstore_pd, 8021 int_x86_avx_maskstore_pd_256>; 8022 8023//===----------------------------------------------------------------------===// 8024// VPERMIL - Permute Single and Double Floating-Point Values 8025// 8026multiclass avx_permil<bits<8> opc_rm, bits<8> opc_rmi, string OpcodeStr, 8027 RegisterClass RC, X86MemOperand x86memop_f, 8028 X86MemOperand x86memop_i, PatFrag i_frag, 8029 ValueType f_vt, ValueType i_vt> { 8030 let Predicates = [HasAVX, NoVLX] in { 8031 def rr : AVX8I<opc_rm, MRMSrcReg, (outs RC:$dst), 8032 (ins RC:$src1, RC:$src2), 8033 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8034 [(set RC:$dst, (f_vt (X86VPermilpv RC:$src1, (i_vt RC:$src2))))]>, VEX_4V, 8035 Sched<[WriteFShuffle]>; 8036 def rm : AVX8I<opc_rm, MRMSrcMem, (outs RC:$dst), 8037 (ins RC:$src1, x86memop_i:$src2), 8038 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8039 [(set RC:$dst, (f_vt (X86VPermilpv RC:$src1, 8040 (i_vt (bitconvert (i_frag addr:$src2))))))]>, VEX_4V, 8041 Sched<[WriteFShuffleLd, ReadAfterLd]>; 8042 8043 def ri : AVXAIi8<opc_rmi, MRMSrcReg, (outs RC:$dst), 8044 (ins RC:$src1, u8imm:$src2), 8045 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8046 [(set RC:$dst, (f_vt (X86VPermilpi RC:$src1, (i8 imm:$src2))))]>, VEX, 8047 Sched<[WriteFShuffle]>; 8048 def mi : AVXAIi8<opc_rmi, MRMSrcMem, (outs RC:$dst), 8049 (ins x86memop_f:$src1, u8imm:$src2), 8050 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8051 [(set RC:$dst, 8052 (f_vt (X86VPermilpi (load addr:$src1), (i8 imm:$src2))))]>, VEX, 8053 Sched<[WriteFShuffleLd]>; 8054 }// Predicates = [HasAVX, NoVLX] 8055} 8056 8057let ExeDomain = SSEPackedSingle in { 8058 defm VPERMILPS : avx_permil<0x0C, 0x04, "vpermilps", VR128, f128mem, i128mem, 8059 loadv2i64, v4f32, v4i32>; 8060 defm VPERMILPSY : avx_permil<0x0C, 0x04, "vpermilps", VR256, f256mem, i256mem, 8061 loadv4i64, v8f32, v8i32>, VEX_L; 8062} 8063let ExeDomain = SSEPackedDouble in { 8064 defm VPERMILPD : avx_permil<0x0D, 0x05, "vpermilpd", VR128, f128mem, i128mem, 8065 loadv2i64, v2f64, v2i64>; 8066 defm VPERMILPDY : avx_permil<0x0D, 0x05, "vpermilpd", VR256, f256mem, i256mem, 8067 loadv4i64, v4f64, v4i64>, VEX_L; 8068} 8069 8070let Predicates = [HasAVX, NoVLX] in { 8071def : Pat<(v8f32 (X86VPermilpv VR256:$src1, (v8i32 VR256:$src2))), 8072 (VPERMILPSYrr VR256:$src1, VR256:$src2)>; 8073def : Pat<(v8f32 (X86VPermilpv VR256:$src1, (bc_v8i32 (loadv4i64 addr:$src2)))), 8074 (VPERMILPSYrm VR256:$src1, addr:$src2)>; 8075def : Pat<(v4f64 (X86VPermilpv VR256:$src1, (v4i64 VR256:$src2))), 8076 (VPERMILPDYrr VR256:$src1, VR256:$src2)>; 8077def : Pat<(v4f64 (X86VPermilpv VR256:$src1, (loadv4i64 addr:$src2))), 8078 (VPERMILPDYrm VR256:$src1, addr:$src2)>; 8079 8080def : Pat<(v8i32 (X86VPermilpi VR256:$src1, (i8 imm:$imm))), 8081 (VPERMILPSYri VR256:$src1, imm:$imm)>; 8082def : Pat<(v4i64 (X86VPermilpi VR256:$src1, (i8 imm:$imm))), 8083 (VPERMILPDYri VR256:$src1, imm:$imm)>; 8084def : Pat<(v8i32 (X86VPermilpi (bc_v8i32 (loadv4i64 addr:$src1)), 8085 (i8 imm:$imm))), 8086 (VPERMILPSYmi addr:$src1, imm:$imm)>; 8087def : Pat<(v4i64 (X86VPermilpi (loadv4i64 addr:$src1), (i8 imm:$imm))), 8088 (VPERMILPDYmi addr:$src1, imm:$imm)>; 8089 8090def : Pat<(v4f32 (X86VPermilpv VR128:$src1, (v4i32 VR128:$src2))), 8091 (VPERMILPSrr VR128:$src1, VR128:$src2)>; 8092def : Pat<(v4f32 (X86VPermilpv VR128:$src1, (bc_v4i32 (loadv2i64 addr:$src2)))), 8093 (VPERMILPSrm VR128:$src1, addr:$src2)>; 8094def : Pat<(v2f64 (X86VPermilpv VR128:$src1, (v2i64 VR128:$src2))), 8095 (VPERMILPDrr VR128:$src1, VR128:$src2)>; 8096def : Pat<(v2f64 (X86VPermilpv VR128:$src1, (loadv2i64 addr:$src2))), 8097 (VPERMILPDrm VR128:$src1, addr:$src2)>; 8098 8099def : Pat<(v2i64 (X86VPermilpi VR128:$src1, (i8 imm:$imm))), 8100 (VPERMILPDri VR128:$src1, imm:$imm)>; 8101def : Pat<(v2i64 (X86VPermilpi (loadv2i64 addr:$src1), (i8 imm:$imm))), 8102 (VPERMILPDmi addr:$src1, imm:$imm)>; 8103} 8104 8105//===----------------------------------------------------------------------===// 8106// VPERM2F128 - Permute Floating-Point Values in 128-bit chunks 8107// 8108let ExeDomain = SSEPackedSingle in { 8109let isCommutable = 1 in 8110def VPERM2F128rr : AVXAIi8<0x06, MRMSrcReg, (outs VR256:$dst), 8111 (ins VR256:$src1, VR256:$src2, u8imm:$src3), 8112 "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8113 [(set VR256:$dst, (v8f32 (X86VPerm2x128 VR256:$src1, VR256:$src2, 8114 (i8 imm:$src3))))]>, VEX_4V, VEX_L, 8115 Sched<[WriteFShuffle]>; 8116def VPERM2F128rm : AVXAIi8<0x06, MRMSrcMem, (outs VR256:$dst), 8117 (ins VR256:$src1, f256mem:$src2, u8imm:$src3), 8118 "vperm2f128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8119 [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv8f32 addr:$src2), 8120 (i8 imm:$src3)))]>, VEX_4V, VEX_L, 8121 Sched<[WriteFShuffleLd, ReadAfterLd]>; 8122} 8123 8124let Predicates = [HasAVX] in { 8125def : Pat<(v4f64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8126 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8127def : Pat<(v4f64 (X86VPerm2x128 VR256:$src1, 8128 (loadv4f64 addr:$src2), (i8 imm:$imm))), 8129 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8130} 8131 8132let Predicates = [HasAVX1Only] in { 8133def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8134 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8135def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8136 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8137def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8138 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8139def : Pat<(v16i16 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8140 (VPERM2F128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8141 8142def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, 8143 (bc_v8i32 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 8144 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8145def : Pat<(v4i64 (X86VPerm2x128 VR256:$src1, 8146 (loadv4i64 addr:$src2), (i8 imm:$imm))), 8147 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8148def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, 8149 (bc_v32i8 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 8150 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8151def : Pat<(v16i16 (X86VPerm2x128 VR256:$src1, 8152 (bc_v16i16 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 8153 (VPERM2F128rm VR256:$src1, addr:$src2, imm:$imm)>; 8154} 8155 8156//===----------------------------------------------------------------------===// 8157// VZERO - Zero YMM registers 8158// 8159let Defs = [YMM0, YMM1, YMM2, YMM3, YMM4, YMM5, YMM6, YMM7, 8160 YMM8, YMM9, YMM10, YMM11, YMM12, YMM13, YMM14, YMM15] in { 8161 // Zero All YMM registers 8162 def VZEROALL : I<0x77, RawFrm, (outs), (ins), "vzeroall", 8163 [(int_x86_avx_vzeroall)]>, PS, VEX, VEX_L, Requires<[HasAVX]>; 8164 8165 // Zero Upper bits of YMM registers 8166 def VZEROUPPER : I<0x77, RawFrm, (outs), (ins), "vzeroupper", 8167 [(int_x86_avx_vzeroupper)]>, PS, VEX, Requires<[HasAVX]>; 8168} 8169 8170//===----------------------------------------------------------------------===// 8171// Half precision conversion instructions 8172//===----------------------------------------------------------------------===// 8173multiclass f16c_ph2ps<RegisterClass RC, X86MemOperand x86memop, Intrinsic Int> { 8174 def rr : I<0x13, MRMSrcReg, (outs RC:$dst), (ins VR128:$src), 8175 "vcvtph2ps\t{$src, $dst|$dst, $src}", 8176 [(set RC:$dst, (Int VR128:$src))]>, 8177 T8PD, VEX, Sched<[WriteCvtF2F]>; 8178 let hasSideEffects = 0, mayLoad = 1 in 8179 def rm : I<0x13, MRMSrcMem, (outs RC:$dst), (ins x86memop:$src), 8180 "vcvtph2ps\t{$src, $dst|$dst, $src}", []>, T8PD, VEX, 8181 Sched<[WriteCvtF2FLd]>; 8182} 8183 8184multiclass f16c_ps2ph<RegisterClass RC, X86MemOperand x86memop, Intrinsic Int> { 8185 def rr : Ii8<0x1D, MRMDestReg, (outs VR128:$dst), 8186 (ins RC:$src1, i32u8imm:$src2), 8187 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", 8188 [(set VR128:$dst, (Int RC:$src1, imm:$src2))]>, 8189 TAPD, VEX, Sched<[WriteCvtF2F]>; 8190 let hasSideEffects = 0, mayStore = 1, 8191 SchedRW = [WriteCvtF2FLd, WriteRMW] in 8192 def mr : Ii8<0x1D, MRMDestMem, (outs), 8193 (ins x86memop:$dst, RC:$src1, i32u8imm:$src2), 8194 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, 8195 TAPD, VEX; 8196} 8197 8198let Predicates = [HasF16C] in { 8199 defm VCVTPH2PS : f16c_ph2ps<VR128, f64mem, int_x86_vcvtph2ps_128>; 8200 defm VCVTPH2PSY : f16c_ph2ps<VR256, f128mem, int_x86_vcvtph2ps_256>, VEX_L; 8201 defm VCVTPS2PH : f16c_ps2ph<VR128, f64mem, int_x86_vcvtps2ph_128>; 8202 defm VCVTPS2PHY : f16c_ps2ph<VR256, f128mem, int_x86_vcvtps2ph_256>, VEX_L; 8203 8204 // Pattern match vcvtph2ps of a scalar i64 load. 8205 def : Pat<(int_x86_vcvtph2ps_128 (vzmovl_v2i64 addr:$src)), 8206 (VCVTPH2PSrm addr:$src)>; 8207 def : Pat<(int_x86_vcvtph2ps_128 (vzload_v2i64 addr:$src)), 8208 (VCVTPH2PSrm addr:$src)>; 8209 def : Pat<(int_x86_vcvtph2ps_128 (bitconvert 8210 (v2i64 (scalar_to_vector (loadi64 addr:$src))))), 8211 (VCVTPH2PSrm addr:$src)>; 8212 8213 def : Pat<(store (f64 (extractelt (bc_v2f64 (v8i16 8214 (int_x86_vcvtps2ph_128 VR128:$src1, i32:$src2))), (iPTR 0))), 8215 addr:$dst), 8216 (VCVTPS2PHmr addr:$dst, VR128:$src1, imm:$src2)>; 8217 def : Pat<(store (i64 (extractelt (bc_v2i64 (v8i16 8218 (int_x86_vcvtps2ph_128 VR128:$src1, i32:$src2))), (iPTR 0))), 8219 addr:$dst), 8220 (VCVTPS2PHmr addr:$dst, VR128:$src1, imm:$src2)>; 8221 def : Pat<(store (v8i16 (int_x86_vcvtps2ph_256 VR256:$src1, i32:$src2)), 8222 addr:$dst), 8223 (VCVTPS2PHYmr addr:$dst, VR256:$src1, imm:$src2)>; 8224} 8225 8226// Patterns for matching conversions from float to half-float and vice versa. 8227let Predicates = [HasF16C] in { 8228 // Use MXCSR.RC for rounding instead of explicitly specifying the default 8229 // rounding mode (Nearest-Even, encoded as 0). Both are equivalent in the 8230 // configurations we support (the default). However, falling back to MXCSR is 8231 // more consistent with other instructions, which are always controlled by it. 8232 // It's encoded as 0b100. 8233 def : Pat<(fp_to_f16 FR32:$src), 8234 (i16 (EXTRACT_SUBREG (VMOVPDI2DIrr (VCVTPS2PHrr 8235 (COPY_TO_REGCLASS FR32:$src, VR128), 4)), sub_16bit))>; 8236 8237 def : Pat<(f16_to_fp GR16:$src), 8238 (f32 (COPY_TO_REGCLASS (VCVTPH2PSrr 8239 (COPY_TO_REGCLASS (MOVSX32rr16 GR16:$src), VR128)), FR32)) >; 8240 8241 def : Pat<(f16_to_fp (i16 (fp_to_f16 FR32:$src))), 8242 (f32 (COPY_TO_REGCLASS (VCVTPH2PSrr 8243 (VCVTPS2PHrr (COPY_TO_REGCLASS FR32:$src, VR128), 4)), FR32)) >; 8244} 8245 8246//===----------------------------------------------------------------------===// 8247// AVX2 Instructions 8248//===----------------------------------------------------------------------===// 8249 8250/// AVX2_binop_rmi - AVX2 binary operator with 8-bit immediate 8251multiclass AVX2_binop_rmi<bits<8> opc, string OpcodeStr, SDNode OpNode, 8252 ValueType OpVT, RegisterClass RC, PatFrag memop_frag, 8253 X86MemOperand x86memop> { 8254 let isCommutable = 1 in 8255 def rri : AVX2AIi8<opc, MRMSrcReg, (outs RC:$dst), 8256 (ins RC:$src1, RC:$src2, u8imm:$src3), 8257 !strconcat(OpcodeStr, 8258 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 8259 [(set RC:$dst, (OpVT (OpNode RC:$src1, RC:$src2, imm:$src3)))]>, 8260 Sched<[WriteBlend]>, VEX_4V; 8261 def rmi : AVX2AIi8<opc, MRMSrcMem, (outs RC:$dst), 8262 (ins RC:$src1, x86memop:$src2, u8imm:$src3), 8263 !strconcat(OpcodeStr, 8264 "\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}"), 8265 [(set RC:$dst, 8266 (OpVT (OpNode RC:$src1, 8267 (bitconvert (memop_frag addr:$src2)), imm:$src3)))]>, 8268 Sched<[WriteBlendLd, ReadAfterLd]>, VEX_4V; 8269} 8270 8271defm VPBLENDD : AVX2_binop_rmi<0x02, "vpblendd", X86Blendi, v4i32, 8272 VR128, loadv2i64, i128mem>; 8273defm VPBLENDDY : AVX2_binop_rmi<0x02, "vpblendd", X86Blendi, v8i32, 8274 VR256, loadv4i64, i256mem>, VEX_L; 8275 8276//===----------------------------------------------------------------------===// 8277// VPBROADCAST - Load from memory and broadcast to all elements of the 8278// destination operand 8279// 8280multiclass avx2_broadcast<bits<8> opc, string OpcodeStr, 8281 X86MemOperand x86memop, PatFrag ld_frag, 8282 ValueType OpVT128, ValueType OpVT256, Predicate prd> { 8283 let Predicates = [HasAVX2, prd] in { 8284 def rr : AVX28I<opc, MRMSrcReg, (outs VR128:$dst), (ins VR128:$src), 8285 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 8286 [(set VR128:$dst, 8287 (OpVT128 (X86VBroadcast (OpVT128 VR128:$src))))]>, 8288 Sched<[WriteShuffle]>, VEX; 8289 def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst), (ins x86memop:$src), 8290 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 8291 [(set VR128:$dst, 8292 (OpVT128 (X86VBroadcast (ld_frag addr:$src))))]>, 8293 Sched<[WriteLoad]>, VEX; 8294 def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), (ins VR128:$src), 8295 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 8296 [(set VR256:$dst, 8297 (OpVT256 (X86VBroadcast (OpVT128 VR128:$src))))]>, 8298 Sched<[WriteShuffle256]>, VEX, VEX_L; 8299 def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), (ins x86memop:$src), 8300 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), 8301 [(set VR256:$dst, 8302 (OpVT256 (X86VBroadcast (ld_frag addr:$src))))]>, 8303 Sched<[WriteLoad]>, VEX, VEX_L; 8304 8305 // Provide aliases for broadcast from the same register class that 8306 // automatically does the extract. 8307 def : Pat<(OpVT256 (X86VBroadcast (OpVT256 VR256:$src))), 8308 (!cast<Instruction>(NAME#"Yrr") 8309 (OpVT128 (EXTRACT_SUBREG (OpVT256 VR256:$src),sub_xmm)))>; 8310 } 8311} 8312 8313defm VPBROADCASTB : avx2_broadcast<0x78, "vpbroadcastb", i8mem, loadi8, 8314 v16i8, v32i8, NoVLX_Or_NoBWI>; 8315defm VPBROADCASTW : avx2_broadcast<0x79, "vpbroadcastw", i16mem, loadi16, 8316 v8i16, v16i16, NoVLX_Or_NoBWI>; 8317defm VPBROADCASTD : avx2_broadcast<0x58, "vpbroadcastd", i32mem, loadi32, 8318 v4i32, v8i32, NoVLX>; 8319defm VPBROADCASTQ : avx2_broadcast<0x59, "vpbroadcastq", i64mem, loadi64, 8320 v2i64, v4i64, NoVLX>; 8321 8322let Predicates = [HasAVX2] in { 8323 // loadi16 is tricky to fold, because !isTypeDesirableForOp, justifiably. 8324 // This means we'll encounter truncated i32 loads; match that here. 8325 def : Pat<(v8i16 (X86VBroadcast (i16 (trunc (i32 (load addr:$src)))))), 8326 (VPBROADCASTWrm addr:$src)>; 8327 def : Pat<(v16i16 (X86VBroadcast (i16 (trunc (i32 (load addr:$src)))))), 8328 (VPBROADCASTWYrm addr:$src)>; 8329 def : Pat<(v8i16 (X86VBroadcast 8330 (i16 (trunc (i32 (zextloadi16 addr:$src)))))), 8331 (VPBROADCASTWrm addr:$src)>; 8332 def : Pat<(v16i16 (X86VBroadcast 8333 (i16 (trunc (i32 (zextloadi16 addr:$src)))))), 8334 (VPBROADCASTWYrm addr:$src)>; 8335 8336 // Provide aliases for broadcast from the same register class that 8337 // automatically does the extract. 8338 def : Pat<(v8f32 (X86VBroadcast (v8f32 VR256:$src))), 8339 (VBROADCASTSSYrr (v4f32 (EXTRACT_SUBREG (v8f32 VR256:$src), 8340 sub_xmm)))>; 8341 def : Pat<(v4f64 (X86VBroadcast (v4f64 VR256:$src))), 8342 (VBROADCASTSDYrr (v2f64 (EXTRACT_SUBREG (v4f64 VR256:$src), 8343 sub_xmm)))>; 8344} 8345 8346let Predicates = [HasAVX2, NoVLX] in { 8347 // Provide fallback in case the load node that is used in the patterns above 8348 // is used by additional users, which prevents the pattern selection. 8349 let AddedComplexity = 20 in { 8350 def : Pat<(v4f32 (X86VBroadcast FR32:$src)), 8351 (VBROADCASTSSrr (COPY_TO_REGCLASS FR32:$src, VR128))>; 8352 def : Pat<(v8f32 (X86VBroadcast FR32:$src)), 8353 (VBROADCASTSSYrr (COPY_TO_REGCLASS FR32:$src, VR128))>; 8354 def : Pat<(v4f64 (X86VBroadcast FR64:$src)), 8355 (VBROADCASTSDYrr (COPY_TO_REGCLASS FR64:$src, VR128))>; 8356 } 8357} 8358 8359let Predicates = [HasAVX2, NoVLX_Or_NoBWI], AddedComplexity = 20 in { 8360 def : Pat<(v16i8 (X86VBroadcast GR8:$src)), 8361 (VPBROADCASTBrr (COPY_TO_REGCLASS 8362 (i32 (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)), 8363 VR128))>; 8364 def : Pat<(v32i8 (X86VBroadcast GR8:$src)), 8365 (VPBROADCASTBYrr (COPY_TO_REGCLASS 8366 (i32 (SUBREG_TO_REG (i32 0), GR8:$src, sub_8bit)), 8367 VR128))>; 8368 8369 def : Pat<(v8i16 (X86VBroadcast GR16:$src)), 8370 (VPBROADCASTWrr (COPY_TO_REGCLASS 8371 (i32 (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit)), 8372 VR128))>; 8373 def : Pat<(v16i16 (X86VBroadcast GR16:$src)), 8374 (VPBROADCASTWYrr (COPY_TO_REGCLASS 8375 (i32 (SUBREG_TO_REG (i32 0), GR16:$src, sub_16bit)), 8376 VR128))>; 8377} 8378let Predicates = [HasAVX2, NoVLX], AddedComplexity = 20 in { 8379 def : Pat<(v4i32 (X86VBroadcast GR32:$src)), 8380 (VBROADCASTSSrr (COPY_TO_REGCLASS GR32:$src, VR128))>; 8381 def : Pat<(v8i32 (X86VBroadcast GR32:$src)), 8382 (VBROADCASTSSYrr (COPY_TO_REGCLASS GR32:$src, VR128))>; 8383 def : Pat<(v4i64 (X86VBroadcast GR64:$src)), 8384 (VBROADCASTSDYrr (COPY_TO_REGCLASS GR64:$src, VR128))>; 8385 8386 // The patterns for VPBROADCASTD are not needed because they would match 8387 // the exact same thing as VBROADCASTSS patterns. 8388 8389 def : Pat<(v2i64 (X86VBroadcast GR64:$src)), 8390 (VPBROADCASTQrr (COPY_TO_REGCLASS GR64:$src, VR128))>; 8391 // The v4i64 pattern is not needed because VBROADCASTSDYrr already match. 8392} 8393 8394// AVX1 broadcast patterns 8395let Predicates = [HasAVX1Only] in { 8396def : Pat<(v8i32 (X86VBroadcast (loadi32 addr:$src))), 8397 (VBROADCASTSSYrm addr:$src)>; 8398def : Pat<(v4i64 (X86VBroadcast (loadi64 addr:$src))), 8399 (VBROADCASTSDYrm addr:$src)>; 8400def : Pat<(v4i32 (X86VBroadcast (loadi32 addr:$src))), 8401 (VBROADCASTSSrm addr:$src)>; 8402} 8403 8404 // Provide fallback in case the load node that is used in the patterns above 8405 // is used by additional users, which prevents the pattern selection. 8406let Predicates = [HasAVX], AddedComplexity = 20 in { 8407 // 128bit broadcasts: 8408 def : Pat<(v2f64 (X86VBroadcast f64:$src)), 8409 (VMOVDDUPrr (COPY_TO_REGCLASS FR64:$src, VR128))>; 8410} 8411 8412let Predicates = [HasAVX, NoVLX], AddedComplexity = 20 in { 8413 def : Pat<(v4f32 (X86VBroadcast FR32:$src)), 8414 (VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0)>; 8415 def : Pat<(v8f32 (X86VBroadcast FR32:$src)), 8416 (VINSERTF128rr (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), 8417 (VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0), sub_xmm), 8418 (VPSHUFDri (COPY_TO_REGCLASS FR32:$src, VR128), 0), 1)>; 8419 def : Pat<(v4f64 (X86VBroadcast FR64:$src)), 8420 (VINSERTF128rr (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), 8421 (VPSHUFDri (COPY_TO_REGCLASS FR64:$src, VR128), 0x44), sub_xmm), 8422 (VPSHUFDri (COPY_TO_REGCLASS FR64:$src, VR128), 0x44), 1)>; 8423 8424 def : Pat<(v4i32 (X86VBroadcast GR32:$src)), 8425 (VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0)>; 8426 def : Pat<(v8i32 (X86VBroadcast GR32:$src)), 8427 (VINSERTF128rr (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), 8428 (VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0), sub_xmm), 8429 (VPSHUFDri (COPY_TO_REGCLASS GR32:$src, VR128), 0), 1)>; 8430 def : Pat<(v4i64 (X86VBroadcast GR64:$src)), 8431 (VINSERTF128rr (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), 8432 (VPSHUFDri (COPY_TO_REGCLASS GR64:$src, VR128), 0x44), sub_xmm), 8433 (VPSHUFDri (COPY_TO_REGCLASS GR64:$src, VR128), 0x44), 1)>; 8434 8435 def : Pat<(v2i64 (X86VBroadcast i64:$src)), 8436 (VMOVDDUPrr (COPY_TO_REGCLASS GR64:$src, VR128))>; 8437} 8438 8439//===----------------------------------------------------------------------===// 8440// VPERM - Permute instructions 8441// 8442 8443multiclass avx2_perm<bits<8> opc, string OpcodeStr, PatFrag mem_frag, 8444 ValueType OpVT, X86FoldableSchedWrite Sched> { 8445 let Predicates = [HasAVX2, NoVLX] in { 8446 def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), 8447 (ins VR256:$src1, VR256:$src2), 8448 !strconcat(OpcodeStr, 8449 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8450 [(set VR256:$dst, 8451 (OpVT (X86VPermv VR256:$src1, VR256:$src2)))]>, 8452 Sched<[Sched]>, VEX_4V, VEX_L; 8453 def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), 8454 (ins VR256:$src1, i256mem:$src2), 8455 !strconcat(OpcodeStr, 8456 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8457 [(set VR256:$dst, 8458 (OpVT (X86VPermv VR256:$src1, 8459 (bitconvert (mem_frag addr:$src2)))))]>, 8460 Sched<[Sched.Folded, ReadAfterLd]>, VEX_4V, VEX_L; 8461 } 8462} 8463 8464defm VPERMD : avx2_perm<0x36, "vpermd", loadv4i64, v8i32, WriteShuffle256>; 8465let ExeDomain = SSEPackedSingle in 8466defm VPERMPS : avx2_perm<0x16, "vpermps", loadv8f32, v8f32, WriteFShuffle256>; 8467 8468multiclass avx2_perm_imm<bits<8> opc, string OpcodeStr, PatFrag mem_frag, 8469 ValueType OpVT, X86FoldableSchedWrite Sched> { 8470 let Predicates = [HasAVX2, NoVLX] in { 8471 def Yri : AVX2AIi8<opc, MRMSrcReg, (outs VR256:$dst), 8472 (ins VR256:$src1, u8imm:$src2), 8473 !strconcat(OpcodeStr, 8474 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8475 [(set VR256:$dst, 8476 (OpVT (X86VPermi VR256:$src1, (i8 imm:$src2))))]>, 8477 Sched<[Sched]>, VEX, VEX_L; 8478 def Ymi : AVX2AIi8<opc, MRMSrcMem, (outs VR256:$dst), 8479 (ins i256mem:$src1, u8imm:$src2), 8480 !strconcat(OpcodeStr, 8481 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8482 [(set VR256:$dst, 8483 (OpVT (X86VPermi (mem_frag addr:$src1), 8484 (i8 imm:$src2))))]>, 8485 Sched<[Sched.Folded, ReadAfterLd]>, VEX, VEX_L; 8486 } 8487} 8488 8489defm VPERMQ : avx2_perm_imm<0x00, "vpermq", loadv4i64, v4i64, 8490 WriteShuffle256>, VEX_W; 8491let ExeDomain = SSEPackedDouble in 8492defm VPERMPD : avx2_perm_imm<0x01, "vpermpd", loadv4f64, v4f64, 8493 WriteFShuffle256>, VEX_W; 8494 8495//===----------------------------------------------------------------------===// 8496// VPERM2I128 - Permute Floating-Point Values in 128-bit chunks 8497// 8498let isCommutable = 1 in 8499def VPERM2I128rr : AVX2AIi8<0x46, MRMSrcReg, (outs VR256:$dst), 8500 (ins VR256:$src1, VR256:$src2, u8imm:$src3), 8501 "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8502 [(set VR256:$dst, (v4i64 (X86VPerm2x128 VR256:$src1, VR256:$src2, 8503 (i8 imm:$src3))))]>, Sched<[WriteShuffle256]>, 8504 VEX_4V, VEX_L; 8505def VPERM2I128rm : AVX2AIi8<0x46, MRMSrcMem, (outs VR256:$dst), 8506 (ins VR256:$src1, f256mem:$src2, u8imm:$src3), 8507 "vperm2i128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8508 [(set VR256:$dst, (X86VPerm2x128 VR256:$src1, (loadv4i64 addr:$src2), 8509 (i8 imm:$src3)))]>, 8510 Sched<[WriteShuffle256Ld, ReadAfterLd]>, VEX_4V, VEX_L; 8511 8512let Predicates = [HasAVX2] in { 8513def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8514 (VPERM2I128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8515def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8516 (VPERM2I128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8517def : Pat<(v16i16 (X86VPerm2x128 VR256:$src1, VR256:$src2, (i8 imm:$imm))), 8518 (VPERM2I128rr VR256:$src1, VR256:$src2, imm:$imm)>; 8519 8520def : Pat<(v32i8 (X86VPerm2x128 VR256:$src1, (bc_v32i8 (loadv4i64 addr:$src2)), 8521 (i8 imm:$imm))), 8522 (VPERM2I128rm VR256:$src1, addr:$src2, imm:$imm)>; 8523def : Pat<(v16i16 (X86VPerm2x128 VR256:$src1, 8524 (bc_v16i16 (loadv4i64 addr:$src2)), (i8 imm:$imm))), 8525 (VPERM2I128rm VR256:$src1, addr:$src2, imm:$imm)>; 8526def : Pat<(v8i32 (X86VPerm2x128 VR256:$src1, (bc_v8i32 (loadv4i64 addr:$src2)), 8527 (i8 imm:$imm))), 8528 (VPERM2I128rm VR256:$src1, addr:$src2, imm:$imm)>; 8529} 8530 8531 8532//===----------------------------------------------------------------------===// 8533// VINSERTI128 - Insert packed integer values 8534// 8535let hasSideEffects = 0 in { 8536def VINSERTI128rr : AVX2AIi8<0x38, MRMSrcReg, (outs VR256:$dst), 8537 (ins VR256:$src1, VR128:$src2, u8imm:$src3), 8538 "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8539 []>, Sched<[WriteShuffle256]>, VEX_4V, VEX_L; 8540let mayLoad = 1 in 8541def VINSERTI128rm : AVX2AIi8<0x38, MRMSrcMem, (outs VR256:$dst), 8542 (ins VR256:$src1, i128mem:$src2, u8imm:$src3), 8543 "vinserti128\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}", 8544 []>, Sched<[WriteShuffle256Ld, ReadAfterLd]>, VEX_4V, VEX_L; 8545} 8546 8547let Predicates = [HasAVX2, NoVLX] in { 8548def : Pat<(vinsert128_insert:$ins (v4i64 VR256:$src1), (v2i64 VR128:$src2), 8549 (iPTR imm)), 8550 (VINSERTI128rr VR256:$src1, VR128:$src2, 8551 (INSERT_get_vinsert128_imm VR256:$ins))>; 8552def : Pat<(vinsert128_insert:$ins (v8i32 VR256:$src1), (v4i32 VR128:$src2), 8553 (iPTR imm)), 8554 (VINSERTI128rr VR256:$src1, VR128:$src2, 8555 (INSERT_get_vinsert128_imm VR256:$ins))>; 8556def : Pat<(vinsert128_insert:$ins (v32i8 VR256:$src1), (v16i8 VR128:$src2), 8557 (iPTR imm)), 8558 (VINSERTI128rr VR256:$src1, VR128:$src2, 8559 (INSERT_get_vinsert128_imm VR256:$ins))>; 8560def : Pat<(vinsert128_insert:$ins (v16i16 VR256:$src1), (v8i16 VR128:$src2), 8561 (iPTR imm)), 8562 (VINSERTI128rr VR256:$src1, VR128:$src2, 8563 (INSERT_get_vinsert128_imm VR256:$ins))>; 8564 8565def : Pat<(vinsert128_insert:$ins (v4i64 VR256:$src1), (loadv2i64 addr:$src2), 8566 (iPTR imm)), 8567 (VINSERTI128rm VR256:$src1, addr:$src2, 8568 (INSERT_get_vinsert128_imm VR256:$ins))>; 8569def : Pat<(vinsert128_insert:$ins (v8i32 VR256:$src1), 8570 (bc_v4i32 (loadv2i64 addr:$src2)), 8571 (iPTR imm)), 8572 (VINSERTI128rm VR256:$src1, addr:$src2, 8573 (INSERT_get_vinsert128_imm VR256:$ins))>; 8574def : Pat<(vinsert128_insert:$ins (v32i8 VR256:$src1), 8575 (bc_v16i8 (loadv2i64 addr:$src2)), 8576 (iPTR imm)), 8577 (VINSERTI128rm VR256:$src1, addr:$src2, 8578 (INSERT_get_vinsert128_imm VR256:$ins))>; 8579def : Pat<(vinsert128_insert:$ins (v16i16 VR256:$src1), 8580 (bc_v8i16 (loadv2i64 addr:$src2)), 8581 (iPTR imm)), 8582 (VINSERTI128rm VR256:$src1, addr:$src2, 8583 (INSERT_get_vinsert128_imm VR256:$ins))>; 8584} 8585 8586//===----------------------------------------------------------------------===// 8587// VEXTRACTI128 - Extract packed integer values 8588// 8589def VEXTRACTI128rr : AVX2AIi8<0x39, MRMDestReg, (outs VR128:$dst), 8590 (ins VR256:$src1, u8imm:$src2), 8591 "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, 8592 Sched<[WriteShuffle256]>, VEX, VEX_L; 8593let hasSideEffects = 0, mayStore = 1 in 8594def VEXTRACTI128mr : AVX2AIi8<0x39, MRMDestMem, (outs), 8595 (ins i128mem:$dst, VR256:$src1, u8imm:$src2), 8596 "vextracti128\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>, 8597 Sched<[WriteStore]>, VEX, VEX_L; 8598 8599let Predicates = [HasAVX2, NoVLX] in { 8600def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 8601 (v2i64 (VEXTRACTI128rr 8602 (v4i64 VR256:$src1), 8603 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8604def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 8605 (v4i32 (VEXTRACTI128rr 8606 (v8i32 VR256:$src1), 8607 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8608def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 8609 (v8i16 (VEXTRACTI128rr 8610 (v16i16 VR256:$src1), 8611 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8612def : Pat<(vextract128_extract:$ext VR256:$src1, (iPTR imm)), 8613 (v16i8 (VEXTRACTI128rr 8614 (v32i8 VR256:$src1), 8615 (EXTRACT_get_vextract128_imm VR128:$ext)))>; 8616 8617def : Pat<(store (v2i64 (vextract128_extract:$ext (v4i64 VR256:$src1), 8618 (iPTR imm))), addr:$dst), 8619 (VEXTRACTI128mr addr:$dst, VR256:$src1, 8620 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8621def : Pat<(store (v4i32 (vextract128_extract:$ext (v8i32 VR256:$src1), 8622 (iPTR imm))), addr:$dst), 8623 (VEXTRACTI128mr addr:$dst, VR256:$src1, 8624 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8625def : Pat<(store (v8i16 (vextract128_extract:$ext (v16i16 VR256:$src1), 8626 (iPTR imm))), addr:$dst), 8627 (VEXTRACTI128mr addr:$dst, VR256:$src1, 8628 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8629def : Pat<(store (v16i8 (vextract128_extract:$ext (v32i8 VR256:$src1), 8630 (iPTR imm))), addr:$dst), 8631 (VEXTRACTI128mr addr:$dst, VR256:$src1, 8632 (EXTRACT_get_vextract128_imm VR128:$ext))>; 8633} 8634 8635//===----------------------------------------------------------------------===// 8636// VPMASKMOV - Conditional SIMD Integer Packed Loads and Stores 8637// 8638multiclass avx2_pmovmask<string OpcodeStr, 8639 Intrinsic IntLd128, Intrinsic IntLd256, 8640 Intrinsic IntSt128, Intrinsic IntSt256> { 8641 def rm : AVX28I<0x8c, MRMSrcMem, (outs VR128:$dst), 8642 (ins VR128:$src1, i128mem:$src2), 8643 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8644 [(set VR128:$dst, (IntLd128 addr:$src2, VR128:$src1))]>, VEX_4V; 8645 def Yrm : AVX28I<0x8c, MRMSrcMem, (outs VR256:$dst), 8646 (ins VR256:$src1, i256mem:$src2), 8647 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8648 [(set VR256:$dst, (IntLd256 addr:$src2, VR256:$src1))]>, 8649 VEX_4V, VEX_L; 8650 def mr : AVX28I<0x8e, MRMDestMem, (outs), 8651 (ins i128mem:$dst, VR128:$src1, VR128:$src2), 8652 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8653 [(IntSt128 addr:$dst, VR128:$src1, VR128:$src2)]>, VEX_4V; 8654 def Ymr : AVX28I<0x8e, MRMDestMem, (outs), 8655 (ins i256mem:$dst, VR256:$src1, VR256:$src2), 8656 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8657 [(IntSt256 addr:$dst, VR256:$src1, VR256:$src2)]>, VEX_4V, VEX_L; 8658} 8659 8660defm VPMASKMOVD : avx2_pmovmask<"vpmaskmovd", 8661 int_x86_avx2_maskload_d, 8662 int_x86_avx2_maskload_d_256, 8663 int_x86_avx2_maskstore_d, 8664 int_x86_avx2_maskstore_d_256>; 8665defm VPMASKMOVQ : avx2_pmovmask<"vpmaskmovq", 8666 int_x86_avx2_maskload_q, 8667 int_x86_avx2_maskload_q_256, 8668 int_x86_avx2_maskstore_q, 8669 int_x86_avx2_maskstore_q_256>, VEX_W; 8670 8671multiclass maskmov_lowering<string InstrStr, RegisterClass RC, ValueType VT, 8672 ValueType MaskVT, string BlendStr, ValueType ZeroVT> { 8673 // masked store 8674 def: Pat<(X86mstore addr:$ptr, (MaskVT RC:$mask), (VT RC:$src)), 8675 (!cast<Instruction>(InstrStr#"mr") addr:$ptr, RC:$mask, RC:$src)>; 8676 // masked load 8677 def: Pat<(VT (masked_load addr:$ptr, (MaskVT RC:$mask), undef)), 8678 (!cast<Instruction>(InstrStr#"rm") RC:$mask, addr:$ptr)>; 8679 def: Pat<(VT (masked_load addr:$ptr, (MaskVT RC:$mask), 8680 (VT (bitconvert (ZeroVT immAllZerosV))))), 8681 (!cast<Instruction>(InstrStr#"rm") RC:$mask, addr:$ptr)>; 8682 def: Pat<(VT (masked_load addr:$ptr, (MaskVT RC:$mask), (VT RC:$src0))), 8683 (!cast<Instruction>(BlendStr#"rr") 8684 RC:$src0, 8685 (!cast<Instruction>(InstrStr#"rm") RC:$mask, addr:$ptr), 8686 RC:$mask)>; 8687} 8688let Predicates = [HasAVX] in { 8689 defm : maskmov_lowering<"VMASKMOVPS", VR128, v4f32, v4i32, "VBLENDVPS", v4i32>; 8690 defm : maskmov_lowering<"VMASKMOVPD", VR128, v2f64, v2i64, "VBLENDVPD", v4i32>; 8691 defm : maskmov_lowering<"VMASKMOVPSY", VR256, v8f32, v8i32, "VBLENDVPSY", v8i32>; 8692 defm : maskmov_lowering<"VMASKMOVPDY", VR256, v4f64, v4i64, "VBLENDVPDY", v8i32>; 8693} 8694let Predicates = [HasAVX1Only] in { 8695 // load/store i32/i64 not supported use ps/pd version 8696 defm : maskmov_lowering<"VMASKMOVPSY", VR256, v8i32, v8i32, "VBLENDVPSY", v8i32>; 8697 defm : maskmov_lowering<"VMASKMOVPDY", VR256, v4i64, v4i64, "VBLENDVPDY", v8i32>; 8698 defm : maskmov_lowering<"VMASKMOVPS", VR128, v4i32, v4i32, "VBLENDVPS", v4i32>; 8699 defm : maskmov_lowering<"VMASKMOVPD", VR128, v2i64, v2i64, "VBLENDVPD", v4i32>; 8700} 8701let Predicates = [HasAVX2] in { 8702 defm : maskmov_lowering<"VPMASKMOVDY", VR256, v8i32, v8i32, "VBLENDVPSY", v8i32>; 8703 defm : maskmov_lowering<"VPMASKMOVQY", VR256, v4i64, v4i64, "VBLENDVPDY", v8i32>; 8704 defm : maskmov_lowering<"VPMASKMOVD", VR128, v4i32, v4i32, "VBLENDVPS", v4i32>; 8705 defm : maskmov_lowering<"VPMASKMOVQ", VR128, v2i64, v2i64, "VBLENDVPD", v4i32>; 8706} 8707//===----------------------------------------------------------------------===// 8708// Variable Bit Shifts 8709// 8710multiclass avx2_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode, 8711 ValueType vt128, ValueType vt256> { 8712 def rr : AVX28I<opc, MRMSrcReg, (outs VR128:$dst), 8713 (ins VR128:$src1, VR128:$src2), 8714 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8715 [(set VR128:$dst, 8716 (vt128 (OpNode VR128:$src1, (vt128 VR128:$src2))))]>, 8717 VEX_4V, Sched<[WriteVarVecShift]>; 8718 def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst), 8719 (ins VR128:$src1, i128mem:$src2), 8720 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8721 [(set VR128:$dst, 8722 (vt128 (OpNode VR128:$src1, 8723 (vt128 (bitconvert (loadv2i64 addr:$src2))))))]>, 8724 VEX_4V, Sched<[WriteVarVecShiftLd, ReadAfterLd]>; 8725 def Yrr : AVX28I<opc, MRMSrcReg, (outs VR256:$dst), 8726 (ins VR256:$src1, VR256:$src2), 8727 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8728 [(set VR256:$dst, 8729 (vt256 (OpNode VR256:$src1, (vt256 VR256:$src2))))]>, 8730 VEX_4V, VEX_L, Sched<[WriteVarVecShift]>; 8731 def Yrm : AVX28I<opc, MRMSrcMem, (outs VR256:$dst), 8732 (ins VR256:$src1, i256mem:$src2), 8733 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"), 8734 [(set VR256:$dst, 8735 (vt256 (OpNode VR256:$src1, 8736 (vt256 (bitconvert (loadv4i64 addr:$src2))))))]>, 8737 VEX_4V, VEX_L, Sched<[WriteVarVecShiftLd, ReadAfterLd]>; 8738} 8739 8740let Predicates = [HasAVX2, NoVLX] in { 8741 defm VPSLLVD : avx2_var_shift<0x47, "vpsllvd", shl, v4i32, v8i32>; 8742 defm VPSLLVQ : avx2_var_shift<0x47, "vpsllvq", shl, v2i64, v4i64>, VEX_W; 8743 defm VPSRLVD : avx2_var_shift<0x45, "vpsrlvd", srl, v4i32, v8i32>; 8744 defm VPSRLVQ : avx2_var_shift<0x45, "vpsrlvq", srl, v2i64, v4i64>, VEX_W; 8745 defm VPSRAVD : avx2_var_shift<0x46, "vpsravd", sra, v4i32, v8i32>; 8746 let isCodeGenOnly = 1 in 8747 defm VPSRAVD_Int : avx2_var_shift<0x46, "vpsravd", X86vsrav, v4i32, v8i32>; 8748} 8749//===----------------------------------------------------------------------===// 8750// VGATHER - GATHER Operations 8751multiclass avx2_gather<bits<8> opc, string OpcodeStr, RegisterClass RC256, 8752 X86MemOperand memop128, X86MemOperand memop256> { 8753 def rm : AVX28I<opc, MRMSrcMem, (outs VR128:$dst, VR128:$mask_wb), 8754 (ins VR128:$src1, memop128:$src2, VR128:$mask), 8755 !strconcat(OpcodeStr, 8756 "\t{$mask, $src2, $dst|$dst, $src2, $mask}"), 8757 []>, VEX_4VOp3; 8758 def Yrm : AVX28I<opc, MRMSrcMem, (outs RC256:$dst, RC256:$mask_wb), 8759 (ins RC256:$src1, memop256:$src2, RC256:$mask), 8760 !strconcat(OpcodeStr, 8761 "\t{$mask, $src2, $dst|$dst, $src2, $mask}"), 8762 []>, VEX_4VOp3, VEX_L; 8763} 8764 8765let mayLoad = 1, hasSideEffects = 0, Constraints 8766 = "@earlyclobber $dst,@earlyclobber $mask_wb, $src1 = $dst, $mask = $mask_wb" 8767 in { 8768 defm VPGATHERDQ : avx2_gather<0x90, "vpgatherdq", VR256, vx128mem, vx256mem>, VEX_W; 8769 defm VPGATHERQQ : avx2_gather<0x91, "vpgatherqq", VR256, vx128mem, vy256mem>, VEX_W; 8770 defm VPGATHERDD : avx2_gather<0x90, "vpgatherdd", VR256, vx128mem, vy256mem>; 8771 defm VPGATHERQD : avx2_gather<0x91, "vpgatherqd", VR128, vx64mem, vy128mem>; 8772 8773 let ExeDomain = SSEPackedDouble in { 8774 defm VGATHERDPD : avx2_gather<0x92, "vgatherdpd", VR256, vx128mem, vx256mem>, VEX_W; 8775 defm VGATHERQPD : avx2_gather<0x93, "vgatherqpd", VR256, vx128mem, vy256mem>, VEX_W; 8776 } 8777 8778 let ExeDomain = SSEPackedSingle in { 8779 defm VGATHERDPS : avx2_gather<0x92, "vgatherdps", VR256, vx128mem, vy256mem>; 8780 defm VGATHERQPS : avx2_gather<0x93, "vgatherqps", VR128, vx64mem, vy128mem>; 8781 } 8782} 8783 8784//===----------------------------------------------------------------------===// 8785// Extra selection patterns for FR128, f128, f128mem 8786 8787// movaps is shorter than movdqa. movaps is in SSE and movdqa is in SSE2. 8788def : Pat<(store (f128 FR128:$src), addr:$dst), 8789 (MOVAPSmr addr:$dst, (COPY_TO_REGCLASS (f128 FR128:$src), VR128))>; 8790 8791def : Pat<(loadf128 addr:$src), 8792 (COPY_TO_REGCLASS (MOVAPSrm addr:$src), FR128)>; 8793 8794// andps is shorter than andpd or pand. andps is SSE and andpd/pand are in SSE2 8795def : Pat<(X86fand FR128:$src1, (loadf128 addr:$src2)), 8796 (COPY_TO_REGCLASS 8797 (ANDPSrm (COPY_TO_REGCLASS FR128:$src1, VR128), f128mem:$src2), 8798 FR128)>; 8799 8800def : Pat<(X86fand FR128:$src1, FR128:$src2), 8801 (COPY_TO_REGCLASS 8802 (ANDPSrr (COPY_TO_REGCLASS FR128:$src1, VR128), 8803 (COPY_TO_REGCLASS FR128:$src2, VR128)), FR128)>; 8804 8805def : Pat<(and FR128:$src1, FR128:$src2), 8806 (COPY_TO_REGCLASS 8807 (ANDPSrr (COPY_TO_REGCLASS FR128:$src1, VR128), 8808 (COPY_TO_REGCLASS FR128:$src2, VR128)), FR128)>; 8809 8810def : Pat<(X86for FR128:$src1, (loadf128 addr:$src2)), 8811 (COPY_TO_REGCLASS 8812 (ORPSrm (COPY_TO_REGCLASS FR128:$src1, VR128), f128mem:$src2), 8813 FR128)>; 8814 8815def : Pat<(X86for FR128:$src1, FR128:$src2), 8816 (COPY_TO_REGCLASS 8817 (ORPSrr (COPY_TO_REGCLASS FR128:$src1, VR128), 8818 (COPY_TO_REGCLASS FR128:$src2, VR128)), FR128)>; 8819 8820def : Pat<(or FR128:$src1, FR128:$src2), 8821 (COPY_TO_REGCLASS 8822 (ORPSrr (COPY_TO_REGCLASS FR128:$src1, VR128), 8823 (COPY_TO_REGCLASS FR128:$src2, VR128)), FR128)>; 8824 8825def : Pat<(X86fxor FR128:$src1, (loadf128 addr:$src2)), 8826 (COPY_TO_REGCLASS 8827 (XORPSrm (COPY_TO_REGCLASS FR128:$src1, VR128), f128mem:$src2), 8828 FR128)>; 8829 8830def : Pat<(X86fxor FR128:$src1, FR128:$src2), 8831 (COPY_TO_REGCLASS 8832 (XORPSrr (COPY_TO_REGCLASS FR128:$src1, VR128), 8833 (COPY_TO_REGCLASS FR128:$src2, VR128)), FR128)>; 8834 8835def : Pat<(xor FR128:$src1, FR128:$src2), 8836 (COPY_TO_REGCLASS 8837 (XORPSrr (COPY_TO_REGCLASS FR128:$src1, VR128), 8838 (COPY_TO_REGCLASS FR128:$src2, VR128)), FR128)>; 8839