1// Copyright 2016 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package arm64 6 7import ( 8 "math" 9 10 "cmd/compile/internal/base" 11 "cmd/compile/internal/ir" 12 "cmd/compile/internal/logopt" 13 "cmd/compile/internal/objw" 14 "cmd/compile/internal/ssa" 15 "cmd/compile/internal/ssagen" 16 "cmd/compile/internal/types" 17 "cmd/internal/obj" 18 "cmd/internal/obj/arm64" 19) 20 21// loadByType returns the load instruction of the given type. 22func loadByType(t *types.Type) obj.As { 23 if t.IsFloat() { 24 switch t.Size() { 25 case 4: 26 return arm64.AFMOVS 27 case 8: 28 return arm64.AFMOVD 29 } 30 } else { 31 switch t.Size() { 32 case 1: 33 if t.IsSigned() { 34 return arm64.AMOVB 35 } else { 36 return arm64.AMOVBU 37 } 38 case 2: 39 if t.IsSigned() { 40 return arm64.AMOVH 41 } else { 42 return arm64.AMOVHU 43 } 44 case 4: 45 if t.IsSigned() { 46 return arm64.AMOVW 47 } else { 48 return arm64.AMOVWU 49 } 50 case 8: 51 return arm64.AMOVD 52 } 53 } 54 panic("bad load type") 55} 56 57// storeByType returns the store instruction of the given type. 58func storeByType(t *types.Type) obj.As { 59 if t.IsFloat() { 60 switch t.Size() { 61 case 4: 62 return arm64.AFMOVS 63 case 8: 64 return arm64.AFMOVD 65 } 66 } else { 67 switch t.Size() { 68 case 1: 69 return arm64.AMOVB 70 case 2: 71 return arm64.AMOVH 72 case 4: 73 return arm64.AMOVW 74 case 8: 75 return arm64.AMOVD 76 } 77 } 78 panic("bad store type") 79} 80 81// makeshift encodes a register shifted by a constant, used as an Offset in Prog. 82func makeshift(v *ssa.Value, reg int16, typ int64, s int64) int64 { 83 if s < 0 || s >= 64 { 84 v.Fatalf("shift out of range: %d", s) 85 } 86 return int64(reg&31)<<16 | typ | (s&63)<<10 87} 88 89// genshift generates a Prog for r = r0 op (r1 shifted by n). 90func genshift(s *ssagen.State, v *ssa.Value, as obj.As, r0, r1, r int16, typ int64, n int64) *obj.Prog { 91 p := s.Prog(as) 92 p.From.Type = obj.TYPE_SHIFT 93 p.From.Offset = makeshift(v, r1, typ, n) 94 p.Reg = r0 95 if r != 0 { 96 p.To.Type = obj.TYPE_REG 97 p.To.Reg = r 98 } 99 return p 100} 101 102// generate the memory operand for the indexed load/store instructions. 103// base and idx are registers. 104func genIndexedOperand(op ssa.Op, base, idx int16) obj.Addr { 105 // Reg: base register, Index: (shifted) index register 106 mop := obj.Addr{Type: obj.TYPE_MEM, Reg: base} 107 switch op { 108 case ssa.OpARM64MOVDloadidx8, ssa.OpARM64MOVDstoreidx8, ssa.OpARM64MOVDstorezeroidx8, 109 ssa.OpARM64FMOVDloadidx8, ssa.OpARM64FMOVDstoreidx8: 110 mop.Index = arm64.REG_LSL | 3<<5 | idx&31 111 case ssa.OpARM64MOVWloadidx4, ssa.OpARM64MOVWUloadidx4, ssa.OpARM64MOVWstoreidx4, ssa.OpARM64MOVWstorezeroidx4, 112 ssa.OpARM64FMOVSloadidx4, ssa.OpARM64FMOVSstoreidx4: 113 mop.Index = arm64.REG_LSL | 2<<5 | idx&31 114 case ssa.OpARM64MOVHloadidx2, ssa.OpARM64MOVHUloadidx2, ssa.OpARM64MOVHstoreidx2, ssa.OpARM64MOVHstorezeroidx2: 115 mop.Index = arm64.REG_LSL | 1<<5 | idx&31 116 default: // not shifted 117 mop.Index = idx 118 } 119 return mop 120} 121 122func ssaGenValue(s *ssagen.State, v *ssa.Value) { 123 switch v.Op { 124 case ssa.OpCopy, ssa.OpARM64MOVDreg: 125 if v.Type.IsMemory() { 126 return 127 } 128 x := v.Args[0].Reg() 129 y := v.Reg() 130 if x == y { 131 return 132 } 133 as := arm64.AMOVD 134 if v.Type.IsFloat() { 135 switch v.Type.Size() { 136 case 4: 137 as = arm64.AFMOVS 138 case 8: 139 as = arm64.AFMOVD 140 default: 141 panic("bad float size") 142 } 143 } 144 p := s.Prog(as) 145 p.From.Type = obj.TYPE_REG 146 p.From.Reg = x 147 p.To.Type = obj.TYPE_REG 148 p.To.Reg = y 149 case ssa.OpARM64MOVDnop: 150 // nothing to do 151 case ssa.OpLoadReg: 152 if v.Type.IsFlags() { 153 v.Fatalf("load flags not implemented: %v", v.LongString()) 154 return 155 } 156 p := s.Prog(loadByType(v.Type)) 157 ssagen.AddrAuto(&p.From, v.Args[0]) 158 p.To.Type = obj.TYPE_REG 159 p.To.Reg = v.Reg() 160 case ssa.OpStoreReg: 161 if v.Type.IsFlags() { 162 v.Fatalf("store flags not implemented: %v", v.LongString()) 163 return 164 } 165 p := s.Prog(storeByType(v.Type)) 166 p.From.Type = obj.TYPE_REG 167 p.From.Reg = v.Args[0].Reg() 168 ssagen.AddrAuto(&p.To, v) 169 case ssa.OpArgIntReg, ssa.OpArgFloatReg: 170 // The assembler needs to wrap the entry safepoint/stack growth code with spill/unspill 171 // The loop only runs once. 172 for _, a := range v.Block.Func.RegArgs { 173 // Pass the spill/unspill information along to the assembler, offset by size of 174 // the saved LR slot. 175 addr := ssagen.SpillSlotAddr(a, arm64.REGSP, base.Ctxt.Arch.FixedFrameSize) 176 s.FuncInfo().AddSpill( 177 obj.RegSpill{Reg: a.Reg, Addr: addr, Unspill: loadByType(a.Type), Spill: storeByType(a.Type)}) 178 } 179 v.Block.Func.RegArgs = nil 180 ssagen.CheckArgReg(v) 181 case ssa.OpARM64ADD, 182 ssa.OpARM64SUB, 183 ssa.OpARM64AND, 184 ssa.OpARM64OR, 185 ssa.OpARM64XOR, 186 ssa.OpARM64BIC, 187 ssa.OpARM64EON, 188 ssa.OpARM64ORN, 189 ssa.OpARM64MUL, 190 ssa.OpARM64MULW, 191 ssa.OpARM64MNEG, 192 ssa.OpARM64MNEGW, 193 ssa.OpARM64MULH, 194 ssa.OpARM64UMULH, 195 ssa.OpARM64MULL, 196 ssa.OpARM64UMULL, 197 ssa.OpARM64DIV, 198 ssa.OpARM64UDIV, 199 ssa.OpARM64DIVW, 200 ssa.OpARM64UDIVW, 201 ssa.OpARM64MOD, 202 ssa.OpARM64UMOD, 203 ssa.OpARM64MODW, 204 ssa.OpARM64UMODW, 205 ssa.OpARM64SLL, 206 ssa.OpARM64SRL, 207 ssa.OpARM64SRA, 208 ssa.OpARM64FADDS, 209 ssa.OpARM64FADDD, 210 ssa.OpARM64FSUBS, 211 ssa.OpARM64FSUBD, 212 ssa.OpARM64FMULS, 213 ssa.OpARM64FMULD, 214 ssa.OpARM64FNMULS, 215 ssa.OpARM64FNMULD, 216 ssa.OpARM64FDIVS, 217 ssa.OpARM64FDIVD, 218 ssa.OpARM64FMINS, 219 ssa.OpARM64FMIND, 220 ssa.OpARM64FMAXS, 221 ssa.OpARM64FMAXD, 222 ssa.OpARM64ROR, 223 ssa.OpARM64RORW: 224 r := v.Reg() 225 r1 := v.Args[0].Reg() 226 r2 := v.Args[1].Reg() 227 p := s.Prog(v.Op.Asm()) 228 p.From.Type = obj.TYPE_REG 229 p.From.Reg = r2 230 p.Reg = r1 231 p.To.Type = obj.TYPE_REG 232 p.To.Reg = r 233 case ssa.OpARM64FMADDS, 234 ssa.OpARM64FMADDD, 235 ssa.OpARM64FNMADDS, 236 ssa.OpARM64FNMADDD, 237 ssa.OpARM64FMSUBS, 238 ssa.OpARM64FMSUBD, 239 ssa.OpARM64FNMSUBS, 240 ssa.OpARM64FNMSUBD, 241 ssa.OpARM64MADD, 242 ssa.OpARM64MADDW, 243 ssa.OpARM64MSUB, 244 ssa.OpARM64MSUBW: 245 rt := v.Reg() 246 ra := v.Args[0].Reg() 247 rm := v.Args[1].Reg() 248 rn := v.Args[2].Reg() 249 p := s.Prog(v.Op.Asm()) 250 p.Reg = ra 251 p.From.Type = obj.TYPE_REG 252 p.From.Reg = rm 253 p.AddRestSourceReg(rn) 254 p.To.Type = obj.TYPE_REG 255 p.To.Reg = rt 256 case ssa.OpARM64ADDconst, 257 ssa.OpARM64SUBconst, 258 ssa.OpARM64ANDconst, 259 ssa.OpARM64ORconst, 260 ssa.OpARM64XORconst, 261 ssa.OpARM64SLLconst, 262 ssa.OpARM64SRLconst, 263 ssa.OpARM64SRAconst, 264 ssa.OpARM64RORconst, 265 ssa.OpARM64RORWconst: 266 p := s.Prog(v.Op.Asm()) 267 p.From.Type = obj.TYPE_CONST 268 p.From.Offset = v.AuxInt 269 p.Reg = v.Args[0].Reg() 270 p.To.Type = obj.TYPE_REG 271 p.To.Reg = v.Reg() 272 case ssa.OpARM64ADDSconstflags: 273 p := s.Prog(v.Op.Asm()) 274 p.From.Type = obj.TYPE_CONST 275 p.From.Offset = v.AuxInt 276 p.Reg = v.Args[0].Reg() 277 p.To.Type = obj.TYPE_REG 278 p.To.Reg = v.Reg0() 279 case ssa.OpARM64ADCzerocarry: 280 p := s.Prog(v.Op.Asm()) 281 p.From.Type = obj.TYPE_REG 282 p.From.Reg = arm64.REGZERO 283 p.Reg = arm64.REGZERO 284 p.To.Type = obj.TYPE_REG 285 p.To.Reg = v.Reg() 286 case ssa.OpARM64ADCSflags, 287 ssa.OpARM64ADDSflags, 288 ssa.OpARM64SBCSflags, 289 ssa.OpARM64SUBSflags: 290 r := v.Reg0() 291 r1 := v.Args[0].Reg() 292 r2 := v.Args[1].Reg() 293 p := s.Prog(v.Op.Asm()) 294 p.From.Type = obj.TYPE_REG 295 p.From.Reg = r2 296 p.Reg = r1 297 p.To.Type = obj.TYPE_REG 298 p.To.Reg = r 299 case ssa.OpARM64NEGSflags: 300 p := s.Prog(v.Op.Asm()) 301 p.From.Type = obj.TYPE_REG 302 p.From.Reg = v.Args[0].Reg() 303 p.To.Type = obj.TYPE_REG 304 p.To.Reg = v.Reg0() 305 case ssa.OpARM64NGCzerocarry: 306 p := s.Prog(v.Op.Asm()) 307 p.From.Type = obj.TYPE_REG 308 p.From.Reg = arm64.REGZERO 309 p.To.Type = obj.TYPE_REG 310 p.To.Reg = v.Reg() 311 case ssa.OpARM64EXTRconst, 312 ssa.OpARM64EXTRWconst: 313 p := s.Prog(v.Op.Asm()) 314 p.From.Type = obj.TYPE_CONST 315 p.From.Offset = v.AuxInt 316 p.AddRestSourceReg(v.Args[0].Reg()) 317 p.Reg = v.Args[1].Reg() 318 p.To.Type = obj.TYPE_REG 319 p.To.Reg = v.Reg() 320 case ssa.OpARM64MVNshiftLL, ssa.OpARM64NEGshiftLL: 321 genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt) 322 case ssa.OpARM64MVNshiftRL, ssa.OpARM64NEGshiftRL: 323 genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt) 324 case ssa.OpARM64MVNshiftRA, ssa.OpARM64NEGshiftRA: 325 genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt) 326 case ssa.OpARM64MVNshiftRO: 327 genshift(s, v, v.Op.Asm(), 0, v.Args[0].Reg(), v.Reg(), arm64.SHIFT_ROR, v.AuxInt) 328 case ssa.OpARM64ADDshiftLL, 329 ssa.OpARM64SUBshiftLL, 330 ssa.OpARM64ANDshiftLL, 331 ssa.OpARM64ORshiftLL, 332 ssa.OpARM64XORshiftLL, 333 ssa.OpARM64EONshiftLL, 334 ssa.OpARM64ORNshiftLL, 335 ssa.OpARM64BICshiftLL: 336 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LL, v.AuxInt) 337 case ssa.OpARM64ADDshiftRL, 338 ssa.OpARM64SUBshiftRL, 339 ssa.OpARM64ANDshiftRL, 340 ssa.OpARM64ORshiftRL, 341 ssa.OpARM64XORshiftRL, 342 ssa.OpARM64EONshiftRL, 343 ssa.OpARM64ORNshiftRL, 344 ssa.OpARM64BICshiftRL: 345 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_LR, v.AuxInt) 346 case ssa.OpARM64ADDshiftRA, 347 ssa.OpARM64SUBshiftRA, 348 ssa.OpARM64ANDshiftRA, 349 ssa.OpARM64ORshiftRA, 350 ssa.OpARM64XORshiftRA, 351 ssa.OpARM64EONshiftRA, 352 ssa.OpARM64ORNshiftRA, 353 ssa.OpARM64BICshiftRA: 354 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_AR, v.AuxInt) 355 case ssa.OpARM64ANDshiftRO, 356 ssa.OpARM64ORshiftRO, 357 ssa.OpARM64XORshiftRO, 358 ssa.OpARM64EONshiftRO, 359 ssa.OpARM64ORNshiftRO, 360 ssa.OpARM64BICshiftRO: 361 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), v.Reg(), arm64.SHIFT_ROR, v.AuxInt) 362 case ssa.OpARM64MOVDconst: 363 p := s.Prog(v.Op.Asm()) 364 p.From.Type = obj.TYPE_CONST 365 p.From.Offset = v.AuxInt 366 p.To.Type = obj.TYPE_REG 367 p.To.Reg = v.Reg() 368 case ssa.OpARM64FMOVSconst, 369 ssa.OpARM64FMOVDconst: 370 p := s.Prog(v.Op.Asm()) 371 p.From.Type = obj.TYPE_FCONST 372 p.From.Val = math.Float64frombits(uint64(v.AuxInt)) 373 p.To.Type = obj.TYPE_REG 374 p.To.Reg = v.Reg() 375 case ssa.OpARM64FCMPS0, 376 ssa.OpARM64FCMPD0: 377 p := s.Prog(v.Op.Asm()) 378 p.From.Type = obj.TYPE_FCONST 379 p.From.Val = math.Float64frombits(0) 380 p.Reg = v.Args[0].Reg() 381 case ssa.OpARM64CMP, 382 ssa.OpARM64CMPW, 383 ssa.OpARM64CMN, 384 ssa.OpARM64CMNW, 385 ssa.OpARM64TST, 386 ssa.OpARM64TSTW, 387 ssa.OpARM64FCMPS, 388 ssa.OpARM64FCMPD: 389 p := s.Prog(v.Op.Asm()) 390 p.From.Type = obj.TYPE_REG 391 p.From.Reg = v.Args[1].Reg() 392 p.Reg = v.Args[0].Reg() 393 case ssa.OpARM64CMPconst, 394 ssa.OpARM64CMPWconst, 395 ssa.OpARM64CMNconst, 396 ssa.OpARM64CMNWconst, 397 ssa.OpARM64TSTconst, 398 ssa.OpARM64TSTWconst: 399 p := s.Prog(v.Op.Asm()) 400 p.From.Type = obj.TYPE_CONST 401 p.From.Offset = v.AuxInt 402 p.Reg = v.Args[0].Reg() 403 case ssa.OpARM64CMPshiftLL, ssa.OpARM64CMNshiftLL, ssa.OpARM64TSTshiftLL: 404 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LL, v.AuxInt) 405 case ssa.OpARM64CMPshiftRL, ssa.OpARM64CMNshiftRL, ssa.OpARM64TSTshiftRL: 406 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_LR, v.AuxInt) 407 case ssa.OpARM64CMPshiftRA, ssa.OpARM64CMNshiftRA, ssa.OpARM64TSTshiftRA: 408 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_AR, v.AuxInt) 409 case ssa.OpARM64TSTshiftRO: 410 genshift(s, v, v.Op.Asm(), v.Args[0].Reg(), v.Args[1].Reg(), 0, arm64.SHIFT_ROR, v.AuxInt) 411 case ssa.OpARM64MOVDaddr: 412 p := s.Prog(arm64.AMOVD) 413 p.From.Type = obj.TYPE_ADDR 414 p.From.Reg = v.Args[0].Reg() 415 p.To.Type = obj.TYPE_REG 416 p.To.Reg = v.Reg() 417 418 var wantreg string 419 // MOVD $sym+off(base), R 420 // the assembler expands it as the following: 421 // - base is SP: add constant offset to SP (R13) 422 // when constant is large, tmp register (R11) may be used 423 // - base is SB: load external address from constant pool (use relocation) 424 switch v.Aux.(type) { 425 default: 426 v.Fatalf("aux is of unknown type %T", v.Aux) 427 case *obj.LSym: 428 wantreg = "SB" 429 ssagen.AddAux(&p.From, v) 430 case *ir.Name: 431 wantreg = "SP" 432 ssagen.AddAux(&p.From, v) 433 case nil: 434 // No sym, just MOVD $off(SP), R 435 wantreg = "SP" 436 p.From.Offset = v.AuxInt 437 } 438 if reg := v.Args[0].RegName(); reg != wantreg { 439 v.Fatalf("bad reg %s for symbol type %T, want %s", reg, v.Aux, wantreg) 440 } 441 case ssa.OpARM64MOVBload, 442 ssa.OpARM64MOVBUload, 443 ssa.OpARM64MOVHload, 444 ssa.OpARM64MOVHUload, 445 ssa.OpARM64MOVWload, 446 ssa.OpARM64MOVWUload, 447 ssa.OpARM64MOVDload, 448 ssa.OpARM64FMOVSload, 449 ssa.OpARM64FMOVDload: 450 p := s.Prog(v.Op.Asm()) 451 p.From.Type = obj.TYPE_MEM 452 p.From.Reg = v.Args[0].Reg() 453 ssagen.AddAux(&p.From, v) 454 p.To.Type = obj.TYPE_REG 455 p.To.Reg = v.Reg() 456 case ssa.OpARM64LDP: 457 p := s.Prog(v.Op.Asm()) 458 p.From.Type = obj.TYPE_MEM 459 p.From.Reg = v.Args[0].Reg() 460 ssagen.AddAux(&p.From, v) 461 p.To.Type = obj.TYPE_REGREG 462 p.To.Reg = v.Reg0() 463 p.To.Offset = int64(v.Reg1()) 464 case ssa.OpARM64MOVBloadidx, 465 ssa.OpARM64MOVBUloadidx, 466 ssa.OpARM64MOVHloadidx, 467 ssa.OpARM64MOVHUloadidx, 468 ssa.OpARM64MOVWloadidx, 469 ssa.OpARM64MOVWUloadidx, 470 ssa.OpARM64MOVDloadidx, 471 ssa.OpARM64FMOVSloadidx, 472 ssa.OpARM64FMOVDloadidx, 473 ssa.OpARM64MOVHloadidx2, 474 ssa.OpARM64MOVHUloadidx2, 475 ssa.OpARM64MOVWloadidx4, 476 ssa.OpARM64MOVWUloadidx4, 477 ssa.OpARM64MOVDloadidx8, 478 ssa.OpARM64FMOVDloadidx8, 479 ssa.OpARM64FMOVSloadidx4: 480 p := s.Prog(v.Op.Asm()) 481 p.From = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg()) 482 p.To.Type = obj.TYPE_REG 483 p.To.Reg = v.Reg() 484 case ssa.OpARM64LDAR, 485 ssa.OpARM64LDARB, 486 ssa.OpARM64LDARW: 487 p := s.Prog(v.Op.Asm()) 488 p.From.Type = obj.TYPE_MEM 489 p.From.Reg = v.Args[0].Reg() 490 ssagen.AddAux(&p.From, v) 491 p.To.Type = obj.TYPE_REG 492 p.To.Reg = v.Reg0() 493 case ssa.OpARM64MOVBstore, 494 ssa.OpARM64MOVHstore, 495 ssa.OpARM64MOVWstore, 496 ssa.OpARM64MOVDstore, 497 ssa.OpARM64FMOVSstore, 498 ssa.OpARM64FMOVDstore, 499 ssa.OpARM64STLRB, 500 ssa.OpARM64STLR, 501 ssa.OpARM64STLRW: 502 p := s.Prog(v.Op.Asm()) 503 p.From.Type = obj.TYPE_REG 504 p.From.Reg = v.Args[1].Reg() 505 p.To.Type = obj.TYPE_MEM 506 p.To.Reg = v.Args[0].Reg() 507 ssagen.AddAux(&p.To, v) 508 case ssa.OpARM64MOVBstoreidx, 509 ssa.OpARM64MOVHstoreidx, 510 ssa.OpARM64MOVWstoreidx, 511 ssa.OpARM64MOVDstoreidx, 512 ssa.OpARM64FMOVSstoreidx, 513 ssa.OpARM64FMOVDstoreidx, 514 ssa.OpARM64MOVHstoreidx2, 515 ssa.OpARM64MOVWstoreidx4, 516 ssa.OpARM64FMOVSstoreidx4, 517 ssa.OpARM64MOVDstoreidx8, 518 ssa.OpARM64FMOVDstoreidx8: 519 p := s.Prog(v.Op.Asm()) 520 p.To = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg()) 521 p.From.Type = obj.TYPE_REG 522 p.From.Reg = v.Args[2].Reg() 523 case ssa.OpARM64STP: 524 p := s.Prog(v.Op.Asm()) 525 p.From.Type = obj.TYPE_REGREG 526 p.From.Reg = v.Args[1].Reg() 527 p.From.Offset = int64(v.Args[2].Reg()) 528 p.To.Type = obj.TYPE_MEM 529 p.To.Reg = v.Args[0].Reg() 530 ssagen.AddAux(&p.To, v) 531 case ssa.OpARM64MOVBstorezero, 532 ssa.OpARM64MOVHstorezero, 533 ssa.OpARM64MOVWstorezero, 534 ssa.OpARM64MOVDstorezero: 535 p := s.Prog(v.Op.Asm()) 536 p.From.Type = obj.TYPE_REG 537 p.From.Reg = arm64.REGZERO 538 p.To.Type = obj.TYPE_MEM 539 p.To.Reg = v.Args[0].Reg() 540 ssagen.AddAux(&p.To, v) 541 case ssa.OpARM64MOVBstorezeroidx, 542 ssa.OpARM64MOVHstorezeroidx, 543 ssa.OpARM64MOVWstorezeroidx, 544 ssa.OpARM64MOVDstorezeroidx, 545 ssa.OpARM64MOVHstorezeroidx2, 546 ssa.OpARM64MOVWstorezeroidx4, 547 ssa.OpARM64MOVDstorezeroidx8: 548 p := s.Prog(v.Op.Asm()) 549 p.To = genIndexedOperand(v.Op, v.Args[0].Reg(), v.Args[1].Reg()) 550 p.From.Type = obj.TYPE_REG 551 p.From.Reg = arm64.REGZERO 552 case ssa.OpARM64MOVQstorezero: 553 p := s.Prog(v.Op.Asm()) 554 p.From.Type = obj.TYPE_REGREG 555 p.From.Reg = arm64.REGZERO 556 p.From.Offset = int64(arm64.REGZERO) 557 p.To.Type = obj.TYPE_MEM 558 p.To.Reg = v.Args[0].Reg() 559 ssagen.AddAux(&p.To, v) 560 case ssa.OpARM64BFI, 561 ssa.OpARM64BFXIL: 562 p := s.Prog(v.Op.Asm()) 563 p.From.Type = obj.TYPE_CONST 564 p.From.Offset = v.AuxInt >> 8 565 p.AddRestSourceConst(v.AuxInt & 0xff) 566 p.Reg = v.Args[1].Reg() 567 p.To.Type = obj.TYPE_REG 568 p.To.Reg = v.Reg() 569 case ssa.OpARM64SBFIZ, 570 ssa.OpARM64SBFX, 571 ssa.OpARM64UBFIZ, 572 ssa.OpARM64UBFX: 573 p := s.Prog(v.Op.Asm()) 574 p.From.Type = obj.TYPE_CONST 575 p.From.Offset = v.AuxInt >> 8 576 p.AddRestSourceConst(v.AuxInt & 0xff) 577 p.Reg = v.Args[0].Reg() 578 p.To.Type = obj.TYPE_REG 579 p.To.Reg = v.Reg() 580 case ssa.OpARM64LoweredAtomicExchange64, 581 ssa.OpARM64LoweredAtomicExchange32: 582 // LDAXR (Rarg0), Rout 583 // STLXR Rarg1, (Rarg0), Rtmp 584 // CBNZ Rtmp, -2(PC) 585 ld := arm64.ALDAXR 586 st := arm64.ASTLXR 587 if v.Op == ssa.OpARM64LoweredAtomicExchange32 { 588 ld = arm64.ALDAXRW 589 st = arm64.ASTLXRW 590 } 591 r0 := v.Args[0].Reg() 592 r1 := v.Args[1].Reg() 593 out := v.Reg0() 594 p := s.Prog(ld) 595 p.From.Type = obj.TYPE_MEM 596 p.From.Reg = r0 597 p.To.Type = obj.TYPE_REG 598 p.To.Reg = out 599 p1 := s.Prog(st) 600 p1.From.Type = obj.TYPE_REG 601 p1.From.Reg = r1 602 p1.To.Type = obj.TYPE_MEM 603 p1.To.Reg = r0 604 p1.RegTo2 = arm64.REGTMP 605 p2 := s.Prog(arm64.ACBNZ) 606 p2.From.Type = obj.TYPE_REG 607 p2.From.Reg = arm64.REGTMP 608 p2.To.Type = obj.TYPE_BRANCH 609 p2.To.SetTarget(p) 610 case ssa.OpARM64LoweredAtomicExchange64Variant, 611 ssa.OpARM64LoweredAtomicExchange32Variant: 612 swap := arm64.ASWPALD 613 if v.Op == ssa.OpARM64LoweredAtomicExchange32Variant { 614 swap = arm64.ASWPALW 615 } 616 r0 := v.Args[0].Reg() 617 r1 := v.Args[1].Reg() 618 out := v.Reg0() 619 620 // SWPALD Rarg1, (Rarg0), Rout 621 p := s.Prog(swap) 622 p.From.Type = obj.TYPE_REG 623 p.From.Reg = r1 624 p.To.Type = obj.TYPE_MEM 625 p.To.Reg = r0 626 p.RegTo2 = out 627 628 case ssa.OpARM64LoweredAtomicAdd64, 629 ssa.OpARM64LoweredAtomicAdd32: 630 // LDAXR (Rarg0), Rout 631 // ADD Rarg1, Rout 632 // STLXR Rout, (Rarg0), Rtmp 633 // CBNZ Rtmp, -3(PC) 634 ld := arm64.ALDAXR 635 st := arm64.ASTLXR 636 if v.Op == ssa.OpARM64LoweredAtomicAdd32 { 637 ld = arm64.ALDAXRW 638 st = arm64.ASTLXRW 639 } 640 r0 := v.Args[0].Reg() 641 r1 := v.Args[1].Reg() 642 out := v.Reg0() 643 p := s.Prog(ld) 644 p.From.Type = obj.TYPE_MEM 645 p.From.Reg = r0 646 p.To.Type = obj.TYPE_REG 647 p.To.Reg = out 648 p1 := s.Prog(arm64.AADD) 649 p1.From.Type = obj.TYPE_REG 650 p1.From.Reg = r1 651 p1.To.Type = obj.TYPE_REG 652 p1.To.Reg = out 653 p2 := s.Prog(st) 654 p2.From.Type = obj.TYPE_REG 655 p2.From.Reg = out 656 p2.To.Type = obj.TYPE_MEM 657 p2.To.Reg = r0 658 p2.RegTo2 = arm64.REGTMP 659 p3 := s.Prog(arm64.ACBNZ) 660 p3.From.Type = obj.TYPE_REG 661 p3.From.Reg = arm64.REGTMP 662 p3.To.Type = obj.TYPE_BRANCH 663 p3.To.SetTarget(p) 664 case ssa.OpARM64LoweredAtomicAdd64Variant, 665 ssa.OpARM64LoweredAtomicAdd32Variant: 666 // LDADDAL Rarg1, (Rarg0), Rout 667 // ADD Rarg1, Rout 668 op := arm64.ALDADDALD 669 if v.Op == ssa.OpARM64LoweredAtomicAdd32Variant { 670 op = arm64.ALDADDALW 671 } 672 r0 := v.Args[0].Reg() 673 r1 := v.Args[1].Reg() 674 out := v.Reg0() 675 p := s.Prog(op) 676 p.From.Type = obj.TYPE_REG 677 p.From.Reg = r1 678 p.To.Type = obj.TYPE_MEM 679 p.To.Reg = r0 680 p.RegTo2 = out 681 p1 := s.Prog(arm64.AADD) 682 p1.From.Type = obj.TYPE_REG 683 p1.From.Reg = r1 684 p1.To.Type = obj.TYPE_REG 685 p1.To.Reg = out 686 case ssa.OpARM64LoweredAtomicCas64, 687 ssa.OpARM64LoweredAtomicCas32: 688 // LDAXR (Rarg0), Rtmp 689 // CMP Rarg1, Rtmp 690 // BNE 3(PC) 691 // STLXR Rarg2, (Rarg0), Rtmp 692 // CBNZ Rtmp, -4(PC) 693 // CSET EQ, Rout 694 ld := arm64.ALDAXR 695 st := arm64.ASTLXR 696 cmp := arm64.ACMP 697 if v.Op == ssa.OpARM64LoweredAtomicCas32 { 698 ld = arm64.ALDAXRW 699 st = arm64.ASTLXRW 700 cmp = arm64.ACMPW 701 } 702 r0 := v.Args[0].Reg() 703 r1 := v.Args[1].Reg() 704 r2 := v.Args[2].Reg() 705 out := v.Reg0() 706 p := s.Prog(ld) 707 p.From.Type = obj.TYPE_MEM 708 p.From.Reg = r0 709 p.To.Type = obj.TYPE_REG 710 p.To.Reg = arm64.REGTMP 711 p1 := s.Prog(cmp) 712 p1.From.Type = obj.TYPE_REG 713 p1.From.Reg = r1 714 p1.Reg = arm64.REGTMP 715 p2 := s.Prog(arm64.ABNE) 716 p2.To.Type = obj.TYPE_BRANCH 717 p3 := s.Prog(st) 718 p3.From.Type = obj.TYPE_REG 719 p3.From.Reg = r2 720 p3.To.Type = obj.TYPE_MEM 721 p3.To.Reg = r0 722 p3.RegTo2 = arm64.REGTMP 723 p4 := s.Prog(arm64.ACBNZ) 724 p4.From.Type = obj.TYPE_REG 725 p4.From.Reg = arm64.REGTMP 726 p4.To.Type = obj.TYPE_BRANCH 727 p4.To.SetTarget(p) 728 p5 := s.Prog(arm64.ACSET) 729 p5.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset 730 p5.From.Offset = int64(arm64.SPOP_EQ) 731 p5.To.Type = obj.TYPE_REG 732 p5.To.Reg = out 733 p2.To.SetTarget(p5) 734 case ssa.OpARM64LoweredAtomicCas64Variant, 735 ssa.OpARM64LoweredAtomicCas32Variant: 736 // Rarg0: ptr 737 // Rarg1: old 738 // Rarg2: new 739 // MOV Rarg1, Rtmp 740 // CASAL Rtmp, (Rarg0), Rarg2 741 // CMP Rarg1, Rtmp 742 // CSET EQ, Rout 743 cas := arm64.ACASALD 744 cmp := arm64.ACMP 745 mov := arm64.AMOVD 746 if v.Op == ssa.OpARM64LoweredAtomicCas32Variant { 747 cas = arm64.ACASALW 748 cmp = arm64.ACMPW 749 mov = arm64.AMOVW 750 } 751 r0 := v.Args[0].Reg() 752 r1 := v.Args[1].Reg() 753 r2 := v.Args[2].Reg() 754 out := v.Reg0() 755 756 // MOV Rarg1, Rtmp 757 p := s.Prog(mov) 758 p.From.Type = obj.TYPE_REG 759 p.From.Reg = r1 760 p.To.Type = obj.TYPE_REG 761 p.To.Reg = arm64.REGTMP 762 763 // CASAL Rtmp, (Rarg0), Rarg2 764 p1 := s.Prog(cas) 765 p1.From.Type = obj.TYPE_REG 766 p1.From.Reg = arm64.REGTMP 767 p1.To.Type = obj.TYPE_MEM 768 p1.To.Reg = r0 769 p1.RegTo2 = r2 770 771 // CMP Rarg1, Rtmp 772 p2 := s.Prog(cmp) 773 p2.From.Type = obj.TYPE_REG 774 p2.From.Reg = r1 775 p2.Reg = arm64.REGTMP 776 777 // CSET EQ, Rout 778 p3 := s.Prog(arm64.ACSET) 779 p3.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset 780 p3.From.Offset = int64(arm64.SPOP_EQ) 781 p3.To.Type = obj.TYPE_REG 782 p3.To.Reg = out 783 784 case ssa.OpARM64LoweredAtomicAnd64, 785 ssa.OpARM64LoweredAtomicOr64, 786 ssa.OpARM64LoweredAtomicAnd32, 787 ssa.OpARM64LoweredAtomicOr32, 788 ssa.OpARM64LoweredAtomicAnd8, 789 ssa.OpARM64LoweredAtomicOr8: 790 // LDAXR[BW] (Rarg0), Rout 791 // AND/OR Rarg1, Rout, tmp1 792 // STLXR[BW] tmp1, (Rarg0), Rtmp 793 // CBNZ Rtmp, -3(PC) 794 ld := arm64.ALDAXR 795 st := arm64.ASTLXR 796 if v.Op == ssa.OpARM64LoweredAtomicAnd32 || v.Op == ssa.OpARM64LoweredAtomicOr32 { 797 ld = arm64.ALDAXRW 798 st = arm64.ASTLXRW 799 } 800 if v.Op == ssa.OpARM64LoweredAtomicAnd8 || v.Op == ssa.OpARM64LoweredAtomicOr8 { 801 ld = arm64.ALDAXRB 802 st = arm64.ASTLXRB 803 } 804 r0 := v.Args[0].Reg() 805 r1 := v.Args[1].Reg() 806 out := v.Reg0() 807 tmp := v.RegTmp() 808 p := s.Prog(ld) 809 p.From.Type = obj.TYPE_MEM 810 p.From.Reg = r0 811 p.To.Type = obj.TYPE_REG 812 p.To.Reg = out 813 p1 := s.Prog(v.Op.Asm()) 814 p1.From.Type = obj.TYPE_REG 815 p1.From.Reg = r1 816 p1.Reg = out 817 p1.To.Type = obj.TYPE_REG 818 p1.To.Reg = tmp 819 p2 := s.Prog(st) 820 p2.From.Type = obj.TYPE_REG 821 p2.From.Reg = tmp 822 p2.To.Type = obj.TYPE_MEM 823 p2.To.Reg = r0 824 p2.RegTo2 = arm64.REGTMP 825 p3 := s.Prog(arm64.ACBNZ) 826 p3.From.Type = obj.TYPE_REG 827 p3.From.Reg = arm64.REGTMP 828 p3.To.Type = obj.TYPE_BRANCH 829 p3.To.SetTarget(p) 830 831 case ssa.OpARM64LoweredAtomicAnd8Variant, 832 ssa.OpARM64LoweredAtomicAnd32Variant, 833 ssa.OpARM64LoweredAtomicAnd64Variant: 834 atomic_clear := arm64.ALDCLRALD 835 if v.Op == ssa.OpARM64LoweredAtomicAnd32Variant { 836 atomic_clear = arm64.ALDCLRALW 837 } 838 if v.Op == ssa.OpARM64LoweredAtomicAnd8Variant { 839 atomic_clear = arm64.ALDCLRALB 840 } 841 r0 := v.Args[0].Reg() 842 r1 := v.Args[1].Reg() 843 out := v.Reg0() 844 845 // MNV Rarg1 Rtemp 846 p := s.Prog(arm64.AMVN) 847 p.From.Type = obj.TYPE_REG 848 p.From.Reg = r1 849 p.To.Type = obj.TYPE_REG 850 p.To.Reg = arm64.REGTMP 851 852 // LDCLRAL[BDW] Rtemp, (Rarg0), Rout 853 p1 := s.Prog(atomic_clear) 854 p1.From.Type = obj.TYPE_REG 855 p1.From.Reg = arm64.REGTMP 856 p1.To.Type = obj.TYPE_MEM 857 p1.To.Reg = r0 858 p1.RegTo2 = out 859 860 case ssa.OpARM64LoweredAtomicOr8Variant, 861 ssa.OpARM64LoweredAtomicOr32Variant, 862 ssa.OpARM64LoweredAtomicOr64Variant: 863 atomic_or := arm64.ALDORALD 864 if v.Op == ssa.OpARM64LoweredAtomicOr32Variant { 865 atomic_or = arm64.ALDORALW 866 } 867 if v.Op == ssa.OpARM64LoweredAtomicOr8Variant { 868 atomic_or = arm64.ALDORALB 869 } 870 r0 := v.Args[0].Reg() 871 r1 := v.Args[1].Reg() 872 out := v.Reg0() 873 874 // LDORAL[BDW] Rarg1, (Rarg0), Rout 875 p := s.Prog(atomic_or) 876 p.From.Type = obj.TYPE_REG 877 p.From.Reg = r1 878 p.To.Type = obj.TYPE_MEM 879 p.To.Reg = r0 880 p.RegTo2 = out 881 882 case ssa.OpARM64MOVBreg, 883 ssa.OpARM64MOVBUreg, 884 ssa.OpARM64MOVHreg, 885 ssa.OpARM64MOVHUreg, 886 ssa.OpARM64MOVWreg, 887 ssa.OpARM64MOVWUreg: 888 a := v.Args[0] 889 for a.Op == ssa.OpCopy || a.Op == ssa.OpARM64MOVDreg { 890 a = a.Args[0] 891 } 892 if a.Op == ssa.OpLoadReg { 893 t := a.Type 894 switch { 895 case v.Op == ssa.OpARM64MOVBreg && t.Size() == 1 && t.IsSigned(), 896 v.Op == ssa.OpARM64MOVBUreg && t.Size() == 1 && !t.IsSigned(), 897 v.Op == ssa.OpARM64MOVHreg && t.Size() == 2 && t.IsSigned(), 898 v.Op == ssa.OpARM64MOVHUreg && t.Size() == 2 && !t.IsSigned(), 899 v.Op == ssa.OpARM64MOVWreg && t.Size() == 4 && t.IsSigned(), 900 v.Op == ssa.OpARM64MOVWUreg && t.Size() == 4 && !t.IsSigned(): 901 // arg is a proper-typed load, already zero/sign-extended, don't extend again 902 if v.Reg() == v.Args[0].Reg() { 903 return 904 } 905 p := s.Prog(arm64.AMOVD) 906 p.From.Type = obj.TYPE_REG 907 p.From.Reg = v.Args[0].Reg() 908 p.To.Type = obj.TYPE_REG 909 p.To.Reg = v.Reg() 910 return 911 default: 912 } 913 } 914 fallthrough 915 case ssa.OpARM64MVN, 916 ssa.OpARM64NEG, 917 ssa.OpARM64FABSD, 918 ssa.OpARM64FMOVDfpgp, 919 ssa.OpARM64FMOVDgpfp, 920 ssa.OpARM64FMOVSfpgp, 921 ssa.OpARM64FMOVSgpfp, 922 ssa.OpARM64FNEGS, 923 ssa.OpARM64FNEGD, 924 ssa.OpARM64FSQRTS, 925 ssa.OpARM64FSQRTD, 926 ssa.OpARM64FCVTZSSW, 927 ssa.OpARM64FCVTZSDW, 928 ssa.OpARM64FCVTZUSW, 929 ssa.OpARM64FCVTZUDW, 930 ssa.OpARM64FCVTZSS, 931 ssa.OpARM64FCVTZSD, 932 ssa.OpARM64FCVTZUS, 933 ssa.OpARM64FCVTZUD, 934 ssa.OpARM64SCVTFWS, 935 ssa.OpARM64SCVTFWD, 936 ssa.OpARM64SCVTFS, 937 ssa.OpARM64SCVTFD, 938 ssa.OpARM64UCVTFWS, 939 ssa.OpARM64UCVTFWD, 940 ssa.OpARM64UCVTFS, 941 ssa.OpARM64UCVTFD, 942 ssa.OpARM64FCVTSD, 943 ssa.OpARM64FCVTDS, 944 ssa.OpARM64REV, 945 ssa.OpARM64REVW, 946 ssa.OpARM64REV16, 947 ssa.OpARM64REV16W, 948 ssa.OpARM64RBIT, 949 ssa.OpARM64RBITW, 950 ssa.OpARM64CLZ, 951 ssa.OpARM64CLZW, 952 ssa.OpARM64FRINTAD, 953 ssa.OpARM64FRINTMD, 954 ssa.OpARM64FRINTND, 955 ssa.OpARM64FRINTPD, 956 ssa.OpARM64FRINTZD: 957 p := s.Prog(v.Op.Asm()) 958 p.From.Type = obj.TYPE_REG 959 p.From.Reg = v.Args[0].Reg() 960 p.To.Type = obj.TYPE_REG 961 p.To.Reg = v.Reg() 962 case ssa.OpARM64LoweredRound32F, ssa.OpARM64LoweredRound64F: 963 // input is already rounded 964 case ssa.OpARM64VCNT: 965 p := s.Prog(v.Op.Asm()) 966 p.From.Type = obj.TYPE_REG 967 p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5) 968 p.To.Type = obj.TYPE_REG 969 p.To.Reg = (v.Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5) 970 case ssa.OpARM64VUADDLV: 971 p := s.Prog(v.Op.Asm()) 972 p.From.Type = obj.TYPE_REG 973 p.From.Reg = (v.Args[0].Reg()-arm64.REG_F0)&31 + arm64.REG_ARNG + ((arm64.ARNG_8B & 15) << 5) 974 p.To.Type = obj.TYPE_REG 975 p.To.Reg = v.Reg() - arm64.REG_F0 + arm64.REG_V0 976 case ssa.OpARM64CSEL, ssa.OpARM64CSEL0: 977 r1 := int16(arm64.REGZERO) 978 if v.Op != ssa.OpARM64CSEL0 { 979 r1 = v.Args[1].Reg() 980 } 981 p := s.Prog(v.Op.Asm()) 982 p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset 983 condCode := condBits[ssa.Op(v.AuxInt)] 984 p.From.Offset = int64(condCode) 985 p.Reg = v.Args[0].Reg() 986 p.AddRestSourceReg(r1) 987 p.To.Type = obj.TYPE_REG 988 p.To.Reg = v.Reg() 989 case ssa.OpARM64CSINC, ssa.OpARM64CSINV, ssa.OpARM64CSNEG: 990 p := s.Prog(v.Op.Asm()) 991 p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset 992 condCode := condBits[ssa.Op(v.AuxInt)] 993 p.From.Offset = int64(condCode) 994 p.Reg = v.Args[0].Reg() 995 p.AddRestSourceReg(v.Args[1].Reg()) 996 p.To.Type = obj.TYPE_REG 997 p.To.Reg = v.Reg() 998 case ssa.OpARM64CSETM: 999 p := s.Prog(arm64.ACSETM) 1000 p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset 1001 condCode := condBits[ssa.Op(v.AuxInt)] 1002 p.From.Offset = int64(condCode) 1003 p.To.Type = obj.TYPE_REG 1004 p.To.Reg = v.Reg() 1005 case ssa.OpARM64DUFFZERO: 1006 // runtime.duffzero expects start address in R20 1007 p := s.Prog(obj.ADUFFZERO) 1008 p.To.Type = obj.TYPE_MEM 1009 p.To.Name = obj.NAME_EXTERN 1010 p.To.Sym = ir.Syms.Duffzero 1011 p.To.Offset = v.AuxInt 1012 case ssa.OpARM64LoweredZero: 1013 // STP.P (ZR,ZR), 16(R16) 1014 // CMP Rarg1, R16 1015 // BLE -2(PC) 1016 // arg1 is the address of the last 16-byte unit to zero 1017 p := s.Prog(arm64.ASTP) 1018 p.Scond = arm64.C_XPOST 1019 p.From.Type = obj.TYPE_REGREG 1020 p.From.Reg = arm64.REGZERO 1021 p.From.Offset = int64(arm64.REGZERO) 1022 p.To.Type = obj.TYPE_MEM 1023 p.To.Reg = arm64.REG_R16 1024 p.To.Offset = 16 1025 p2 := s.Prog(arm64.ACMP) 1026 p2.From.Type = obj.TYPE_REG 1027 p2.From.Reg = v.Args[1].Reg() 1028 p2.Reg = arm64.REG_R16 1029 p3 := s.Prog(arm64.ABLE) 1030 p3.To.Type = obj.TYPE_BRANCH 1031 p3.To.SetTarget(p) 1032 case ssa.OpARM64DUFFCOPY: 1033 p := s.Prog(obj.ADUFFCOPY) 1034 p.To.Type = obj.TYPE_MEM 1035 p.To.Name = obj.NAME_EXTERN 1036 p.To.Sym = ir.Syms.Duffcopy 1037 p.To.Offset = v.AuxInt 1038 case ssa.OpARM64LoweredMove: 1039 // LDP.P 16(R16), (R25, Rtmp) 1040 // STP.P (R25, Rtmp), 16(R17) 1041 // CMP Rarg2, R16 1042 // BLE -3(PC) 1043 // arg2 is the address of the last element of src 1044 p := s.Prog(arm64.ALDP) 1045 p.Scond = arm64.C_XPOST 1046 p.From.Type = obj.TYPE_MEM 1047 p.From.Reg = arm64.REG_R16 1048 p.From.Offset = 16 1049 p.To.Type = obj.TYPE_REGREG 1050 p.To.Reg = arm64.REG_R25 1051 p.To.Offset = int64(arm64.REGTMP) 1052 p2 := s.Prog(arm64.ASTP) 1053 p2.Scond = arm64.C_XPOST 1054 p2.From.Type = obj.TYPE_REGREG 1055 p2.From.Reg = arm64.REG_R25 1056 p2.From.Offset = int64(arm64.REGTMP) 1057 p2.To.Type = obj.TYPE_MEM 1058 p2.To.Reg = arm64.REG_R17 1059 p2.To.Offset = 16 1060 p3 := s.Prog(arm64.ACMP) 1061 p3.From.Type = obj.TYPE_REG 1062 p3.From.Reg = v.Args[2].Reg() 1063 p3.Reg = arm64.REG_R16 1064 p4 := s.Prog(arm64.ABLE) 1065 p4.To.Type = obj.TYPE_BRANCH 1066 p4.To.SetTarget(p) 1067 case ssa.OpARM64CALLstatic, ssa.OpARM64CALLclosure, ssa.OpARM64CALLinter: 1068 s.Call(v) 1069 case ssa.OpARM64CALLtail: 1070 s.TailCall(v) 1071 case ssa.OpARM64LoweredWB: 1072 p := s.Prog(obj.ACALL) 1073 p.To.Type = obj.TYPE_MEM 1074 p.To.Name = obj.NAME_EXTERN 1075 // AuxInt encodes how many buffer entries we need. 1076 p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1] 1077 1078 case ssa.OpARM64LoweredPanicBoundsA, ssa.OpARM64LoweredPanicBoundsB, ssa.OpARM64LoweredPanicBoundsC: 1079 p := s.Prog(obj.ACALL) 1080 p.To.Type = obj.TYPE_MEM 1081 p.To.Name = obj.NAME_EXTERN 1082 p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt] 1083 s.UseArgs(16) // space used in callee args area by assembly stubs 1084 case ssa.OpARM64LoweredNilCheck: 1085 // Issue a load which will fault if arg is nil. 1086 p := s.Prog(arm64.AMOVB) 1087 p.From.Type = obj.TYPE_MEM 1088 p.From.Reg = v.Args[0].Reg() 1089 ssagen.AddAux(&p.From, v) 1090 p.To.Type = obj.TYPE_REG 1091 p.To.Reg = arm64.REGTMP 1092 if logopt.Enabled() { 1093 logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name) 1094 } 1095 if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Line==1 in generated wrappers 1096 base.WarnfAt(v.Pos, "generated nil check") 1097 } 1098 case ssa.OpARM64Equal, 1099 ssa.OpARM64NotEqual, 1100 ssa.OpARM64LessThan, 1101 ssa.OpARM64LessEqual, 1102 ssa.OpARM64GreaterThan, 1103 ssa.OpARM64GreaterEqual, 1104 ssa.OpARM64LessThanU, 1105 ssa.OpARM64LessEqualU, 1106 ssa.OpARM64GreaterThanU, 1107 ssa.OpARM64GreaterEqualU, 1108 ssa.OpARM64LessThanF, 1109 ssa.OpARM64LessEqualF, 1110 ssa.OpARM64GreaterThanF, 1111 ssa.OpARM64GreaterEqualF, 1112 ssa.OpARM64NotLessThanF, 1113 ssa.OpARM64NotLessEqualF, 1114 ssa.OpARM64NotGreaterThanF, 1115 ssa.OpARM64NotGreaterEqualF, 1116 ssa.OpARM64LessThanNoov, 1117 ssa.OpARM64GreaterEqualNoov: 1118 // generate boolean values using CSET 1119 p := s.Prog(arm64.ACSET) 1120 p.From.Type = obj.TYPE_SPECIAL // assembler encodes conditional bits in Offset 1121 condCode := condBits[v.Op] 1122 p.From.Offset = int64(condCode) 1123 p.To.Type = obj.TYPE_REG 1124 p.To.Reg = v.Reg() 1125 case ssa.OpARM64PRFM: 1126 p := s.Prog(v.Op.Asm()) 1127 p.From.Type = obj.TYPE_MEM 1128 p.From.Reg = v.Args[0].Reg() 1129 p.To.Type = obj.TYPE_CONST 1130 p.To.Offset = v.AuxInt 1131 case ssa.OpARM64LoweredGetClosurePtr: 1132 // Closure pointer is R26 (arm64.REGCTXT). 1133 ssagen.CheckLoweredGetClosurePtr(v) 1134 case ssa.OpARM64LoweredGetCallerSP: 1135 // caller's SP is FixedFrameSize below the address of the first arg 1136 p := s.Prog(arm64.AMOVD) 1137 p.From.Type = obj.TYPE_ADDR 1138 p.From.Offset = -base.Ctxt.Arch.FixedFrameSize 1139 p.From.Name = obj.NAME_PARAM 1140 p.To.Type = obj.TYPE_REG 1141 p.To.Reg = v.Reg() 1142 case ssa.OpARM64LoweredGetCallerPC: 1143 p := s.Prog(obj.AGETCALLERPC) 1144 p.To.Type = obj.TYPE_REG 1145 p.To.Reg = v.Reg() 1146 case ssa.OpARM64DMB: 1147 p := s.Prog(v.Op.Asm()) 1148 p.From.Type = obj.TYPE_CONST 1149 p.From.Offset = v.AuxInt 1150 case ssa.OpARM64FlagConstant: 1151 v.Fatalf("FlagConstant op should never make it to codegen %v", v.LongString()) 1152 case ssa.OpARM64InvertFlags: 1153 v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString()) 1154 case ssa.OpClobber: 1155 // MOVW $0xdeaddead, REGTMP 1156 // MOVW REGTMP, (slot) 1157 // MOVW REGTMP, 4(slot) 1158 p := s.Prog(arm64.AMOVW) 1159 p.From.Type = obj.TYPE_CONST 1160 p.From.Offset = 0xdeaddead 1161 p.To.Type = obj.TYPE_REG 1162 p.To.Reg = arm64.REGTMP 1163 p = s.Prog(arm64.AMOVW) 1164 p.From.Type = obj.TYPE_REG 1165 p.From.Reg = arm64.REGTMP 1166 p.To.Type = obj.TYPE_MEM 1167 p.To.Reg = arm64.REGSP 1168 ssagen.AddAux(&p.To, v) 1169 p = s.Prog(arm64.AMOVW) 1170 p.From.Type = obj.TYPE_REG 1171 p.From.Reg = arm64.REGTMP 1172 p.To.Type = obj.TYPE_MEM 1173 p.To.Reg = arm64.REGSP 1174 ssagen.AddAux2(&p.To, v, v.AuxInt+4) 1175 case ssa.OpClobberReg: 1176 x := uint64(0xdeaddeaddeaddead) 1177 p := s.Prog(arm64.AMOVD) 1178 p.From.Type = obj.TYPE_CONST 1179 p.From.Offset = int64(x) 1180 p.To.Type = obj.TYPE_REG 1181 p.To.Reg = v.Reg() 1182 default: 1183 v.Fatalf("genValue not implemented: %s", v.LongString()) 1184 } 1185} 1186 1187var condBits = map[ssa.Op]arm64.SpecialOperand{ 1188 ssa.OpARM64Equal: arm64.SPOP_EQ, 1189 ssa.OpARM64NotEqual: arm64.SPOP_NE, 1190 ssa.OpARM64LessThan: arm64.SPOP_LT, 1191 ssa.OpARM64LessThanU: arm64.SPOP_LO, 1192 ssa.OpARM64LessEqual: arm64.SPOP_LE, 1193 ssa.OpARM64LessEqualU: arm64.SPOP_LS, 1194 ssa.OpARM64GreaterThan: arm64.SPOP_GT, 1195 ssa.OpARM64GreaterThanU: arm64.SPOP_HI, 1196 ssa.OpARM64GreaterEqual: arm64.SPOP_GE, 1197 ssa.OpARM64GreaterEqualU: arm64.SPOP_HS, 1198 ssa.OpARM64LessThanF: arm64.SPOP_MI, // Less than 1199 ssa.OpARM64LessEqualF: arm64.SPOP_LS, // Less than or equal to 1200 ssa.OpARM64GreaterThanF: arm64.SPOP_GT, // Greater than 1201 ssa.OpARM64GreaterEqualF: arm64.SPOP_GE, // Greater than or equal to 1202 1203 // The following condition codes have unordered to handle comparisons related to NaN. 1204 ssa.OpARM64NotLessThanF: arm64.SPOP_PL, // Greater than, equal to, or unordered 1205 ssa.OpARM64NotLessEqualF: arm64.SPOP_HI, // Greater than or unordered 1206 ssa.OpARM64NotGreaterThanF: arm64.SPOP_LE, // Less than, equal to or unordered 1207 ssa.OpARM64NotGreaterEqualF: arm64.SPOP_LT, // Less than or unordered 1208 1209 ssa.OpARM64LessThanNoov: arm64.SPOP_MI, // Less than but without honoring overflow 1210 ssa.OpARM64GreaterEqualNoov: arm64.SPOP_PL, // Greater than or equal to but without honoring overflow 1211} 1212 1213var blockJump = map[ssa.BlockKind]struct { 1214 asm, invasm obj.As 1215}{ 1216 ssa.BlockARM64EQ: {arm64.ABEQ, arm64.ABNE}, 1217 ssa.BlockARM64NE: {arm64.ABNE, arm64.ABEQ}, 1218 ssa.BlockARM64LT: {arm64.ABLT, arm64.ABGE}, 1219 ssa.BlockARM64GE: {arm64.ABGE, arm64.ABLT}, 1220 ssa.BlockARM64LE: {arm64.ABLE, arm64.ABGT}, 1221 ssa.BlockARM64GT: {arm64.ABGT, arm64.ABLE}, 1222 ssa.BlockARM64ULT: {arm64.ABLO, arm64.ABHS}, 1223 ssa.BlockARM64UGE: {arm64.ABHS, arm64.ABLO}, 1224 ssa.BlockARM64UGT: {arm64.ABHI, arm64.ABLS}, 1225 ssa.BlockARM64ULE: {arm64.ABLS, arm64.ABHI}, 1226 ssa.BlockARM64Z: {arm64.ACBZ, arm64.ACBNZ}, 1227 ssa.BlockARM64NZ: {arm64.ACBNZ, arm64.ACBZ}, 1228 ssa.BlockARM64ZW: {arm64.ACBZW, arm64.ACBNZW}, 1229 ssa.BlockARM64NZW: {arm64.ACBNZW, arm64.ACBZW}, 1230 ssa.BlockARM64TBZ: {arm64.ATBZ, arm64.ATBNZ}, 1231 ssa.BlockARM64TBNZ: {arm64.ATBNZ, arm64.ATBZ}, 1232 ssa.BlockARM64FLT: {arm64.ABMI, arm64.ABPL}, 1233 ssa.BlockARM64FGE: {arm64.ABGE, arm64.ABLT}, 1234 ssa.BlockARM64FLE: {arm64.ABLS, arm64.ABHI}, 1235 ssa.BlockARM64FGT: {arm64.ABGT, arm64.ABLE}, 1236 ssa.BlockARM64LTnoov: {arm64.ABMI, arm64.ABPL}, 1237 ssa.BlockARM64GEnoov: {arm64.ABPL, arm64.ABMI}, 1238} 1239 1240// To model a 'LEnoov' ('<=' without overflow checking) branching. 1241var leJumps = [2][2]ssagen.IndexJump{ 1242 {{Jump: arm64.ABEQ, Index: 0}, {Jump: arm64.ABPL, Index: 1}}, // next == b.Succs[0] 1243 {{Jump: arm64.ABMI, Index: 0}, {Jump: arm64.ABEQ, Index: 0}}, // next == b.Succs[1] 1244} 1245 1246// To model a 'GTnoov' ('>' without overflow checking) branching. 1247var gtJumps = [2][2]ssagen.IndexJump{ 1248 {{Jump: arm64.ABMI, Index: 1}, {Jump: arm64.ABEQ, Index: 1}}, // next == b.Succs[0] 1249 {{Jump: arm64.ABEQ, Index: 1}, {Jump: arm64.ABPL, Index: 0}}, // next == b.Succs[1] 1250} 1251 1252func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) { 1253 switch b.Kind { 1254 case ssa.BlockPlain: 1255 if b.Succs[0].Block() != next { 1256 p := s.Prog(obj.AJMP) 1257 p.To.Type = obj.TYPE_BRANCH 1258 s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) 1259 } 1260 1261 case ssa.BlockDefer: 1262 // defer returns in R0: 1263 // 0 if we should continue executing 1264 // 1 if we should jump to deferreturn call 1265 p := s.Prog(arm64.ACMP) 1266 p.From.Type = obj.TYPE_CONST 1267 p.From.Offset = 0 1268 p.Reg = arm64.REG_R0 1269 p = s.Prog(arm64.ABNE) 1270 p.To.Type = obj.TYPE_BRANCH 1271 s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[1].Block()}) 1272 if b.Succs[0].Block() != next { 1273 p := s.Prog(obj.AJMP) 1274 p.To.Type = obj.TYPE_BRANCH 1275 s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()}) 1276 } 1277 1278 case ssa.BlockExit, ssa.BlockRetJmp: 1279 1280 case ssa.BlockRet: 1281 s.Prog(obj.ARET) 1282 1283 case ssa.BlockARM64EQ, ssa.BlockARM64NE, 1284 ssa.BlockARM64LT, ssa.BlockARM64GE, 1285 ssa.BlockARM64LE, ssa.BlockARM64GT, 1286 ssa.BlockARM64ULT, ssa.BlockARM64UGT, 1287 ssa.BlockARM64ULE, ssa.BlockARM64UGE, 1288 ssa.BlockARM64Z, ssa.BlockARM64NZ, 1289 ssa.BlockARM64ZW, ssa.BlockARM64NZW, 1290 ssa.BlockARM64FLT, ssa.BlockARM64FGE, 1291 ssa.BlockARM64FLE, ssa.BlockARM64FGT, 1292 ssa.BlockARM64LTnoov, ssa.BlockARM64GEnoov: 1293 jmp := blockJump[b.Kind] 1294 var p *obj.Prog 1295 switch next { 1296 case b.Succs[0].Block(): 1297 p = s.Br(jmp.invasm, b.Succs[1].Block()) 1298 case b.Succs[1].Block(): 1299 p = s.Br(jmp.asm, b.Succs[0].Block()) 1300 default: 1301 if b.Likely != ssa.BranchUnlikely { 1302 p = s.Br(jmp.asm, b.Succs[0].Block()) 1303 s.Br(obj.AJMP, b.Succs[1].Block()) 1304 } else { 1305 p = s.Br(jmp.invasm, b.Succs[1].Block()) 1306 s.Br(obj.AJMP, b.Succs[0].Block()) 1307 } 1308 } 1309 if !b.Controls[0].Type.IsFlags() { 1310 p.From.Type = obj.TYPE_REG 1311 p.From.Reg = b.Controls[0].Reg() 1312 } 1313 case ssa.BlockARM64TBZ, ssa.BlockARM64TBNZ: 1314 jmp := blockJump[b.Kind] 1315 var p *obj.Prog 1316 switch next { 1317 case b.Succs[0].Block(): 1318 p = s.Br(jmp.invasm, b.Succs[1].Block()) 1319 case b.Succs[1].Block(): 1320 p = s.Br(jmp.asm, b.Succs[0].Block()) 1321 default: 1322 if b.Likely != ssa.BranchUnlikely { 1323 p = s.Br(jmp.asm, b.Succs[0].Block()) 1324 s.Br(obj.AJMP, b.Succs[1].Block()) 1325 } else { 1326 p = s.Br(jmp.invasm, b.Succs[1].Block()) 1327 s.Br(obj.AJMP, b.Succs[0].Block()) 1328 } 1329 } 1330 p.From.Offset = b.AuxInt 1331 p.From.Type = obj.TYPE_CONST 1332 p.Reg = b.Controls[0].Reg() 1333 1334 case ssa.BlockARM64LEnoov: 1335 s.CombJump(b, next, &leJumps) 1336 case ssa.BlockARM64GTnoov: 1337 s.CombJump(b, next, >Jumps) 1338 1339 case ssa.BlockARM64JUMPTABLE: 1340 // MOVD (TABLE)(IDX<<3), Rtmp 1341 // JMP (Rtmp) 1342 p := s.Prog(arm64.AMOVD) 1343 p.From = genIndexedOperand(ssa.OpARM64MOVDloadidx8, b.Controls[1].Reg(), b.Controls[0].Reg()) 1344 p.To.Type = obj.TYPE_REG 1345 p.To.Reg = arm64.REGTMP 1346 p = s.Prog(obj.AJMP) 1347 p.To.Type = obj.TYPE_MEM 1348 p.To.Reg = arm64.REGTMP 1349 // Save jump tables for later resolution of the target blocks. 1350 s.JumpTables = append(s.JumpTables, b) 1351 1352 default: 1353 b.Fatalf("branch not implemented: %s", b.LongString()) 1354 } 1355} 1356 1357func loadRegResult(s *ssagen.State, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog { 1358 p := s.Prog(loadByType(t)) 1359 p.From.Type = obj.TYPE_MEM 1360 p.From.Name = obj.NAME_AUTO 1361 p.From.Sym = n.Linksym() 1362 p.From.Offset = n.FrameOffset() + off 1363 p.To.Type = obj.TYPE_REG 1364 p.To.Reg = reg 1365 return p 1366} 1367 1368func spillArgReg(pp *objw.Progs, p *obj.Prog, f *ssa.Func, t *types.Type, reg int16, n *ir.Name, off int64) *obj.Prog { 1369 p = pp.Append(p, storeByType(t), obj.TYPE_REG, reg, 0, obj.TYPE_MEM, 0, n.FrameOffset()+off) 1370 p.To.Name = obj.NAME_PARAM 1371 p.To.Sym = n.Linksym() 1372 p.Pos = p.Pos.WithNotStmt() 1373 return p 1374} 1375