1// Copyright 2016 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package s390x
6
7import (
8	"math"
9
10	"cmd/compile/internal/base"
11	"cmd/compile/internal/ir"
12	"cmd/compile/internal/logopt"
13	"cmd/compile/internal/ssa"
14	"cmd/compile/internal/ssagen"
15	"cmd/compile/internal/types"
16	"cmd/internal/obj"
17	"cmd/internal/obj/s390x"
18)
19
20// ssaMarkMoves marks any MOVXconst ops that need to avoid clobbering flags.
21func ssaMarkMoves(s *ssagen.State, b *ssa.Block) {
22	flive := b.FlagsLiveAtEnd
23	for _, c := range b.ControlValues() {
24		flive = c.Type.IsFlags() || flive
25	}
26	for i := len(b.Values) - 1; i >= 0; i-- {
27		v := b.Values[i]
28		if flive && v.Op == ssa.OpS390XMOVDconst {
29			// The "mark" is any non-nil Aux value.
30			v.Aux = ssa.AuxMark
31		}
32		if v.Type.IsFlags() {
33			flive = false
34		}
35		for _, a := range v.Args {
36			if a.Type.IsFlags() {
37				flive = true
38			}
39		}
40	}
41}
42
43// loadByType returns the load instruction of the given type.
44func loadByType(t *types.Type) obj.As {
45	if t.IsFloat() {
46		switch t.Size() {
47		case 4:
48			return s390x.AFMOVS
49		case 8:
50			return s390x.AFMOVD
51		}
52	} else {
53		switch t.Size() {
54		case 1:
55			if t.IsSigned() {
56				return s390x.AMOVB
57			} else {
58				return s390x.AMOVBZ
59			}
60		case 2:
61			if t.IsSigned() {
62				return s390x.AMOVH
63			} else {
64				return s390x.AMOVHZ
65			}
66		case 4:
67			if t.IsSigned() {
68				return s390x.AMOVW
69			} else {
70				return s390x.AMOVWZ
71			}
72		case 8:
73			return s390x.AMOVD
74		}
75	}
76	panic("bad load type")
77}
78
79// storeByType returns the store instruction of the given type.
80func storeByType(t *types.Type) obj.As {
81	width := t.Size()
82	if t.IsFloat() {
83		switch width {
84		case 4:
85			return s390x.AFMOVS
86		case 8:
87			return s390x.AFMOVD
88		}
89	} else {
90		switch width {
91		case 1:
92			return s390x.AMOVB
93		case 2:
94			return s390x.AMOVH
95		case 4:
96			return s390x.AMOVW
97		case 8:
98			return s390x.AMOVD
99		}
100	}
101	panic("bad store type")
102}
103
104// moveByType returns the reg->reg move instruction of the given type.
105func moveByType(t *types.Type) obj.As {
106	if t.IsFloat() {
107		return s390x.AFMOVD
108	} else {
109		switch t.Size() {
110		case 1:
111			if t.IsSigned() {
112				return s390x.AMOVB
113			} else {
114				return s390x.AMOVBZ
115			}
116		case 2:
117			if t.IsSigned() {
118				return s390x.AMOVH
119			} else {
120				return s390x.AMOVHZ
121			}
122		case 4:
123			if t.IsSigned() {
124				return s390x.AMOVW
125			} else {
126				return s390x.AMOVWZ
127			}
128		case 8:
129			return s390x.AMOVD
130		}
131	}
132	panic("bad load type")
133}
134
135// opregreg emits instructions for
136//
137//	dest := dest(To) op src(From)
138//
139// and also returns the created obj.Prog so it
140// may be further adjusted (offset, scale, etc).
141func opregreg(s *ssagen.State, op obj.As, dest, src int16) *obj.Prog {
142	p := s.Prog(op)
143	p.From.Type = obj.TYPE_REG
144	p.To.Type = obj.TYPE_REG
145	p.To.Reg = dest
146	p.From.Reg = src
147	return p
148}
149
150// opregregimm emits instructions for
151//
152//	dest := src(From) op off
153//
154// and also returns the created obj.Prog so it
155// may be further adjusted (offset, scale, etc).
156func opregregimm(s *ssagen.State, op obj.As, dest, src int16, off int64) *obj.Prog {
157	p := s.Prog(op)
158	p.From.Type = obj.TYPE_CONST
159	p.From.Offset = off
160	p.Reg = src
161	p.To.Reg = dest
162	p.To.Type = obj.TYPE_REG
163	return p
164}
165
166func ssaGenValue(s *ssagen.State, v *ssa.Value) {
167	switch v.Op {
168	case ssa.OpS390XSLD, ssa.OpS390XSLW,
169		ssa.OpS390XSRD, ssa.OpS390XSRW,
170		ssa.OpS390XSRAD, ssa.OpS390XSRAW,
171		ssa.OpS390XRLLG, ssa.OpS390XRLL:
172		r := v.Reg()
173		r1 := v.Args[0].Reg()
174		r2 := v.Args[1].Reg()
175		if r2 == s390x.REG_R0 {
176			v.Fatalf("cannot use R0 as shift value %s", v.LongString())
177		}
178		p := opregreg(s, v.Op.Asm(), r, r2)
179		if r != r1 {
180			p.Reg = r1
181		}
182	case ssa.OpS390XRXSBG:
183		r2 := v.Args[1].Reg()
184		i := v.Aux.(s390x.RotateParams)
185		p := s.Prog(v.Op.Asm())
186		p.From = obj.Addr{Type: obj.TYPE_CONST, Offset: int64(i.Start)}
187		p.AddRestSourceArgs([]obj.Addr{
188			{Type: obj.TYPE_CONST, Offset: int64(i.End)},
189			{Type: obj.TYPE_CONST, Offset: int64(i.Amount)},
190			{Type: obj.TYPE_REG, Reg: r2},
191		})
192		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: v.Reg()}
193	case ssa.OpS390XRISBGZ:
194		r1 := v.Reg()
195		r2 := v.Args[0].Reg()
196		i := v.Aux.(s390x.RotateParams)
197		p := s.Prog(v.Op.Asm())
198		p.From = obj.Addr{Type: obj.TYPE_CONST, Offset: int64(i.Start)}
199		p.AddRestSourceArgs([]obj.Addr{
200			{Type: obj.TYPE_CONST, Offset: int64(i.End)},
201			{Type: obj.TYPE_CONST, Offset: int64(i.Amount)},
202			{Type: obj.TYPE_REG, Reg: r2},
203		})
204		p.To = obj.Addr{Type: obj.TYPE_REG, Reg: r1}
205	case ssa.OpS390XADD, ssa.OpS390XADDW,
206		ssa.OpS390XSUB, ssa.OpS390XSUBW,
207		ssa.OpS390XAND, ssa.OpS390XANDW,
208		ssa.OpS390XOR, ssa.OpS390XORW,
209		ssa.OpS390XXOR, ssa.OpS390XXORW:
210		r := v.Reg()
211		r1 := v.Args[0].Reg()
212		r2 := v.Args[1].Reg()
213		p := opregreg(s, v.Op.Asm(), r, r2)
214		if r != r1 {
215			p.Reg = r1
216		}
217	case ssa.OpS390XADDC:
218		r1 := v.Reg0()
219		r2 := v.Args[0].Reg()
220		r3 := v.Args[1].Reg()
221		if r1 == r2 {
222			r2, r3 = r3, r2
223		}
224		p := opregreg(s, v.Op.Asm(), r1, r2)
225		if r3 != r1 {
226			p.Reg = r3
227		}
228	case ssa.OpS390XSUBC:
229		r1 := v.Reg0()
230		r2 := v.Args[0].Reg()
231		r3 := v.Args[1].Reg()
232		p := opregreg(s, v.Op.Asm(), r1, r3)
233		if r1 != r2 {
234			p.Reg = r2
235		}
236	case ssa.OpS390XADDE, ssa.OpS390XSUBE:
237		r2 := v.Args[1].Reg()
238		opregreg(s, v.Op.Asm(), v.Reg0(), r2)
239	case ssa.OpS390XADDCconst:
240		r1 := v.Reg0()
241		r3 := v.Args[0].Reg()
242		i2 := int64(int16(v.AuxInt))
243		opregregimm(s, v.Op.Asm(), r1, r3, i2)
244	// 2-address opcode arithmetic
245	case ssa.OpS390XMULLD, ssa.OpS390XMULLW,
246		ssa.OpS390XMULHD, ssa.OpS390XMULHDU,
247		ssa.OpS390XFMULS, ssa.OpS390XFMUL, ssa.OpS390XFDIVS, ssa.OpS390XFDIV:
248		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
249	case ssa.OpS390XFSUBS, ssa.OpS390XFSUB,
250		ssa.OpS390XFADDS, ssa.OpS390XFADD:
251		opregreg(s, v.Op.Asm(), v.Reg0(), v.Args[1].Reg())
252	case ssa.OpS390XMLGR:
253		// MLGR Rx R3 -> R2:R3
254		r0 := v.Args[0].Reg()
255		r1 := v.Args[1].Reg()
256		if r1 != s390x.REG_R3 {
257			v.Fatalf("We require the multiplcand to be stored in R3 for MLGR %s", v.LongString())
258		}
259		p := s.Prog(s390x.AMLGR)
260		p.From.Type = obj.TYPE_REG
261		p.From.Reg = r0
262		p.To.Reg = s390x.REG_R2
263		p.To.Type = obj.TYPE_REG
264	case ssa.OpS390XFMADD, ssa.OpS390XFMADDS,
265		ssa.OpS390XFMSUB, ssa.OpS390XFMSUBS:
266		r1 := v.Args[1].Reg()
267		r2 := v.Args[2].Reg()
268		p := s.Prog(v.Op.Asm())
269		p.From.Type = obj.TYPE_REG
270		p.From.Reg = r1
271		p.Reg = r2
272		p.To.Type = obj.TYPE_REG
273		p.To.Reg = v.Reg()
274	case ssa.OpS390XFIDBR:
275		switch v.AuxInt {
276		case 0, 1, 3, 4, 5, 6, 7:
277			opregregimm(s, v.Op.Asm(), v.Reg(), v.Args[0].Reg(), v.AuxInt)
278		default:
279			v.Fatalf("invalid FIDBR mask: %v", v.AuxInt)
280		}
281	case ssa.OpS390XCPSDR:
282		p := opregreg(s, v.Op.Asm(), v.Reg(), v.Args[1].Reg())
283		p.Reg = v.Args[0].Reg()
284	case ssa.OpS390XDIVD, ssa.OpS390XDIVW,
285		ssa.OpS390XDIVDU, ssa.OpS390XDIVWU,
286		ssa.OpS390XMODD, ssa.OpS390XMODW,
287		ssa.OpS390XMODDU, ssa.OpS390XMODWU:
288
289		// TODO(mundaym): use the temp registers every time like x86 does with AX?
290		dividend := v.Args[0].Reg()
291		divisor := v.Args[1].Reg()
292
293		// CPU faults upon signed overflow, which occurs when most
294		// negative int is divided by -1.
295		var j *obj.Prog
296		if v.Op == ssa.OpS390XDIVD || v.Op == ssa.OpS390XDIVW ||
297			v.Op == ssa.OpS390XMODD || v.Op == ssa.OpS390XMODW {
298
299			var c *obj.Prog
300			c = s.Prog(s390x.ACMP)
301			j = s.Prog(s390x.ABEQ)
302
303			c.From.Type = obj.TYPE_REG
304			c.From.Reg = divisor
305			c.To.Type = obj.TYPE_CONST
306			c.To.Offset = -1
307
308			j.To.Type = obj.TYPE_BRANCH
309
310		}
311
312		p := s.Prog(v.Op.Asm())
313		p.From.Type = obj.TYPE_REG
314		p.From.Reg = divisor
315		p.Reg = 0
316		p.To.Type = obj.TYPE_REG
317		p.To.Reg = dividend
318
319		// signed division, rest of the check for -1 case
320		if j != nil {
321			j2 := s.Prog(s390x.ABR)
322			j2.To.Type = obj.TYPE_BRANCH
323
324			var n *obj.Prog
325			if v.Op == ssa.OpS390XDIVD || v.Op == ssa.OpS390XDIVW {
326				// n * -1 = -n
327				n = s.Prog(s390x.ANEG)
328				n.To.Type = obj.TYPE_REG
329				n.To.Reg = dividend
330			} else {
331				// n % -1 == 0
332				n = s.Prog(s390x.AXOR)
333				n.From.Type = obj.TYPE_REG
334				n.From.Reg = dividend
335				n.To.Type = obj.TYPE_REG
336				n.To.Reg = dividend
337			}
338
339			j.To.SetTarget(n)
340			j2.To.SetTarget(s.Pc())
341		}
342	case ssa.OpS390XADDconst, ssa.OpS390XADDWconst:
343		opregregimm(s, v.Op.Asm(), v.Reg(), v.Args[0].Reg(), v.AuxInt)
344	case ssa.OpS390XMULLDconst, ssa.OpS390XMULLWconst,
345		ssa.OpS390XSUBconst, ssa.OpS390XSUBWconst,
346		ssa.OpS390XANDconst, ssa.OpS390XANDWconst,
347		ssa.OpS390XORconst, ssa.OpS390XORWconst,
348		ssa.OpS390XXORconst, ssa.OpS390XXORWconst:
349		p := s.Prog(v.Op.Asm())
350		p.From.Type = obj.TYPE_CONST
351		p.From.Offset = v.AuxInt
352		p.To.Type = obj.TYPE_REG
353		p.To.Reg = v.Reg()
354	case ssa.OpS390XSLDconst, ssa.OpS390XSLWconst,
355		ssa.OpS390XSRDconst, ssa.OpS390XSRWconst,
356		ssa.OpS390XSRADconst, ssa.OpS390XSRAWconst,
357		ssa.OpS390XRLLconst:
358		p := s.Prog(v.Op.Asm())
359		p.From.Type = obj.TYPE_CONST
360		p.From.Offset = v.AuxInt
361		r := v.Reg()
362		r1 := v.Args[0].Reg()
363		if r != r1 {
364			p.Reg = r1
365		}
366		p.To.Type = obj.TYPE_REG
367		p.To.Reg = r
368	case ssa.OpS390XMOVDaddridx:
369		r := v.Args[0].Reg()
370		i := v.Args[1].Reg()
371		p := s.Prog(s390x.AMOVD)
372		p.From.Scale = 1
373		if i == s390x.REGSP {
374			r, i = i, r
375		}
376		p.From.Type = obj.TYPE_ADDR
377		p.From.Reg = r
378		p.From.Index = i
379		ssagen.AddAux(&p.From, v)
380		p.To.Type = obj.TYPE_REG
381		p.To.Reg = v.Reg()
382	case ssa.OpS390XMOVDaddr:
383		p := s.Prog(s390x.AMOVD)
384		p.From.Type = obj.TYPE_ADDR
385		p.From.Reg = v.Args[0].Reg()
386		ssagen.AddAux(&p.From, v)
387		p.To.Type = obj.TYPE_REG
388		p.To.Reg = v.Reg()
389	case ssa.OpS390XCMP, ssa.OpS390XCMPW, ssa.OpS390XCMPU, ssa.OpS390XCMPWU:
390		opregreg(s, v.Op.Asm(), v.Args[1].Reg(), v.Args[0].Reg())
391	case ssa.OpS390XFCMPS, ssa.OpS390XFCMP:
392		opregreg(s, v.Op.Asm(), v.Args[1].Reg(), v.Args[0].Reg())
393	case ssa.OpS390XCMPconst, ssa.OpS390XCMPWconst:
394		p := s.Prog(v.Op.Asm())
395		p.From.Type = obj.TYPE_REG
396		p.From.Reg = v.Args[0].Reg()
397		p.To.Type = obj.TYPE_CONST
398		p.To.Offset = v.AuxInt
399	case ssa.OpS390XCMPUconst, ssa.OpS390XCMPWUconst:
400		p := s.Prog(v.Op.Asm())
401		p.From.Type = obj.TYPE_REG
402		p.From.Reg = v.Args[0].Reg()
403		p.To.Type = obj.TYPE_CONST
404		p.To.Offset = int64(uint32(v.AuxInt))
405	case ssa.OpS390XMOVDconst:
406		x := v.Reg()
407		p := s.Prog(v.Op.Asm())
408		p.From.Type = obj.TYPE_CONST
409		p.From.Offset = v.AuxInt
410		p.To.Type = obj.TYPE_REG
411		p.To.Reg = x
412	case ssa.OpS390XFMOVSconst, ssa.OpS390XFMOVDconst:
413		x := v.Reg()
414		p := s.Prog(v.Op.Asm())
415		p.From.Type = obj.TYPE_FCONST
416		p.From.Val = math.Float64frombits(uint64(v.AuxInt))
417		p.To.Type = obj.TYPE_REG
418		p.To.Reg = x
419	case ssa.OpS390XADDWload, ssa.OpS390XADDload,
420		ssa.OpS390XMULLWload, ssa.OpS390XMULLDload,
421		ssa.OpS390XSUBWload, ssa.OpS390XSUBload,
422		ssa.OpS390XANDWload, ssa.OpS390XANDload,
423		ssa.OpS390XORWload, ssa.OpS390XORload,
424		ssa.OpS390XXORWload, ssa.OpS390XXORload:
425		p := s.Prog(v.Op.Asm())
426		p.From.Type = obj.TYPE_MEM
427		p.From.Reg = v.Args[1].Reg()
428		ssagen.AddAux(&p.From, v)
429		p.To.Type = obj.TYPE_REG
430		p.To.Reg = v.Reg()
431	case ssa.OpS390XMOVDload,
432		ssa.OpS390XMOVWZload, ssa.OpS390XMOVHZload, ssa.OpS390XMOVBZload,
433		ssa.OpS390XMOVDBRload, ssa.OpS390XMOVWBRload, ssa.OpS390XMOVHBRload,
434		ssa.OpS390XMOVBload, ssa.OpS390XMOVHload, ssa.OpS390XMOVWload,
435		ssa.OpS390XFMOVSload, ssa.OpS390XFMOVDload:
436		p := s.Prog(v.Op.Asm())
437		p.From.Type = obj.TYPE_MEM
438		p.From.Reg = v.Args[0].Reg()
439		ssagen.AddAux(&p.From, v)
440		p.To.Type = obj.TYPE_REG
441		p.To.Reg = v.Reg()
442	case ssa.OpS390XMOVBZloadidx, ssa.OpS390XMOVHZloadidx, ssa.OpS390XMOVWZloadidx,
443		ssa.OpS390XMOVBloadidx, ssa.OpS390XMOVHloadidx, ssa.OpS390XMOVWloadidx, ssa.OpS390XMOVDloadidx,
444		ssa.OpS390XMOVHBRloadidx, ssa.OpS390XMOVWBRloadidx, ssa.OpS390XMOVDBRloadidx,
445		ssa.OpS390XFMOVSloadidx, ssa.OpS390XFMOVDloadidx:
446		r := v.Args[0].Reg()
447		i := v.Args[1].Reg()
448		if i == s390x.REGSP {
449			r, i = i, r
450		}
451		p := s.Prog(v.Op.Asm())
452		p.From.Type = obj.TYPE_MEM
453		p.From.Reg = r
454		p.From.Scale = 1
455		p.From.Index = i
456		ssagen.AddAux(&p.From, v)
457		p.To.Type = obj.TYPE_REG
458		p.To.Reg = v.Reg()
459	case ssa.OpS390XMOVBstore, ssa.OpS390XMOVHstore, ssa.OpS390XMOVWstore, ssa.OpS390XMOVDstore,
460		ssa.OpS390XMOVHBRstore, ssa.OpS390XMOVWBRstore, ssa.OpS390XMOVDBRstore,
461		ssa.OpS390XFMOVSstore, ssa.OpS390XFMOVDstore:
462		p := s.Prog(v.Op.Asm())
463		p.From.Type = obj.TYPE_REG
464		p.From.Reg = v.Args[1].Reg()
465		p.To.Type = obj.TYPE_MEM
466		p.To.Reg = v.Args[0].Reg()
467		ssagen.AddAux(&p.To, v)
468	case ssa.OpS390XMOVBstoreidx, ssa.OpS390XMOVHstoreidx, ssa.OpS390XMOVWstoreidx, ssa.OpS390XMOVDstoreidx,
469		ssa.OpS390XMOVHBRstoreidx, ssa.OpS390XMOVWBRstoreidx, ssa.OpS390XMOVDBRstoreidx,
470		ssa.OpS390XFMOVSstoreidx, ssa.OpS390XFMOVDstoreidx:
471		r := v.Args[0].Reg()
472		i := v.Args[1].Reg()
473		if i == s390x.REGSP {
474			r, i = i, r
475		}
476		p := s.Prog(v.Op.Asm())
477		p.From.Type = obj.TYPE_REG
478		p.From.Reg = v.Args[2].Reg()
479		p.To.Type = obj.TYPE_MEM
480		p.To.Reg = r
481		p.To.Scale = 1
482		p.To.Index = i
483		ssagen.AddAux(&p.To, v)
484	case ssa.OpS390XMOVDstoreconst, ssa.OpS390XMOVWstoreconst, ssa.OpS390XMOVHstoreconst, ssa.OpS390XMOVBstoreconst:
485		p := s.Prog(v.Op.Asm())
486		p.From.Type = obj.TYPE_CONST
487		sc := v.AuxValAndOff()
488		p.From.Offset = sc.Val64()
489		p.To.Type = obj.TYPE_MEM
490		p.To.Reg = v.Args[0].Reg()
491		ssagen.AddAux2(&p.To, v, sc.Off64())
492	case ssa.OpS390XMOVBreg, ssa.OpS390XMOVHreg, ssa.OpS390XMOVWreg,
493		ssa.OpS390XMOVBZreg, ssa.OpS390XMOVHZreg, ssa.OpS390XMOVWZreg,
494		ssa.OpS390XLDGR, ssa.OpS390XLGDR,
495		ssa.OpS390XCEFBRA, ssa.OpS390XCDFBRA, ssa.OpS390XCEGBRA, ssa.OpS390XCDGBRA,
496		ssa.OpS390XCFEBRA, ssa.OpS390XCFDBRA, ssa.OpS390XCGEBRA, ssa.OpS390XCGDBRA,
497		ssa.OpS390XCELFBR, ssa.OpS390XCDLFBR, ssa.OpS390XCELGBR, ssa.OpS390XCDLGBR,
498		ssa.OpS390XCLFEBR, ssa.OpS390XCLFDBR, ssa.OpS390XCLGEBR, ssa.OpS390XCLGDBR,
499		ssa.OpS390XLDEBR, ssa.OpS390XLEDBR,
500		ssa.OpS390XFNEG, ssa.OpS390XFNEGS,
501		ssa.OpS390XLPDFR, ssa.OpS390XLNDFR:
502		opregreg(s, v.Op.Asm(), v.Reg(), v.Args[0].Reg())
503	case ssa.OpS390XCLEAR:
504		p := s.Prog(v.Op.Asm())
505		p.From.Type = obj.TYPE_CONST
506		sc := v.AuxValAndOff()
507		p.From.Offset = sc.Val64()
508		p.To.Type = obj.TYPE_MEM
509		p.To.Reg = v.Args[0].Reg()
510		ssagen.AddAux2(&p.To, v, sc.Off64())
511	case ssa.OpCopy:
512		if v.Type.IsMemory() {
513			return
514		}
515		x := v.Args[0].Reg()
516		y := v.Reg()
517		if x != y {
518			opregreg(s, moveByType(v.Type), y, x)
519		}
520	case ssa.OpLoadReg:
521		if v.Type.IsFlags() {
522			v.Fatalf("load flags not implemented: %v", v.LongString())
523			return
524		}
525		p := s.Prog(loadByType(v.Type))
526		ssagen.AddrAuto(&p.From, v.Args[0])
527		p.To.Type = obj.TYPE_REG
528		p.To.Reg = v.Reg()
529	case ssa.OpStoreReg:
530		if v.Type.IsFlags() {
531			v.Fatalf("store flags not implemented: %v", v.LongString())
532			return
533		}
534		p := s.Prog(storeByType(v.Type))
535		p.From.Type = obj.TYPE_REG
536		p.From.Reg = v.Args[0].Reg()
537		ssagen.AddrAuto(&p.To, v)
538	case ssa.OpS390XLoweredGetClosurePtr:
539		// Closure pointer is R12 (already)
540		ssagen.CheckLoweredGetClosurePtr(v)
541	case ssa.OpS390XLoweredRound32F, ssa.OpS390XLoweredRound64F:
542		// input is already rounded
543	case ssa.OpS390XLoweredGetG:
544		r := v.Reg()
545		p := s.Prog(s390x.AMOVD)
546		p.From.Type = obj.TYPE_REG
547		p.From.Reg = s390x.REGG
548		p.To.Type = obj.TYPE_REG
549		p.To.Reg = r
550	case ssa.OpS390XLoweredGetCallerSP:
551		// caller's SP is FixedFrameSize below the address of the first arg
552		p := s.Prog(s390x.AMOVD)
553		p.From.Type = obj.TYPE_ADDR
554		p.From.Offset = -base.Ctxt.Arch.FixedFrameSize
555		p.From.Name = obj.NAME_PARAM
556		p.To.Type = obj.TYPE_REG
557		p.To.Reg = v.Reg()
558	case ssa.OpS390XLoweredGetCallerPC:
559		p := s.Prog(obj.AGETCALLERPC)
560		p.To.Type = obj.TYPE_REG
561		p.To.Reg = v.Reg()
562	case ssa.OpS390XCALLstatic, ssa.OpS390XCALLclosure, ssa.OpS390XCALLinter:
563		s.Call(v)
564	case ssa.OpS390XCALLtail:
565		s.TailCall(v)
566	case ssa.OpS390XLoweredWB:
567		p := s.Prog(obj.ACALL)
568		p.To.Type = obj.TYPE_MEM
569		p.To.Name = obj.NAME_EXTERN
570		// AuxInt encodes how many buffer entries we need.
571		p.To.Sym = ir.Syms.GCWriteBarrier[v.AuxInt-1]
572	case ssa.OpS390XLoweredPanicBoundsA, ssa.OpS390XLoweredPanicBoundsB, ssa.OpS390XLoweredPanicBoundsC:
573		p := s.Prog(obj.ACALL)
574		p.To.Type = obj.TYPE_MEM
575		p.To.Name = obj.NAME_EXTERN
576		p.To.Sym = ssagen.BoundsCheckFunc[v.AuxInt]
577		s.UseArgs(16) // space used in callee args area by assembly stubs
578	case ssa.OpS390XFLOGR, ssa.OpS390XPOPCNT,
579		ssa.OpS390XNEG, ssa.OpS390XNEGW,
580		ssa.OpS390XMOVWBR, ssa.OpS390XMOVDBR:
581		p := s.Prog(v.Op.Asm())
582		p.From.Type = obj.TYPE_REG
583		p.From.Reg = v.Args[0].Reg()
584		p.To.Type = obj.TYPE_REG
585		p.To.Reg = v.Reg()
586	case ssa.OpS390XNOT, ssa.OpS390XNOTW:
587		v.Fatalf("NOT/NOTW generated %s", v.LongString())
588	case ssa.OpS390XSumBytes2, ssa.OpS390XSumBytes4, ssa.OpS390XSumBytes8:
589		v.Fatalf("SumBytes generated %s", v.LongString())
590	case ssa.OpS390XLOCGR:
591		p := s.Prog(v.Op.Asm())
592		p.From.Type = obj.TYPE_CONST
593		p.From.Offset = int64(v.Aux.(s390x.CCMask))
594		p.Reg = v.Args[1].Reg()
595		p.To.Type = obj.TYPE_REG
596		p.To.Reg = v.Reg()
597	case ssa.OpS390XFSQRTS, ssa.OpS390XFSQRT:
598		p := s.Prog(v.Op.Asm())
599		p.From.Type = obj.TYPE_REG
600		p.From.Reg = v.Args[0].Reg()
601		p.To.Type = obj.TYPE_REG
602		p.To.Reg = v.Reg()
603	case ssa.OpS390XLTDBR, ssa.OpS390XLTEBR:
604		opregreg(s, v.Op.Asm(), v.Args[0].Reg(), v.Args[0].Reg())
605	case ssa.OpS390XInvertFlags:
606		v.Fatalf("InvertFlags should never make it to codegen %v", v.LongString())
607	case ssa.OpS390XFlagEQ, ssa.OpS390XFlagLT, ssa.OpS390XFlagGT, ssa.OpS390XFlagOV:
608		v.Fatalf("Flag* ops should never make it to codegen %v", v.LongString())
609	case ssa.OpS390XAddTupleFirst32, ssa.OpS390XAddTupleFirst64:
610		v.Fatalf("AddTupleFirst* should never make it to codegen %v", v.LongString())
611	case ssa.OpS390XLoweredNilCheck:
612		// Issue a load which will fault if the input is nil.
613		p := s.Prog(s390x.AMOVBZ)
614		p.From.Type = obj.TYPE_MEM
615		p.From.Reg = v.Args[0].Reg()
616		ssagen.AddAux(&p.From, v)
617		p.To.Type = obj.TYPE_REG
618		p.To.Reg = s390x.REGTMP
619		if logopt.Enabled() {
620			logopt.LogOpt(v.Pos, "nilcheck", "genssa", v.Block.Func.Name)
621		}
622		if base.Debug.Nil != 0 && v.Pos.Line() > 1 { // v.Pos.Line()==1 in generated wrappers
623			base.WarnfAt(v.Pos, "generated nil check")
624		}
625	case ssa.OpS390XMVC:
626		vo := v.AuxValAndOff()
627		p := s.Prog(s390x.AMVC)
628		p.From.Type = obj.TYPE_CONST
629		p.From.Offset = vo.Val64()
630		p.AddRestSource(obj.Addr{
631			Type:   obj.TYPE_MEM,
632			Reg:    v.Args[1].Reg(),
633			Offset: vo.Off64(),
634		})
635		p.To.Type = obj.TYPE_MEM
636		p.To.Reg = v.Args[0].Reg()
637		p.To.Offset = vo.Off64()
638	case ssa.OpS390XSTMG2, ssa.OpS390XSTMG3, ssa.OpS390XSTMG4,
639		ssa.OpS390XSTM2, ssa.OpS390XSTM3, ssa.OpS390XSTM4:
640		for i := 2; i < len(v.Args)-1; i++ {
641			if v.Args[i].Reg() != v.Args[i-1].Reg()+1 {
642				v.Fatalf("invalid store multiple %s", v.LongString())
643			}
644		}
645		p := s.Prog(v.Op.Asm())
646		p.From.Type = obj.TYPE_REG
647		p.From.Reg = v.Args[1].Reg()
648		p.Reg = v.Args[len(v.Args)-2].Reg()
649		p.To.Type = obj.TYPE_MEM
650		p.To.Reg = v.Args[0].Reg()
651		ssagen.AddAux(&p.To, v)
652	case ssa.OpS390XLoweredMove:
653		// Inputs must be valid pointers to memory,
654		// so adjust arg0 and arg1 as part of the expansion.
655		// arg2 should be src+size,
656		//
657		// mvc: MVC  $256, 0(R2), 0(R1)
658		//      MOVD $256(R1), R1
659		//      MOVD $256(R2), R2
660		//      CMP  R2, Rarg2
661		//      BNE  mvc
662		//      MVC  $rem, 0(R2), 0(R1) // if rem > 0
663		// arg2 is the last address to move in the loop + 256
664		mvc := s.Prog(s390x.AMVC)
665		mvc.From.Type = obj.TYPE_CONST
666		mvc.From.Offset = 256
667		mvc.AddRestSource(obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()})
668		mvc.To.Type = obj.TYPE_MEM
669		mvc.To.Reg = v.Args[0].Reg()
670
671		for i := 0; i < 2; i++ {
672			movd := s.Prog(s390x.AMOVD)
673			movd.From.Type = obj.TYPE_ADDR
674			movd.From.Reg = v.Args[i].Reg()
675			movd.From.Offset = 256
676			movd.To.Type = obj.TYPE_REG
677			movd.To.Reg = v.Args[i].Reg()
678		}
679
680		cmpu := s.Prog(s390x.ACMPU)
681		cmpu.From.Reg = v.Args[1].Reg()
682		cmpu.From.Type = obj.TYPE_REG
683		cmpu.To.Reg = v.Args[2].Reg()
684		cmpu.To.Type = obj.TYPE_REG
685
686		bne := s.Prog(s390x.ABLT)
687		bne.To.Type = obj.TYPE_BRANCH
688		bne.To.SetTarget(mvc)
689
690		if v.AuxInt > 0 {
691			mvc := s.Prog(s390x.AMVC)
692			mvc.From.Type = obj.TYPE_CONST
693			mvc.From.Offset = v.AuxInt
694			mvc.AddRestSource(obj.Addr{Type: obj.TYPE_MEM, Reg: v.Args[1].Reg()})
695			mvc.To.Type = obj.TYPE_MEM
696			mvc.To.Reg = v.Args[0].Reg()
697		}
698	case ssa.OpS390XLoweredZero:
699		// Input must be valid pointers to memory,
700		// so adjust arg0 as part of the expansion.
701		// arg1 should be src+size,
702		//
703		// clear: CLEAR $256, 0(R1)
704		//        MOVD  $256(R1), R1
705		//        CMP   R1, Rarg1
706		//        BNE   clear
707		//        CLEAR $rem, 0(R1) // if rem > 0
708		// arg1 is the last address to zero in the loop + 256
709		clear := s.Prog(s390x.ACLEAR)
710		clear.From.Type = obj.TYPE_CONST
711		clear.From.Offset = 256
712		clear.To.Type = obj.TYPE_MEM
713		clear.To.Reg = v.Args[0].Reg()
714
715		movd := s.Prog(s390x.AMOVD)
716		movd.From.Type = obj.TYPE_ADDR
717		movd.From.Reg = v.Args[0].Reg()
718		movd.From.Offset = 256
719		movd.To.Type = obj.TYPE_REG
720		movd.To.Reg = v.Args[0].Reg()
721
722		cmpu := s.Prog(s390x.ACMPU)
723		cmpu.From.Reg = v.Args[0].Reg()
724		cmpu.From.Type = obj.TYPE_REG
725		cmpu.To.Reg = v.Args[1].Reg()
726		cmpu.To.Type = obj.TYPE_REG
727
728		bne := s.Prog(s390x.ABLT)
729		bne.To.Type = obj.TYPE_BRANCH
730		bne.To.SetTarget(clear)
731
732		if v.AuxInt > 0 {
733			clear := s.Prog(s390x.ACLEAR)
734			clear.From.Type = obj.TYPE_CONST
735			clear.From.Offset = v.AuxInt
736			clear.To.Type = obj.TYPE_MEM
737			clear.To.Reg = v.Args[0].Reg()
738		}
739	case ssa.OpS390XMOVBZatomicload, ssa.OpS390XMOVWZatomicload, ssa.OpS390XMOVDatomicload:
740		p := s.Prog(v.Op.Asm())
741		p.From.Type = obj.TYPE_MEM
742		p.From.Reg = v.Args[0].Reg()
743		ssagen.AddAux(&p.From, v)
744		p.To.Type = obj.TYPE_REG
745		p.To.Reg = v.Reg0()
746	case ssa.OpS390XMOVBatomicstore, ssa.OpS390XMOVWatomicstore, ssa.OpS390XMOVDatomicstore:
747		p := s.Prog(v.Op.Asm())
748		p.From.Type = obj.TYPE_REG
749		p.From.Reg = v.Args[1].Reg()
750		p.To.Type = obj.TYPE_MEM
751		p.To.Reg = v.Args[0].Reg()
752		ssagen.AddAux(&p.To, v)
753	case ssa.OpS390XLAN, ssa.OpS390XLAO:
754		// LA(N|O) Ry, TMP, 0(Rx)
755		op := s.Prog(v.Op.Asm())
756		op.From.Type = obj.TYPE_REG
757		op.From.Reg = v.Args[1].Reg()
758		op.Reg = s390x.REGTMP
759		op.To.Type = obj.TYPE_MEM
760		op.To.Reg = v.Args[0].Reg()
761	case ssa.OpS390XLANfloor, ssa.OpS390XLAOfloor:
762		r := v.Args[0].Reg() // clobbered, assumed R1 in comments
763
764		// Round ptr down to nearest multiple of 4.
765		// ANDW $~3, R1
766		ptr := s.Prog(s390x.AANDW)
767		ptr.From.Type = obj.TYPE_CONST
768		ptr.From.Offset = 0xfffffffc
769		ptr.To.Type = obj.TYPE_REG
770		ptr.To.Reg = r
771
772		// Redirect output of LA(N|O) into R1 since it is clobbered anyway.
773		// LA(N|O) Rx, R1, 0(R1)
774		op := s.Prog(v.Op.Asm())
775		op.From.Type = obj.TYPE_REG
776		op.From.Reg = v.Args[1].Reg()
777		op.Reg = r
778		op.To.Type = obj.TYPE_MEM
779		op.To.Reg = r
780	case ssa.OpS390XLAA, ssa.OpS390XLAAG:
781		p := s.Prog(v.Op.Asm())
782		p.Reg = v.Reg0()
783		p.From.Type = obj.TYPE_REG
784		p.From.Reg = v.Args[1].Reg()
785		p.To.Type = obj.TYPE_MEM
786		p.To.Reg = v.Args[0].Reg()
787		ssagen.AddAux(&p.To, v)
788	case ssa.OpS390XLoweredAtomicCas32, ssa.OpS390XLoweredAtomicCas64:
789		// Convert the flags output of CS{,G} into a bool.
790		//    CS{,G} arg1, arg2, arg0
791		//    MOVD   $0, ret
792		//    BNE    2(PC)
793		//    MOVD   $1, ret
794		//    NOP (so the BNE has somewhere to land)
795
796		// CS{,G} arg1, arg2, arg0
797		cs := s.Prog(v.Op.Asm())
798		cs.From.Type = obj.TYPE_REG
799		cs.From.Reg = v.Args[1].Reg() // old
800		cs.Reg = v.Args[2].Reg()      // new
801		cs.To.Type = obj.TYPE_MEM
802		cs.To.Reg = v.Args[0].Reg()
803		ssagen.AddAux(&cs.To, v)
804
805		// MOVD $0, ret
806		movd := s.Prog(s390x.AMOVD)
807		movd.From.Type = obj.TYPE_CONST
808		movd.From.Offset = 0
809		movd.To.Type = obj.TYPE_REG
810		movd.To.Reg = v.Reg0()
811
812		// BNE 2(PC)
813		bne := s.Prog(s390x.ABNE)
814		bne.To.Type = obj.TYPE_BRANCH
815
816		// MOVD $1, ret
817		movd = s.Prog(s390x.AMOVD)
818		movd.From.Type = obj.TYPE_CONST
819		movd.From.Offset = 1
820		movd.To.Type = obj.TYPE_REG
821		movd.To.Reg = v.Reg0()
822
823		// NOP (so the BNE has somewhere to land)
824		nop := s.Prog(obj.ANOP)
825		bne.To.SetTarget(nop)
826	case ssa.OpS390XLoweredAtomicExchange32, ssa.OpS390XLoweredAtomicExchange64:
827		// Loop until the CS{,G} succeeds.
828		//     MOV{WZ,D} arg0, ret
829		// cs: CS{,G}    ret, arg1, arg0
830		//     BNE       cs
831
832		// MOV{WZ,D} arg0, ret
833		load := s.Prog(loadByType(v.Type.FieldType(0)))
834		load.From.Type = obj.TYPE_MEM
835		load.From.Reg = v.Args[0].Reg()
836		load.To.Type = obj.TYPE_REG
837		load.To.Reg = v.Reg0()
838		ssagen.AddAux(&load.From, v)
839
840		// CS{,G} ret, arg1, arg0
841		cs := s.Prog(v.Op.Asm())
842		cs.From.Type = obj.TYPE_REG
843		cs.From.Reg = v.Reg0()   // old
844		cs.Reg = v.Args[1].Reg() // new
845		cs.To.Type = obj.TYPE_MEM
846		cs.To.Reg = v.Args[0].Reg()
847		ssagen.AddAux(&cs.To, v)
848
849		// BNE cs
850		bne := s.Prog(s390x.ABNE)
851		bne.To.Type = obj.TYPE_BRANCH
852		bne.To.SetTarget(cs)
853	case ssa.OpS390XSYNC:
854		s.Prog(s390x.ASYNC)
855	case ssa.OpClobber, ssa.OpClobberReg:
856		// TODO: implement for clobberdead experiment. Nop is ok for now.
857	default:
858		v.Fatalf("genValue not implemented: %s", v.LongString())
859	}
860}
861
862func blockAsm(b *ssa.Block) obj.As {
863	switch b.Kind {
864	case ssa.BlockS390XBRC:
865		return s390x.ABRC
866	case ssa.BlockS390XCRJ:
867		return s390x.ACRJ
868	case ssa.BlockS390XCGRJ:
869		return s390x.ACGRJ
870	case ssa.BlockS390XCLRJ:
871		return s390x.ACLRJ
872	case ssa.BlockS390XCLGRJ:
873		return s390x.ACLGRJ
874	case ssa.BlockS390XCIJ:
875		return s390x.ACIJ
876	case ssa.BlockS390XCGIJ:
877		return s390x.ACGIJ
878	case ssa.BlockS390XCLIJ:
879		return s390x.ACLIJ
880	case ssa.BlockS390XCLGIJ:
881		return s390x.ACLGIJ
882	}
883	b.Fatalf("blockAsm not implemented: %s", b.LongString())
884	panic("unreachable")
885}
886
887func ssaGenBlock(s *ssagen.State, b, next *ssa.Block) {
888	// Handle generic blocks first.
889	switch b.Kind {
890	case ssa.BlockPlain:
891		if b.Succs[0].Block() != next {
892			p := s.Prog(s390x.ABR)
893			p.To.Type = obj.TYPE_BRANCH
894			s.Branches = append(s.Branches, ssagen.Branch{P: p, B: b.Succs[0].Block()})
895		}
896		return
897	case ssa.BlockDefer:
898		// defer returns in R3:
899		// 0 if we should continue executing
900		// 1 if we should jump to deferreturn call
901		p := s.Br(s390x.ACIJ, b.Succs[1].Block())
902		p.From.Type = obj.TYPE_CONST
903		p.From.Offset = int64(s390x.NotEqual & s390x.NotUnordered) // unordered is not possible
904		p.Reg = s390x.REG_R3
905		p.AddRestSourceConst(0)
906		if b.Succs[0].Block() != next {
907			s.Br(s390x.ABR, b.Succs[0].Block())
908		}
909		return
910	case ssa.BlockExit, ssa.BlockRetJmp:
911		return
912	case ssa.BlockRet:
913		s.Prog(obj.ARET)
914		return
915	}
916
917	// Handle s390x-specific blocks. These blocks all have a
918	// condition code mask in the Aux value and 2 successors.
919	succs := [...]*ssa.Block{b.Succs[0].Block(), b.Succs[1].Block()}
920	mask := b.Aux.(s390x.CCMask)
921
922	// TODO: take into account Likely property for forward/backward
923	// branches. We currently can't do this because we don't know
924	// whether a block has already been emitted. In general forward
925	// branches are assumed 'not taken' and backward branches are
926	// assumed 'taken'.
927	if next == succs[0] {
928		succs[0], succs[1] = succs[1], succs[0]
929		mask = mask.Inverse()
930	}
931
932	p := s.Br(blockAsm(b), succs[0])
933	switch b.Kind {
934	case ssa.BlockS390XBRC:
935		p.From.Type = obj.TYPE_CONST
936		p.From.Offset = int64(mask)
937	case ssa.BlockS390XCGRJ, ssa.BlockS390XCRJ,
938		ssa.BlockS390XCLGRJ, ssa.BlockS390XCLRJ:
939		p.From.Type = obj.TYPE_CONST
940		p.From.Offset = int64(mask & s390x.NotUnordered) // unordered is not possible
941		p.Reg = b.Controls[0].Reg()
942		p.AddRestSourceReg(b.Controls[1].Reg())
943	case ssa.BlockS390XCGIJ, ssa.BlockS390XCIJ:
944		p.From.Type = obj.TYPE_CONST
945		p.From.Offset = int64(mask & s390x.NotUnordered) // unordered is not possible
946		p.Reg = b.Controls[0].Reg()
947		p.AddRestSourceConst(int64(int8(b.AuxInt)))
948	case ssa.BlockS390XCLGIJ, ssa.BlockS390XCLIJ:
949		p.From.Type = obj.TYPE_CONST
950		p.From.Offset = int64(mask & s390x.NotUnordered) // unordered is not possible
951		p.Reg = b.Controls[0].Reg()
952		p.AddRestSourceConst(int64(uint8(b.AuxInt)))
953	default:
954		b.Fatalf("branch not implemented: %s", b.LongString())
955	}
956	if next != succs[1] {
957		s.Br(s390x.ABR, succs[1])
958	}
959}
960