1// Copyright 2017 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5// Package cpu implements processor feature detection
6// used by the Go standard library.
7package cpu
8
9import _ "unsafe" // for linkname
10
11// DebugOptions is set to true by the runtime if the OS supports reading
12// GODEBUG early in runtime startup.
13// This should not be changed after it is initialized.
14var DebugOptions bool
15
16// CacheLinePad is used to pad structs to avoid false sharing.
17type CacheLinePad struct{ _ [CacheLinePadSize]byte }
18
19// CacheLineSize is the CPU's assumed cache line size.
20// There is currently no runtime detection of the real cache line size
21// so we use the constant per GOARCH CacheLinePadSize as an approximation.
22var CacheLineSize uintptr = CacheLinePadSize
23
24// The booleans in X86 contain the correspondingly named cpuid feature bit.
25// HasAVX and HasAVX2 are only set if the OS does support XMM and YMM registers
26// in addition to the cpuid feature bit being set.
27// The struct is padded to avoid false sharing.
28var X86 struct {
29	_            CacheLinePad
30	HasAES       bool
31	HasADX       bool
32	HasAVX       bool
33	HasAVX2      bool
34	HasAVX512F   bool
35	HasAVX512BW  bool
36	HasAVX512VL  bool
37	HasBMI1      bool
38	HasBMI2      bool
39	HasERMS      bool
40	HasFMA       bool
41	HasOSXSAVE   bool
42	HasPCLMULQDQ bool
43	HasPOPCNT    bool
44	HasRDTSCP    bool
45	HasSHA       bool
46	HasSSE3      bool
47	HasSSSE3     bool
48	HasSSE41     bool
49	HasSSE42     bool
50	_            CacheLinePad
51}
52
53// The booleans in ARM contain the correspondingly named cpu feature bit.
54// The struct is padded to avoid false sharing.
55var ARM struct {
56	_            CacheLinePad
57	HasVFPv4     bool
58	HasIDIVA     bool
59	HasV7Atomics bool
60	_            CacheLinePad
61}
62
63// The booleans in ARM64 contain the correspondingly named cpu feature bit.
64// The struct is padded to avoid false sharing.
65var ARM64 struct {
66	_          CacheLinePad
67	HasAES     bool
68	HasPMULL   bool
69	HasSHA1    bool
70	HasSHA2    bool
71	HasSHA512  bool
72	HasCRC32   bool
73	HasATOMICS bool
74	HasCPUID   bool
75	IsNeoverse bool
76	_          CacheLinePad
77}
78
79var MIPS64X struct {
80	_      CacheLinePad
81	HasMSA bool // MIPS SIMD architecture
82	_      CacheLinePad
83}
84
85// For ppc64(le), it is safe to check only for ISA level starting on ISA v3.00,
86// since there are no optional categories. There are some exceptions that also
87// require kernel support to work (darn, scv), so there are feature bits for
88// those as well. The minimum processor requirement is POWER8 (ISA 2.07).
89// The struct is padded to avoid false sharing.
90var PPC64 struct {
91	_         CacheLinePad
92	HasDARN   bool // Hardware random number generator (requires kernel enablement)
93	HasSCV    bool // Syscall vectored (requires kernel enablement)
94	IsPOWER8  bool // ISA v2.07 (POWER8)
95	IsPOWER9  bool // ISA v3.00 (POWER9)
96	IsPOWER10 bool // ISA v3.1  (POWER10)
97	_         CacheLinePad
98}
99
100var S390X struct {
101	_         CacheLinePad
102	HasZARCH  bool // z architecture mode is active [mandatory]
103	HasSTFLE  bool // store facility list extended [mandatory]
104	HasLDISP  bool // long (20-bit) displacements [mandatory]
105	HasEIMM   bool // 32-bit immediates [mandatory]
106	HasDFP    bool // decimal floating point
107	HasETF3EH bool // ETF-3 enhanced
108	HasMSA    bool // message security assist (CPACF)
109	HasAES    bool // KM-AES{128,192,256} functions
110	HasAESCBC bool // KMC-AES{128,192,256} functions
111	HasAESCTR bool // KMCTR-AES{128,192,256} functions
112	HasAESGCM bool // KMA-GCM-AES{128,192,256} functions
113	HasGHASH  bool // KIMD-GHASH function
114	HasSHA1   bool // K{I,L}MD-SHA-1 functions
115	HasSHA256 bool // K{I,L}MD-SHA-256 functions
116	HasSHA512 bool // K{I,L}MD-SHA-512 functions
117	HasSHA3   bool // K{I,L}MD-SHA3-{224,256,384,512} and K{I,L}MD-SHAKE-{128,256} functions
118	HasVX     bool // vector facility. Note: the runtime sets this when it processes auxv records.
119	HasVXE    bool // vector-enhancements facility 1
120	HasKDSA   bool // elliptic curve functions
121	HasECDSA  bool // NIST curves
122	HasEDDSA  bool // Edwards curves
123	_         CacheLinePad
124}
125
126// CPU feature variables are accessed by assembly code in various packages.
127//go:linkname X86
128//go:linkname ARM
129//go:linkname ARM64
130//go:linkname MIPS64X
131//go:linkname PPC64
132//go:linkname S390X
133
134// Initialize examines the processor and sets the relevant variables above.
135// This is called by the runtime package early in program initialization,
136// before normal init functions are run. env is set by runtime if the OS supports
137// cpu feature options in GODEBUG.
138func Initialize(env string) {
139	doinit()
140	processOptions(env)
141}
142
143// options contains the cpu debug options that can be used in GODEBUG.
144// Options are arch dependent and are added by the arch specific doinit functions.
145// Features that are mandatory for the specific GOARCH should not be added to options
146// (e.g. SSE2 on amd64).
147var options []option
148
149// Option names should be lower case. e.g. avx instead of AVX.
150type option struct {
151	Name      string
152	Feature   *bool
153	Specified bool // whether feature value was specified in GODEBUG
154	Enable    bool // whether feature should be enabled
155}
156
157// processOptions enables or disables CPU feature values based on the parsed env string.
158// The env string is expected to be of the form cpu.feature1=value1,cpu.feature2=value2...
159// where feature names is one of the architecture specific list stored in the
160// cpu packages options variable and values are either 'on' or 'off'.
161// If env contains cpu.all=off then all cpu features referenced through the options
162// variable are disabled. Other feature names and values result in warning messages.
163func processOptions(env string) {
164field:
165	for env != "" {
166		field := ""
167		i := indexByte(env, ',')
168		if i < 0 {
169			field, env = env, ""
170		} else {
171			field, env = env[:i], env[i+1:]
172		}
173		if len(field) < 4 || field[:4] != "cpu." {
174			continue
175		}
176		i = indexByte(field, '=')
177		if i < 0 {
178			print("GODEBUG: no value specified for \"", field, "\"\n")
179			continue
180		}
181		key, value := field[4:i], field[i+1:] // e.g. "SSE2", "on"
182
183		var enable bool
184		switch value {
185		case "on":
186			enable = true
187		case "off":
188			enable = false
189		default:
190			print("GODEBUG: value \"", value, "\" not supported for cpu option \"", key, "\"\n")
191			continue field
192		}
193
194		if key == "all" {
195			for i := range options {
196				options[i].Specified = true
197				options[i].Enable = enable
198			}
199			continue field
200		}
201
202		for i := range options {
203			if options[i].Name == key {
204				options[i].Specified = true
205				options[i].Enable = enable
206				continue field
207			}
208		}
209
210		print("GODEBUG: unknown cpu feature \"", key, "\"\n")
211	}
212
213	for _, o := range options {
214		if !o.Specified {
215			continue
216		}
217
218		if o.Enable && !*o.Feature {
219			print("GODEBUG: can not enable \"", o.Name, "\", missing CPU support\n")
220			continue
221		}
222
223		*o.Feature = o.Enable
224	}
225}
226
227// indexByte returns the index of the first instance of c in s,
228// or -1 if c is not present in s.
229// indexByte is semantically the same as [strings.IndexByte].
230// We copy this function because "internal/cpu" should not have external dependencies.
231func indexByte(s string, c byte) int {
232	for i := 0; i < len(s); i++ {
233		if s[i] == c {
234			return i
235		}
236	}
237	return -1
238}
239