1// Copyright 2009 The Go Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style
3// license that can be found in the LICENSE file.
4
5package runtime
6
7import (
8	"internal/abi"
9	"internal/goarch"
10	"internal/runtime/atomic"
11	"unsafe"
12)
13
14// TODO(brainman): should not need those
15const (
16	_NSIG = 65
17)
18
19//go:cgo_import_dynamic runtime._AddVectoredContinueHandler AddVectoredContinueHandler%2 "kernel32.dll"
20//go:cgo_import_dynamic runtime._AddVectoredExceptionHandler AddVectoredExceptionHandler%2 "kernel32.dll"
21//go:cgo_import_dynamic runtime._CloseHandle CloseHandle%1 "kernel32.dll"
22//go:cgo_import_dynamic runtime._CreateEventA CreateEventA%4 "kernel32.dll"
23//go:cgo_import_dynamic runtime._CreateIoCompletionPort CreateIoCompletionPort%4 "kernel32.dll"
24//go:cgo_import_dynamic runtime._CreateThread CreateThread%6 "kernel32.dll"
25//go:cgo_import_dynamic runtime._CreateWaitableTimerA CreateWaitableTimerA%3 "kernel32.dll"
26//go:cgo_import_dynamic runtime._CreateWaitableTimerExW CreateWaitableTimerExW%4 "kernel32.dll"
27//go:cgo_import_dynamic runtime._DuplicateHandle DuplicateHandle%7 "kernel32.dll"
28//go:cgo_import_dynamic runtime._ExitProcess ExitProcess%1 "kernel32.dll"
29//go:cgo_import_dynamic runtime._FreeEnvironmentStringsW FreeEnvironmentStringsW%1 "kernel32.dll"
30//go:cgo_import_dynamic runtime._GetConsoleMode GetConsoleMode%2 "kernel32.dll"
31//go:cgo_import_dynamic runtime._GetCurrentThreadId GetCurrentThreadId%0 "kernel32.dll"
32//go:cgo_import_dynamic runtime._GetEnvironmentStringsW GetEnvironmentStringsW%0 "kernel32.dll"
33//go:cgo_import_dynamic runtime._GetErrorMode GetErrorMode%0 "kernel32.dll"
34//go:cgo_import_dynamic runtime._GetProcAddress GetProcAddress%2 "kernel32.dll"
35//go:cgo_import_dynamic runtime._GetProcessAffinityMask GetProcessAffinityMask%3 "kernel32.dll"
36//go:cgo_import_dynamic runtime._GetQueuedCompletionStatusEx GetQueuedCompletionStatusEx%6 "kernel32.dll"
37//go:cgo_import_dynamic runtime._GetStdHandle GetStdHandle%1 "kernel32.dll"
38//go:cgo_import_dynamic runtime._GetSystemDirectoryA GetSystemDirectoryA%2 "kernel32.dll"
39//go:cgo_import_dynamic runtime._GetSystemInfo GetSystemInfo%1 "kernel32.dll"
40//go:cgo_import_dynamic runtime._GetThreadContext GetThreadContext%2 "kernel32.dll"
41//go:cgo_import_dynamic runtime._SetThreadContext SetThreadContext%2 "kernel32.dll"
42//go:cgo_import_dynamic runtime._LoadLibraryExW LoadLibraryExW%3 "kernel32.dll"
43//go:cgo_import_dynamic runtime._LoadLibraryW LoadLibraryW%1 "kernel32.dll"
44//go:cgo_import_dynamic runtime._PostQueuedCompletionStatus PostQueuedCompletionStatus%4 "kernel32.dll"
45//go:cgo_import_dynamic runtime._QueryPerformanceCounter QueryPerformanceCounter%1 "kernel32.dll"
46//go:cgo_import_dynamic runtime._QueryPerformanceFrequency QueryPerformanceFrequency%1 "kernel32.dll"
47//go:cgo_import_dynamic runtime._RaiseFailFastException RaiseFailFastException%3 "kernel32.dll"
48//go:cgo_import_dynamic runtime._ResumeThread ResumeThread%1 "kernel32.dll"
49//go:cgo_import_dynamic runtime._RtlLookupFunctionEntry RtlLookupFunctionEntry%3 "kernel32.dll"
50//go:cgo_import_dynamic runtime._RtlVirtualUnwind  RtlVirtualUnwind%8 "kernel32.dll"
51//go:cgo_import_dynamic runtime._SetConsoleCtrlHandler SetConsoleCtrlHandler%2 "kernel32.dll"
52//go:cgo_import_dynamic runtime._SetErrorMode SetErrorMode%1 "kernel32.dll"
53//go:cgo_import_dynamic runtime._SetEvent SetEvent%1 "kernel32.dll"
54//go:cgo_import_dynamic runtime._SetProcessPriorityBoost SetProcessPriorityBoost%2 "kernel32.dll"
55//go:cgo_import_dynamic runtime._SetThreadPriority SetThreadPriority%2 "kernel32.dll"
56//go:cgo_import_dynamic runtime._SetUnhandledExceptionFilter SetUnhandledExceptionFilter%1 "kernel32.dll"
57//go:cgo_import_dynamic runtime._SetWaitableTimer SetWaitableTimer%6 "kernel32.dll"
58//go:cgo_import_dynamic runtime._SuspendThread SuspendThread%1 "kernel32.dll"
59//go:cgo_import_dynamic runtime._SwitchToThread SwitchToThread%0 "kernel32.dll"
60//go:cgo_import_dynamic runtime._TlsAlloc TlsAlloc%0 "kernel32.dll"
61//go:cgo_import_dynamic runtime._VirtualAlloc VirtualAlloc%4 "kernel32.dll"
62//go:cgo_import_dynamic runtime._VirtualFree VirtualFree%3 "kernel32.dll"
63//go:cgo_import_dynamic runtime._VirtualQuery VirtualQuery%3 "kernel32.dll"
64//go:cgo_import_dynamic runtime._WaitForSingleObject WaitForSingleObject%2 "kernel32.dll"
65//go:cgo_import_dynamic runtime._WaitForMultipleObjects WaitForMultipleObjects%4 "kernel32.dll"
66//go:cgo_import_dynamic runtime._WerGetFlags WerGetFlags%2 "kernel32.dll"
67//go:cgo_import_dynamic runtime._WerSetFlags WerSetFlags%1 "kernel32.dll"
68//go:cgo_import_dynamic runtime._WriteConsoleW WriteConsoleW%5 "kernel32.dll"
69//go:cgo_import_dynamic runtime._WriteFile WriteFile%5 "kernel32.dll"
70
71type stdFunction unsafe.Pointer
72
73var (
74	// Following syscalls are available on every Windows PC.
75	// All these variables are set by the Windows executable
76	// loader before the Go program starts.
77	_AddVectoredContinueHandler,
78	_AddVectoredExceptionHandler,
79	_CloseHandle,
80	_CreateEventA,
81	_CreateIoCompletionPort,
82	_CreateThread,
83	_CreateWaitableTimerA,
84	_CreateWaitableTimerExW,
85	_DuplicateHandle,
86	_ExitProcess,
87	_FreeEnvironmentStringsW,
88	_GetConsoleMode,
89	_GetCurrentThreadId,
90	_GetEnvironmentStringsW,
91	_GetErrorMode,
92	_GetProcAddress,
93	_GetProcessAffinityMask,
94	_GetQueuedCompletionStatusEx,
95	_GetStdHandle,
96	_GetSystemDirectoryA,
97	_GetSystemInfo,
98	_GetThreadContext,
99	_SetThreadContext,
100	_LoadLibraryExW,
101	_LoadLibraryW,
102	_PostQueuedCompletionStatus,
103	_QueryPerformanceCounter,
104	_QueryPerformanceFrequency,
105	_RaiseFailFastException,
106	_ResumeThread,
107	_RtlLookupFunctionEntry,
108	_RtlVirtualUnwind,
109	_SetConsoleCtrlHandler,
110	_SetErrorMode,
111	_SetEvent,
112	_SetProcessPriorityBoost,
113	_SetThreadPriority,
114	_SetUnhandledExceptionFilter,
115	_SetWaitableTimer,
116	_SuspendThread,
117	_SwitchToThread,
118	_TlsAlloc,
119	_VirtualAlloc,
120	_VirtualFree,
121	_VirtualQuery,
122	_WaitForSingleObject,
123	_WaitForMultipleObjects,
124	_WerGetFlags,
125	_WerSetFlags,
126	_WriteConsoleW,
127	_WriteFile,
128	_ stdFunction
129
130	// Use ProcessPrng to generate cryptographically random data.
131	_ProcessPrng stdFunction
132
133	// Load ntdll.dll manually during startup, otherwise Mingw
134	// links wrong printf function to cgo executable (see issue
135	// 12030 for details).
136	_NtCreateWaitCompletionPacket    stdFunction
137	_NtAssociateWaitCompletionPacket stdFunction
138	_NtCancelWaitCompletionPacket    stdFunction
139	_RtlGetCurrentPeb                stdFunction
140	_RtlGetVersion                   stdFunction
141
142	// These are from non-kernel32.dll, so we prefer to LoadLibraryEx them.
143	_timeBeginPeriod,
144	_timeEndPeriod,
145	_ stdFunction
146)
147
148var (
149	bcryptprimitivesdll = [...]uint16{'b', 'c', 'r', 'y', 'p', 't', 'p', 'r', 'i', 'm', 'i', 't', 'i', 'v', 'e', 's', '.', 'd', 'l', 'l', 0}
150	ntdlldll            = [...]uint16{'n', 't', 'd', 'l', 'l', '.', 'd', 'l', 'l', 0}
151	powrprofdll         = [...]uint16{'p', 'o', 'w', 'r', 'p', 'r', 'o', 'f', '.', 'd', 'l', 'l', 0}
152	winmmdll            = [...]uint16{'w', 'i', 'n', 'm', 'm', '.', 'd', 'l', 'l', 0}
153)
154
155// Function to be called by windows CreateThread
156// to start new os thread.
157func tstart_stdcall(newm *m)
158
159// Init-time helper
160func wintls()
161
162type mOS struct {
163	threadLock mutex   // protects "thread" and prevents closing
164	thread     uintptr // thread handle
165
166	waitsema   uintptr // semaphore for parking on locks
167	resumesema uintptr // semaphore to indicate suspend/resume
168
169	highResTimer   uintptr // high resolution timer handle used in usleep
170	waitIocpTimer  uintptr // high resolution timer handle used in netpoll
171	waitIocpHandle uintptr // wait completion handle used in netpoll
172
173	// preemptExtLock synchronizes preemptM with entry/exit from
174	// external C code.
175	//
176	// This protects against races between preemptM calling
177	// SuspendThread and external code on this thread calling
178	// ExitProcess. If these happen concurrently, it's possible to
179	// exit the suspending thread and suspend the exiting thread,
180	// leading to deadlock.
181	//
182	// 0 indicates this M is not being preempted or in external
183	// code. Entering external code CASes this from 0 to 1. If
184	// this fails, a preemption is in progress, so the thread must
185	// wait for the preemption. preemptM also CASes this from 0 to
186	// 1. If this fails, the preemption fails (as it would if the
187	// PC weren't in Go code). The value is reset to 0 when
188	// returning from external code or after a preemption is
189	// complete.
190	//
191	// TODO(austin): We may not need this if preemption were more
192	// tightly synchronized on the G/P status and preemption
193	// blocked transition into _Gsyscall/_Psyscall.
194	preemptExtLock uint32
195}
196
197// Stubs so tests can link correctly. These should never be called.
198func open(name *byte, mode, perm int32) int32 {
199	throw("unimplemented")
200	return -1
201}
202func closefd(fd int32) int32 {
203	throw("unimplemented")
204	return -1
205}
206func read(fd int32, p unsafe.Pointer, n int32) int32 {
207	throw("unimplemented")
208	return -1
209}
210
211type sigset struct{}
212
213// Call a Windows function with stdcall conventions,
214// and switch to os stack during the call.
215func asmstdcall(fn unsafe.Pointer)
216
217var asmstdcallAddr unsafe.Pointer
218
219type winlibcall libcall
220
221func windowsFindfunc(lib uintptr, name []byte) stdFunction {
222	if name[len(name)-1] != 0 {
223		throw("usage")
224	}
225	f := stdcall2(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0])))
226	return stdFunction(unsafe.Pointer(f))
227}
228
229const _MAX_PATH = 260 // https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation
230var sysDirectory [_MAX_PATH + 1]byte
231var sysDirectoryLen uintptr
232
233func initSysDirectory() {
234	l := stdcall2(_GetSystemDirectoryA, uintptr(unsafe.Pointer(&sysDirectory[0])), uintptr(len(sysDirectory)-1))
235	if l == 0 || l > uintptr(len(sysDirectory)-1) {
236		throw("Unable to determine system directory")
237	}
238	sysDirectory[l] = '\\'
239	sysDirectoryLen = l + 1
240}
241
242//go:linkname windows_GetSystemDirectory internal/syscall/windows.GetSystemDirectory
243func windows_GetSystemDirectory() string {
244	return unsafe.String(&sysDirectory[0], sysDirectoryLen)
245}
246
247func windowsLoadSystemLib(name []uint16) uintptr {
248	return stdcall3(_LoadLibraryExW, uintptr(unsafe.Pointer(&name[0])), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32)
249}
250
251//go:linkname windows_QueryPerformanceCounter internal/syscall/windows.QueryPerformanceCounter
252func windows_QueryPerformanceCounter() int64 {
253	var counter int64
254	stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter)))
255	return counter
256}
257
258//go:linkname windows_QueryPerformanceFrequency internal/syscall/windows.QueryPerformanceFrequency
259func windows_QueryPerformanceFrequency() int64 {
260	var frequency int64
261	stdcall1(_QueryPerformanceFrequency, uintptr(unsafe.Pointer(&frequency)))
262	return frequency
263}
264
265func loadOptionalSyscalls() {
266	bcryptPrimitives := windowsLoadSystemLib(bcryptprimitivesdll[:])
267	if bcryptPrimitives == 0 {
268		throw("bcryptprimitives.dll not found")
269	}
270	_ProcessPrng = windowsFindfunc(bcryptPrimitives, []byte("ProcessPrng\000"))
271
272	n32 := windowsLoadSystemLib(ntdlldll[:])
273	if n32 == 0 {
274		throw("ntdll.dll not found")
275	}
276	_NtCreateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCreateWaitCompletionPacket\000"))
277	if _NtCreateWaitCompletionPacket != nil {
278		// These functions should exists if NtCreateWaitCompletionPacket exists.
279		_NtAssociateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtAssociateWaitCompletionPacket\000"))
280		if _NtAssociateWaitCompletionPacket == nil {
281			throw("NtCreateWaitCompletionPacket exists but NtAssociateWaitCompletionPacket does not")
282		}
283		_NtCancelWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCancelWaitCompletionPacket\000"))
284		if _NtCancelWaitCompletionPacket == nil {
285			throw("NtCreateWaitCompletionPacket exists but NtCancelWaitCompletionPacket does not")
286		}
287	}
288	_RtlGetCurrentPeb = windowsFindfunc(n32, []byte("RtlGetCurrentPeb\000"))
289	_RtlGetVersion = windowsFindfunc(n32, []byte("RtlGetVersion\000"))
290}
291
292func monitorSuspendResume() {
293	const (
294		_DEVICE_NOTIFY_CALLBACK = 2
295	)
296	type _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS struct {
297		callback uintptr
298		context  uintptr
299	}
300
301	powrprof := windowsLoadSystemLib(powrprofdll[:])
302	if powrprof == 0 {
303		return // Running on Windows 7, where we don't need it anyway.
304	}
305	powerRegisterSuspendResumeNotification := windowsFindfunc(powrprof, []byte("PowerRegisterSuspendResumeNotification\000"))
306	if powerRegisterSuspendResumeNotification == nil {
307		return // Running on Windows 7, where we don't need it anyway.
308	}
309	var fn any = func(context uintptr, changeType uint32, setting uintptr) uintptr {
310		for mp := (*m)(atomic.Loadp(unsafe.Pointer(&allm))); mp != nil; mp = mp.alllink {
311			if mp.resumesema != 0 {
312				stdcall1(_SetEvent, mp.resumesema)
313			}
314		}
315		return 0
316	}
317	params := _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS{
318		callback: compileCallback(*efaceOf(&fn), true),
319	}
320	handle := uintptr(0)
321	stdcall3(powerRegisterSuspendResumeNotification, _DEVICE_NOTIFY_CALLBACK,
322		uintptr(unsafe.Pointer(&params)), uintptr(unsafe.Pointer(&handle)))
323}
324
325func getproccount() int32 {
326	var mask, sysmask uintptr
327	ret := stdcall3(_GetProcessAffinityMask, currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask)))
328	if ret != 0 {
329		n := 0
330		maskbits := int(unsafe.Sizeof(mask) * 8)
331		for i := 0; i < maskbits; i++ {
332			if mask&(1<<uint(i)) != 0 {
333				n++
334			}
335		}
336		if n != 0 {
337			return int32(n)
338		}
339	}
340	// use GetSystemInfo if GetProcessAffinityMask fails
341	var info systeminfo
342	stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
343	return int32(info.dwnumberofprocessors)
344}
345
346func getPageSize() uintptr {
347	var info systeminfo
348	stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info)))
349	return uintptr(info.dwpagesize)
350}
351
352const (
353	currentProcess = ^uintptr(0) // -1 = current process
354	currentThread  = ^uintptr(1) // -2 = current thread
355)
356
357// in sys_windows_386.s and sys_windows_amd64.s:
358func getlasterror() uint32
359
360var timeBeginPeriodRetValue uint32
361
362// osRelaxMinNS indicates that sysmon shouldn't osRelax if the next
363// timer is less than 60 ms from now. Since osRelaxing may reduce
364// timer resolution to 15.6 ms, this keeps timer error under roughly 1
365// part in 4.
366const osRelaxMinNS = 60 * 1e6
367
368// osRelax is called by the scheduler when transitioning to and from
369// all Ps being idle.
370//
371// Some versions of Windows have high resolution timer. For those
372// versions osRelax is noop.
373// For Windows versions without high resolution timer, osRelax
374// adjusts the system-wide timer resolution. Go needs a
375// high resolution timer while running and there's little extra cost
376// if we're already using the CPU, but if all Ps are idle there's no
377// need to consume extra power to drive the high-res timer.
378func osRelax(relax bool) uint32 {
379	if haveHighResTimer {
380		// If the high resolution timer is available, the runtime uses the timer
381		// to sleep for short durations. This means there's no need to adjust
382		// the global clock frequency.
383		return 0
384	}
385
386	if relax {
387		return uint32(stdcall1(_timeEndPeriod, 1))
388	} else {
389		return uint32(stdcall1(_timeBeginPeriod, 1))
390	}
391}
392
393// haveHighResTimer indicates that the CreateWaitableTimerEx
394// CREATE_WAITABLE_TIMER_HIGH_RESOLUTION flag is available.
395var haveHighResTimer = false
396
397// haveHighResSleep indicates that NtCreateWaitCompletionPacket
398// exists and haveHighResTimer is true.
399// NtCreateWaitCompletionPacket has been available since Windows 10,
400// but has just been publicly documented, so some platforms, like Wine,
401// doesn't support it yet.
402var haveHighResSleep = false
403
404// createHighResTimer calls CreateWaitableTimerEx with
405// CREATE_WAITABLE_TIMER_HIGH_RESOLUTION flag to create high
406// resolution timer. createHighResTimer returns new timer
407// handle or 0, if CreateWaitableTimerEx failed.
408func createHighResTimer() uintptr {
409	const (
410		// As per @jstarks, see
411		// https://github.com/golang/go/issues/8687#issuecomment-656259353
412		_CREATE_WAITABLE_TIMER_HIGH_RESOLUTION = 0x00000002
413
414		_SYNCHRONIZE        = 0x00100000
415		_TIMER_QUERY_STATE  = 0x0001
416		_TIMER_MODIFY_STATE = 0x0002
417	)
418	return stdcall4(_CreateWaitableTimerExW, 0, 0,
419		_CREATE_WAITABLE_TIMER_HIGH_RESOLUTION,
420		_SYNCHRONIZE|_TIMER_QUERY_STATE|_TIMER_MODIFY_STATE)
421}
422
423func initHighResTimer() {
424	h := createHighResTimer()
425	if h != 0 {
426		haveHighResTimer = true
427		haveHighResSleep = _NtCreateWaitCompletionPacket != nil
428		stdcall1(_CloseHandle, h)
429	} else {
430		// Only load winmm.dll if we need it.
431		// This avoids a dependency on winmm.dll for Go programs
432		// that run on new Windows versions.
433		m32 := windowsLoadSystemLib(winmmdll[:])
434		if m32 == 0 {
435			print("runtime: LoadLibraryExW failed; errno=", getlasterror(), "\n")
436			throw("winmm.dll not found")
437		}
438		_timeBeginPeriod = windowsFindfunc(m32, []byte("timeBeginPeriod\000"))
439		_timeEndPeriod = windowsFindfunc(m32, []byte("timeEndPeriod\000"))
440		if _timeBeginPeriod == nil || _timeEndPeriod == nil {
441			print("runtime: GetProcAddress failed; errno=", getlasterror(), "\n")
442			throw("timeBegin/EndPeriod not found")
443		}
444	}
445}
446
447//go:linkname canUseLongPaths internal/syscall/windows.CanUseLongPaths
448var canUseLongPaths bool
449
450// initLongPathSupport enables long path support.
451func initLongPathSupport() {
452	const (
453		IsLongPathAwareProcess = 0x80
454		PebBitFieldOffset      = 3
455	)
456
457	// Check that we're ≥ 10.0.15063.
458	info := _OSVERSIONINFOW{}
459	info.osVersionInfoSize = uint32(unsafe.Sizeof(info))
460	stdcall1(_RtlGetVersion, uintptr(unsafe.Pointer(&info)))
461	if info.majorVersion < 10 || (info.majorVersion == 10 && info.minorVersion == 0 && info.buildNumber < 15063) {
462		return
463	}
464
465	// Set the IsLongPathAwareProcess flag of the PEB's bit field.
466	// This flag is not documented, but it's known to be used
467	// by Windows to enable long path support.
468	bitField := (*byte)(unsafe.Pointer(stdcall0(_RtlGetCurrentPeb) + PebBitFieldOffset))
469	*bitField |= IsLongPathAwareProcess
470
471	canUseLongPaths = true
472}
473
474func osinit() {
475	asmstdcallAddr = unsafe.Pointer(abi.FuncPCABI0(asmstdcall))
476
477	loadOptionalSyscalls()
478
479	preventErrorDialogs()
480
481	initExceptionHandler()
482
483	initHighResTimer()
484	timeBeginPeriodRetValue = osRelax(false)
485
486	initSysDirectory()
487	initLongPathSupport()
488
489	ncpu = getproccount()
490
491	physPageSize = getPageSize()
492
493	// Windows dynamic priority boosting assumes that a process has different types
494	// of dedicated threads -- GUI, IO, computational, etc. Go processes use
495	// equivalent threads that all do a mix of GUI, IO, computations, etc.
496	// In such context dynamic priority boosting does nothing but harm, so we turn it off.
497	stdcall2(_SetProcessPriorityBoost, currentProcess, 1)
498}
499
500//go:nosplit
501func readRandom(r []byte) int {
502	n := 0
503	if stdcall2(_ProcessPrng, uintptr(unsafe.Pointer(&r[0])), uintptr(len(r)))&0xff != 0 {
504		n = len(r)
505	}
506	return n
507}
508
509func goenvs() {
510	// strings is a pointer to environment variable pairs in the form:
511	//     "envA=valA\x00envB=valB\x00\x00" (in UTF-16)
512	// Two consecutive zero bytes end the list.
513	strings := unsafe.Pointer(stdcall0(_GetEnvironmentStringsW))
514	p := (*[1 << 24]uint16)(strings)[:]
515
516	n := 0
517	for from, i := 0, 0; true; i++ {
518		if p[i] == 0 {
519			// empty string marks the end
520			if i == from {
521				break
522			}
523			from = i + 1
524			n++
525		}
526	}
527	envs = make([]string, n)
528
529	for i := range envs {
530		envs[i] = gostringw(&p[0])
531		for p[0] != 0 {
532			p = p[1:]
533		}
534		p = p[1:] // skip nil byte
535	}
536
537	stdcall1(_FreeEnvironmentStringsW, uintptr(strings))
538
539	// We call these all the way here, late in init, so that malloc works
540	// for the callback functions these generate.
541	var fn any = ctrlHandler
542	ctrlHandlerPC := compileCallback(*efaceOf(&fn), true)
543	stdcall2(_SetConsoleCtrlHandler, ctrlHandlerPC, 1)
544
545	monitorSuspendResume()
546}
547
548// exiting is set to non-zero when the process is exiting.
549var exiting uint32
550
551//go:nosplit
552func exit(code int32) {
553	// Disallow thread suspension for preemption. Otherwise,
554	// ExitProcess and SuspendThread can race: SuspendThread
555	// queues a suspension request for this thread, ExitProcess
556	// kills the suspending thread, and then this thread suspends.
557	lock(&suspendLock)
558	atomic.Store(&exiting, 1)
559	stdcall1(_ExitProcess, uintptr(code))
560}
561
562// write1 must be nosplit because it's used as a last resort in
563// functions like badmorestackg0. In such cases, we'll always take the
564// ASCII path.
565//
566//go:nosplit
567func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 {
568	const (
569		_STD_OUTPUT_HANDLE = ^uintptr(10) // -11
570		_STD_ERROR_HANDLE  = ^uintptr(11) // -12
571	)
572	var handle uintptr
573	switch fd {
574	case 1:
575		handle = stdcall1(_GetStdHandle, _STD_OUTPUT_HANDLE)
576	case 2:
577		handle = stdcall1(_GetStdHandle, _STD_ERROR_HANDLE)
578	default:
579		// assume fd is real windows handle.
580		handle = fd
581	}
582	isASCII := true
583	b := (*[1 << 30]byte)(buf)[:n]
584	for _, x := range b {
585		if x >= 0x80 {
586			isASCII = false
587			break
588		}
589	}
590
591	if !isASCII {
592		var m uint32
593		isConsole := stdcall2(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0
594		// If this is a console output, various non-unicode code pages can be in use.
595		// Use the dedicated WriteConsole call to ensure unicode is printed correctly.
596		if isConsole {
597			return int32(writeConsole(handle, buf, n))
598		}
599	}
600	var written uint32
601	stdcall5(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0)
602	return int32(written)
603}
604
605var (
606	utf16ConsoleBack     [1000]uint16
607	utf16ConsoleBackLock mutex
608)
609
610// writeConsole writes bufLen bytes from buf to the console File.
611// It returns the number of bytes written.
612func writeConsole(handle uintptr, buf unsafe.Pointer, bufLen int32) int {
613	const surr2 = (surrogateMin + surrogateMax + 1) / 2
614
615	// Do not use defer for unlock. May cause issues when printing a panic.
616	lock(&utf16ConsoleBackLock)
617
618	b := (*[1 << 30]byte)(buf)[:bufLen]
619	s := *(*string)(unsafe.Pointer(&b))
620
621	utf16tmp := utf16ConsoleBack[:]
622
623	total := len(s)
624	w := 0
625	for _, r := range s {
626		if w >= len(utf16tmp)-2 {
627			writeConsoleUTF16(handle, utf16tmp[:w])
628			w = 0
629		}
630		if r < 0x10000 {
631			utf16tmp[w] = uint16(r)
632			w++
633		} else {
634			r -= 0x10000
635			utf16tmp[w] = surrogateMin + uint16(r>>10)&0x3ff
636			utf16tmp[w+1] = surr2 + uint16(r)&0x3ff
637			w += 2
638		}
639	}
640	writeConsoleUTF16(handle, utf16tmp[:w])
641	unlock(&utf16ConsoleBackLock)
642	return total
643}
644
645// writeConsoleUTF16 is the dedicated windows calls that correctly prints
646// to the console regardless of the current code page. Input is utf-16 code points.
647// The handle must be a console handle.
648func writeConsoleUTF16(handle uintptr, b []uint16) {
649	l := uint32(len(b))
650	if l == 0 {
651		return
652	}
653	var written uint32
654	stdcall5(_WriteConsoleW,
655		handle,
656		uintptr(unsafe.Pointer(&b[0])),
657		uintptr(l),
658		uintptr(unsafe.Pointer(&written)),
659		0,
660	)
661	return
662}
663
664//go:nosplit
665func semasleep(ns int64) int32 {
666	const (
667		_WAIT_ABANDONED = 0x00000080
668		_WAIT_OBJECT_0  = 0x00000000
669		_WAIT_TIMEOUT   = 0x00000102
670		_WAIT_FAILED    = 0xFFFFFFFF
671	)
672
673	var result uintptr
674	if ns < 0 {
675		result = stdcall2(_WaitForSingleObject, getg().m.waitsema, uintptr(_INFINITE))
676	} else {
677		start := nanotime()
678		elapsed := int64(0)
679		for {
680			ms := int64(timediv(ns-elapsed, 1000000, nil))
681			if ms == 0 {
682				ms = 1
683			}
684			result = stdcall4(_WaitForMultipleObjects, 2,
685				uintptr(unsafe.Pointer(&[2]uintptr{getg().m.waitsema, getg().m.resumesema})),
686				0, uintptr(ms))
687			if result != _WAIT_OBJECT_0+1 {
688				// Not a suspend/resume event
689				break
690			}
691			elapsed = nanotime() - start
692			if elapsed >= ns {
693				return -1
694			}
695		}
696	}
697	switch result {
698	case _WAIT_OBJECT_0: // Signaled
699		return 0
700
701	case _WAIT_TIMEOUT:
702		return -1
703
704	case _WAIT_ABANDONED:
705		systemstack(func() {
706			throw("runtime.semasleep wait_abandoned")
707		})
708
709	case _WAIT_FAILED:
710		systemstack(func() {
711			print("runtime: waitforsingleobject wait_failed; errno=", getlasterror(), "\n")
712			throw("runtime.semasleep wait_failed")
713		})
714
715	default:
716		systemstack(func() {
717			print("runtime: waitforsingleobject unexpected; result=", result, "\n")
718			throw("runtime.semasleep unexpected")
719		})
720	}
721
722	return -1 // unreachable
723}
724
725//go:nosplit
726func semawakeup(mp *m) {
727	if stdcall1(_SetEvent, mp.waitsema) == 0 {
728		systemstack(func() {
729			print("runtime: setevent failed; errno=", getlasterror(), "\n")
730			throw("runtime.semawakeup")
731		})
732	}
733}
734
735//go:nosplit
736func semacreate(mp *m) {
737	if mp.waitsema != 0 {
738		return
739	}
740	mp.waitsema = stdcall4(_CreateEventA, 0, 0, 0, 0)
741	if mp.waitsema == 0 {
742		systemstack(func() {
743			print("runtime: createevent failed; errno=", getlasterror(), "\n")
744			throw("runtime.semacreate")
745		})
746	}
747	mp.resumesema = stdcall4(_CreateEventA, 0, 0, 0, 0)
748	if mp.resumesema == 0 {
749		systemstack(func() {
750			print("runtime: createevent failed; errno=", getlasterror(), "\n")
751			throw("runtime.semacreate")
752		})
753		stdcall1(_CloseHandle, mp.waitsema)
754		mp.waitsema = 0
755	}
756}
757
758// May run with m.p==nil, so write barriers are not allowed. This
759// function is called by newosproc0, so it is also required to
760// operate without stack guards.
761//
762//go:nowritebarrierrec
763//go:nosplit
764func newosproc(mp *m) {
765	// We pass 0 for the stack size to use the default for this binary.
766	thandle := stdcall6(_CreateThread, 0, 0,
767		abi.FuncPCABI0(tstart_stdcall), uintptr(unsafe.Pointer(mp)),
768		0, 0)
769
770	if thandle == 0 {
771		if atomic.Load(&exiting) != 0 {
772			// CreateThread may fail if called
773			// concurrently with ExitProcess. If this
774			// happens, just freeze this thread and let
775			// the process exit. See issue #18253.
776			lock(&deadlock)
777			lock(&deadlock)
778		}
779		print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", getlasterror(), ")\n")
780		throw("runtime.newosproc")
781	}
782
783	// Close thandle to avoid leaking the thread object if it exits.
784	stdcall1(_CloseHandle, thandle)
785}
786
787// Used by the C library build mode. On Linux this function would allocate a
788// stack, but that's not necessary for Windows. No stack guards are present
789// and the GC has not been initialized, so write barriers will fail.
790//
791//go:nowritebarrierrec
792//go:nosplit
793func newosproc0(mp *m, stk unsafe.Pointer) {
794	// TODO: this is completely broken. The args passed to newosproc0 (in asm_amd64.s)
795	// are stacksize and function, not *m and stack.
796	// Check os_linux.go for an implementation that might actually work.
797	throw("bad newosproc0")
798}
799
800func exitThread(wait *atomic.Uint32) {
801	// We should never reach exitThread on Windows because we let
802	// the OS clean up threads.
803	throw("exitThread")
804}
805
806// Called to initialize a new m (including the bootstrap m).
807// Called on the parent thread (main thread in case of bootstrap), can allocate memory.
808func mpreinit(mp *m) {
809}
810
811//go:nosplit
812func sigsave(p *sigset) {
813}
814
815//go:nosplit
816func msigrestore(sigmask sigset) {
817}
818
819//go:nosplit
820//go:nowritebarrierrec
821func clearSignalHandlers() {
822}
823
824//go:nosplit
825func sigblock(exiting bool) {
826}
827
828// Called to initialize a new m (including the bootstrap m).
829// Called on the new thread, cannot allocate Go memory.
830func minit() {
831	var thandle uintptr
832	if stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 {
833		print("runtime.minit: duplicatehandle failed; errno=", getlasterror(), "\n")
834		throw("runtime.minit: duplicatehandle failed")
835	}
836
837	mp := getg().m
838	lock(&mp.threadLock)
839	mp.thread = thandle
840	mp.procid = uint64(stdcall0(_GetCurrentThreadId))
841
842	// Configure usleep timer, if possible.
843	if mp.highResTimer == 0 && haveHighResTimer {
844		mp.highResTimer = createHighResTimer()
845		if mp.highResTimer == 0 {
846			print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n")
847			throw("CreateWaitableTimerEx when creating timer failed")
848		}
849	}
850	if mp.waitIocpHandle == 0 && haveHighResSleep {
851		mp.waitIocpTimer = createHighResTimer()
852		if mp.waitIocpTimer == 0 {
853			print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n")
854			throw("CreateWaitableTimerEx when creating timer failed")
855		}
856		const GENERIC_ALL = 0x10000000
857		errno := stdcall3(_NtCreateWaitCompletionPacket, uintptr(unsafe.Pointer(&mp.waitIocpHandle)), GENERIC_ALL, 0)
858		if mp.waitIocpHandle == 0 {
859			print("runtime: NtCreateWaitCompletionPacket failed; errno=", errno, "\n")
860			throw("NtCreateWaitCompletionPacket failed")
861		}
862	}
863	unlock(&mp.threadLock)
864
865	// Query the true stack base from the OS. Currently we're
866	// running on a small assumed stack.
867	var mbi memoryBasicInformation
868	res := stdcall3(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi))
869	if res == 0 {
870		print("runtime: VirtualQuery failed; errno=", getlasterror(), "\n")
871		throw("VirtualQuery for stack base failed")
872	}
873	// The system leaves an 8K PAGE_GUARD region at the bottom of
874	// the stack (in theory VirtualQuery isn't supposed to include
875	// that, but it does). Add an additional 8K of slop for
876	// calling C functions that don't have stack checks and for
877	// lastcontinuehandler. We shouldn't be anywhere near this
878	// bound anyway.
879	base := mbi.allocationBase + 16<<10
880	// Sanity check the stack bounds.
881	g0 := getg()
882	if base > g0.stack.hi || g0.stack.hi-base > 64<<20 {
883		print("runtime: g0 stack [", hex(base), ",", hex(g0.stack.hi), ")\n")
884		throw("bad g0 stack")
885	}
886	g0.stack.lo = base
887	g0.stackguard0 = g0.stack.lo + stackGuard
888	g0.stackguard1 = g0.stackguard0
889	// Sanity check the SP.
890	stackcheck()
891}
892
893// Called from dropm to undo the effect of an minit.
894//
895//go:nosplit
896func unminit() {
897	mp := getg().m
898	lock(&mp.threadLock)
899	if mp.thread != 0 {
900		stdcall1(_CloseHandle, mp.thread)
901		mp.thread = 0
902	}
903	unlock(&mp.threadLock)
904
905	mp.procid = 0
906}
907
908// Called from exitm, but not from drop, to undo the effect of thread-owned
909// resources in minit, semacreate, or elsewhere. Do not take locks after calling this.
910//
911//go:nosplit
912func mdestroy(mp *m) {
913	if mp.highResTimer != 0 {
914		stdcall1(_CloseHandle, mp.highResTimer)
915		mp.highResTimer = 0
916	}
917	if mp.waitIocpTimer != 0 {
918		stdcall1(_CloseHandle, mp.waitIocpTimer)
919		mp.waitIocpTimer = 0
920	}
921	if mp.waitIocpHandle != 0 {
922		stdcall1(_CloseHandle, mp.waitIocpHandle)
923		mp.waitIocpHandle = 0
924	}
925	if mp.waitsema != 0 {
926		stdcall1(_CloseHandle, mp.waitsema)
927		mp.waitsema = 0
928	}
929	if mp.resumesema != 0 {
930		stdcall1(_CloseHandle, mp.resumesema)
931		mp.resumesema = 0
932	}
933}
934
935// asmstdcall_trampoline calls asmstdcall converting from Go to C calling convention.
936func asmstdcall_trampoline(args unsafe.Pointer)
937
938// stdcall_no_g calls asmstdcall on os stack without using g.
939//
940//go:nosplit
941func stdcall_no_g(fn stdFunction, n int, args uintptr) uintptr {
942	libcall := libcall{
943		fn:   uintptr(unsafe.Pointer(fn)),
944		n:    uintptr(n),
945		args: args,
946	}
947	asmstdcall_trampoline(noescape(unsafe.Pointer(&libcall)))
948	return libcall.r1
949}
950
951// Calling stdcall on os stack.
952// May run during STW, so write barriers are not allowed.
953//
954//go:nowritebarrier
955//go:nosplit
956func stdcall(fn stdFunction) uintptr {
957	gp := getg()
958	mp := gp.m
959	mp.libcall.fn = uintptr(unsafe.Pointer(fn))
960	resetLibcall := false
961	if mp.profilehz != 0 && mp.libcallsp == 0 {
962		// leave pc/sp for cpu profiler
963		mp.libcallg.set(gp)
964		mp.libcallpc = getcallerpc()
965		// sp must be the last, because once async cpu profiler finds
966		// all three values to be non-zero, it will use them
967		mp.libcallsp = getcallersp()
968		resetLibcall = true // See comment in sys_darwin.go:libcCall
969	}
970	asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall))
971	if resetLibcall {
972		mp.libcallsp = 0
973	}
974	return mp.libcall.r1
975}
976
977//go:nosplit
978func stdcall0(fn stdFunction) uintptr {
979	mp := getg().m
980	mp.libcall.n = 0
981	mp.libcall.args = 0
982	return stdcall(fn)
983}
984
985//go:nosplit
986//go:cgo_unsafe_args
987func stdcall1(fn stdFunction, a0 uintptr) uintptr {
988	mp := getg().m
989	mp.libcall.n = 1
990	mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
991	return stdcall(fn)
992}
993
994//go:nosplit
995//go:cgo_unsafe_args
996func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr {
997	mp := getg().m
998	mp.libcall.n = 2
999	mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1000	return stdcall(fn)
1001}
1002
1003//go:nosplit
1004//go:cgo_unsafe_args
1005func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr {
1006	mp := getg().m
1007	mp.libcall.n = 3
1008	mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1009	return stdcall(fn)
1010}
1011
1012//go:nosplit
1013//go:cgo_unsafe_args
1014func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr {
1015	mp := getg().m
1016	mp.libcall.n = 4
1017	mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1018	return stdcall(fn)
1019}
1020
1021//go:nosplit
1022//go:cgo_unsafe_args
1023func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr {
1024	mp := getg().m
1025	mp.libcall.n = 5
1026	mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1027	return stdcall(fn)
1028}
1029
1030//go:nosplit
1031//go:cgo_unsafe_args
1032func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr {
1033	mp := getg().m
1034	mp.libcall.n = 6
1035	mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1036	return stdcall(fn)
1037}
1038
1039//go:nosplit
1040//go:cgo_unsafe_args
1041func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr {
1042	mp := getg().m
1043	mp.libcall.n = 7
1044	mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1045	return stdcall(fn)
1046}
1047
1048//go:nosplit
1049//go:cgo_unsafe_args
1050func stdcall8(fn stdFunction, a0, a1, a2, a3, a4, a5, a6, a7 uintptr) uintptr {
1051	mp := getg().m
1052	mp.libcall.n = 8
1053	mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0)))
1054	return stdcall(fn)
1055}
1056
1057// These must run on the system stack only.
1058
1059//go:nosplit
1060func osyield_no_g() {
1061	stdcall_no_g(_SwitchToThread, 0, 0)
1062}
1063
1064//go:nosplit
1065func osyield() {
1066	systemstack(func() {
1067		stdcall0(_SwitchToThread)
1068	})
1069}
1070
1071//go:nosplit
1072func usleep_no_g(us uint32) {
1073	timeout := uintptr(us) / 1000 // ms units
1074	args := [...]uintptr{_INVALID_HANDLE_VALUE, timeout}
1075	stdcall_no_g(_WaitForSingleObject, len(args), uintptr(noescape(unsafe.Pointer(&args[0]))))
1076}
1077
1078//go:nosplit
1079func usleep(us uint32) {
1080	systemstack(func() {
1081		var h, timeout uintptr
1082		// If the high-res timer is available and its handle has been allocated for this m, use it.
1083		// Otherwise fall back to the low-res one, which doesn't need a handle.
1084		if haveHighResTimer && getg().m.highResTimer != 0 {
1085			h = getg().m.highResTimer
1086			dt := -10 * int64(us) // relative sleep (negative), 100ns units
1087			stdcall6(_SetWaitableTimer, h, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0)
1088			timeout = _INFINITE
1089		} else {
1090			h = _INVALID_HANDLE_VALUE
1091			timeout = uintptr(us) / 1000 // ms units
1092		}
1093		stdcall2(_WaitForSingleObject, h, timeout)
1094	})
1095}
1096
1097func ctrlHandler(_type uint32) uintptr {
1098	var s uint32
1099
1100	switch _type {
1101	case _CTRL_C_EVENT, _CTRL_BREAK_EVENT:
1102		s = _SIGINT
1103	case _CTRL_CLOSE_EVENT, _CTRL_LOGOFF_EVENT, _CTRL_SHUTDOWN_EVENT:
1104		s = _SIGTERM
1105	default:
1106		return 0
1107	}
1108
1109	if sigsend(s) {
1110		if s == _SIGTERM {
1111			// Windows terminates the process after this handler returns.
1112			// Block indefinitely to give signal handlers a chance to clean up,
1113			// but make sure to be properly parked first, so the rest of the
1114			// program can continue executing.
1115			block()
1116		}
1117		return 1
1118	}
1119	return 0
1120}
1121
1122// called from zcallback_windows_*.s to sys_windows_*.s
1123func callbackasm1()
1124
1125var profiletimer uintptr
1126
1127func profilem(mp *m, thread uintptr) {
1128	// Align Context to 16 bytes.
1129	var c *context
1130	var cbuf [unsafe.Sizeof(*c) + 15]byte
1131	c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15))
1132
1133	c.contextflags = _CONTEXT_CONTROL
1134	stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c)))
1135
1136	gp := gFromSP(mp, c.sp())
1137
1138	sigprof(c.ip(), c.sp(), c.lr(), gp, mp)
1139}
1140
1141func gFromSP(mp *m, sp uintptr) *g {
1142	if gp := mp.g0; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
1143		return gp
1144	}
1145	if gp := mp.gsignal; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
1146		return gp
1147	}
1148	if gp := mp.curg; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi {
1149		return gp
1150	}
1151	return nil
1152}
1153
1154func profileLoop() {
1155	stdcall2(_SetThreadPriority, currentThread, _THREAD_PRIORITY_HIGHEST)
1156
1157	for {
1158		stdcall2(_WaitForSingleObject, profiletimer, _INFINITE)
1159		first := (*m)(atomic.Loadp(unsafe.Pointer(&allm)))
1160		for mp := first; mp != nil; mp = mp.alllink {
1161			if mp == getg().m {
1162				// Don't profile ourselves.
1163				continue
1164			}
1165
1166			lock(&mp.threadLock)
1167			// Do not profile threads blocked on Notes,
1168			// this includes idle worker threads,
1169			// idle timer thread, idle heap scavenger, etc.
1170			if mp.thread == 0 || mp.profilehz == 0 || mp.blocked {
1171				unlock(&mp.threadLock)
1172				continue
1173			}
1174			// Acquire our own handle to the thread.
1175			var thread uintptr
1176			if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 {
1177				print("runtime: duplicatehandle failed; errno=", getlasterror(), "\n")
1178				throw("duplicatehandle failed")
1179			}
1180			unlock(&mp.threadLock)
1181
1182			// mp may exit between the DuplicateHandle
1183			// above and the SuspendThread. The handle
1184			// will remain valid, but SuspendThread may
1185			// fail.
1186			if int32(stdcall1(_SuspendThread, thread)) == -1 {
1187				// The thread no longer exists.
1188				stdcall1(_CloseHandle, thread)
1189				continue
1190			}
1191			if mp.profilehz != 0 && !mp.blocked {
1192				// Pass the thread handle in case mp
1193				// was in the process of shutting down.
1194				profilem(mp, thread)
1195			}
1196			stdcall1(_ResumeThread, thread)
1197			stdcall1(_CloseHandle, thread)
1198		}
1199	}
1200}
1201
1202func setProcessCPUProfiler(hz int32) {
1203	if profiletimer == 0 {
1204		var timer uintptr
1205		if haveHighResTimer {
1206			timer = createHighResTimer()
1207		} else {
1208			timer = stdcall3(_CreateWaitableTimerA, 0, 0, 0)
1209		}
1210		atomic.Storeuintptr(&profiletimer, timer)
1211		newm(profileLoop, nil, -1)
1212	}
1213}
1214
1215func setThreadCPUProfiler(hz int32) {
1216	ms := int32(0)
1217	due := ^int64(^uint64(1 << 63))
1218	if hz > 0 {
1219		ms = 1000 / hz
1220		if ms == 0 {
1221			ms = 1
1222		}
1223		due = int64(ms) * -10000
1224	}
1225	stdcall6(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0)
1226	atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz))
1227}
1228
1229const preemptMSupported = true
1230
1231// suspendLock protects simultaneous SuspendThread operations from
1232// suspending each other.
1233var suspendLock mutex
1234
1235func preemptM(mp *m) {
1236	if mp == getg().m {
1237		throw("self-preempt")
1238	}
1239
1240	// Synchronize with external code that may try to ExitProcess.
1241	if !atomic.Cas(&mp.preemptExtLock, 0, 1) {
1242		// External code is running. Fail the preemption
1243		// attempt.
1244		mp.preemptGen.Add(1)
1245		return
1246	}
1247
1248	// Acquire our own handle to mp's thread.
1249	lock(&mp.threadLock)
1250	if mp.thread == 0 {
1251		// The M hasn't been minit'd yet (or was just unminit'd).
1252		unlock(&mp.threadLock)
1253		atomic.Store(&mp.preemptExtLock, 0)
1254		mp.preemptGen.Add(1)
1255		return
1256	}
1257	var thread uintptr
1258	if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 {
1259		print("runtime.preemptM: duplicatehandle failed; errno=", getlasterror(), "\n")
1260		throw("runtime.preemptM: duplicatehandle failed")
1261	}
1262	unlock(&mp.threadLock)
1263
1264	// Prepare thread context buffer. This must be aligned to 16 bytes.
1265	var c *context
1266	var cbuf [unsafe.Sizeof(*c) + 15]byte
1267	c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15))
1268	c.contextflags = _CONTEXT_CONTROL
1269
1270	// Serialize thread suspension. SuspendThread is asynchronous,
1271	// so it's otherwise possible for two threads to suspend each
1272	// other and deadlock. We must hold this lock until after
1273	// GetThreadContext, since that blocks until the thread is
1274	// actually suspended.
1275	lock(&suspendLock)
1276
1277	// Suspend the thread.
1278	if int32(stdcall1(_SuspendThread, thread)) == -1 {
1279		unlock(&suspendLock)
1280		stdcall1(_CloseHandle, thread)
1281		atomic.Store(&mp.preemptExtLock, 0)
1282		// The thread no longer exists. This shouldn't be
1283		// possible, but just acknowledge the request.
1284		mp.preemptGen.Add(1)
1285		return
1286	}
1287
1288	// We have to be very careful between this point and once
1289	// we've shown mp is at an async safe-point. This is like a
1290	// signal handler in the sense that mp could have been doing
1291	// anything when we stopped it, including holding arbitrary
1292	// locks.
1293
1294	// We have to get the thread context before inspecting the M
1295	// because SuspendThread only requests a suspend.
1296	// GetThreadContext actually blocks until it's suspended.
1297	stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c)))
1298
1299	unlock(&suspendLock)
1300
1301	// Does it want a preemption and is it safe to preempt?
1302	gp := gFromSP(mp, c.sp())
1303	if gp != nil && wantAsyncPreempt(gp) {
1304		if ok, newpc := isAsyncSafePoint(gp, c.ip(), c.sp(), c.lr()); ok {
1305			// Inject call to asyncPreempt
1306			targetPC := abi.FuncPCABI0(asyncPreempt)
1307			switch GOARCH {
1308			default:
1309				throw("unsupported architecture")
1310			case "386", "amd64":
1311				// Make it look like the thread called targetPC.
1312				sp := c.sp()
1313				sp -= goarch.PtrSize
1314				*(*uintptr)(unsafe.Pointer(sp)) = newpc
1315				c.set_sp(sp)
1316				c.set_ip(targetPC)
1317
1318			case "arm":
1319				// Push LR. The injected call is responsible
1320				// for restoring LR. gentraceback is aware of
1321				// this extra slot. See sigctxt.pushCall in
1322				// signal_arm.go, which is similar except we
1323				// subtract 1 from IP here.
1324				sp := c.sp()
1325				sp -= goarch.PtrSize
1326				c.set_sp(sp)
1327				*(*uint32)(unsafe.Pointer(sp)) = uint32(c.lr())
1328				c.set_lr(newpc - 1)
1329				c.set_ip(targetPC)
1330
1331			case "arm64":
1332				// Push LR. The injected call is responsible
1333				// for restoring LR. gentraceback is aware of
1334				// this extra slot. See sigctxt.pushCall in
1335				// signal_arm64.go.
1336				sp := c.sp() - 16 // SP needs 16-byte alignment
1337				c.set_sp(sp)
1338				*(*uint64)(unsafe.Pointer(sp)) = uint64(c.lr())
1339				c.set_lr(newpc)
1340				c.set_ip(targetPC)
1341			}
1342			stdcall2(_SetThreadContext, thread, uintptr(unsafe.Pointer(c)))
1343		}
1344	}
1345
1346	atomic.Store(&mp.preemptExtLock, 0)
1347
1348	// Acknowledge the preemption.
1349	mp.preemptGen.Add(1)
1350
1351	stdcall1(_ResumeThread, thread)
1352	stdcall1(_CloseHandle, thread)
1353}
1354
1355// osPreemptExtEnter is called before entering external code that may
1356// call ExitProcess.
1357//
1358// This must be nosplit because it may be called from a syscall with
1359// untyped stack slots, so the stack must not be grown or scanned.
1360//
1361//go:nosplit
1362func osPreemptExtEnter(mp *m) {
1363	for !atomic.Cas(&mp.preemptExtLock, 0, 1) {
1364		// An asynchronous preemption is in progress. It's not
1365		// safe to enter external code because it may call
1366		// ExitProcess and deadlock with SuspendThread.
1367		// Ideally we would do the preemption ourselves, but
1368		// can't since there may be untyped syscall arguments
1369		// on the stack. Instead, just wait and encourage the
1370		// SuspendThread APC to run. The preemption should be
1371		// done shortly.
1372		osyield()
1373	}
1374	// Asynchronous preemption is now blocked.
1375}
1376
1377// osPreemptExtExit is called after returning from external code that
1378// may call ExitProcess.
1379//
1380// See osPreemptExtEnter for why this is nosplit.
1381//
1382//go:nosplit
1383func osPreemptExtExit(mp *m) {
1384	atomic.Store(&mp.preemptExtLock, 0)
1385}
1386