1// Copyright 2009 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package runtime 6 7import ( 8 "internal/abi" 9 "internal/goarch" 10 "internal/runtime/atomic" 11 "unsafe" 12) 13 14// TODO(brainman): should not need those 15const ( 16 _NSIG = 65 17) 18 19//go:cgo_import_dynamic runtime._AddVectoredContinueHandler AddVectoredContinueHandler%2 "kernel32.dll" 20//go:cgo_import_dynamic runtime._AddVectoredExceptionHandler AddVectoredExceptionHandler%2 "kernel32.dll" 21//go:cgo_import_dynamic runtime._CloseHandle CloseHandle%1 "kernel32.dll" 22//go:cgo_import_dynamic runtime._CreateEventA CreateEventA%4 "kernel32.dll" 23//go:cgo_import_dynamic runtime._CreateIoCompletionPort CreateIoCompletionPort%4 "kernel32.dll" 24//go:cgo_import_dynamic runtime._CreateThread CreateThread%6 "kernel32.dll" 25//go:cgo_import_dynamic runtime._CreateWaitableTimerA CreateWaitableTimerA%3 "kernel32.dll" 26//go:cgo_import_dynamic runtime._CreateWaitableTimerExW CreateWaitableTimerExW%4 "kernel32.dll" 27//go:cgo_import_dynamic runtime._DuplicateHandle DuplicateHandle%7 "kernel32.dll" 28//go:cgo_import_dynamic runtime._ExitProcess ExitProcess%1 "kernel32.dll" 29//go:cgo_import_dynamic runtime._FreeEnvironmentStringsW FreeEnvironmentStringsW%1 "kernel32.dll" 30//go:cgo_import_dynamic runtime._GetConsoleMode GetConsoleMode%2 "kernel32.dll" 31//go:cgo_import_dynamic runtime._GetCurrentThreadId GetCurrentThreadId%0 "kernel32.dll" 32//go:cgo_import_dynamic runtime._GetEnvironmentStringsW GetEnvironmentStringsW%0 "kernel32.dll" 33//go:cgo_import_dynamic runtime._GetErrorMode GetErrorMode%0 "kernel32.dll" 34//go:cgo_import_dynamic runtime._GetProcAddress GetProcAddress%2 "kernel32.dll" 35//go:cgo_import_dynamic runtime._GetProcessAffinityMask GetProcessAffinityMask%3 "kernel32.dll" 36//go:cgo_import_dynamic runtime._GetQueuedCompletionStatusEx GetQueuedCompletionStatusEx%6 "kernel32.dll" 37//go:cgo_import_dynamic runtime._GetStdHandle GetStdHandle%1 "kernel32.dll" 38//go:cgo_import_dynamic runtime._GetSystemDirectoryA GetSystemDirectoryA%2 "kernel32.dll" 39//go:cgo_import_dynamic runtime._GetSystemInfo GetSystemInfo%1 "kernel32.dll" 40//go:cgo_import_dynamic runtime._GetThreadContext GetThreadContext%2 "kernel32.dll" 41//go:cgo_import_dynamic runtime._SetThreadContext SetThreadContext%2 "kernel32.dll" 42//go:cgo_import_dynamic runtime._LoadLibraryExW LoadLibraryExW%3 "kernel32.dll" 43//go:cgo_import_dynamic runtime._LoadLibraryW LoadLibraryW%1 "kernel32.dll" 44//go:cgo_import_dynamic runtime._PostQueuedCompletionStatus PostQueuedCompletionStatus%4 "kernel32.dll" 45//go:cgo_import_dynamic runtime._QueryPerformanceCounter QueryPerformanceCounter%1 "kernel32.dll" 46//go:cgo_import_dynamic runtime._QueryPerformanceFrequency QueryPerformanceFrequency%1 "kernel32.dll" 47//go:cgo_import_dynamic runtime._RaiseFailFastException RaiseFailFastException%3 "kernel32.dll" 48//go:cgo_import_dynamic runtime._ResumeThread ResumeThread%1 "kernel32.dll" 49//go:cgo_import_dynamic runtime._RtlLookupFunctionEntry RtlLookupFunctionEntry%3 "kernel32.dll" 50//go:cgo_import_dynamic runtime._RtlVirtualUnwind RtlVirtualUnwind%8 "kernel32.dll" 51//go:cgo_import_dynamic runtime._SetConsoleCtrlHandler SetConsoleCtrlHandler%2 "kernel32.dll" 52//go:cgo_import_dynamic runtime._SetErrorMode SetErrorMode%1 "kernel32.dll" 53//go:cgo_import_dynamic runtime._SetEvent SetEvent%1 "kernel32.dll" 54//go:cgo_import_dynamic runtime._SetProcessPriorityBoost SetProcessPriorityBoost%2 "kernel32.dll" 55//go:cgo_import_dynamic runtime._SetThreadPriority SetThreadPriority%2 "kernel32.dll" 56//go:cgo_import_dynamic runtime._SetUnhandledExceptionFilter SetUnhandledExceptionFilter%1 "kernel32.dll" 57//go:cgo_import_dynamic runtime._SetWaitableTimer SetWaitableTimer%6 "kernel32.dll" 58//go:cgo_import_dynamic runtime._SuspendThread SuspendThread%1 "kernel32.dll" 59//go:cgo_import_dynamic runtime._SwitchToThread SwitchToThread%0 "kernel32.dll" 60//go:cgo_import_dynamic runtime._TlsAlloc TlsAlloc%0 "kernel32.dll" 61//go:cgo_import_dynamic runtime._VirtualAlloc VirtualAlloc%4 "kernel32.dll" 62//go:cgo_import_dynamic runtime._VirtualFree VirtualFree%3 "kernel32.dll" 63//go:cgo_import_dynamic runtime._VirtualQuery VirtualQuery%3 "kernel32.dll" 64//go:cgo_import_dynamic runtime._WaitForSingleObject WaitForSingleObject%2 "kernel32.dll" 65//go:cgo_import_dynamic runtime._WaitForMultipleObjects WaitForMultipleObjects%4 "kernel32.dll" 66//go:cgo_import_dynamic runtime._WerGetFlags WerGetFlags%2 "kernel32.dll" 67//go:cgo_import_dynamic runtime._WerSetFlags WerSetFlags%1 "kernel32.dll" 68//go:cgo_import_dynamic runtime._WriteConsoleW WriteConsoleW%5 "kernel32.dll" 69//go:cgo_import_dynamic runtime._WriteFile WriteFile%5 "kernel32.dll" 70 71type stdFunction unsafe.Pointer 72 73var ( 74 // Following syscalls are available on every Windows PC. 75 // All these variables are set by the Windows executable 76 // loader before the Go program starts. 77 _AddVectoredContinueHandler, 78 _AddVectoredExceptionHandler, 79 _CloseHandle, 80 _CreateEventA, 81 _CreateIoCompletionPort, 82 _CreateThread, 83 _CreateWaitableTimerA, 84 _CreateWaitableTimerExW, 85 _DuplicateHandle, 86 _ExitProcess, 87 _FreeEnvironmentStringsW, 88 _GetConsoleMode, 89 _GetCurrentThreadId, 90 _GetEnvironmentStringsW, 91 _GetErrorMode, 92 _GetProcAddress, 93 _GetProcessAffinityMask, 94 _GetQueuedCompletionStatusEx, 95 _GetStdHandle, 96 _GetSystemDirectoryA, 97 _GetSystemInfo, 98 _GetThreadContext, 99 _SetThreadContext, 100 _LoadLibraryExW, 101 _LoadLibraryW, 102 _PostQueuedCompletionStatus, 103 _QueryPerformanceCounter, 104 _QueryPerformanceFrequency, 105 _RaiseFailFastException, 106 _ResumeThread, 107 _RtlLookupFunctionEntry, 108 _RtlVirtualUnwind, 109 _SetConsoleCtrlHandler, 110 _SetErrorMode, 111 _SetEvent, 112 _SetProcessPriorityBoost, 113 _SetThreadPriority, 114 _SetUnhandledExceptionFilter, 115 _SetWaitableTimer, 116 _SuspendThread, 117 _SwitchToThread, 118 _TlsAlloc, 119 _VirtualAlloc, 120 _VirtualFree, 121 _VirtualQuery, 122 _WaitForSingleObject, 123 _WaitForMultipleObjects, 124 _WerGetFlags, 125 _WerSetFlags, 126 _WriteConsoleW, 127 _WriteFile, 128 _ stdFunction 129 130 // Use ProcessPrng to generate cryptographically random data. 131 _ProcessPrng stdFunction 132 133 // Load ntdll.dll manually during startup, otherwise Mingw 134 // links wrong printf function to cgo executable (see issue 135 // 12030 for details). 136 _NtCreateWaitCompletionPacket stdFunction 137 _NtAssociateWaitCompletionPacket stdFunction 138 _NtCancelWaitCompletionPacket stdFunction 139 _RtlGetCurrentPeb stdFunction 140 _RtlGetVersion stdFunction 141 142 // These are from non-kernel32.dll, so we prefer to LoadLibraryEx them. 143 _timeBeginPeriod, 144 _timeEndPeriod, 145 _ stdFunction 146) 147 148var ( 149 bcryptprimitivesdll = [...]uint16{'b', 'c', 'r', 'y', 'p', 't', 'p', 'r', 'i', 'm', 'i', 't', 'i', 'v', 'e', 's', '.', 'd', 'l', 'l', 0} 150 ntdlldll = [...]uint16{'n', 't', 'd', 'l', 'l', '.', 'd', 'l', 'l', 0} 151 powrprofdll = [...]uint16{'p', 'o', 'w', 'r', 'p', 'r', 'o', 'f', '.', 'd', 'l', 'l', 0} 152 winmmdll = [...]uint16{'w', 'i', 'n', 'm', 'm', '.', 'd', 'l', 'l', 0} 153) 154 155// Function to be called by windows CreateThread 156// to start new os thread. 157func tstart_stdcall(newm *m) 158 159// Init-time helper 160func wintls() 161 162type mOS struct { 163 threadLock mutex // protects "thread" and prevents closing 164 thread uintptr // thread handle 165 166 waitsema uintptr // semaphore for parking on locks 167 resumesema uintptr // semaphore to indicate suspend/resume 168 169 highResTimer uintptr // high resolution timer handle used in usleep 170 waitIocpTimer uintptr // high resolution timer handle used in netpoll 171 waitIocpHandle uintptr // wait completion handle used in netpoll 172 173 // preemptExtLock synchronizes preemptM with entry/exit from 174 // external C code. 175 // 176 // This protects against races between preemptM calling 177 // SuspendThread and external code on this thread calling 178 // ExitProcess. If these happen concurrently, it's possible to 179 // exit the suspending thread and suspend the exiting thread, 180 // leading to deadlock. 181 // 182 // 0 indicates this M is not being preempted or in external 183 // code. Entering external code CASes this from 0 to 1. If 184 // this fails, a preemption is in progress, so the thread must 185 // wait for the preemption. preemptM also CASes this from 0 to 186 // 1. If this fails, the preemption fails (as it would if the 187 // PC weren't in Go code). The value is reset to 0 when 188 // returning from external code or after a preemption is 189 // complete. 190 // 191 // TODO(austin): We may not need this if preemption were more 192 // tightly synchronized on the G/P status and preemption 193 // blocked transition into _Gsyscall/_Psyscall. 194 preemptExtLock uint32 195} 196 197// Stubs so tests can link correctly. These should never be called. 198func open(name *byte, mode, perm int32) int32 { 199 throw("unimplemented") 200 return -1 201} 202func closefd(fd int32) int32 { 203 throw("unimplemented") 204 return -1 205} 206func read(fd int32, p unsafe.Pointer, n int32) int32 { 207 throw("unimplemented") 208 return -1 209} 210 211type sigset struct{} 212 213// Call a Windows function with stdcall conventions, 214// and switch to os stack during the call. 215func asmstdcall(fn unsafe.Pointer) 216 217var asmstdcallAddr unsafe.Pointer 218 219type winlibcall libcall 220 221func windowsFindfunc(lib uintptr, name []byte) stdFunction { 222 if name[len(name)-1] != 0 { 223 throw("usage") 224 } 225 f := stdcall2(_GetProcAddress, lib, uintptr(unsafe.Pointer(&name[0]))) 226 return stdFunction(unsafe.Pointer(f)) 227} 228 229const _MAX_PATH = 260 // https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation 230var sysDirectory [_MAX_PATH + 1]byte 231var sysDirectoryLen uintptr 232 233func initSysDirectory() { 234 l := stdcall2(_GetSystemDirectoryA, uintptr(unsafe.Pointer(&sysDirectory[0])), uintptr(len(sysDirectory)-1)) 235 if l == 0 || l > uintptr(len(sysDirectory)-1) { 236 throw("Unable to determine system directory") 237 } 238 sysDirectory[l] = '\\' 239 sysDirectoryLen = l + 1 240} 241 242//go:linkname windows_GetSystemDirectory internal/syscall/windows.GetSystemDirectory 243func windows_GetSystemDirectory() string { 244 return unsafe.String(&sysDirectory[0], sysDirectoryLen) 245} 246 247func windowsLoadSystemLib(name []uint16) uintptr { 248 return stdcall3(_LoadLibraryExW, uintptr(unsafe.Pointer(&name[0])), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32) 249} 250 251//go:linkname windows_QueryPerformanceCounter internal/syscall/windows.QueryPerformanceCounter 252func windows_QueryPerformanceCounter() int64 { 253 var counter int64 254 stdcall1(_QueryPerformanceCounter, uintptr(unsafe.Pointer(&counter))) 255 return counter 256} 257 258//go:linkname windows_QueryPerformanceFrequency internal/syscall/windows.QueryPerformanceFrequency 259func windows_QueryPerformanceFrequency() int64 { 260 var frequency int64 261 stdcall1(_QueryPerformanceFrequency, uintptr(unsafe.Pointer(&frequency))) 262 return frequency 263} 264 265func loadOptionalSyscalls() { 266 bcryptPrimitives := windowsLoadSystemLib(bcryptprimitivesdll[:]) 267 if bcryptPrimitives == 0 { 268 throw("bcryptprimitives.dll not found") 269 } 270 _ProcessPrng = windowsFindfunc(bcryptPrimitives, []byte("ProcessPrng\000")) 271 272 n32 := windowsLoadSystemLib(ntdlldll[:]) 273 if n32 == 0 { 274 throw("ntdll.dll not found") 275 } 276 _NtCreateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCreateWaitCompletionPacket\000")) 277 if _NtCreateWaitCompletionPacket != nil { 278 // These functions should exists if NtCreateWaitCompletionPacket exists. 279 _NtAssociateWaitCompletionPacket = windowsFindfunc(n32, []byte("NtAssociateWaitCompletionPacket\000")) 280 if _NtAssociateWaitCompletionPacket == nil { 281 throw("NtCreateWaitCompletionPacket exists but NtAssociateWaitCompletionPacket does not") 282 } 283 _NtCancelWaitCompletionPacket = windowsFindfunc(n32, []byte("NtCancelWaitCompletionPacket\000")) 284 if _NtCancelWaitCompletionPacket == nil { 285 throw("NtCreateWaitCompletionPacket exists but NtCancelWaitCompletionPacket does not") 286 } 287 } 288 _RtlGetCurrentPeb = windowsFindfunc(n32, []byte("RtlGetCurrentPeb\000")) 289 _RtlGetVersion = windowsFindfunc(n32, []byte("RtlGetVersion\000")) 290} 291 292func monitorSuspendResume() { 293 const ( 294 _DEVICE_NOTIFY_CALLBACK = 2 295 ) 296 type _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS struct { 297 callback uintptr 298 context uintptr 299 } 300 301 powrprof := windowsLoadSystemLib(powrprofdll[:]) 302 if powrprof == 0 { 303 return // Running on Windows 7, where we don't need it anyway. 304 } 305 powerRegisterSuspendResumeNotification := windowsFindfunc(powrprof, []byte("PowerRegisterSuspendResumeNotification\000")) 306 if powerRegisterSuspendResumeNotification == nil { 307 return // Running on Windows 7, where we don't need it anyway. 308 } 309 var fn any = func(context uintptr, changeType uint32, setting uintptr) uintptr { 310 for mp := (*m)(atomic.Loadp(unsafe.Pointer(&allm))); mp != nil; mp = mp.alllink { 311 if mp.resumesema != 0 { 312 stdcall1(_SetEvent, mp.resumesema) 313 } 314 } 315 return 0 316 } 317 params := _DEVICE_NOTIFY_SUBSCRIBE_PARAMETERS{ 318 callback: compileCallback(*efaceOf(&fn), true), 319 } 320 handle := uintptr(0) 321 stdcall3(powerRegisterSuspendResumeNotification, _DEVICE_NOTIFY_CALLBACK, 322 uintptr(unsafe.Pointer(¶ms)), uintptr(unsafe.Pointer(&handle))) 323} 324 325func getproccount() int32 { 326 var mask, sysmask uintptr 327 ret := stdcall3(_GetProcessAffinityMask, currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask))) 328 if ret != 0 { 329 n := 0 330 maskbits := int(unsafe.Sizeof(mask) * 8) 331 for i := 0; i < maskbits; i++ { 332 if mask&(1<<uint(i)) != 0 { 333 n++ 334 } 335 } 336 if n != 0 { 337 return int32(n) 338 } 339 } 340 // use GetSystemInfo if GetProcessAffinityMask fails 341 var info systeminfo 342 stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) 343 return int32(info.dwnumberofprocessors) 344} 345 346func getPageSize() uintptr { 347 var info systeminfo 348 stdcall1(_GetSystemInfo, uintptr(unsafe.Pointer(&info))) 349 return uintptr(info.dwpagesize) 350} 351 352const ( 353 currentProcess = ^uintptr(0) // -1 = current process 354 currentThread = ^uintptr(1) // -2 = current thread 355) 356 357// in sys_windows_386.s and sys_windows_amd64.s: 358func getlasterror() uint32 359 360var timeBeginPeriodRetValue uint32 361 362// osRelaxMinNS indicates that sysmon shouldn't osRelax if the next 363// timer is less than 60 ms from now. Since osRelaxing may reduce 364// timer resolution to 15.6 ms, this keeps timer error under roughly 1 365// part in 4. 366const osRelaxMinNS = 60 * 1e6 367 368// osRelax is called by the scheduler when transitioning to and from 369// all Ps being idle. 370// 371// Some versions of Windows have high resolution timer. For those 372// versions osRelax is noop. 373// For Windows versions without high resolution timer, osRelax 374// adjusts the system-wide timer resolution. Go needs a 375// high resolution timer while running and there's little extra cost 376// if we're already using the CPU, but if all Ps are idle there's no 377// need to consume extra power to drive the high-res timer. 378func osRelax(relax bool) uint32 { 379 if haveHighResTimer { 380 // If the high resolution timer is available, the runtime uses the timer 381 // to sleep for short durations. This means there's no need to adjust 382 // the global clock frequency. 383 return 0 384 } 385 386 if relax { 387 return uint32(stdcall1(_timeEndPeriod, 1)) 388 } else { 389 return uint32(stdcall1(_timeBeginPeriod, 1)) 390 } 391} 392 393// haveHighResTimer indicates that the CreateWaitableTimerEx 394// CREATE_WAITABLE_TIMER_HIGH_RESOLUTION flag is available. 395var haveHighResTimer = false 396 397// haveHighResSleep indicates that NtCreateWaitCompletionPacket 398// exists and haveHighResTimer is true. 399// NtCreateWaitCompletionPacket has been available since Windows 10, 400// but has just been publicly documented, so some platforms, like Wine, 401// doesn't support it yet. 402var haveHighResSleep = false 403 404// createHighResTimer calls CreateWaitableTimerEx with 405// CREATE_WAITABLE_TIMER_HIGH_RESOLUTION flag to create high 406// resolution timer. createHighResTimer returns new timer 407// handle or 0, if CreateWaitableTimerEx failed. 408func createHighResTimer() uintptr { 409 const ( 410 // As per @jstarks, see 411 // https://github.com/golang/go/issues/8687#issuecomment-656259353 412 _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION = 0x00000002 413 414 _SYNCHRONIZE = 0x00100000 415 _TIMER_QUERY_STATE = 0x0001 416 _TIMER_MODIFY_STATE = 0x0002 417 ) 418 return stdcall4(_CreateWaitableTimerExW, 0, 0, 419 _CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, 420 _SYNCHRONIZE|_TIMER_QUERY_STATE|_TIMER_MODIFY_STATE) 421} 422 423func initHighResTimer() { 424 h := createHighResTimer() 425 if h != 0 { 426 haveHighResTimer = true 427 haveHighResSleep = _NtCreateWaitCompletionPacket != nil 428 stdcall1(_CloseHandle, h) 429 } else { 430 // Only load winmm.dll if we need it. 431 // This avoids a dependency on winmm.dll for Go programs 432 // that run on new Windows versions. 433 m32 := windowsLoadSystemLib(winmmdll[:]) 434 if m32 == 0 { 435 print("runtime: LoadLibraryExW failed; errno=", getlasterror(), "\n") 436 throw("winmm.dll not found") 437 } 438 _timeBeginPeriod = windowsFindfunc(m32, []byte("timeBeginPeriod\000")) 439 _timeEndPeriod = windowsFindfunc(m32, []byte("timeEndPeriod\000")) 440 if _timeBeginPeriod == nil || _timeEndPeriod == nil { 441 print("runtime: GetProcAddress failed; errno=", getlasterror(), "\n") 442 throw("timeBegin/EndPeriod not found") 443 } 444 } 445} 446 447//go:linkname canUseLongPaths internal/syscall/windows.CanUseLongPaths 448var canUseLongPaths bool 449 450// initLongPathSupport enables long path support. 451func initLongPathSupport() { 452 const ( 453 IsLongPathAwareProcess = 0x80 454 PebBitFieldOffset = 3 455 ) 456 457 // Check that we're ≥ 10.0.15063. 458 info := _OSVERSIONINFOW{} 459 info.osVersionInfoSize = uint32(unsafe.Sizeof(info)) 460 stdcall1(_RtlGetVersion, uintptr(unsafe.Pointer(&info))) 461 if info.majorVersion < 10 || (info.majorVersion == 10 && info.minorVersion == 0 && info.buildNumber < 15063) { 462 return 463 } 464 465 // Set the IsLongPathAwareProcess flag of the PEB's bit field. 466 // This flag is not documented, but it's known to be used 467 // by Windows to enable long path support. 468 bitField := (*byte)(unsafe.Pointer(stdcall0(_RtlGetCurrentPeb) + PebBitFieldOffset)) 469 *bitField |= IsLongPathAwareProcess 470 471 canUseLongPaths = true 472} 473 474func osinit() { 475 asmstdcallAddr = unsafe.Pointer(abi.FuncPCABI0(asmstdcall)) 476 477 loadOptionalSyscalls() 478 479 preventErrorDialogs() 480 481 initExceptionHandler() 482 483 initHighResTimer() 484 timeBeginPeriodRetValue = osRelax(false) 485 486 initSysDirectory() 487 initLongPathSupport() 488 489 ncpu = getproccount() 490 491 physPageSize = getPageSize() 492 493 // Windows dynamic priority boosting assumes that a process has different types 494 // of dedicated threads -- GUI, IO, computational, etc. Go processes use 495 // equivalent threads that all do a mix of GUI, IO, computations, etc. 496 // In such context dynamic priority boosting does nothing but harm, so we turn it off. 497 stdcall2(_SetProcessPriorityBoost, currentProcess, 1) 498} 499 500//go:nosplit 501func readRandom(r []byte) int { 502 n := 0 503 if stdcall2(_ProcessPrng, uintptr(unsafe.Pointer(&r[0])), uintptr(len(r)))&0xff != 0 { 504 n = len(r) 505 } 506 return n 507} 508 509func goenvs() { 510 // strings is a pointer to environment variable pairs in the form: 511 // "envA=valA\x00envB=valB\x00\x00" (in UTF-16) 512 // Two consecutive zero bytes end the list. 513 strings := unsafe.Pointer(stdcall0(_GetEnvironmentStringsW)) 514 p := (*[1 << 24]uint16)(strings)[:] 515 516 n := 0 517 for from, i := 0, 0; true; i++ { 518 if p[i] == 0 { 519 // empty string marks the end 520 if i == from { 521 break 522 } 523 from = i + 1 524 n++ 525 } 526 } 527 envs = make([]string, n) 528 529 for i := range envs { 530 envs[i] = gostringw(&p[0]) 531 for p[0] != 0 { 532 p = p[1:] 533 } 534 p = p[1:] // skip nil byte 535 } 536 537 stdcall1(_FreeEnvironmentStringsW, uintptr(strings)) 538 539 // We call these all the way here, late in init, so that malloc works 540 // for the callback functions these generate. 541 var fn any = ctrlHandler 542 ctrlHandlerPC := compileCallback(*efaceOf(&fn), true) 543 stdcall2(_SetConsoleCtrlHandler, ctrlHandlerPC, 1) 544 545 monitorSuspendResume() 546} 547 548// exiting is set to non-zero when the process is exiting. 549var exiting uint32 550 551//go:nosplit 552func exit(code int32) { 553 // Disallow thread suspension for preemption. Otherwise, 554 // ExitProcess and SuspendThread can race: SuspendThread 555 // queues a suspension request for this thread, ExitProcess 556 // kills the suspending thread, and then this thread suspends. 557 lock(&suspendLock) 558 atomic.Store(&exiting, 1) 559 stdcall1(_ExitProcess, uintptr(code)) 560} 561 562// write1 must be nosplit because it's used as a last resort in 563// functions like badmorestackg0. In such cases, we'll always take the 564// ASCII path. 565// 566//go:nosplit 567func write1(fd uintptr, buf unsafe.Pointer, n int32) int32 { 568 const ( 569 _STD_OUTPUT_HANDLE = ^uintptr(10) // -11 570 _STD_ERROR_HANDLE = ^uintptr(11) // -12 571 ) 572 var handle uintptr 573 switch fd { 574 case 1: 575 handle = stdcall1(_GetStdHandle, _STD_OUTPUT_HANDLE) 576 case 2: 577 handle = stdcall1(_GetStdHandle, _STD_ERROR_HANDLE) 578 default: 579 // assume fd is real windows handle. 580 handle = fd 581 } 582 isASCII := true 583 b := (*[1 << 30]byte)(buf)[:n] 584 for _, x := range b { 585 if x >= 0x80 { 586 isASCII = false 587 break 588 } 589 } 590 591 if !isASCII { 592 var m uint32 593 isConsole := stdcall2(_GetConsoleMode, handle, uintptr(unsafe.Pointer(&m))) != 0 594 // If this is a console output, various non-unicode code pages can be in use. 595 // Use the dedicated WriteConsole call to ensure unicode is printed correctly. 596 if isConsole { 597 return int32(writeConsole(handle, buf, n)) 598 } 599 } 600 var written uint32 601 stdcall5(_WriteFile, handle, uintptr(buf), uintptr(n), uintptr(unsafe.Pointer(&written)), 0) 602 return int32(written) 603} 604 605var ( 606 utf16ConsoleBack [1000]uint16 607 utf16ConsoleBackLock mutex 608) 609 610// writeConsole writes bufLen bytes from buf to the console File. 611// It returns the number of bytes written. 612func writeConsole(handle uintptr, buf unsafe.Pointer, bufLen int32) int { 613 const surr2 = (surrogateMin + surrogateMax + 1) / 2 614 615 // Do not use defer for unlock. May cause issues when printing a panic. 616 lock(&utf16ConsoleBackLock) 617 618 b := (*[1 << 30]byte)(buf)[:bufLen] 619 s := *(*string)(unsafe.Pointer(&b)) 620 621 utf16tmp := utf16ConsoleBack[:] 622 623 total := len(s) 624 w := 0 625 for _, r := range s { 626 if w >= len(utf16tmp)-2 { 627 writeConsoleUTF16(handle, utf16tmp[:w]) 628 w = 0 629 } 630 if r < 0x10000 { 631 utf16tmp[w] = uint16(r) 632 w++ 633 } else { 634 r -= 0x10000 635 utf16tmp[w] = surrogateMin + uint16(r>>10)&0x3ff 636 utf16tmp[w+1] = surr2 + uint16(r)&0x3ff 637 w += 2 638 } 639 } 640 writeConsoleUTF16(handle, utf16tmp[:w]) 641 unlock(&utf16ConsoleBackLock) 642 return total 643} 644 645// writeConsoleUTF16 is the dedicated windows calls that correctly prints 646// to the console regardless of the current code page. Input is utf-16 code points. 647// The handle must be a console handle. 648func writeConsoleUTF16(handle uintptr, b []uint16) { 649 l := uint32(len(b)) 650 if l == 0 { 651 return 652 } 653 var written uint32 654 stdcall5(_WriteConsoleW, 655 handle, 656 uintptr(unsafe.Pointer(&b[0])), 657 uintptr(l), 658 uintptr(unsafe.Pointer(&written)), 659 0, 660 ) 661 return 662} 663 664//go:nosplit 665func semasleep(ns int64) int32 { 666 const ( 667 _WAIT_ABANDONED = 0x00000080 668 _WAIT_OBJECT_0 = 0x00000000 669 _WAIT_TIMEOUT = 0x00000102 670 _WAIT_FAILED = 0xFFFFFFFF 671 ) 672 673 var result uintptr 674 if ns < 0 { 675 result = stdcall2(_WaitForSingleObject, getg().m.waitsema, uintptr(_INFINITE)) 676 } else { 677 start := nanotime() 678 elapsed := int64(0) 679 for { 680 ms := int64(timediv(ns-elapsed, 1000000, nil)) 681 if ms == 0 { 682 ms = 1 683 } 684 result = stdcall4(_WaitForMultipleObjects, 2, 685 uintptr(unsafe.Pointer(&[2]uintptr{getg().m.waitsema, getg().m.resumesema})), 686 0, uintptr(ms)) 687 if result != _WAIT_OBJECT_0+1 { 688 // Not a suspend/resume event 689 break 690 } 691 elapsed = nanotime() - start 692 if elapsed >= ns { 693 return -1 694 } 695 } 696 } 697 switch result { 698 case _WAIT_OBJECT_0: // Signaled 699 return 0 700 701 case _WAIT_TIMEOUT: 702 return -1 703 704 case _WAIT_ABANDONED: 705 systemstack(func() { 706 throw("runtime.semasleep wait_abandoned") 707 }) 708 709 case _WAIT_FAILED: 710 systemstack(func() { 711 print("runtime: waitforsingleobject wait_failed; errno=", getlasterror(), "\n") 712 throw("runtime.semasleep wait_failed") 713 }) 714 715 default: 716 systemstack(func() { 717 print("runtime: waitforsingleobject unexpected; result=", result, "\n") 718 throw("runtime.semasleep unexpected") 719 }) 720 } 721 722 return -1 // unreachable 723} 724 725//go:nosplit 726func semawakeup(mp *m) { 727 if stdcall1(_SetEvent, mp.waitsema) == 0 { 728 systemstack(func() { 729 print("runtime: setevent failed; errno=", getlasterror(), "\n") 730 throw("runtime.semawakeup") 731 }) 732 } 733} 734 735//go:nosplit 736func semacreate(mp *m) { 737 if mp.waitsema != 0 { 738 return 739 } 740 mp.waitsema = stdcall4(_CreateEventA, 0, 0, 0, 0) 741 if mp.waitsema == 0 { 742 systemstack(func() { 743 print("runtime: createevent failed; errno=", getlasterror(), "\n") 744 throw("runtime.semacreate") 745 }) 746 } 747 mp.resumesema = stdcall4(_CreateEventA, 0, 0, 0, 0) 748 if mp.resumesema == 0 { 749 systemstack(func() { 750 print("runtime: createevent failed; errno=", getlasterror(), "\n") 751 throw("runtime.semacreate") 752 }) 753 stdcall1(_CloseHandle, mp.waitsema) 754 mp.waitsema = 0 755 } 756} 757 758// May run with m.p==nil, so write barriers are not allowed. This 759// function is called by newosproc0, so it is also required to 760// operate without stack guards. 761// 762//go:nowritebarrierrec 763//go:nosplit 764func newosproc(mp *m) { 765 // We pass 0 for the stack size to use the default for this binary. 766 thandle := stdcall6(_CreateThread, 0, 0, 767 abi.FuncPCABI0(tstart_stdcall), uintptr(unsafe.Pointer(mp)), 768 0, 0) 769 770 if thandle == 0 { 771 if atomic.Load(&exiting) != 0 { 772 // CreateThread may fail if called 773 // concurrently with ExitProcess. If this 774 // happens, just freeze this thread and let 775 // the process exit. See issue #18253. 776 lock(&deadlock) 777 lock(&deadlock) 778 } 779 print("runtime: failed to create new OS thread (have ", mcount(), " already; errno=", getlasterror(), ")\n") 780 throw("runtime.newosproc") 781 } 782 783 // Close thandle to avoid leaking the thread object if it exits. 784 stdcall1(_CloseHandle, thandle) 785} 786 787// Used by the C library build mode. On Linux this function would allocate a 788// stack, but that's not necessary for Windows. No stack guards are present 789// and the GC has not been initialized, so write barriers will fail. 790// 791//go:nowritebarrierrec 792//go:nosplit 793func newosproc0(mp *m, stk unsafe.Pointer) { 794 // TODO: this is completely broken. The args passed to newosproc0 (in asm_amd64.s) 795 // are stacksize and function, not *m and stack. 796 // Check os_linux.go for an implementation that might actually work. 797 throw("bad newosproc0") 798} 799 800func exitThread(wait *atomic.Uint32) { 801 // We should never reach exitThread on Windows because we let 802 // the OS clean up threads. 803 throw("exitThread") 804} 805 806// Called to initialize a new m (including the bootstrap m). 807// Called on the parent thread (main thread in case of bootstrap), can allocate memory. 808func mpreinit(mp *m) { 809} 810 811//go:nosplit 812func sigsave(p *sigset) { 813} 814 815//go:nosplit 816func msigrestore(sigmask sigset) { 817} 818 819//go:nosplit 820//go:nowritebarrierrec 821func clearSignalHandlers() { 822} 823 824//go:nosplit 825func sigblock(exiting bool) { 826} 827 828// Called to initialize a new m (including the bootstrap m). 829// Called on the new thread, cannot allocate Go memory. 830func minit() { 831 var thandle uintptr 832 if stdcall7(_DuplicateHandle, currentProcess, currentThread, currentProcess, uintptr(unsafe.Pointer(&thandle)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { 833 print("runtime.minit: duplicatehandle failed; errno=", getlasterror(), "\n") 834 throw("runtime.minit: duplicatehandle failed") 835 } 836 837 mp := getg().m 838 lock(&mp.threadLock) 839 mp.thread = thandle 840 mp.procid = uint64(stdcall0(_GetCurrentThreadId)) 841 842 // Configure usleep timer, if possible. 843 if mp.highResTimer == 0 && haveHighResTimer { 844 mp.highResTimer = createHighResTimer() 845 if mp.highResTimer == 0 { 846 print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n") 847 throw("CreateWaitableTimerEx when creating timer failed") 848 } 849 } 850 if mp.waitIocpHandle == 0 && haveHighResSleep { 851 mp.waitIocpTimer = createHighResTimer() 852 if mp.waitIocpTimer == 0 { 853 print("runtime: CreateWaitableTimerEx failed; errno=", getlasterror(), "\n") 854 throw("CreateWaitableTimerEx when creating timer failed") 855 } 856 const GENERIC_ALL = 0x10000000 857 errno := stdcall3(_NtCreateWaitCompletionPacket, uintptr(unsafe.Pointer(&mp.waitIocpHandle)), GENERIC_ALL, 0) 858 if mp.waitIocpHandle == 0 { 859 print("runtime: NtCreateWaitCompletionPacket failed; errno=", errno, "\n") 860 throw("NtCreateWaitCompletionPacket failed") 861 } 862 } 863 unlock(&mp.threadLock) 864 865 // Query the true stack base from the OS. Currently we're 866 // running on a small assumed stack. 867 var mbi memoryBasicInformation 868 res := stdcall3(_VirtualQuery, uintptr(unsafe.Pointer(&mbi)), uintptr(unsafe.Pointer(&mbi)), unsafe.Sizeof(mbi)) 869 if res == 0 { 870 print("runtime: VirtualQuery failed; errno=", getlasterror(), "\n") 871 throw("VirtualQuery for stack base failed") 872 } 873 // The system leaves an 8K PAGE_GUARD region at the bottom of 874 // the stack (in theory VirtualQuery isn't supposed to include 875 // that, but it does). Add an additional 8K of slop for 876 // calling C functions that don't have stack checks and for 877 // lastcontinuehandler. We shouldn't be anywhere near this 878 // bound anyway. 879 base := mbi.allocationBase + 16<<10 880 // Sanity check the stack bounds. 881 g0 := getg() 882 if base > g0.stack.hi || g0.stack.hi-base > 64<<20 { 883 print("runtime: g0 stack [", hex(base), ",", hex(g0.stack.hi), ")\n") 884 throw("bad g0 stack") 885 } 886 g0.stack.lo = base 887 g0.stackguard0 = g0.stack.lo + stackGuard 888 g0.stackguard1 = g0.stackguard0 889 // Sanity check the SP. 890 stackcheck() 891} 892 893// Called from dropm to undo the effect of an minit. 894// 895//go:nosplit 896func unminit() { 897 mp := getg().m 898 lock(&mp.threadLock) 899 if mp.thread != 0 { 900 stdcall1(_CloseHandle, mp.thread) 901 mp.thread = 0 902 } 903 unlock(&mp.threadLock) 904 905 mp.procid = 0 906} 907 908// Called from exitm, but not from drop, to undo the effect of thread-owned 909// resources in minit, semacreate, or elsewhere. Do not take locks after calling this. 910// 911//go:nosplit 912func mdestroy(mp *m) { 913 if mp.highResTimer != 0 { 914 stdcall1(_CloseHandle, mp.highResTimer) 915 mp.highResTimer = 0 916 } 917 if mp.waitIocpTimer != 0 { 918 stdcall1(_CloseHandle, mp.waitIocpTimer) 919 mp.waitIocpTimer = 0 920 } 921 if mp.waitIocpHandle != 0 { 922 stdcall1(_CloseHandle, mp.waitIocpHandle) 923 mp.waitIocpHandle = 0 924 } 925 if mp.waitsema != 0 { 926 stdcall1(_CloseHandle, mp.waitsema) 927 mp.waitsema = 0 928 } 929 if mp.resumesema != 0 { 930 stdcall1(_CloseHandle, mp.resumesema) 931 mp.resumesema = 0 932 } 933} 934 935// asmstdcall_trampoline calls asmstdcall converting from Go to C calling convention. 936func asmstdcall_trampoline(args unsafe.Pointer) 937 938// stdcall_no_g calls asmstdcall on os stack without using g. 939// 940//go:nosplit 941func stdcall_no_g(fn stdFunction, n int, args uintptr) uintptr { 942 libcall := libcall{ 943 fn: uintptr(unsafe.Pointer(fn)), 944 n: uintptr(n), 945 args: args, 946 } 947 asmstdcall_trampoline(noescape(unsafe.Pointer(&libcall))) 948 return libcall.r1 949} 950 951// Calling stdcall on os stack. 952// May run during STW, so write barriers are not allowed. 953// 954//go:nowritebarrier 955//go:nosplit 956func stdcall(fn stdFunction) uintptr { 957 gp := getg() 958 mp := gp.m 959 mp.libcall.fn = uintptr(unsafe.Pointer(fn)) 960 resetLibcall := false 961 if mp.profilehz != 0 && mp.libcallsp == 0 { 962 // leave pc/sp for cpu profiler 963 mp.libcallg.set(gp) 964 mp.libcallpc = getcallerpc() 965 // sp must be the last, because once async cpu profiler finds 966 // all three values to be non-zero, it will use them 967 mp.libcallsp = getcallersp() 968 resetLibcall = true // See comment in sys_darwin.go:libcCall 969 } 970 asmcgocall(asmstdcallAddr, unsafe.Pointer(&mp.libcall)) 971 if resetLibcall { 972 mp.libcallsp = 0 973 } 974 return mp.libcall.r1 975} 976 977//go:nosplit 978func stdcall0(fn stdFunction) uintptr { 979 mp := getg().m 980 mp.libcall.n = 0 981 mp.libcall.args = 0 982 return stdcall(fn) 983} 984 985//go:nosplit 986//go:cgo_unsafe_args 987func stdcall1(fn stdFunction, a0 uintptr) uintptr { 988 mp := getg().m 989 mp.libcall.n = 1 990 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 991 return stdcall(fn) 992} 993 994//go:nosplit 995//go:cgo_unsafe_args 996func stdcall2(fn stdFunction, a0, a1 uintptr) uintptr { 997 mp := getg().m 998 mp.libcall.n = 2 999 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1000 return stdcall(fn) 1001} 1002 1003//go:nosplit 1004//go:cgo_unsafe_args 1005func stdcall3(fn stdFunction, a0, a1, a2 uintptr) uintptr { 1006 mp := getg().m 1007 mp.libcall.n = 3 1008 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1009 return stdcall(fn) 1010} 1011 1012//go:nosplit 1013//go:cgo_unsafe_args 1014func stdcall4(fn stdFunction, a0, a1, a2, a3 uintptr) uintptr { 1015 mp := getg().m 1016 mp.libcall.n = 4 1017 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1018 return stdcall(fn) 1019} 1020 1021//go:nosplit 1022//go:cgo_unsafe_args 1023func stdcall5(fn stdFunction, a0, a1, a2, a3, a4 uintptr) uintptr { 1024 mp := getg().m 1025 mp.libcall.n = 5 1026 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1027 return stdcall(fn) 1028} 1029 1030//go:nosplit 1031//go:cgo_unsafe_args 1032func stdcall6(fn stdFunction, a0, a1, a2, a3, a4, a5 uintptr) uintptr { 1033 mp := getg().m 1034 mp.libcall.n = 6 1035 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1036 return stdcall(fn) 1037} 1038 1039//go:nosplit 1040//go:cgo_unsafe_args 1041func stdcall7(fn stdFunction, a0, a1, a2, a3, a4, a5, a6 uintptr) uintptr { 1042 mp := getg().m 1043 mp.libcall.n = 7 1044 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1045 return stdcall(fn) 1046} 1047 1048//go:nosplit 1049//go:cgo_unsafe_args 1050func stdcall8(fn stdFunction, a0, a1, a2, a3, a4, a5, a6, a7 uintptr) uintptr { 1051 mp := getg().m 1052 mp.libcall.n = 8 1053 mp.libcall.args = uintptr(noescape(unsafe.Pointer(&a0))) 1054 return stdcall(fn) 1055} 1056 1057// These must run on the system stack only. 1058 1059//go:nosplit 1060func osyield_no_g() { 1061 stdcall_no_g(_SwitchToThread, 0, 0) 1062} 1063 1064//go:nosplit 1065func osyield() { 1066 systemstack(func() { 1067 stdcall0(_SwitchToThread) 1068 }) 1069} 1070 1071//go:nosplit 1072func usleep_no_g(us uint32) { 1073 timeout := uintptr(us) / 1000 // ms units 1074 args := [...]uintptr{_INVALID_HANDLE_VALUE, timeout} 1075 stdcall_no_g(_WaitForSingleObject, len(args), uintptr(noescape(unsafe.Pointer(&args[0])))) 1076} 1077 1078//go:nosplit 1079func usleep(us uint32) { 1080 systemstack(func() { 1081 var h, timeout uintptr 1082 // If the high-res timer is available and its handle has been allocated for this m, use it. 1083 // Otherwise fall back to the low-res one, which doesn't need a handle. 1084 if haveHighResTimer && getg().m.highResTimer != 0 { 1085 h = getg().m.highResTimer 1086 dt := -10 * int64(us) // relative sleep (negative), 100ns units 1087 stdcall6(_SetWaitableTimer, h, uintptr(unsafe.Pointer(&dt)), 0, 0, 0, 0) 1088 timeout = _INFINITE 1089 } else { 1090 h = _INVALID_HANDLE_VALUE 1091 timeout = uintptr(us) / 1000 // ms units 1092 } 1093 stdcall2(_WaitForSingleObject, h, timeout) 1094 }) 1095} 1096 1097func ctrlHandler(_type uint32) uintptr { 1098 var s uint32 1099 1100 switch _type { 1101 case _CTRL_C_EVENT, _CTRL_BREAK_EVENT: 1102 s = _SIGINT 1103 case _CTRL_CLOSE_EVENT, _CTRL_LOGOFF_EVENT, _CTRL_SHUTDOWN_EVENT: 1104 s = _SIGTERM 1105 default: 1106 return 0 1107 } 1108 1109 if sigsend(s) { 1110 if s == _SIGTERM { 1111 // Windows terminates the process after this handler returns. 1112 // Block indefinitely to give signal handlers a chance to clean up, 1113 // but make sure to be properly parked first, so the rest of the 1114 // program can continue executing. 1115 block() 1116 } 1117 return 1 1118 } 1119 return 0 1120} 1121 1122// called from zcallback_windows_*.s to sys_windows_*.s 1123func callbackasm1() 1124 1125var profiletimer uintptr 1126 1127func profilem(mp *m, thread uintptr) { 1128 // Align Context to 16 bytes. 1129 var c *context 1130 var cbuf [unsafe.Sizeof(*c) + 15]byte 1131 c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) 1132 1133 c.contextflags = _CONTEXT_CONTROL 1134 stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) 1135 1136 gp := gFromSP(mp, c.sp()) 1137 1138 sigprof(c.ip(), c.sp(), c.lr(), gp, mp) 1139} 1140 1141func gFromSP(mp *m, sp uintptr) *g { 1142 if gp := mp.g0; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { 1143 return gp 1144 } 1145 if gp := mp.gsignal; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { 1146 return gp 1147 } 1148 if gp := mp.curg; gp != nil && gp.stack.lo < sp && sp < gp.stack.hi { 1149 return gp 1150 } 1151 return nil 1152} 1153 1154func profileLoop() { 1155 stdcall2(_SetThreadPriority, currentThread, _THREAD_PRIORITY_HIGHEST) 1156 1157 for { 1158 stdcall2(_WaitForSingleObject, profiletimer, _INFINITE) 1159 first := (*m)(atomic.Loadp(unsafe.Pointer(&allm))) 1160 for mp := first; mp != nil; mp = mp.alllink { 1161 if mp == getg().m { 1162 // Don't profile ourselves. 1163 continue 1164 } 1165 1166 lock(&mp.threadLock) 1167 // Do not profile threads blocked on Notes, 1168 // this includes idle worker threads, 1169 // idle timer thread, idle heap scavenger, etc. 1170 if mp.thread == 0 || mp.profilehz == 0 || mp.blocked { 1171 unlock(&mp.threadLock) 1172 continue 1173 } 1174 // Acquire our own handle to the thread. 1175 var thread uintptr 1176 if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { 1177 print("runtime: duplicatehandle failed; errno=", getlasterror(), "\n") 1178 throw("duplicatehandle failed") 1179 } 1180 unlock(&mp.threadLock) 1181 1182 // mp may exit between the DuplicateHandle 1183 // above and the SuspendThread. The handle 1184 // will remain valid, but SuspendThread may 1185 // fail. 1186 if int32(stdcall1(_SuspendThread, thread)) == -1 { 1187 // The thread no longer exists. 1188 stdcall1(_CloseHandle, thread) 1189 continue 1190 } 1191 if mp.profilehz != 0 && !mp.blocked { 1192 // Pass the thread handle in case mp 1193 // was in the process of shutting down. 1194 profilem(mp, thread) 1195 } 1196 stdcall1(_ResumeThread, thread) 1197 stdcall1(_CloseHandle, thread) 1198 } 1199 } 1200} 1201 1202func setProcessCPUProfiler(hz int32) { 1203 if profiletimer == 0 { 1204 var timer uintptr 1205 if haveHighResTimer { 1206 timer = createHighResTimer() 1207 } else { 1208 timer = stdcall3(_CreateWaitableTimerA, 0, 0, 0) 1209 } 1210 atomic.Storeuintptr(&profiletimer, timer) 1211 newm(profileLoop, nil, -1) 1212 } 1213} 1214 1215func setThreadCPUProfiler(hz int32) { 1216 ms := int32(0) 1217 due := ^int64(^uint64(1 << 63)) 1218 if hz > 0 { 1219 ms = 1000 / hz 1220 if ms == 0 { 1221 ms = 1 1222 } 1223 due = int64(ms) * -10000 1224 } 1225 stdcall6(_SetWaitableTimer, profiletimer, uintptr(unsafe.Pointer(&due)), uintptr(ms), 0, 0, 0) 1226 atomic.Store((*uint32)(unsafe.Pointer(&getg().m.profilehz)), uint32(hz)) 1227} 1228 1229const preemptMSupported = true 1230 1231// suspendLock protects simultaneous SuspendThread operations from 1232// suspending each other. 1233var suspendLock mutex 1234 1235func preemptM(mp *m) { 1236 if mp == getg().m { 1237 throw("self-preempt") 1238 } 1239 1240 // Synchronize with external code that may try to ExitProcess. 1241 if !atomic.Cas(&mp.preemptExtLock, 0, 1) { 1242 // External code is running. Fail the preemption 1243 // attempt. 1244 mp.preemptGen.Add(1) 1245 return 1246 } 1247 1248 // Acquire our own handle to mp's thread. 1249 lock(&mp.threadLock) 1250 if mp.thread == 0 { 1251 // The M hasn't been minit'd yet (or was just unminit'd). 1252 unlock(&mp.threadLock) 1253 atomic.Store(&mp.preemptExtLock, 0) 1254 mp.preemptGen.Add(1) 1255 return 1256 } 1257 var thread uintptr 1258 if stdcall7(_DuplicateHandle, currentProcess, mp.thread, currentProcess, uintptr(unsafe.Pointer(&thread)), 0, 0, _DUPLICATE_SAME_ACCESS) == 0 { 1259 print("runtime.preemptM: duplicatehandle failed; errno=", getlasterror(), "\n") 1260 throw("runtime.preemptM: duplicatehandle failed") 1261 } 1262 unlock(&mp.threadLock) 1263 1264 // Prepare thread context buffer. This must be aligned to 16 bytes. 1265 var c *context 1266 var cbuf [unsafe.Sizeof(*c) + 15]byte 1267 c = (*context)(unsafe.Pointer((uintptr(unsafe.Pointer(&cbuf[15]))) &^ 15)) 1268 c.contextflags = _CONTEXT_CONTROL 1269 1270 // Serialize thread suspension. SuspendThread is asynchronous, 1271 // so it's otherwise possible for two threads to suspend each 1272 // other and deadlock. We must hold this lock until after 1273 // GetThreadContext, since that blocks until the thread is 1274 // actually suspended. 1275 lock(&suspendLock) 1276 1277 // Suspend the thread. 1278 if int32(stdcall1(_SuspendThread, thread)) == -1 { 1279 unlock(&suspendLock) 1280 stdcall1(_CloseHandle, thread) 1281 atomic.Store(&mp.preemptExtLock, 0) 1282 // The thread no longer exists. This shouldn't be 1283 // possible, but just acknowledge the request. 1284 mp.preemptGen.Add(1) 1285 return 1286 } 1287 1288 // We have to be very careful between this point and once 1289 // we've shown mp is at an async safe-point. This is like a 1290 // signal handler in the sense that mp could have been doing 1291 // anything when we stopped it, including holding arbitrary 1292 // locks. 1293 1294 // We have to get the thread context before inspecting the M 1295 // because SuspendThread only requests a suspend. 1296 // GetThreadContext actually blocks until it's suspended. 1297 stdcall2(_GetThreadContext, thread, uintptr(unsafe.Pointer(c))) 1298 1299 unlock(&suspendLock) 1300 1301 // Does it want a preemption and is it safe to preempt? 1302 gp := gFromSP(mp, c.sp()) 1303 if gp != nil && wantAsyncPreempt(gp) { 1304 if ok, newpc := isAsyncSafePoint(gp, c.ip(), c.sp(), c.lr()); ok { 1305 // Inject call to asyncPreempt 1306 targetPC := abi.FuncPCABI0(asyncPreempt) 1307 switch GOARCH { 1308 default: 1309 throw("unsupported architecture") 1310 case "386", "amd64": 1311 // Make it look like the thread called targetPC. 1312 sp := c.sp() 1313 sp -= goarch.PtrSize 1314 *(*uintptr)(unsafe.Pointer(sp)) = newpc 1315 c.set_sp(sp) 1316 c.set_ip(targetPC) 1317 1318 case "arm": 1319 // Push LR. The injected call is responsible 1320 // for restoring LR. gentraceback is aware of 1321 // this extra slot. See sigctxt.pushCall in 1322 // signal_arm.go, which is similar except we 1323 // subtract 1 from IP here. 1324 sp := c.sp() 1325 sp -= goarch.PtrSize 1326 c.set_sp(sp) 1327 *(*uint32)(unsafe.Pointer(sp)) = uint32(c.lr()) 1328 c.set_lr(newpc - 1) 1329 c.set_ip(targetPC) 1330 1331 case "arm64": 1332 // Push LR. The injected call is responsible 1333 // for restoring LR. gentraceback is aware of 1334 // this extra slot. See sigctxt.pushCall in 1335 // signal_arm64.go. 1336 sp := c.sp() - 16 // SP needs 16-byte alignment 1337 c.set_sp(sp) 1338 *(*uint64)(unsafe.Pointer(sp)) = uint64(c.lr()) 1339 c.set_lr(newpc) 1340 c.set_ip(targetPC) 1341 } 1342 stdcall2(_SetThreadContext, thread, uintptr(unsafe.Pointer(c))) 1343 } 1344 } 1345 1346 atomic.Store(&mp.preemptExtLock, 0) 1347 1348 // Acknowledge the preemption. 1349 mp.preemptGen.Add(1) 1350 1351 stdcall1(_ResumeThread, thread) 1352 stdcall1(_CloseHandle, thread) 1353} 1354 1355// osPreemptExtEnter is called before entering external code that may 1356// call ExitProcess. 1357// 1358// This must be nosplit because it may be called from a syscall with 1359// untyped stack slots, so the stack must not be grown or scanned. 1360// 1361//go:nosplit 1362func osPreemptExtEnter(mp *m) { 1363 for !atomic.Cas(&mp.preemptExtLock, 0, 1) { 1364 // An asynchronous preemption is in progress. It's not 1365 // safe to enter external code because it may call 1366 // ExitProcess and deadlock with SuspendThread. 1367 // Ideally we would do the preemption ourselves, but 1368 // can't since there may be untyped syscall arguments 1369 // on the stack. Instead, just wait and encourage the 1370 // SuspendThread APC to run. The preemption should be 1371 // done shortly. 1372 osyield() 1373 } 1374 // Asynchronous preemption is now blocked. 1375} 1376 1377// osPreemptExtExit is called after returning from external code that 1378// may call ExitProcess. 1379// 1380// See osPreemptExtEnter for why this is nosplit. 1381// 1382//go:nosplit 1383func osPreemptExtExit(mp *m) { 1384 atomic.Store(&mp.preemptExtLock, 0) 1385} 1386