1// Copyright 2014 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5package runtime 6 7import ( 8 "internal/abi" 9 "internal/goarch" 10 "unsafe" 11) 12 13// cbs stores all registered Go callbacks. 14var cbs struct { 15 lock mutex // use cbsLock / cbsUnlock for race instrumentation. 16 ctxt [cb_max]winCallback 17 index map[winCallbackKey]int 18 n int 19} 20 21func cbsLock() { 22 lock(&cbs.lock) 23 // compileCallback is used by goenvs prior to completion of schedinit. 24 // raceacquire involves a racecallback to get the proc, which is not 25 // safe prior to scheduler initialization. Thus avoid instrumentation 26 // until then. 27 if raceenabled && mainStarted { 28 raceacquire(unsafe.Pointer(&cbs.lock)) 29 } 30} 31 32func cbsUnlock() { 33 if raceenabled && mainStarted { 34 racerelease(unsafe.Pointer(&cbs.lock)) 35 } 36 unlock(&cbs.lock) 37} 38 39// winCallback records information about a registered Go callback. 40type winCallback struct { 41 fn *funcval // Go function 42 retPop uintptr // For 386 cdecl, how many bytes to pop on return 43 abiMap abiDesc 44} 45 46// abiPartKind is the action an abiPart should take. 47type abiPartKind int 48 49const ( 50 abiPartBad abiPartKind = iota 51 abiPartStack // Move a value from memory to the stack. 52 abiPartReg // Move a value from memory to a register. 53) 54 55// abiPart encodes a step in translating between calling ABIs. 56type abiPart struct { 57 kind abiPartKind 58 srcStackOffset uintptr 59 dstStackOffset uintptr // used if kind == abiPartStack 60 dstRegister int // used if kind == abiPartReg 61 len uintptr 62} 63 64func (a *abiPart) tryMerge(b abiPart) bool { 65 if a.kind != abiPartStack || b.kind != abiPartStack { 66 return false 67 } 68 if a.srcStackOffset+a.len == b.srcStackOffset && a.dstStackOffset+a.len == b.dstStackOffset { 69 a.len += b.len 70 return true 71 } 72 return false 73} 74 75// abiDesc specifies how to translate from a C frame to a Go 76// frame. This does not specify how to translate back because 77// the result is always a uintptr. If the C ABI is fastcall, 78// this assumes the four fastcall registers were first spilled 79// to the shadow space. 80type abiDesc struct { 81 parts []abiPart 82 83 srcStackSize uintptr // stdcall/fastcall stack space tracking 84 dstStackSize uintptr // Go stack space used 85 dstSpill uintptr // Extra stack space for argument spill slots 86 dstRegisters int // Go ABI int argument registers used 87 88 // retOffset is the offset of the uintptr-sized result in the Go 89 // frame. 90 retOffset uintptr 91} 92 93func (p *abiDesc) assignArg(t *_type) { 94 if t.Size_ > goarch.PtrSize { 95 // We don't support this right now. In 96 // stdcall/cdecl, 64-bit ints and doubles are 97 // passed as two words (little endian); and 98 // structs are pushed on the stack. In 99 // fastcall, arguments larger than the word 100 // size are passed by reference. On arm, 101 // 8-byte aligned arguments round up to the 102 // next even register and can be split across 103 // registers and the stack. 104 panic("compileCallback: argument size is larger than uintptr") 105 } 106 if k := t.Kind_ & abi.KindMask; GOARCH != "386" && (k == abi.Float32 || k == abi.Float64) { 107 // In fastcall, floating-point arguments in 108 // the first four positions are passed in 109 // floating-point registers, which we don't 110 // currently spill. arm passes floating-point 111 // arguments in VFP registers, which we also 112 // don't support. 113 // So basically we only support 386. 114 panic("compileCallback: float arguments not supported") 115 } 116 117 if t.Size_ == 0 { 118 // The Go ABI aligns for zero-sized types. 119 p.dstStackSize = alignUp(p.dstStackSize, uintptr(t.Align_)) 120 return 121 } 122 123 // In the C ABI, we're already on a word boundary. 124 // Also, sub-word-sized fastcall register arguments 125 // are stored to the least-significant bytes of the 126 // argument word and all supported Windows 127 // architectures are little endian, so srcStackOffset 128 // is already pointing to the right place for smaller 129 // arguments. The same is true on arm. 130 131 oldParts := p.parts 132 if p.tryRegAssignArg(t, 0) { 133 // Account for spill space. 134 // 135 // TODO(mknyszek): Remove this when we no longer have 136 // caller reserved spill space. 137 p.dstSpill = alignUp(p.dstSpill, uintptr(t.Align_)) 138 p.dstSpill += t.Size_ 139 } else { 140 // Register assignment failed. 141 // Undo the work and stack assign. 142 p.parts = oldParts 143 144 // The Go ABI aligns arguments. 145 p.dstStackSize = alignUp(p.dstStackSize, uintptr(t.Align_)) 146 147 // Copy just the size of the argument. Note that this 148 // could be a small by-value struct, but C and Go 149 // struct layouts are compatible, so we can copy these 150 // directly, too. 151 part := abiPart{ 152 kind: abiPartStack, 153 srcStackOffset: p.srcStackSize, 154 dstStackOffset: p.dstStackSize, 155 len: t.Size_, 156 } 157 // Add this step to the adapter. 158 if len(p.parts) == 0 || !p.parts[len(p.parts)-1].tryMerge(part) { 159 p.parts = append(p.parts, part) 160 } 161 // The Go ABI packs arguments. 162 p.dstStackSize += t.Size_ 163 } 164 165 // cdecl, stdcall, fastcall, and arm pad arguments to word size. 166 // TODO(rsc): On arm and arm64 do we need to skip the caller's saved LR? 167 p.srcStackSize += goarch.PtrSize 168} 169 170// tryRegAssignArg tries to register-assign a value of type t. 171// If this type is nested in an aggregate type, then offset is the 172// offset of this type within its parent type. 173// Assumes t.size <= goarch.PtrSize and t.size != 0. 174// 175// Returns whether the assignment succeeded. 176func (p *abiDesc) tryRegAssignArg(t *_type, offset uintptr) bool { 177 switch k := t.Kind_ & abi.KindMask; k { 178 case abi.Bool, abi.Int, abi.Int8, abi.Int16, abi.Int32, abi.Uint, abi.Uint8, abi.Uint16, abi.Uint32, abi.Uintptr, abi.Pointer, abi.UnsafePointer: 179 // Assign a register for all these types. 180 return p.assignReg(t.Size_, offset) 181 case abi.Int64, abi.Uint64: 182 // Only register-assign if the registers are big enough. 183 if goarch.PtrSize == 8 { 184 return p.assignReg(t.Size_, offset) 185 } 186 case abi.Array: 187 at := (*arraytype)(unsafe.Pointer(t)) 188 if at.Len == 1 { 189 return p.tryRegAssignArg(at.Elem, offset) // TODO fix when runtime is fully commoned up w/ abi.Type 190 } 191 case abi.Struct: 192 st := (*structtype)(unsafe.Pointer(t)) 193 for i := range st.Fields { 194 f := &st.Fields[i] 195 if !p.tryRegAssignArg(f.Typ, offset+f.Offset) { 196 return false 197 } 198 } 199 return true 200 } 201 // Pointer-sized types such as maps and channels are currently 202 // not supported. 203 panic("compileCallback: type " + toRType(t).string() + " is currently not supported for use in system callbacks") 204} 205 206// assignReg attempts to assign a single register for an 207// argument with the given size, at the given offset into the 208// value in the C ABI space. 209// 210// Returns whether the assignment was successful. 211func (p *abiDesc) assignReg(size, offset uintptr) bool { 212 if p.dstRegisters >= intArgRegs { 213 return false 214 } 215 p.parts = append(p.parts, abiPart{ 216 kind: abiPartReg, 217 srcStackOffset: p.srcStackSize + offset, 218 dstRegister: p.dstRegisters, 219 len: size, 220 }) 221 p.dstRegisters++ 222 return true 223} 224 225type winCallbackKey struct { 226 fn *funcval 227 cdecl bool 228} 229 230func callbackasm() 231 232// callbackasmAddr returns address of runtime.callbackasm 233// function adjusted by i. 234// On x86 and amd64, runtime.callbackasm is a series of CALL instructions, 235// and we want callback to arrive at 236// correspondent call instruction instead of start of 237// runtime.callbackasm. 238// On ARM, runtime.callbackasm is a series of mov and branch instructions. 239// R12 is loaded with the callback index. Each entry is two instructions, 240// hence 8 bytes. 241func callbackasmAddr(i int) uintptr { 242 var entrySize int 243 switch GOARCH { 244 default: 245 panic("unsupported architecture") 246 case "386", "amd64": 247 entrySize = 5 248 case "arm", "arm64": 249 // On ARM and ARM64, each entry is a MOV instruction 250 // followed by a branch instruction 251 entrySize = 8 252 } 253 return abi.FuncPCABI0(callbackasm) + uintptr(i*entrySize) 254} 255 256const callbackMaxFrame = 64 * goarch.PtrSize 257 258// compileCallback converts a Go function fn into a C function pointer 259// that can be passed to Windows APIs. 260// 261// On 386, if cdecl is true, the returned C function will use the 262// cdecl calling convention; otherwise, it will use stdcall. On amd64, 263// it always uses fastcall. On arm, it always uses the ARM convention. 264// 265//go:linkname compileCallback syscall.compileCallback 266func compileCallback(fn eface, cdecl bool) (code uintptr) { 267 if GOARCH != "386" { 268 // cdecl is only meaningful on 386. 269 cdecl = false 270 } 271 272 if fn._type == nil || (fn._type.Kind_&abi.KindMask) != abi.Func { 273 panic("compileCallback: expected function with one uintptr-sized result") 274 } 275 ft := (*functype)(unsafe.Pointer(fn._type)) 276 277 // Check arguments and construct ABI translation. 278 var abiMap abiDesc 279 for _, t := range ft.InSlice() { 280 abiMap.assignArg(t) 281 } 282 // The Go ABI aligns the result to the word size. src is 283 // already aligned. 284 abiMap.dstStackSize = alignUp(abiMap.dstStackSize, goarch.PtrSize) 285 abiMap.retOffset = abiMap.dstStackSize 286 287 if len(ft.OutSlice()) != 1 { 288 panic("compileCallback: expected function with one uintptr-sized result") 289 } 290 if ft.OutSlice()[0].Size_ != goarch.PtrSize { 291 panic("compileCallback: expected function with one uintptr-sized result") 292 } 293 if k := ft.OutSlice()[0].Kind_ & abi.KindMask; k == abi.Float32 || k == abi.Float64 { 294 // In cdecl and stdcall, float results are returned in 295 // ST(0). In fastcall, they're returned in XMM0. 296 // Either way, it's not AX. 297 panic("compileCallback: float results not supported") 298 } 299 if intArgRegs == 0 { 300 // Make room for the uintptr-sized result. 301 // If there are argument registers, the return value will 302 // be passed in the first register. 303 abiMap.dstStackSize += goarch.PtrSize 304 } 305 306 // TODO(mknyszek): Remove dstSpill from this calculation when we no longer have 307 // caller reserved spill space. 308 frameSize := alignUp(abiMap.dstStackSize, goarch.PtrSize) 309 frameSize += abiMap.dstSpill 310 if frameSize > callbackMaxFrame { 311 panic("compileCallback: function argument frame too large") 312 } 313 314 // For cdecl, the callee is responsible for popping its 315 // arguments from the C stack. 316 var retPop uintptr 317 if cdecl { 318 retPop = abiMap.srcStackSize 319 } 320 321 key := winCallbackKey{(*funcval)(fn.data), cdecl} 322 323 cbsLock() 324 325 // Check if this callback is already registered. 326 if n, ok := cbs.index[key]; ok { 327 cbsUnlock() 328 return callbackasmAddr(n) 329 } 330 331 // Register the callback. 332 if cbs.index == nil { 333 cbs.index = make(map[winCallbackKey]int) 334 } 335 n := cbs.n 336 if n >= len(cbs.ctxt) { 337 cbsUnlock() 338 throw("too many callback functions") 339 } 340 c := winCallback{key.fn, retPop, abiMap} 341 cbs.ctxt[n] = c 342 cbs.index[key] = n 343 cbs.n++ 344 345 cbsUnlock() 346 return callbackasmAddr(n) 347} 348 349type callbackArgs struct { 350 index uintptr 351 // args points to the argument block. 352 // 353 // For cdecl and stdcall, all arguments are on the stack. 354 // 355 // For fastcall, the trampoline spills register arguments to 356 // the reserved spill slots below the stack arguments, 357 // resulting in a layout equivalent to stdcall. 358 // 359 // For arm, the trampoline stores the register arguments just 360 // below the stack arguments, so again we can treat it as one 361 // big stack arguments frame. 362 args unsafe.Pointer 363 // Below are out-args from callbackWrap 364 result uintptr 365 retPop uintptr // For 386 cdecl, how many bytes to pop on return 366} 367 368// callbackWrap is called by callbackasm to invoke a registered C callback. 369func callbackWrap(a *callbackArgs) { 370 c := cbs.ctxt[a.index] 371 a.retPop = c.retPop 372 373 // Convert from C to Go ABI. 374 var regs abi.RegArgs 375 var frame [callbackMaxFrame]byte 376 goArgs := unsafe.Pointer(&frame) 377 for _, part := range c.abiMap.parts { 378 switch part.kind { 379 case abiPartStack: 380 memmove(add(goArgs, part.dstStackOffset), add(a.args, part.srcStackOffset), part.len) 381 case abiPartReg: 382 goReg := unsafe.Pointer(®s.Ints[part.dstRegister]) 383 memmove(goReg, add(a.args, part.srcStackOffset), part.len) 384 default: 385 panic("bad ABI description") 386 } 387 } 388 389 // TODO(mknyszek): Remove this when we no longer have 390 // caller reserved spill space. 391 frameSize := alignUp(c.abiMap.dstStackSize, goarch.PtrSize) 392 frameSize += c.abiMap.dstSpill 393 394 // Even though this is copying back results, we can pass a nil 395 // type because those results must not require write barriers. 396 reflectcall(nil, unsafe.Pointer(c.fn), noescape(goArgs), uint32(c.abiMap.dstStackSize), uint32(c.abiMap.retOffset), uint32(frameSize), ®s) 397 398 // Extract the result. 399 // 400 // There's always exactly one return value, one pointer in size. 401 // If it's on the stack, then we will have reserved space for it 402 // at the end of the frame, otherwise it was passed in a register. 403 if c.abiMap.dstStackSize != c.abiMap.retOffset { 404 a.result = *(*uintptr)(unsafe.Pointer(&frame[c.abiMap.retOffset])) 405 } else { 406 var zero int 407 // On architectures with no registers, Ints[0] would be a compile error, 408 // so we use a dynamic index. These architectures will never take this 409 // branch, so this won't cause a runtime panic. 410 a.result = regs.Ints[zero] 411 } 412} 413 414const _LOAD_LIBRARY_SEARCH_SYSTEM32 = 0x00000800 415 416//go:linkname syscall_loadsystemlibrary syscall.loadsystemlibrary 417func syscall_loadsystemlibrary(filename *uint16) (handle, err uintptr) { 418 handle, _, err = syscall_SyscallN(uintptr(unsafe.Pointer(_LoadLibraryExW)), uintptr(unsafe.Pointer(filename)), 0, _LOAD_LIBRARY_SEARCH_SYSTEM32) 419 KeepAlive(filename) 420 if handle != 0 { 421 err = 0 422 } 423 return 424} 425 426// golang.org/x/sys linknames syscall.loadlibrary 427// (in addition to standard package syscall). 428// Do not remove or change the type signature. 429// 430//go:linkname syscall_loadlibrary syscall.loadlibrary 431func syscall_loadlibrary(filename *uint16) (handle, err uintptr) { 432 handle, _, err = syscall_SyscallN(uintptr(unsafe.Pointer(_LoadLibraryW)), uintptr(unsafe.Pointer(filename))) 433 KeepAlive(filename) 434 if handle != 0 { 435 err = 0 436 } 437 return 438} 439 440// golang.org/x/sys linknames syscall.getprocaddress 441// (in addition to standard package syscall). 442// Do not remove or change the type signature. 443// 444//go:linkname syscall_getprocaddress syscall.getprocaddress 445func syscall_getprocaddress(handle uintptr, procname *byte) (outhandle, err uintptr) { 446 outhandle, _, err = syscall_SyscallN(uintptr(unsafe.Pointer(_GetProcAddress)), handle, uintptr(unsafe.Pointer(procname))) 447 KeepAlive(procname) 448 if outhandle != 0 { 449 err = 0 450 } 451 return 452} 453 454//go:linkname syscall_Syscall syscall.Syscall 455//go:nosplit 456func syscall_Syscall(fn, nargs, a1, a2, a3 uintptr) (r1, r2, err uintptr) { 457 args := [...]uintptr{a1, a2, a3} 458 return syscall_SyscallN(fn, args[:nargs]...) 459} 460 461//go:linkname syscall_Syscall6 syscall.Syscall6 462//go:nosplit 463func syscall_Syscall6(fn, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2, err uintptr) { 464 args := [...]uintptr{a1, a2, a3, a4, a5, a6} 465 return syscall_SyscallN(fn, args[:nargs]...) 466} 467 468//go:linkname syscall_Syscall9 syscall.Syscall9 469//go:nosplit 470func syscall_Syscall9(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2, err uintptr) { 471 args := [...]uintptr{a1, a2, a3, a4, a5, a6, a7, a8, a9} 472 return syscall_SyscallN(fn, args[:nargs]...) 473} 474 475//go:linkname syscall_Syscall12 syscall.Syscall12 476//go:nosplit 477func syscall_Syscall12(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12 uintptr) (r1, r2, err uintptr) { 478 args := [...]uintptr{a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12} 479 return syscall_SyscallN(fn, args[:nargs]...) 480} 481 482//go:linkname syscall_Syscall15 syscall.Syscall15 483//go:nosplit 484func syscall_Syscall15(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15 uintptr) (r1, r2, err uintptr) { 485 args := [...]uintptr{a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15} 486 return syscall_SyscallN(fn, args[:nargs]...) 487} 488 489//go:linkname syscall_Syscall18 syscall.Syscall18 490//go:nosplit 491func syscall_Syscall18(fn, nargs, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18 uintptr) (r1, r2, err uintptr) { 492 args := [...]uintptr{a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18} 493 return syscall_SyscallN(fn, args[:nargs]...) 494} 495 496// maxArgs should be divisible by 2, as Windows stack 497// must be kept 16-byte aligned on syscall entry. 498// 499// Although it only permits maximum 42 parameters, it 500// is arguably large enough. 501const maxArgs = 42 502 503//go:linkname syscall_SyscallN syscall.SyscallN 504//go:nosplit 505func syscall_SyscallN(fn uintptr, args ...uintptr) (r1, r2, err uintptr) { 506 if len(args) > maxArgs { 507 panic("runtime: SyscallN has too many arguments") 508 } 509 510 // The cgocall parameters are stored in m instead of in 511 // the stack because the stack can move during fn if it 512 // calls back into Go. 513 c := &getg().m.winsyscall 514 c.fn = fn 515 c.n = uintptr(len(args)) 516 if c.n != 0 { 517 c.args = uintptr(noescape(unsafe.Pointer(&args[0]))) 518 } 519 cgocall(asmstdcallAddr, unsafe.Pointer(c)) 520 // cgocall may reschedule us on to a different M, 521 // but it copies the return values into the new M's 522 // so we can read them from there. 523 c = &getg().m.winsyscall 524 return c.r1, c.r2, c.err 525} 526