1// Copyright 2022 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5#include "go_asm.h" 6#include "go_tls.h" 7#include "funcdata.h" 8#include "textflag.h" 9 10#define REGCTXT R29 11 12TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 13 // R3 = stack; R4 = argc; R5 = argv 14 15 ADDV $-24, R3 16 MOVW R4, 8(R3) // argc 17 MOVV R5, 16(R3) // argv 18 19 // create istack out of the given (operating system) stack. 20 // _cgo_init may update stackguard. 21 MOVV $runtime·g0(SB), g 22 MOVV $(-64*1024), R30 23 ADDV R30, R3, R19 24 MOVV R19, g_stackguard0(g) 25 MOVV R19, g_stackguard1(g) 26 MOVV R19, (g_stack+stack_lo)(g) 27 MOVV R3, (g_stack+stack_hi)(g) 28 29 // if there is a _cgo_init, call it using the gcc ABI. 30 MOVV _cgo_init(SB), R25 31 BEQ R25, nocgo 32 33 MOVV R0, R7 // arg 3: not used 34 MOVV R0, R6 // arg 2: not used 35 MOVV $setg_gcc<>(SB), R5 // arg 1: setg 36 MOVV g, R4 // arg 0: G 37 JAL (R25) 38 39nocgo: 40 // update stackguard after _cgo_init 41 MOVV (g_stack+stack_lo)(g), R19 42 ADDV $const_stackGuard, R19 43 MOVV R19, g_stackguard0(g) 44 MOVV R19, g_stackguard1(g) 45 46 // set the per-goroutine and per-mach "registers" 47 MOVV $runtime·m0(SB), R19 48 49 // save m->g0 = g0 50 MOVV g, m_g0(R19) 51 // save m0 to g0->m 52 MOVV R19, g_m(g) 53 54 JAL runtime·check(SB) 55 56 // args are already prepared 57 JAL runtime·args(SB) 58 JAL runtime·osinit(SB) 59 JAL runtime·schedinit(SB) 60 61 // create a new goroutine to start program 62 MOVV $runtime·mainPC(SB), R19 // entry 63 ADDV $-16, R3 64 MOVV R19, 8(R3) 65 MOVV R0, 0(R3) 66 JAL runtime·newproc(SB) 67 ADDV $16, R3 68 69 // start this M 70 JAL runtime·mstart(SB) 71 72 MOVV R0, 1(R0) 73 RET 74 75DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB) 76GLOBL runtime·mainPC(SB),RODATA,$8 77 78TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 79 BREAK 80 RET 81 82TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 83 RET 84 85TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 86 JAL runtime·mstart0(SB) 87 RET // not reached 88 89// func cputicks() int64 90TEXT runtime·cputicks(SB),NOSPLIT,$0-8 91 RDTIMED R0, R4 92 MOVV R4, ret+0(FP) 93 RET 94 95/* 96 * go-routine 97 */ 98 99// void gogo(Gobuf*) 100// restore state from Gobuf; longjmp 101TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 102 MOVV buf+0(FP), R4 103 MOVV gobuf_g(R4), R5 104 MOVV 0(R5), R0 // make sure g != nil 105 JMP gogo<>(SB) 106 107TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 108 MOVV R5, g 109 JAL runtime·save_g(SB) 110 111 MOVV gobuf_sp(R4), R3 112 MOVV gobuf_lr(R4), R1 113 MOVV gobuf_ret(R4), R19 114 MOVV gobuf_ctxt(R4), REGCTXT 115 MOVV R0, gobuf_sp(R4) 116 MOVV R0, gobuf_ret(R4) 117 MOVV R0, gobuf_lr(R4) 118 MOVV R0, gobuf_ctxt(R4) 119 MOVV gobuf_pc(R4), R6 120 JMP (R6) 121 122// void mcall(fn func(*g)) 123// Switch to m->g0's stack, call fn(g). 124// Fn must never return. It should gogo(&g->sched) 125// to keep running g. 126TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8 127 MOVV R4, REGCTXT 128 // Save caller state in g->sched 129 MOVV R3, (g_sched+gobuf_sp)(g) 130 MOVV R1, (g_sched+gobuf_pc)(g) 131 MOVV R0, (g_sched+gobuf_lr)(g) 132 133 // Switch to m->g0 & its stack, call fn. 134 MOVV g, R4 // arg = g 135 MOVV g_m(g), R20 136 MOVV m_g0(R20), g 137 JAL runtime·save_g(SB) 138 BNE g, R4, 2(PC) 139 JMP runtime·badmcall(SB) 140 MOVV 0(REGCTXT), R20 // code pointer 141 MOVV (g_sched+gobuf_sp)(g), R3 // sp = m->g0->sched.sp 142 ADDV $-16, R3 143 MOVV R4, 8(R3) 144 MOVV R0, 0(R3) 145 JAL (R20) 146 JMP runtime·badmcall2(SB) 147 148// systemstack_switch is a dummy routine that systemstack leaves at the bottom 149// of the G stack. We need to distinguish the routine that 150// lives at the bottom of the G stack from the one that lives 151// at the top of the system stack because the one at the top of 152// the system stack terminates the stack walk (see topofstack()). 153TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 154 UNDEF 155 JAL (R1) // make sure this function is not leaf 156 RET 157 158// func systemstack(fn func()) 159TEXT runtime·systemstack(SB), NOSPLIT, $0-8 160 MOVV fn+0(FP), R19 // R19 = fn 161 MOVV R19, REGCTXT // context 162 MOVV g_m(g), R4 // R4 = m 163 164 MOVV m_gsignal(R4), R5 // R5 = gsignal 165 BEQ g, R5, noswitch 166 167 MOVV m_g0(R4), R5 // R5 = g0 168 BEQ g, R5, noswitch 169 170 MOVV m_curg(R4), R6 171 BEQ g, R6, switch 172 173 // Bad: g is not gsignal, not g0, not curg. What is it? 174 // Hide call from linker nosplit analysis. 175 MOVV $runtime·badsystemstack(SB), R7 176 JAL (R7) 177 JAL runtime·abort(SB) 178 179switch: 180 // save our state in g->sched. Pretend to 181 // be systemstack_switch if the G stack is scanned. 182 JAL gosave_systemstack_switch<>(SB) 183 184 // switch to g0 185 MOVV R5, g 186 JAL runtime·save_g(SB) 187 MOVV (g_sched+gobuf_sp)(g), R19 188 MOVV R19, R3 189 190 // call target function 191 MOVV 0(REGCTXT), R6 // code pointer 192 JAL (R6) 193 194 // switch back to g 195 MOVV g_m(g), R4 196 MOVV m_curg(R4), g 197 JAL runtime·save_g(SB) 198 MOVV (g_sched+gobuf_sp)(g), R3 199 MOVV R0, (g_sched+gobuf_sp)(g) 200 RET 201 202noswitch: 203 // already on m stack, just call directly 204 // Using a tail call here cleans up tracebacks since we won't stop 205 // at an intermediate systemstack. 206 MOVV 0(REGCTXT), R4 // code pointer 207 MOVV 0(R3), R1 // restore LR 208 ADDV $8, R3 209 JMP (R4) 210 211// func switchToCrashStack0(fn func()) 212TEXT runtime·switchToCrashStack0(SB), NOSPLIT, $0-8 213 MOVV fn+0(FP), REGCTXT // context register 214 MOVV g_m(g), R4 // curm 215 216 // set g to gcrash 217 MOVV $runtime·gcrash(SB), g // g = &gcrash 218 JAL runtime·save_g(SB) 219 MOVV R4, g_m(g) // g.m = curm 220 MOVV g, m_g0(R4) // curm.g0 = g 221 222 // switch to crashstack 223 MOVV (g_stack+stack_hi)(g), R4 224 ADDV $(-4*8), R4, R3 225 226 // call target function 227 MOVV 0(REGCTXT), R6 228 JAL (R6) 229 230 // should never return 231 JAL runtime·abort(SB) 232 UNDEF 233 234/* 235 * support for morestack 236 */ 237 238// Called during function prolog when more stack is needed. 239// Caller has already loaded: 240// loong64: R31: LR 241// 242// The traceback routines see morestack on a g0 as being 243// the top of a stack (for example, morestack calling newstack 244// calling the scheduler calling newm calling gc), so we must 245// record an argument size. For that purpose, it has no arguments. 246TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 247 // Called from f. 248 // Set g->sched to context in f. 249 MOVV R3, (g_sched+gobuf_sp)(g) 250 MOVV R1, (g_sched+gobuf_pc)(g) 251 MOVV R31, (g_sched+gobuf_lr)(g) 252 MOVV REGCTXT, (g_sched+gobuf_ctxt)(g) 253 254 // Cannot grow scheduler stack (m->g0). 255 MOVV g_m(g), R7 256 MOVV m_g0(R7), R8 257 BNE g, R8, 3(PC) 258 JAL runtime·badmorestackg0(SB) 259 JAL runtime·abort(SB) 260 261 // Cannot grow signal stack (m->gsignal). 262 MOVV m_gsignal(R7), R8 263 BNE g, R8, 3(PC) 264 JAL runtime·badmorestackgsignal(SB) 265 JAL runtime·abort(SB) 266 267 // Called from f. 268 // Set m->morebuf to f's caller. 269 MOVV R31, (m_morebuf+gobuf_pc)(R7) // f's caller's PC 270 MOVV R3, (m_morebuf+gobuf_sp)(R7) // f's caller's SP 271 MOVV g, (m_morebuf+gobuf_g)(R7) 272 273 // Call newstack on m->g0's stack. 274 MOVV m_g0(R7), g 275 JAL runtime·save_g(SB) 276 MOVV (g_sched+gobuf_sp)(g), R3 277 // Create a stack frame on g0 to call newstack. 278 MOVV R0, -8(R3) // Zero saved LR in frame 279 ADDV $-8, R3 280 JAL runtime·newstack(SB) 281 282 // Not reached, but make sure the return PC from the call to newstack 283 // is still in this function, and not the beginning of the next. 284 UNDEF 285 286TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 287 // Force SPWRITE. This function doesn't actually write SP, 288 // but it is called with a special calling convention where 289 // the caller doesn't save LR on stack but passes it as a 290 // register (R5), and the unwinder currently doesn't understand. 291 // Make it SPWRITE to stop unwinding. (See issue 54332) 292 MOVV R3, R3 293 294 MOVV R0, REGCTXT 295 JMP runtime·morestack(SB) 296 297// reflectcall: call a function with the given argument list 298// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs). 299// we don't have variable-sized frames, so we use a small number 300// of constant-sized-frame functions to encode a few bits of size in the pc. 301// Caution: ugly multiline assembly macros in your future! 302 303#define DISPATCH(NAME,MAXSIZE) \ 304 MOVV $MAXSIZE, R30; \ 305 SGTU R19, R30, R30; \ 306 BNE R30, 3(PC); \ 307 MOVV $NAME(SB), R4; \ 308 JMP (R4) 309// Note: can't just "BR NAME(SB)" - bad inlining results. 310 311TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48 312 MOVWU frameSize+32(FP), R19 313 DISPATCH(runtime·call32, 32) 314 DISPATCH(runtime·call64, 64) 315 DISPATCH(runtime·call128, 128) 316 DISPATCH(runtime·call256, 256) 317 DISPATCH(runtime·call512, 512) 318 DISPATCH(runtime·call1024, 1024) 319 DISPATCH(runtime·call2048, 2048) 320 DISPATCH(runtime·call4096, 4096) 321 DISPATCH(runtime·call8192, 8192) 322 DISPATCH(runtime·call16384, 16384) 323 DISPATCH(runtime·call32768, 32768) 324 DISPATCH(runtime·call65536, 65536) 325 DISPATCH(runtime·call131072, 131072) 326 DISPATCH(runtime·call262144, 262144) 327 DISPATCH(runtime·call524288, 524288) 328 DISPATCH(runtime·call1048576, 1048576) 329 DISPATCH(runtime·call2097152, 2097152) 330 DISPATCH(runtime·call4194304, 4194304) 331 DISPATCH(runtime·call8388608, 8388608) 332 DISPATCH(runtime·call16777216, 16777216) 333 DISPATCH(runtime·call33554432, 33554432) 334 DISPATCH(runtime·call67108864, 67108864) 335 DISPATCH(runtime·call134217728, 134217728) 336 DISPATCH(runtime·call268435456, 268435456) 337 DISPATCH(runtime·call536870912, 536870912) 338 DISPATCH(runtime·call1073741824, 1073741824) 339 MOVV $runtime·badreflectcall(SB), R4 340 JMP (R4) 341 342#define CALLFN(NAME,MAXSIZE) \ 343TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \ 344 NO_LOCAL_POINTERS; \ 345 /* copy arguments to stack */ \ 346 MOVV arg+16(FP), R4; \ 347 MOVWU argsize+24(FP), R5; \ 348 MOVV R3, R12; \ 349 ADDV $8, R12; \ 350 ADDV R12, R5; \ 351 BEQ R12, R5, 6(PC); \ 352 MOVBU (R4), R6; \ 353 ADDV $1, R4; \ 354 MOVBU R6, (R12); \ 355 ADDV $1, R12; \ 356 JMP -5(PC); \ 357 /* set up argument registers */ \ 358 MOVV regArgs+40(FP), R25; \ 359 JAL ·unspillArgs(SB); \ 360 /* call function */ \ 361 MOVV f+8(FP), REGCTXT; \ 362 MOVV (REGCTXT), R25; \ 363 PCDATA $PCDATA_StackMapIndex, $0; \ 364 JAL (R25); \ 365 /* copy return values back */ \ 366 MOVV regArgs+40(FP), R25; \ 367 JAL ·spillArgs(SB); \ 368 MOVV argtype+0(FP), R7; \ 369 MOVV arg+16(FP), R4; \ 370 MOVWU n+24(FP), R5; \ 371 MOVWU retoffset+28(FP), R6; \ 372 ADDV $8, R3, R12; \ 373 ADDV R6, R12; \ 374 ADDV R6, R4; \ 375 SUBVU R6, R5; \ 376 JAL callRet<>(SB); \ 377 RET 378 379// callRet copies return values back at the end of call*. This is a 380// separate function so it can allocate stack space for the arguments 381// to reflectcallmove. It does not follow the Go ABI; it expects its 382// arguments in registers. 383TEXT callRet<>(SB), NOSPLIT, $40-0 384 NO_LOCAL_POINTERS 385 MOVV R7, 8(R3) 386 MOVV R4, 16(R3) 387 MOVV R12, 24(R3) 388 MOVV R5, 32(R3) 389 MOVV R25, 40(R3) 390 JAL runtime·reflectcallmove(SB) 391 RET 392 393CALLFN(·call16, 16) 394CALLFN(·call32, 32) 395CALLFN(·call64, 64) 396CALLFN(·call128, 128) 397CALLFN(·call256, 256) 398CALLFN(·call512, 512) 399CALLFN(·call1024, 1024) 400CALLFN(·call2048, 2048) 401CALLFN(·call4096, 4096) 402CALLFN(·call8192, 8192) 403CALLFN(·call16384, 16384) 404CALLFN(·call32768, 32768) 405CALLFN(·call65536, 65536) 406CALLFN(·call131072, 131072) 407CALLFN(·call262144, 262144) 408CALLFN(·call524288, 524288) 409CALLFN(·call1048576, 1048576) 410CALLFN(·call2097152, 2097152) 411CALLFN(·call4194304, 4194304) 412CALLFN(·call8388608, 8388608) 413CALLFN(·call16777216, 16777216) 414CALLFN(·call33554432, 33554432) 415CALLFN(·call67108864, 67108864) 416CALLFN(·call134217728, 134217728) 417CALLFN(·call268435456, 268435456) 418CALLFN(·call536870912, 536870912) 419CALLFN(·call1073741824, 1073741824) 420 421TEXT runtime·procyield(SB),NOSPLIT,$0-0 422 RET 423 424// Save state of caller into g->sched. 425// but using fake PC from systemstack_switch. 426// Must only be called from functions with no locals ($0) 427// or else unwinding from systemstack_switch is incorrect. 428// Smashes R19. 429TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 430 MOVV $runtime·systemstack_switch(SB), R19 431 ADDV $8, R19 432 MOVV R19, (g_sched+gobuf_pc)(g) 433 MOVV R3, (g_sched+gobuf_sp)(g) 434 MOVV R0, (g_sched+gobuf_lr)(g) 435 MOVV R0, (g_sched+gobuf_ret)(g) 436 // Assert ctxt is zero. See func save. 437 MOVV (g_sched+gobuf_ctxt)(g), R19 438 BEQ R19, 2(PC) 439 JAL runtime·abort(SB) 440 RET 441 442// func asmcgocall(fn, arg unsafe.Pointer) int32 443// Call fn(arg) on the scheduler stack, 444// aligned appropriately for the gcc ABI. 445// See cgocall.go for more details. 446TEXT ·asmcgocall(SB),NOSPLIT,$0-20 447 MOVV fn+0(FP), R25 448 MOVV arg+8(FP), R4 449 450 MOVV R3, R12 // save original stack pointer 451 MOVV g, R13 452 453 // Figure out if we need to switch to m->g0 stack. 454 // We get called to create new OS threads too, and those 455 // come in on the m->g0 stack already. 456 MOVV g_m(g), R5 457 MOVV m_gsignal(R5), R6 458 BEQ R6, g, g0 459 MOVV m_g0(R5), R6 460 BEQ R6, g, g0 461 462 JAL gosave_systemstack_switch<>(SB) 463 MOVV R6, g 464 JAL runtime·save_g(SB) 465 MOVV (g_sched+gobuf_sp)(g), R3 466 467 // Now on a scheduling stack (a pthread-created stack). 468g0: 469 // Save room for two of our pointers. 470 ADDV $-16, R3 471 MOVV R13, 0(R3) // save old g on stack 472 MOVV (g_stack+stack_hi)(R13), R13 473 SUBVU R12, R13 474 MOVV R13, 8(R3) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 475 JAL (R25) 476 477 // Restore g, stack pointer. R4 is return value. 478 MOVV 0(R3), g 479 JAL runtime·save_g(SB) 480 MOVV (g_stack+stack_hi)(g), R5 481 MOVV 8(R3), R6 482 SUBVU R6, R5 483 MOVV R5, R3 484 485 MOVW R4, ret+16(FP) 486 RET 487 488// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) 489// See cgocall.go for more details. 490TEXT ·cgocallback(SB),NOSPLIT,$24-24 491 NO_LOCAL_POINTERS 492 493 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. 494 // It is used to dropm while thread is exiting. 495 MOVV fn+0(FP), R5 496 BNE R5, loadg 497 // Restore the g from frame. 498 MOVV frame+8(FP), g 499 JMP dropm 500 501loadg: 502 // Load m and g from thread-local storage. 503 MOVB runtime·iscgo(SB), R19 504 BEQ R19, nocgo 505 JAL runtime·load_g(SB) 506nocgo: 507 508 // If g is nil, Go did not create the current thread, 509 // or if this thread never called into Go on pthread platforms. 510 // Call needm to obtain one for temporary use. 511 // In this case, we're running on the thread stack, so there's 512 // lots of space, but the linker doesn't know. Hide the call from 513 // the linker analysis by using an indirect call. 514 BEQ g, needm 515 516 MOVV g_m(g), R12 517 MOVV R12, savedm-8(SP) 518 JMP havem 519 520needm: 521 MOVV g, savedm-8(SP) // g is zero, so is m. 522 MOVV $runtime·needAndBindM(SB), R4 523 JAL (R4) 524 525 // Set m->sched.sp = SP, so that if a panic happens 526 // during the function we are about to execute, it will 527 // have a valid SP to run on the g0 stack. 528 // The next few lines (after the havem label) 529 // will save this SP onto the stack and then write 530 // the same SP back to m->sched.sp. That seems redundant, 531 // but if an unrecovered panic happens, unwindm will 532 // restore the g->sched.sp from the stack location 533 // and then systemstack will try to use it. If we don't set it here, 534 // that restored SP will be uninitialized (typically 0) and 535 // will not be usable. 536 MOVV g_m(g), R12 537 MOVV m_g0(R12), R19 538 MOVV R3, (g_sched+gobuf_sp)(R19) 539 540havem: 541 // Now there's a valid m, and we're running on its m->g0. 542 // Save current m->g0->sched.sp on stack and then set it to SP. 543 // Save current sp in m->g0->sched.sp in preparation for 544 // switch back to m->curg stack. 545 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R29) aka savedsp-16(SP). 546 MOVV m_g0(R12), R19 547 MOVV (g_sched+gobuf_sp)(R19), R13 548 MOVV R13, savedsp-24(SP) // must match frame size 549 MOVV R3, (g_sched+gobuf_sp)(R19) 550 551 // Switch to m->curg stack and call runtime.cgocallbackg. 552 // Because we are taking over the execution of m->curg 553 // but *not* resuming what had been running, we need to 554 // save that information (m->curg->sched) so we can restore it. 555 // We can restore m->curg->sched.sp easily, because calling 556 // runtime.cgocallbackg leaves SP unchanged upon return. 557 // To save m->curg->sched.pc, we push it onto the stack. 558 // This has the added benefit that it looks to the traceback 559 // routine like cgocallbackg is going to return to that 560 // PC (because the frame we allocate below has the same 561 // size as cgocallback_gofunc's frame declared above) 562 // so that the traceback will seamlessly trace back into 563 // the earlier calls. 564 MOVV m_curg(R12), g 565 JAL runtime·save_g(SB) 566 MOVV (g_sched+gobuf_sp)(g), R13 // prepare stack as R13 567 MOVV (g_sched+gobuf_pc)(g), R4 568 MOVV R4, -(24+8)(R13) // "saved LR"; must match frame size 569 MOVV fn+0(FP), R5 570 MOVV frame+8(FP), R6 571 MOVV ctxt+16(FP), R7 572 MOVV $-(24+8)(R13), R3 573 MOVV R5, 8(R3) 574 MOVV R6, 16(R3) 575 MOVV R7, 24(R3) 576 JAL runtime·cgocallbackg(SB) 577 578 // Restore g->sched (== m->curg->sched) from saved values. 579 MOVV 0(R3), R4 580 MOVV R4, (g_sched+gobuf_pc)(g) 581 MOVV $(24+8)(R3), R13 // must match frame size 582 MOVV R13, (g_sched+gobuf_sp)(g) 583 584 // Switch back to m->g0's stack and restore m->g0->sched.sp. 585 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 586 // so we do not have to restore it.) 587 MOVV g_m(g), R12 588 MOVV m_g0(R12), g 589 JAL runtime·save_g(SB) 590 MOVV (g_sched+gobuf_sp)(g), R3 591 MOVV savedsp-24(SP), R13 // must match frame size 592 MOVV R13, (g_sched+gobuf_sp)(g) 593 594 // If the m on entry was nil, we called needm above to borrow an m, 595 // 1. for the duration of the call on non-pthread platforms, 596 // 2. or the duration of the C thread alive on pthread platforms. 597 // If the m on entry wasn't nil, 598 // 1. the thread might be a Go thread, 599 // 2. or it wasn't the first call from a C thread on pthread platforms, 600 // since then we skip dropm to resue the m in the first call. 601 MOVV savedm-8(SP), R12 602 BNE R12, droppedm 603 604 // Skip dropm to reuse it in the next call, when a pthread key has been created. 605 MOVV _cgo_pthread_key_created(SB), R12 606 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. 607 BEQ R12, dropm 608 MOVV (R12), R12 609 BNE R12, droppedm 610 611dropm: 612 MOVV $runtime·dropm(SB), R4 613 JAL (R4) 614droppedm: 615 616 // Done! 617 RET 618 619// void setg(G*); set g. for use by needm. 620TEXT runtime·setg(SB), NOSPLIT, $0-8 621 MOVV gg+0(FP), g 622 // This only happens if iscgo, so jump straight to save_g 623 JAL runtime·save_g(SB) 624 RET 625 626// void setg_gcc(G*); set g called from gcc with g in R19 627TEXT setg_gcc<>(SB),NOSPLIT,$0-0 628 MOVV R19, g 629 JAL runtime·save_g(SB) 630 RET 631 632TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 633 MOVW (R0), R0 634 UNDEF 635 636// AES hashing not implemented for loong64 637TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32 638 JMP runtime·memhashFallback<ABIInternal>(SB) 639TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24 640 JMP runtime·strhashFallback<ABIInternal>(SB) 641TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24 642 JMP runtime·memhash32Fallback<ABIInternal>(SB) 643TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24 644 JMP runtime·memhash64Fallback<ABIInternal>(SB) 645 646TEXT runtime·return0(SB), NOSPLIT, $0 647 MOVW $0, R19 648 RET 649 650// Called from cgo wrappers, this function returns g->m->curg.stack.hi. 651// Must obey the gcc calling convention. 652TEXT _cgo_topofstack(SB),NOSPLIT,$16 653 // g (R22) and REGTMP (R30) might be clobbered by load_g. They 654 // are callee-save in the gcc calling convention, so save them. 655 MOVV R30, savedREGTMP-16(SP) 656 MOVV g, savedG-8(SP) 657 658 JAL runtime·load_g(SB) 659 MOVV g_m(g), R19 660 MOVV m_curg(R19), R19 661 MOVV (g_stack+stack_hi)(R19), R4 // return value in R4 662 663 MOVV savedG-8(SP), g 664 MOVV savedREGTMP-16(SP), R30 665 RET 666 667// The top-most function running on a goroutine 668// returns to goexit+PCQuantum. 669TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 670 NOOP 671 JAL runtime·goexit1(SB) // does not return 672 // traceback from goexit1 must hit code range of goexit 673 NOOP 674 675// This is called from .init_array and follows the platform, not Go, ABI. 676TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0 677 ADDV $-0x10, R3 678 MOVV R30, 8(R3) // The access to global variables below implicitly uses R30, which is callee-save 679 MOVV runtime·lastmoduledatap(SB), R12 680 MOVV R4, moduledata_next(R12) 681 MOVV R4, runtime·lastmoduledatap(SB) 682 MOVV 8(R3), R30 683 ADDV $0x10, R3 684 RET 685 686TEXT ·checkASM(SB),NOSPLIT,$0-1 687 MOVW $1, R19 688 MOVB R19, ret+0(FP) 689 RET 690 691// spillArgs stores return values from registers to a *internal/abi.RegArgs in R25. 692TEXT ·spillArgs(SB),NOSPLIT,$0-0 693 MOVV R4, (0*8)(R25) 694 MOVV R5, (1*8)(R25) 695 MOVV R6, (2*8)(R25) 696 MOVV R7, (3*8)(R25) 697 MOVV R8, (4*8)(R25) 698 MOVV R9, (5*8)(R25) 699 MOVV R10, (6*8)(R25) 700 MOVV R11, (7*8)(R25) 701 MOVV R12, (8*8)(R25) 702 MOVV R13, (9*8)(R25) 703 MOVV R14, (10*8)(R25) 704 MOVV R15, (11*8)(R25) 705 MOVV R16, (12*8)(R25) 706 MOVV R17, (13*8)(R25) 707 MOVV R18, (14*8)(R25) 708 MOVV R19, (15*8)(R25) 709 MOVD F0, (16*8)(R25) 710 MOVD F1, (17*8)(R25) 711 MOVD F2, (18*8)(R25) 712 MOVD F3, (19*8)(R25) 713 MOVD F4, (20*8)(R25) 714 MOVD F5, (21*8)(R25) 715 MOVD F6, (22*8)(R25) 716 MOVD F7, (23*8)(R25) 717 MOVD F8, (24*8)(R25) 718 MOVD F9, (25*8)(R25) 719 MOVD F10, (26*8)(R25) 720 MOVD F11, (27*8)(R25) 721 MOVD F12, (28*8)(R25) 722 MOVD F13, (29*8)(R25) 723 MOVD F14, (30*8)(R25) 724 MOVD F15, (31*8)(R25) 725 RET 726 727// unspillArgs loads args into registers from a *internal/abi.RegArgs in R25. 728TEXT ·unspillArgs(SB),NOSPLIT,$0-0 729 MOVV (0*8)(R25), R4 730 MOVV (1*8)(R25), R5 731 MOVV (2*8)(R25), R6 732 MOVV (3*8)(R25), R7 733 MOVV (4*8)(R25), R8 734 MOVV (5*8)(R25), R9 735 MOVV (6*8)(R25), R10 736 MOVV (7*8)(R25), R11 737 MOVV (8*8)(R25), R12 738 MOVV (9*8)(R25), R13 739 MOVV (10*8)(R25), R14 740 MOVV (11*8)(R25), R15 741 MOVV (12*8)(R25), R16 742 MOVV (13*8)(R25), R17 743 MOVV (14*8)(R25), R18 744 MOVV (15*8)(R25), R19 745 MOVD (16*8)(R25), F0 746 MOVD (17*8)(R25), F1 747 MOVD (18*8)(R25), F2 748 MOVD (19*8)(R25), F3 749 MOVD (20*8)(R25), F4 750 MOVD (21*8)(R25), F5 751 MOVD (22*8)(R25), F6 752 MOVD (23*8)(R25), F7 753 MOVD (24*8)(R25), F8 754 MOVD (25*8)(R25), F9 755 MOVD (26*8)(R25), F10 756 MOVD (27*8)(R25), F11 757 MOVD (28*8)(R25), F12 758 MOVD (29*8)(R25), F13 759 MOVD (30*8)(R25), F14 760 MOVD (31*8)(R25), F15 761 RET 762 763// gcWriteBarrier informs the GC about heap pointer writes. 764// 765// gcWriteBarrier does NOT follow the Go ABI. It accepts the 766// number of bytes of buffer needed in R29, and returns a pointer 767// to the buffer space in R29. 768// It clobbers R30 (the linker temp register). 769// The act of CALLing gcWriteBarrier will clobber R1 (LR). 770// It does not clobber any other general-purpose registers, 771// but may clobber others (e.g., floating point registers). 772TEXT gcWriteBarrier<>(SB),NOSPLIT,$216 773 // Save the registers clobbered by the fast path. 774 MOVV R19, 208(R3) 775 MOVV R13, 216(R3) 776retry: 777 MOVV g_m(g), R19 778 MOVV m_p(R19), R19 779 MOVV (p_wbBuf+wbBuf_next)(R19), R13 780 MOVV (p_wbBuf+wbBuf_end)(R19), R30 // R30 is linker temp register 781 // Increment wbBuf.next position. 782 ADDV R29, R13 783 // Is the buffer full? 784 BLTU R30, R13, flush 785 // Commit to the larger buffer. 786 MOVV R13, (p_wbBuf+wbBuf_next)(R19) 787 // Make return value (the original next position) 788 SUBV R29, R13, R29 789 // Restore registers. 790 MOVV 208(R3), R19 791 MOVV 216(R3), R13 792 RET 793 794flush: 795 // Save all general purpose registers since these could be 796 // clobbered by wbBufFlush and were not saved by the caller. 797 MOVV R27, 8(R3) 798 MOVV R28, 16(R3) 799 // R1 is LR, which was saved by the prologue. 800 MOVV R2, 24(R3) 801 // R3 is SP. 802 MOVV R4, 32(R3) 803 MOVV R5, 40(R3) 804 MOVV R6, 48(R3) 805 MOVV R7, 56(R3) 806 MOVV R8, 64(R3) 807 MOVV R9, 72(R3) 808 MOVV R10, 80(R3) 809 MOVV R11, 88(R3) 810 MOVV R12, 96(R3) 811 // R13 already saved 812 MOVV R14, 104(R3) 813 MOVV R15, 112(R3) 814 MOVV R16, 120(R3) 815 MOVV R17, 128(R3) 816 MOVV R18, 136(R3) 817 // R19 already saved 818 MOVV R20, 144(R3) 819 MOVV R21, 152(R3) 820 // R22 is g. 821 MOVV R23, 160(R3) 822 MOVV R24, 168(R3) 823 MOVV R25, 176(R3) 824 MOVV R26, 184(R3) 825 // R27 already saved 826 // R28 already saved. 827 MOVV R29, 192(R3) 828 // R30 is tmp register. 829 MOVV R31, 200(R3) 830 831 CALL runtime·wbBufFlush(SB) 832 833 MOVV 8(R3), R27 834 MOVV 16(R3), R28 835 MOVV 24(R3), R2 836 MOVV 32(R3), R4 837 MOVV 40(R3), R5 838 MOVV 48(R3), R6 839 MOVV 56(R3), R7 840 MOVV 64(R3), R8 841 MOVV 72(R3), R9 842 MOVV 80(R3), R10 843 MOVV 88(R3), R11 844 MOVV 96(R3), R12 845 MOVV 104(R3), R14 846 MOVV 112(R3), R15 847 MOVV 120(R3), R16 848 MOVV 128(R3), R17 849 MOVV 136(R3), R18 850 MOVV 144(R3), R20 851 MOVV 152(R3), R21 852 MOVV 160(R3), R23 853 MOVV 168(R3), R24 854 MOVV 176(R3), R25 855 MOVV 184(R3), R26 856 MOVV 192(R3), R29 857 MOVV 200(R3), R31 858 JMP retry 859 860TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0 861 MOVV $8, R29 862 JMP gcWriteBarrier<>(SB) 863TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0 864 MOVV $16, R29 865 JMP gcWriteBarrier<>(SB) 866TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0 867 MOVV $24, R29 868 JMP gcWriteBarrier<>(SB) 869TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0 870 MOVV $32, R29 871 JMP gcWriteBarrier<>(SB) 872TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0 873 MOVV $40, R29 874 JMP gcWriteBarrier<>(SB) 875TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0 876 MOVV $48, R29 877 JMP gcWriteBarrier<>(SB) 878TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0 879 MOVV $56, R29 880 JMP gcWriteBarrier<>(SB) 881TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0 882 MOVV $64, R29 883 JMP gcWriteBarrier<>(SB) 884 885// Note: these functions use a special calling convention to save generated code space. 886// Arguments are passed in registers, but the space for those arguments are allocated 887// in the caller's stack frame. These stubs write the args into that stack space and 888// then tail call to the corresponding runtime handler. 889// The tail call makes these stubs disappear in backtraces. 890TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16 891 MOVV R20, R4 892 MOVV R21, R5 893 JMP runtime·goPanicIndex<ABIInternal>(SB) 894TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16 895 MOVV R20, R4 896 MOVV R21, R5 897 JMP runtime·goPanicIndexU<ABIInternal>(SB) 898TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16 899 MOVV R21, R4 900 MOVV R23, R5 901 JMP runtime·goPanicSliceAlen<ABIInternal>(SB) 902TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16 903 MOVV R21, R4 904 MOVV R23, R5 905 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB) 906TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16 907 MOVV R21, R4 908 MOVV R23, R5 909 JMP runtime·goPanicSliceAcap<ABIInternal>(SB) 910TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16 911 MOVV R21, R4 912 MOVV R23, R5 913 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB) 914TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16 915 MOVV R20, R4 916 MOVV R21, R5 917 JMP runtime·goPanicSliceB<ABIInternal>(SB) 918TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16 919 MOVV R20, R4 920 MOVV R21, R5 921 JMP runtime·goPanicSliceBU<ABIInternal>(SB) 922TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16 923 MOVV R23, R4 924 MOVV R24, R5 925 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB) 926TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16 927 MOVV R23, R4 928 MOVV R24, R5 929 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB) 930TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16 931 MOVV R23, R4 932 MOVV R24, R5 933 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB) 934TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16 935 MOVV R23, R4 936 MOVV R24, R5 937 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB) 938TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16 939 MOVV R21, R4 940 MOVV R23, R5 941 JMP runtime·goPanicSlice3B<ABIInternal>(SB) 942TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16 943 MOVV R21, R4 944 MOVV R23, R5 945 JMP runtime·goPanicSlice3BU<ABIInternal>(SB) 946TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16 947 MOVV R20, R4 948 MOVV R21, R5 949 JMP runtime·goPanicSlice3C<ABIInternal>(SB) 950TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16 951 MOVV R20, R4 952 MOVV R21, R5 953 JMP runtime·goPanicSlice3CU<ABIInternal>(SB) 954TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16 955 MOVV R23, R4 956 MOVV R24, R5 957 JMP runtime·goPanicSliceConvert<ABIInternal>(SB) 958