1// Copyright 2017 The Go Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style 3// license that can be found in the LICENSE file. 4 5#include "go_asm.h" 6#include "funcdata.h" 7#include "textflag.h" 8 9// func rt0_go() 10TEXT runtime·rt0_go(SB),NOSPLIT|TOPFRAME,$0 11 // X2 = stack; A0 = argc; A1 = argv 12 SUB $24, X2 13 MOV A0, 8(X2) // argc 14 MOV A1, 16(X2) // argv 15 16 // create istack out of the given (operating system) stack. 17 // _cgo_init may update stackguard. 18 MOV $runtime·g0(SB), g 19 MOV $(-64*1024), T0 20 ADD T0, X2, T1 21 MOV T1, g_stackguard0(g) 22 MOV T1, g_stackguard1(g) 23 MOV T1, (g_stack+stack_lo)(g) 24 MOV X2, (g_stack+stack_hi)(g) 25 26 // if there is a _cgo_init, call it using the gcc ABI. 27 MOV _cgo_init(SB), T0 28 BEQ T0, ZERO, nocgo 29 30 MOV ZERO, A3 // arg 3: not used 31 MOV ZERO, A2 // arg 2: not used 32 MOV $setg_gcc<>(SB), A1 // arg 1: setg 33 MOV g, A0 // arg 0: G 34 JALR RA, T0 35 36nocgo: 37 // update stackguard after _cgo_init 38 MOV (g_stack+stack_lo)(g), T0 39 ADD $const_stackGuard, T0 40 MOV T0, g_stackguard0(g) 41 MOV T0, g_stackguard1(g) 42 43 // set the per-goroutine and per-mach "registers" 44 MOV $runtime·m0(SB), T0 45 46 // save m->g0 = g0 47 MOV g, m_g0(T0) 48 // save m0 to g0->m 49 MOV T0, g_m(g) 50 51 CALL runtime·check(SB) 52 53 // args are already prepared 54 CALL runtime·args(SB) 55 CALL runtime·osinit(SB) 56 CALL runtime·schedinit(SB) 57 58 // create a new goroutine to start program 59 MOV $runtime·mainPC(SB), T0 // entry 60 SUB $16, X2 61 MOV T0, 8(X2) 62 MOV ZERO, 0(X2) 63 CALL runtime·newproc(SB) 64 ADD $16, X2 65 66 // start this M 67 CALL runtime·mstart(SB) 68 69 WORD $0 // crash if reached 70 RET 71 72TEXT runtime·mstart(SB),NOSPLIT|TOPFRAME,$0 73 CALL runtime·mstart0(SB) 74 RET // not reached 75 76// void setg_gcc(G*); set g called from gcc with g in A0 77TEXT setg_gcc<>(SB),NOSPLIT,$0-0 78 MOV A0, g 79 CALL runtime·save_g(SB) 80 RET 81 82// func cputicks() int64 83TEXT runtime·cputicks(SB),NOSPLIT,$0-8 84 // RDTIME to emulate cpu ticks 85 // RDCYCLE reads counter that is per HART(core) based 86 // according to the riscv manual, see issue 46737 87 RDTIME A0 88 MOV A0, ret+0(FP) 89 RET 90 91// systemstack_switch is a dummy routine that systemstack leaves at the bottom 92// of the G stack. We need to distinguish the routine that 93// lives at the bottom of the G stack from the one that lives 94// at the top of the system stack because the one at the top of 95// the system stack terminates the stack walk (see topofstack()). 96TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0 97 UNDEF 98 JALR RA, ZERO // make sure this function is not leaf 99 RET 100 101// func systemstack(fn func()) 102TEXT runtime·systemstack(SB), NOSPLIT, $0-8 103 MOV fn+0(FP), CTXT // CTXT = fn 104 MOV g_m(g), T0 // T0 = m 105 106 MOV m_gsignal(T0), T1 // T1 = gsignal 107 BEQ g, T1, noswitch 108 109 MOV m_g0(T0), T1 // T1 = g0 110 BEQ g, T1, noswitch 111 112 MOV m_curg(T0), T2 113 BEQ g, T2, switch 114 115 // Bad: g is not gsignal, not g0, not curg. What is it? 116 // Hide call from linker nosplit analysis. 117 MOV $runtime·badsystemstack(SB), T1 118 JALR RA, T1 119 120switch: 121 // save our state in g->sched. Pretend to 122 // be systemstack_switch if the G stack is scanned. 123 CALL gosave_systemstack_switch<>(SB) 124 125 // switch to g0 126 MOV T1, g 127 CALL runtime·save_g(SB) 128 MOV (g_sched+gobuf_sp)(g), T0 129 MOV T0, X2 130 131 // call target function 132 MOV 0(CTXT), T1 // code pointer 133 JALR RA, T1 134 135 // switch back to g 136 MOV g_m(g), T0 137 MOV m_curg(T0), g 138 CALL runtime·save_g(SB) 139 MOV (g_sched+gobuf_sp)(g), X2 140 MOV ZERO, (g_sched+gobuf_sp)(g) 141 RET 142 143noswitch: 144 // already on m stack, just call directly 145 // Using a tail call here cleans up tracebacks since we won't stop 146 // at an intermediate systemstack. 147 MOV 0(CTXT), T1 // code pointer 148 ADD $8, X2 149 JMP (T1) 150 151// func switchToCrashStack0(fn func()) 152TEXT runtime·switchToCrashStack0<ABIInternal>(SB), NOSPLIT, $0-8 153 MOV X10, CTXT // context register 154 MOV g_m(g), X11 // curm 155 156 // set g to gcrash 157 MOV $runtime·gcrash(SB), g // g = &gcrash 158 CALL runtime·save_g(SB) // clobbers X31 159 MOV X11, g_m(g) // g.m = curm 160 MOV g, m_g0(X11) // curm.g0 = g 161 162 // switch to crashstack 163 MOV (g_stack+stack_hi)(g), X11 164 SUB $(4*8), X11 165 MOV X11, X2 166 167 // call target function 168 MOV 0(CTXT), X10 169 JALR X1, X10 170 171 // should never return 172 CALL runtime·abort(SB) 173 UNDEF 174 175/* 176 * support for morestack 177 */ 178 179// Called during function prolog when more stack is needed. 180// Called with return address (i.e. caller's PC) in X5 (aka T0), 181// and the LR register contains the caller's LR. 182// 183// The traceback routines see morestack on a g0 as being 184// the top of a stack (for example, morestack calling newstack 185// calling the scheduler calling newm calling gc), so we must 186// record an argument size. For that purpose, it has no arguments. 187 188// func morestack() 189TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0 190 // Called from f. 191 // Set g->sched to context in f. 192 MOV X2, (g_sched+gobuf_sp)(g) 193 MOV T0, (g_sched+gobuf_pc)(g) 194 MOV RA, (g_sched+gobuf_lr)(g) 195 MOV CTXT, (g_sched+gobuf_ctxt)(g) 196 197 // Cannot grow scheduler stack (m->g0). 198 MOV g_m(g), A0 199 MOV m_g0(A0), A1 200 BNE g, A1, 3(PC) 201 CALL runtime·badmorestackg0(SB) 202 CALL runtime·abort(SB) 203 204 // Cannot grow signal stack (m->gsignal). 205 MOV m_gsignal(A0), A1 206 BNE g, A1, 3(PC) 207 CALL runtime·badmorestackgsignal(SB) 208 CALL runtime·abort(SB) 209 210 // Called from f. 211 // Set m->morebuf to f's caller. 212 MOV RA, (m_morebuf+gobuf_pc)(A0) // f's caller's PC 213 MOV X2, (m_morebuf+gobuf_sp)(A0) // f's caller's SP 214 MOV g, (m_morebuf+gobuf_g)(A0) 215 216 // Call newstack on m->g0's stack. 217 MOV m_g0(A0), g 218 CALL runtime·save_g(SB) 219 MOV (g_sched+gobuf_sp)(g), X2 220 // Create a stack frame on g0 to call newstack. 221 MOV ZERO, -8(X2) // Zero saved LR in frame 222 SUB $8, X2 223 CALL runtime·newstack(SB) 224 225 // Not reached, but make sure the return PC from the call to newstack 226 // is still in this function, and not the beginning of the next. 227 UNDEF 228 229// func morestack_noctxt() 230TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0 231 // Force SPWRITE. This function doesn't actually write SP, 232 // but it is called with a special calling convention where 233 // the caller doesn't save LR on stack but passes it as a 234 // register, and the unwinder currently doesn't understand. 235 // Make it SPWRITE to stop unwinding. (See issue 54332) 236 MOV X2, X2 237 238 MOV ZERO, CTXT 239 JMP runtime·morestack(SB) 240 241// AES hashing not implemented for riscv64 242TEXT runtime·memhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-32 243 JMP runtime·memhashFallback<ABIInternal>(SB) 244TEXT runtime·strhash<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24 245 JMP runtime·strhashFallback<ABIInternal>(SB) 246TEXT runtime·memhash32<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24 247 JMP runtime·memhash32Fallback<ABIInternal>(SB) 248TEXT runtime·memhash64<ABIInternal>(SB),NOSPLIT|NOFRAME,$0-24 249 JMP runtime·memhash64Fallback<ABIInternal>(SB) 250 251// func return0() 252TEXT runtime·return0(SB), NOSPLIT, $0 253 MOV $0, A0 254 RET 255 256// restore state from Gobuf; longjmp 257 258// func gogo(buf *gobuf) 259TEXT runtime·gogo(SB), NOSPLIT|NOFRAME, $0-8 260 MOV buf+0(FP), T0 261 MOV gobuf_g(T0), T1 262 MOV 0(T1), ZERO // make sure g != nil 263 JMP gogo<>(SB) 264 265TEXT gogo<>(SB), NOSPLIT|NOFRAME, $0 266 MOV T1, g 267 CALL runtime·save_g(SB) 268 269 MOV gobuf_sp(T0), X2 270 MOV gobuf_lr(T0), RA 271 MOV gobuf_ret(T0), A0 272 MOV gobuf_ctxt(T0), CTXT 273 MOV ZERO, gobuf_sp(T0) 274 MOV ZERO, gobuf_ret(T0) 275 MOV ZERO, gobuf_lr(T0) 276 MOV ZERO, gobuf_ctxt(T0) 277 MOV gobuf_pc(T0), T0 278 JALR ZERO, T0 279 280// func procyield(cycles uint32) 281TEXT runtime·procyield(SB),NOSPLIT,$0-0 282 RET 283 284// Switch to m->g0's stack, call fn(g). 285// Fn must never return. It should gogo(&g->sched) 286// to keep running g. 287 288// func mcall(fn func(*g)) 289TEXT runtime·mcall<ABIInternal>(SB), NOSPLIT|NOFRAME, $0-8 290 MOV X10, CTXT 291 292 // Save caller state in g->sched 293 MOV X2, (g_sched+gobuf_sp)(g) 294 MOV RA, (g_sched+gobuf_pc)(g) 295 MOV ZERO, (g_sched+gobuf_lr)(g) 296 297 // Switch to m->g0 & its stack, call fn. 298 MOV g, X10 299 MOV g_m(g), T1 300 MOV m_g0(T1), g 301 CALL runtime·save_g(SB) 302 BNE g, X10, 2(PC) 303 JMP runtime·badmcall(SB) 304 MOV 0(CTXT), T1 // code pointer 305 MOV (g_sched+gobuf_sp)(g), X2 // sp = m->g0->sched.sp 306 // we don't need special macro for regabi since arg0(X10) = g 307 SUB $16, X2 308 MOV X10, 8(X2) // setup g 309 MOV ZERO, 0(X2) // clear return address 310 JALR RA, T1 311 JMP runtime·badmcall2(SB) 312 313// Save state of caller into g->sched, 314// but using fake PC from systemstack_switch. 315// Must only be called from functions with no locals ($0) 316// or else unwinding from systemstack_switch is incorrect. 317// Smashes X31. 318TEXT gosave_systemstack_switch<>(SB),NOSPLIT|NOFRAME,$0 319 MOV $runtime·systemstack_switch(SB), X31 320 ADD $8, X31 // get past prologue 321 MOV X31, (g_sched+gobuf_pc)(g) 322 MOV X2, (g_sched+gobuf_sp)(g) 323 MOV ZERO, (g_sched+gobuf_lr)(g) 324 MOV ZERO, (g_sched+gobuf_ret)(g) 325 // Assert ctxt is zero. See func save. 326 MOV (g_sched+gobuf_ctxt)(g), X31 327 BEQ ZERO, X31, 2(PC) 328 CALL runtime·abort(SB) 329 RET 330 331// func asmcgocall_no_g(fn, arg unsafe.Pointer) 332// Call fn(arg) aligned appropriately for the gcc ABI. 333// Called on a system stack, and there may be no g yet (during needm). 334TEXT ·asmcgocall_no_g(SB),NOSPLIT,$0-16 335 MOV fn+0(FP), X5 336 MOV arg+8(FP), X10 337 JALR RA, (X5) 338 RET 339 340// func asmcgocall(fn, arg unsafe.Pointer) int32 341// Call fn(arg) on the scheduler stack, 342// aligned appropriately for the gcc ABI. 343// See cgocall.go for more details. 344TEXT ·asmcgocall(SB),NOSPLIT,$0-20 345 MOV fn+0(FP), X5 346 MOV arg+8(FP), X10 347 348 MOV X2, X8 // save original stack pointer 349 MOV g, X9 350 351 // Figure out if we need to switch to m->g0 stack. 352 // We get called to create new OS threads too, and those 353 // come in on the m->g0 stack already. Or we might already 354 // be on the m->gsignal stack. 355 MOV g_m(g), X6 356 MOV m_gsignal(X6), X7 357 BEQ X7, g, g0 358 MOV m_g0(X6), X7 359 BEQ X7, g, g0 360 361 CALL gosave_systemstack_switch<>(SB) 362 MOV X7, g 363 CALL runtime·save_g(SB) 364 MOV (g_sched+gobuf_sp)(g), X2 365 366 // Now on a scheduling stack (a pthread-created stack). 367g0: 368 // Save room for two of our pointers. 369 SUB $16, X2 370 MOV X9, 0(X2) // save old g on stack 371 MOV (g_stack+stack_hi)(X9), X9 372 SUB X8, X9, X8 373 MOV X8, 8(X2) // save depth in old g stack (can't just save SP, as stack might be copied during a callback) 374 375 JALR RA, (X5) 376 377 // Restore g, stack pointer. X10 is return value. 378 MOV 0(X2), g 379 CALL runtime·save_g(SB) 380 MOV (g_stack+stack_hi)(g), X5 381 MOV 8(X2), X6 382 SUB X6, X5, X6 383 MOV X6, X2 384 385 MOVW X10, ret+16(FP) 386 RET 387 388// func asminit() 389TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0 390 RET 391 392// reflectcall: call a function with the given argument list 393// func call(stackArgsType *_type, f *FuncVal, stackArgs *byte, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs). 394// we don't have variable-sized frames, so we use a small number 395// of constant-sized-frame functions to encode a few bits of size in the pc. 396// Caution: ugly multiline assembly macros in your future! 397 398#define DISPATCH(NAME,MAXSIZE) \ 399 MOV $MAXSIZE, T1 \ 400 BLTU T1, T0, 3(PC) \ 401 MOV $NAME(SB), T2; \ 402 JALR ZERO, T2 403// Note: can't just "BR NAME(SB)" - bad inlining results. 404 405// func call(stackArgsType *rtype, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs). 406TEXT reflect·call(SB), NOSPLIT, $0-0 407 JMP ·reflectcall(SB) 408 409// func call(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs). 410TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-48 411 MOVWU frameSize+32(FP), T0 412 DISPATCH(runtime·call16, 16) 413 DISPATCH(runtime·call32, 32) 414 DISPATCH(runtime·call64, 64) 415 DISPATCH(runtime·call128, 128) 416 DISPATCH(runtime·call256, 256) 417 DISPATCH(runtime·call512, 512) 418 DISPATCH(runtime·call1024, 1024) 419 DISPATCH(runtime·call2048, 2048) 420 DISPATCH(runtime·call4096, 4096) 421 DISPATCH(runtime·call8192, 8192) 422 DISPATCH(runtime·call16384, 16384) 423 DISPATCH(runtime·call32768, 32768) 424 DISPATCH(runtime·call65536, 65536) 425 DISPATCH(runtime·call131072, 131072) 426 DISPATCH(runtime·call262144, 262144) 427 DISPATCH(runtime·call524288, 524288) 428 DISPATCH(runtime·call1048576, 1048576) 429 DISPATCH(runtime·call2097152, 2097152) 430 DISPATCH(runtime·call4194304, 4194304) 431 DISPATCH(runtime·call8388608, 8388608) 432 DISPATCH(runtime·call16777216, 16777216) 433 DISPATCH(runtime·call33554432, 33554432) 434 DISPATCH(runtime·call67108864, 67108864) 435 DISPATCH(runtime·call134217728, 134217728) 436 DISPATCH(runtime·call268435456, 268435456) 437 DISPATCH(runtime·call536870912, 536870912) 438 DISPATCH(runtime·call1073741824, 1073741824) 439 MOV $runtime·badreflectcall(SB), T2 440 JALR ZERO, T2 441 442#define CALLFN(NAME,MAXSIZE) \ 443TEXT NAME(SB), WRAPPER, $MAXSIZE-48; \ 444 NO_LOCAL_POINTERS; \ 445 /* copy arguments to stack */ \ 446 MOV stackArgs+16(FP), A1; \ 447 MOVWU stackArgsSize+24(FP), A2; \ 448 MOV X2, A3; \ 449 ADD $8, A3; \ 450 ADD A3, A2; \ 451 BEQ A3, A2, 6(PC); \ 452 MOVBU (A1), A4; \ 453 ADD $1, A1; \ 454 MOVB A4, (A3); \ 455 ADD $1, A3; \ 456 JMP -5(PC); \ 457 /* set up argument registers */ \ 458 MOV regArgs+40(FP), X25; \ 459 CALL ·unspillArgs(SB); \ 460 /* call function */ \ 461 MOV f+8(FP), CTXT; \ 462 MOV (CTXT), X25; \ 463 PCDATA $PCDATA_StackMapIndex, $0; \ 464 JALR RA, X25; \ 465 /* copy return values back */ \ 466 MOV regArgs+40(FP), X25; \ 467 CALL ·spillArgs(SB); \ 468 MOV stackArgsType+0(FP), A5; \ 469 MOV stackArgs+16(FP), A1; \ 470 MOVWU stackArgsSize+24(FP), A2; \ 471 MOVWU stackRetOffset+28(FP), A4; \ 472 ADD $8, X2, A3; \ 473 ADD A4, A3; \ 474 ADD A4, A1; \ 475 SUB A4, A2; \ 476 CALL callRet<>(SB); \ 477 RET 478 479// callRet copies return values back at the end of call*. This is a 480// separate function so it can allocate stack space for the arguments 481// to reflectcallmove. It does not follow the Go ABI; it expects its 482// arguments in registers. 483TEXT callRet<>(SB), NOSPLIT, $40-0 484 NO_LOCAL_POINTERS 485 MOV A5, 8(X2) 486 MOV A1, 16(X2) 487 MOV A3, 24(X2) 488 MOV A2, 32(X2) 489 MOV X25, 40(X2) 490 CALL runtime·reflectcallmove(SB) 491 RET 492 493CALLFN(·call16, 16) 494CALLFN(·call32, 32) 495CALLFN(·call64, 64) 496CALLFN(·call128, 128) 497CALLFN(·call256, 256) 498CALLFN(·call512, 512) 499CALLFN(·call1024, 1024) 500CALLFN(·call2048, 2048) 501CALLFN(·call4096, 4096) 502CALLFN(·call8192, 8192) 503CALLFN(·call16384, 16384) 504CALLFN(·call32768, 32768) 505CALLFN(·call65536, 65536) 506CALLFN(·call131072, 131072) 507CALLFN(·call262144, 262144) 508CALLFN(·call524288, 524288) 509CALLFN(·call1048576, 1048576) 510CALLFN(·call2097152, 2097152) 511CALLFN(·call4194304, 4194304) 512CALLFN(·call8388608, 8388608) 513CALLFN(·call16777216, 16777216) 514CALLFN(·call33554432, 33554432) 515CALLFN(·call67108864, 67108864) 516CALLFN(·call134217728, 134217728) 517CALLFN(·call268435456, 268435456) 518CALLFN(·call536870912, 536870912) 519CALLFN(·call1073741824, 1073741824) 520 521// Called from cgo wrappers, this function returns g->m->curg.stack.hi. 522// Must obey the gcc calling convention. 523TEXT _cgo_topofstack(SB),NOSPLIT,$8 524 // g (X27) and REG_TMP (X31) might be clobbered by load_g. 525 // X27 is callee-save in the gcc calling convention, so save it. 526 MOV g, savedX27-8(SP) 527 528 CALL runtime·load_g(SB) 529 MOV g_m(g), X5 530 MOV m_curg(X5), X5 531 MOV (g_stack+stack_hi)(X5), X10 // return value in X10 532 533 MOV savedX27-8(SP), g 534 RET 535 536// func goexit(neverCallThisFunction) 537// The top-most function running on a goroutine 538// returns to goexit+PCQuantum. 539TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0 540 MOV ZERO, ZERO // NOP 541 JMP runtime·goexit1(SB) // does not return 542 // traceback from goexit1 must hit code range of goexit 543 MOV ZERO, ZERO // NOP 544 545// func cgocallback(fn, frame unsafe.Pointer, ctxt uintptr) 546// See cgocall.go for more details. 547TEXT ·cgocallback(SB),NOSPLIT,$24-24 548 NO_LOCAL_POINTERS 549 550 // Skip cgocallbackg, just dropm when fn is nil, and frame is the saved g. 551 // It is used to dropm while thread is exiting. 552 MOV fn+0(FP), X7 553 BNE ZERO, X7, loadg 554 // Restore the g from frame. 555 MOV frame+8(FP), g 556 JMP dropm 557 558loadg: 559 // Load m and g from thread-local storage. 560 MOVBU runtime·iscgo(SB), X5 561 BEQ ZERO, X5, nocgo 562 CALL runtime·load_g(SB) 563nocgo: 564 565 // If g is nil, Go did not create the current thread, 566 // or if this thread never called into Go on pthread platforms. 567 // Call needm to obtain one for temporary use. 568 // In this case, we're running on the thread stack, so there's 569 // lots of space, but the linker doesn't know. Hide the call from 570 // the linker analysis by using an indirect call. 571 BEQ ZERO, g, needm 572 573 MOV g_m(g), X5 574 MOV X5, savedm-8(SP) 575 JMP havem 576 577needm: 578 MOV g, savedm-8(SP) // g is zero, so is m. 579 MOV $runtime·needAndBindM(SB), X6 580 JALR RA, X6 581 582 // Set m->sched.sp = SP, so that if a panic happens 583 // during the function we are about to execute, it will 584 // have a valid SP to run on the g0 stack. 585 // The next few lines (after the havem label) 586 // will save this SP onto the stack and then write 587 // the same SP back to m->sched.sp. That seems redundant, 588 // but if an unrecovered panic happens, unwindm will 589 // restore the g->sched.sp from the stack location 590 // and then systemstack will try to use it. If we don't set it here, 591 // that restored SP will be uninitialized (typically 0) and 592 // will not be usable. 593 MOV g_m(g), X5 594 MOV m_g0(X5), X6 595 MOV X2, (g_sched+gobuf_sp)(X6) 596 597havem: 598 // Now there's a valid m, and we're running on its m->g0. 599 // Save current m->g0->sched.sp on stack and then set it to SP. 600 // Save current sp in m->g0->sched.sp in preparation for 601 // switch back to m->curg stack. 602 // NOTE: unwindm knows that the saved g->sched.sp is at 8(X2) aka savedsp-24(SP). 603 MOV m_g0(X5), X6 604 MOV (g_sched+gobuf_sp)(X6), X7 605 MOV X7, savedsp-24(SP) // must match frame size 606 MOV X2, (g_sched+gobuf_sp)(X6) 607 608 // Switch to m->curg stack and call runtime.cgocallbackg. 609 // Because we are taking over the execution of m->curg 610 // but *not* resuming what had been running, we need to 611 // save that information (m->curg->sched) so we can restore it. 612 // We can restore m->curg->sched.sp easily, because calling 613 // runtime.cgocallbackg leaves SP unchanged upon return. 614 // To save m->curg->sched.pc, we push it onto the curg stack and 615 // open a frame the same size as cgocallback's g0 frame. 616 // Once we switch to the curg stack, the pushed PC will appear 617 // to be the return PC of cgocallback, so that the traceback 618 // will seamlessly trace back into the earlier calls. 619 MOV m_curg(X5), g 620 CALL runtime·save_g(SB) 621 MOV (g_sched+gobuf_sp)(g), X6 // prepare stack as X6 622 MOV (g_sched+gobuf_pc)(g), X7 623 MOV X7, -(24+8)(X6) // "saved LR"; must match frame size 624 // Gather our arguments into registers. 625 MOV fn+0(FP), X7 626 MOV frame+8(FP), X8 627 MOV ctxt+16(FP), X9 628 MOV $-(24+8)(X6), X2 // switch stack; must match frame size 629 MOV X7, 8(X2) 630 MOV X8, 16(X2) 631 MOV X9, 24(X2) 632 CALL runtime·cgocallbackg(SB) 633 634 // Restore g->sched (== m->curg->sched) from saved values. 635 MOV 0(X2), X7 636 MOV X7, (g_sched+gobuf_pc)(g) 637 MOV $(24+8)(X2), X6 // must match frame size 638 MOV X6, (g_sched+gobuf_sp)(g) 639 640 // Switch back to m->g0's stack and restore m->g0->sched.sp. 641 // (Unlike m->curg, the g0 goroutine never uses sched.pc, 642 // so we do not have to restore it.) 643 MOV g_m(g), X5 644 MOV m_g0(X5), g 645 CALL runtime·save_g(SB) 646 MOV (g_sched+gobuf_sp)(g), X2 647 MOV savedsp-24(SP), X6 // must match frame size 648 MOV X6, (g_sched+gobuf_sp)(g) 649 650 // If the m on entry was nil, we called needm above to borrow an m, 651 // 1. for the duration of the call on non-pthread platforms, 652 // 2. or the duration of the C thread alive on pthread platforms. 653 // If the m on entry wasn't nil, 654 // 1. the thread might be a Go thread, 655 // 2. or it wasn't the first call from a C thread on pthread platforms, 656 // since then we skip dropm to reuse the m in the first call. 657 MOV savedm-8(SP), X5 658 BNE ZERO, X5, droppedm 659 660 // Skip dropm to reuse it in the next call, when a pthread key has been created. 661 MOV _cgo_pthread_key_created(SB), X5 662 // It means cgo is disabled when _cgo_pthread_key_created is a nil pointer, need dropm. 663 BEQ ZERO, X5, dropm 664 MOV (X5), X5 665 BNE ZERO, X5, droppedm 666 667dropm: 668 MOV $runtime·dropm(SB), X6 669 JALR RA, X6 670droppedm: 671 672 // Done! 673 RET 674 675TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0 676 EBREAK 677 RET 678 679TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0 680 EBREAK 681 RET 682 683// void setg(G*); set g. for use by needm. 684TEXT runtime·setg(SB), NOSPLIT, $0-8 685 MOV gg+0(FP), g 686 // This only happens if iscgo, so jump straight to save_g 687 CALL runtime·save_g(SB) 688 RET 689 690TEXT ·checkASM(SB),NOSPLIT,$0-1 691 MOV $1, T0 692 MOV T0, ret+0(FP) 693 RET 694 695// spillArgs stores return values from registers to a *internal/abi.RegArgs in X25. 696TEXT ·spillArgs(SB),NOSPLIT,$0-0 697 MOV X10, (0*8)(X25) 698 MOV X11, (1*8)(X25) 699 MOV X12, (2*8)(X25) 700 MOV X13, (3*8)(X25) 701 MOV X14, (4*8)(X25) 702 MOV X15, (5*8)(X25) 703 MOV X16, (6*8)(X25) 704 MOV X17, (7*8)(X25) 705 MOV X8, (8*8)(X25) 706 MOV X9, (9*8)(X25) 707 MOV X18, (10*8)(X25) 708 MOV X19, (11*8)(X25) 709 MOV X20, (12*8)(X25) 710 MOV X21, (13*8)(X25) 711 MOV X22, (14*8)(X25) 712 MOV X23, (15*8)(X25) 713 MOVD F10, (16*8)(X25) 714 MOVD F11, (17*8)(X25) 715 MOVD F12, (18*8)(X25) 716 MOVD F13, (19*8)(X25) 717 MOVD F14, (20*8)(X25) 718 MOVD F15, (21*8)(X25) 719 MOVD F16, (22*8)(X25) 720 MOVD F17, (23*8)(X25) 721 MOVD F8, (24*8)(X25) 722 MOVD F9, (25*8)(X25) 723 MOVD F18, (26*8)(X25) 724 MOVD F19, (27*8)(X25) 725 MOVD F20, (28*8)(X25) 726 MOVD F21, (29*8)(X25) 727 MOVD F22, (30*8)(X25) 728 MOVD F23, (31*8)(X25) 729 RET 730 731// unspillArgs loads args into registers from a *internal/abi.RegArgs in X25. 732TEXT ·unspillArgs(SB),NOSPLIT,$0-0 733 MOV (0*8)(X25), X10 734 MOV (1*8)(X25), X11 735 MOV (2*8)(X25), X12 736 MOV (3*8)(X25), X13 737 MOV (4*8)(X25), X14 738 MOV (5*8)(X25), X15 739 MOV (6*8)(X25), X16 740 MOV (7*8)(X25), X17 741 MOV (8*8)(X25), X8 742 MOV (9*8)(X25), X9 743 MOV (10*8)(X25), X18 744 MOV (11*8)(X25), X19 745 MOV (12*8)(X25), X20 746 MOV (13*8)(X25), X21 747 MOV (14*8)(X25), X22 748 MOV (15*8)(X25), X23 749 MOVD (16*8)(X25), F10 750 MOVD (17*8)(X25), F11 751 MOVD (18*8)(X25), F12 752 MOVD (19*8)(X25), F13 753 MOVD (20*8)(X25), F14 754 MOVD (21*8)(X25), F15 755 MOVD (22*8)(X25), F16 756 MOVD (23*8)(X25), F17 757 MOVD (24*8)(X25), F8 758 MOVD (25*8)(X25), F9 759 MOVD (26*8)(X25), F18 760 MOVD (27*8)(X25), F19 761 MOVD (28*8)(X25), F20 762 MOVD (29*8)(X25), F21 763 MOVD (30*8)(X25), F22 764 MOVD (31*8)(X25), F23 765 RET 766 767// gcWriteBarrier informs the GC about heap pointer writes. 768// 769// gcWriteBarrier does NOT follow the Go ABI. It accepts the 770// number of bytes of buffer needed in X24, and returns a pointer 771// to the buffer space in X24. 772// It clobbers X31 aka T6 (the linker temp register - REG_TMP). 773// The act of CALLing gcWriteBarrier will clobber RA (LR). 774// It does not clobber any other general-purpose registers, 775// but may clobber others (e.g., floating point registers). 776TEXT gcWriteBarrier<>(SB),NOSPLIT,$208 777 // Save the registers clobbered by the fast path. 778 MOV A0, 24*8(X2) 779 MOV A1, 25*8(X2) 780retry: 781 MOV g_m(g), A0 782 MOV m_p(A0), A0 783 MOV (p_wbBuf+wbBuf_next)(A0), A1 784 MOV (p_wbBuf+wbBuf_end)(A0), T6 // T6 is linker temp register (REG_TMP) 785 // Increment wbBuf.next position. 786 ADD X24, A1 787 // Is the buffer full? 788 BLTU T6, A1, flush 789 // Commit to the larger buffer. 790 MOV A1, (p_wbBuf+wbBuf_next)(A0) 791 // Make the return value (the original next position) 792 SUB X24, A1, X24 793 // Restore registers. 794 MOV 24*8(X2), A0 795 MOV 25*8(X2), A1 796 RET 797 798flush: 799 // Save all general purpose registers since these could be 800 // clobbered by wbBufFlush and were not saved by the caller. 801 MOV T0, 1*8(X2) 802 MOV T1, 2*8(X2) 803 // X0 is zero register 804 // X1 is LR, saved by prologue 805 // X2 is SP 806 // X3 is GP 807 // X4 is TP 808 MOV X7, 3*8(X2) 809 MOV X8, 4*8(X2) 810 MOV X9, 5*8(X2) 811 // X10 already saved (A0) 812 // X11 already saved (A1) 813 MOV X12, 6*8(X2) 814 MOV X13, 7*8(X2) 815 MOV X14, 8*8(X2) 816 MOV X15, 9*8(X2) 817 MOV X16, 10*8(X2) 818 MOV X17, 11*8(X2) 819 MOV X18, 12*8(X2) 820 MOV X19, 13*8(X2) 821 MOV X20, 14*8(X2) 822 MOV X21, 15*8(X2) 823 MOV X22, 16*8(X2) 824 MOV X23, 17*8(X2) 825 MOV X24, 18*8(X2) 826 MOV X25, 19*8(X2) 827 MOV X26, 20*8(X2) 828 // X27 is g. 829 MOV X28, 21*8(X2) 830 MOV X29, 22*8(X2) 831 MOV X30, 23*8(X2) 832 // X31 is tmp register. 833 834 CALL runtime·wbBufFlush(SB) 835 836 MOV 1*8(X2), T0 837 MOV 2*8(X2), T1 838 MOV 3*8(X2), X7 839 MOV 4*8(X2), X8 840 MOV 5*8(X2), X9 841 MOV 6*8(X2), X12 842 MOV 7*8(X2), X13 843 MOV 8*8(X2), X14 844 MOV 9*8(X2), X15 845 MOV 10*8(X2), X16 846 MOV 11*8(X2), X17 847 MOV 12*8(X2), X18 848 MOV 13*8(X2), X19 849 MOV 14*8(X2), X20 850 MOV 15*8(X2), X21 851 MOV 16*8(X2), X22 852 MOV 17*8(X2), X23 853 MOV 18*8(X2), X24 854 MOV 19*8(X2), X25 855 MOV 20*8(X2), X26 856 MOV 21*8(X2), X28 857 MOV 22*8(X2), X29 858 MOV 23*8(X2), X30 859 860 JMP retry 861 862TEXT runtime·gcWriteBarrier1<ABIInternal>(SB),NOSPLIT,$0 863 MOV $8, X24 864 JMP gcWriteBarrier<>(SB) 865TEXT runtime·gcWriteBarrier2<ABIInternal>(SB),NOSPLIT,$0 866 MOV $16, X24 867 JMP gcWriteBarrier<>(SB) 868TEXT runtime·gcWriteBarrier3<ABIInternal>(SB),NOSPLIT,$0 869 MOV $24, X24 870 JMP gcWriteBarrier<>(SB) 871TEXT runtime·gcWriteBarrier4<ABIInternal>(SB),NOSPLIT,$0 872 MOV $32, X24 873 JMP gcWriteBarrier<>(SB) 874TEXT runtime·gcWriteBarrier5<ABIInternal>(SB),NOSPLIT,$0 875 MOV $40, X24 876 JMP gcWriteBarrier<>(SB) 877TEXT runtime·gcWriteBarrier6<ABIInternal>(SB),NOSPLIT,$0 878 MOV $48, X24 879 JMP gcWriteBarrier<>(SB) 880TEXT runtime·gcWriteBarrier7<ABIInternal>(SB),NOSPLIT,$0 881 MOV $56, X24 882 JMP gcWriteBarrier<>(SB) 883TEXT runtime·gcWriteBarrier8<ABIInternal>(SB),NOSPLIT,$0 884 MOV $64, X24 885 JMP gcWriteBarrier<>(SB) 886 887// Note: these functions use a special calling convention to save generated code space. 888// Arguments are passed in registers (ssa/gen/RISCV64Ops.go), but the space for those 889// arguments are allocated in the caller's stack frame. 890// These stubs write the args into that stack space and then tail call to the 891// corresponding runtime handler. 892// The tail call makes these stubs disappear in backtraces. 893TEXT runtime·panicIndex<ABIInternal>(SB),NOSPLIT,$0-16 894 MOV T0, X10 895 MOV T1, X11 896 JMP runtime·goPanicIndex<ABIInternal>(SB) 897TEXT runtime·panicIndexU<ABIInternal>(SB),NOSPLIT,$0-16 898 MOV T0, X10 899 MOV T1, X11 900 JMP runtime·goPanicIndexU<ABIInternal>(SB) 901TEXT runtime·panicSliceAlen<ABIInternal>(SB),NOSPLIT,$0-16 902 MOV T1, X10 903 MOV T2, X11 904 JMP runtime·goPanicSliceAlen<ABIInternal>(SB) 905TEXT runtime·panicSliceAlenU<ABIInternal>(SB),NOSPLIT,$0-16 906 MOV T1, X10 907 MOV T2, X11 908 JMP runtime·goPanicSliceAlenU<ABIInternal>(SB) 909TEXT runtime·panicSliceAcap<ABIInternal>(SB),NOSPLIT,$0-16 910 MOV T1, X10 911 MOV T2, X11 912 JMP runtime·goPanicSliceAcap<ABIInternal>(SB) 913TEXT runtime·panicSliceAcapU<ABIInternal>(SB),NOSPLIT,$0-16 914 MOV T1, X10 915 MOV T2, X11 916 JMP runtime·goPanicSliceAcapU<ABIInternal>(SB) 917TEXT runtime·panicSliceB<ABIInternal>(SB),NOSPLIT,$0-16 918 MOV T0, X10 919 MOV T1, X11 920 JMP runtime·goPanicSliceB<ABIInternal>(SB) 921TEXT runtime·panicSliceBU<ABIInternal>(SB),NOSPLIT,$0-16 922 MOV T0, X10 923 MOV T1, X11 924 JMP runtime·goPanicSliceBU<ABIInternal>(SB) 925TEXT runtime·panicSlice3Alen<ABIInternal>(SB),NOSPLIT,$0-16 926 MOV T2, X10 927 MOV T3, X11 928 JMP runtime·goPanicSlice3Alen<ABIInternal>(SB) 929TEXT runtime·panicSlice3AlenU<ABIInternal>(SB),NOSPLIT,$0-16 930 MOV T2, X10 931 MOV T3, X11 932 JMP runtime·goPanicSlice3AlenU<ABIInternal>(SB) 933TEXT runtime·panicSlice3Acap<ABIInternal>(SB),NOSPLIT,$0-16 934 MOV T2, X10 935 MOV T3, X11 936 JMP runtime·goPanicSlice3Acap<ABIInternal>(SB) 937TEXT runtime·panicSlice3AcapU<ABIInternal>(SB),NOSPLIT,$0-16 938 MOV T2, X10 939 MOV T3, X11 940 JMP runtime·goPanicSlice3AcapU<ABIInternal>(SB) 941TEXT runtime·panicSlice3B<ABIInternal>(SB),NOSPLIT,$0-16 942 MOV T1, X10 943 MOV T2, X11 944 JMP runtime·goPanicSlice3B<ABIInternal>(SB) 945TEXT runtime·panicSlice3BU<ABIInternal>(SB),NOSPLIT,$0-16 946 MOV T1, X10 947 MOV T2, X11 948 JMP runtime·goPanicSlice3BU<ABIInternal>(SB) 949TEXT runtime·panicSlice3C<ABIInternal>(SB),NOSPLIT,$0-16 950 MOV T0, X10 951 MOV T1, X11 952 JMP runtime·goPanicSlice3C<ABIInternal>(SB) 953TEXT runtime·panicSlice3CU<ABIInternal>(SB),NOSPLIT,$0-16 954 MOV T0, X10 955 MOV T1, X11 956 JMP runtime·goPanicSlice3CU<ABIInternal>(SB) 957TEXT runtime·panicSliceConvert<ABIInternal>(SB),NOSPLIT,$0-16 958 MOV T2, X10 959 MOV T3, X11 960 JMP runtime·goPanicSliceConvert<ABIInternal>(SB) 961 962DATA runtime·mainPC+0(SB)/8,$runtime·main<ABIInternal>(SB) 963GLOBL runtime·mainPC(SB),RODATA,$8 964