Text file src/runtime/asm_ppc64x.s
1 // Copyright 2014 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 // +build ppc64 ppc64le
6
7 #include "go_asm.h"
8 #include "go_tls.h"
9 #include "funcdata.h"
10 #include "textflag.h"
11 #include "asm_ppc64x.h"
12
13 #ifdef GOOS_aix
14 #define cgoCalleeStackSize 48
15 #else
16 #define cgoCalleeStackSize 32
17 #endif
18
19 TEXT runtime·rt0_go(SB),NOSPLIT,$0
20 // R1 = stack; R3 = argc; R4 = argv; R13 = C TLS base pointer
21
22 // initialize essential registers
23 BL runtime·reginit(SB)
24
25 SUB $(FIXED_FRAME+16), R1
26 MOVD R2, 24(R1) // stash the TOC pointer away again now we've created a new frame
27 MOVW R3, FIXED_FRAME+0(R1) // argc
28 MOVD R4, FIXED_FRAME+8(R1) // argv
29
30 // create istack out of the given (operating system) stack.
31 // _cgo_init may update stackguard.
32 MOVD $runtime·g0(SB), g
33 BL runtime·save_g(SB)
34 MOVD $(-64*1024), R31
35 ADD R31, R1, R3
36 MOVD R3, g_stackguard0(g)
37 MOVD R3, g_stackguard1(g)
38 MOVD R3, (g_stack+stack_lo)(g)
39 MOVD R1, (g_stack+stack_hi)(g)
40
41 // if there is a _cgo_init, call it using the gcc ABI.
42 MOVD _cgo_init(SB), R12
43 CMP R0, R12
44 BEQ nocgo
45 #ifdef GOARCH_ppc64
46 // ppc64 use elf ABI v1. we must get the real entry address from
47 // first slot of the function descriptor before call.
48 MOVD 8(R12), R2
49 MOVD (R12), R12
50 #endif
51 MOVD R12, CTR // r12 = "global function entry point"
52 MOVD R13, R5 // arg 2: TLS base pointer
53 MOVD $setg_gcc<>(SB), R4 // arg 1: setg
54 MOVD g, R3 // arg 0: G
55 // C functions expect 32 (48 for AIX) bytes of space on caller
56 // stack frame and a 16-byte aligned R1
57 MOVD R1, R14 // save current stack
58 SUB $cgoCalleeStackSize, R1 // reserve the callee area
59 RLDCR $0, R1, $~15, R1 // 16-byte align
60 BL (CTR) // may clobber R0, R3-R12
61 MOVD R14, R1 // restore stack
62 #ifndef GOOS_aix
63 MOVD 24(R1), R2
64 #endif
65 XOR R0, R0 // fix R0
66
67 nocgo:
68 // update stackguard after _cgo_init
69 MOVD (g_stack+stack_lo)(g), R3
70 ADD $const__StackGuard, R3
71 MOVD R3, g_stackguard0(g)
72 MOVD R3, g_stackguard1(g)
73
74 // set the per-goroutine and per-mach "registers"
75 MOVD $runtime·m0(SB), R3
76
77 // save m->g0 = g0
78 MOVD g, m_g0(R3)
79 // save m0 to g0->m
80 MOVD R3, g_m(g)
81
82 BL runtime·check(SB)
83
84 // args are already prepared
85 BL runtime·args(SB)
86 BL runtime·osinit(SB)
87 BL runtime·schedinit(SB)
88
89 // create a new goroutine to start program
90 MOVD $runtime·mainPC(SB), R3 // entry
91 MOVDU R3, -8(R1)
92 MOVDU R0, -8(R1)
93 MOVDU R0, -8(R1)
94 MOVDU R0, -8(R1)
95 MOVDU R0, -8(R1)
96 MOVDU R0, -8(R1)
97 BL runtime·newproc(SB)
98 ADD $(16+FIXED_FRAME), R1
99
100 // start this M
101 BL runtime·mstart(SB)
102
103 MOVD R0, 0(R0)
104 RET
105
106 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
107 GLOBL runtime·mainPC(SB),RODATA,$8
108
109 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
110 MOVD R0, 0(R0) // TODO: TD
111 RET
112
113 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
114 RET
115
116 // Any changes must be reflected to runtime/cgo/gcc_aix_ppc64.S:.crosscall_ppc64
117 TEXT _cgo_reginit(SB),NOSPLIT|NOFRAME,$0-0
118 // crosscall_ppc64 and crosscall2 need to reginit, but can't
119 // get at the 'runtime.reginit' symbol.
120 BR runtime·reginit(SB)
121
122 TEXT runtime·reginit(SB),NOSPLIT|NOFRAME,$0-0
123 // set R0 to zero, it's expected by the toolchain
124 XOR R0, R0
125 RET
126
127 /*
128 * go-routine
129 */
130
131 // void gosave(Gobuf*)
132 // save state in Gobuf; setjmp
133 TEXT runtime·gosave(SB), NOSPLIT|NOFRAME, $0-8
134 MOVD buf+0(FP), R3
135 MOVD R1, gobuf_sp(R3)
136 MOVD LR, R31
137 MOVD R31, gobuf_pc(R3)
138 MOVD g, gobuf_g(R3)
139 MOVD R0, gobuf_lr(R3)
140 MOVD R0, gobuf_ret(R3)
141 // Assert ctxt is zero. See func save.
142 MOVD gobuf_ctxt(R3), R3
143 CMP R0, R3
144 BEQ 2(PC)
145 BL runtime·badctxt(SB)
146 RET
147
148 // void gogo(Gobuf*)
149 // restore state from Gobuf; longjmp
150 TEXT runtime·gogo(SB), NOSPLIT, $16-8
151 MOVD buf+0(FP), R5
152 MOVD gobuf_g(R5), g // make sure g is not nil
153 BL runtime·save_g(SB)
154
155 MOVD 0(g), R4
156 MOVD gobuf_sp(R5), R1
157 MOVD gobuf_lr(R5), R31
158 #ifndef GOOS_aix
159 MOVD 24(R1), R2 // restore R2
160 #endif
161 MOVD R31, LR
162 MOVD gobuf_ret(R5), R3
163 MOVD gobuf_ctxt(R5), R11
164 MOVD R0, gobuf_sp(R5)
165 MOVD R0, gobuf_ret(R5)
166 MOVD R0, gobuf_lr(R5)
167 MOVD R0, gobuf_ctxt(R5)
168 CMP R0, R0 // set condition codes for == test, needed by stack split
169 MOVD gobuf_pc(R5), R12
170 MOVD R12, CTR
171 BR (CTR)
172
173 // void mcall(fn func(*g))
174 // Switch to m->g0's stack, call fn(g).
175 // Fn must never return. It should gogo(&g->sched)
176 // to keep running g.
177 TEXT runtime·mcall(SB), NOSPLIT|NOFRAME, $0-8
178 // Save caller state in g->sched
179 MOVD R1, (g_sched+gobuf_sp)(g)
180 MOVD LR, R31
181 MOVD R31, (g_sched+gobuf_pc)(g)
182 MOVD R0, (g_sched+gobuf_lr)(g)
183 MOVD g, (g_sched+gobuf_g)(g)
184
185 // Switch to m->g0 & its stack, call fn.
186 MOVD g, R3
187 MOVD g_m(g), R8
188 MOVD m_g0(R8), g
189 BL runtime·save_g(SB)
190 CMP g, R3
191 BNE 2(PC)
192 BR runtime·badmcall(SB)
193 MOVD fn+0(FP), R11 // context
194 MOVD 0(R11), R12 // code pointer
195 MOVD R12, CTR
196 MOVD (g_sched+gobuf_sp)(g), R1 // sp = m->g0->sched.sp
197 MOVDU R3, -8(R1)
198 MOVDU R0, -8(R1)
199 MOVDU R0, -8(R1)
200 MOVDU R0, -8(R1)
201 MOVDU R0, -8(R1)
202 BL (CTR)
203 MOVD 24(R1), R2
204 BR runtime·badmcall2(SB)
205
206 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
207 // of the G stack. We need to distinguish the routine that
208 // lives at the bottom of the G stack from the one that lives
209 // at the top of the system stack because the one at the top of
210 // the system stack terminates the stack walk (see topofstack()).
211 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
212 // We have several undefs here so that 16 bytes past
213 // $runtime·systemstack_switch lies within them whether or not the
214 // instructions that derive r2 from r12 are there.
215 UNDEF
216 UNDEF
217 UNDEF
218 BL (LR) // make sure this function is not leaf
219 RET
220
221 // func systemstack(fn func())
222 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
223 MOVD fn+0(FP), R3 // R3 = fn
224 MOVD R3, R11 // context
225 MOVD g_m(g), R4 // R4 = m
226
227 MOVD m_gsignal(R4), R5 // R5 = gsignal
228 CMP g, R5
229 BEQ noswitch
230
231 MOVD m_g0(R4), R5 // R5 = g0
232 CMP g, R5
233 BEQ noswitch
234
235 MOVD m_curg(R4), R6
236 CMP g, R6
237 BEQ switch
238
239 // Bad: g is not gsignal, not g0, not curg. What is it?
240 // Hide call from linker nosplit analysis.
241 MOVD $runtime·badsystemstack(SB), R12
242 MOVD R12, CTR
243 BL (CTR)
244 BL runtime·abort(SB)
245
246 switch:
247 // save our state in g->sched. Pretend to
248 // be systemstack_switch if the G stack is scanned.
249 MOVD $runtime·systemstack_switch(SB), R6
250 ADD $16, R6 // get past prologue (including r2-setting instructions when they're there)
251 MOVD R6, (g_sched+gobuf_pc)(g)
252 MOVD R1, (g_sched+gobuf_sp)(g)
253 MOVD R0, (g_sched+gobuf_lr)(g)
254 MOVD g, (g_sched+gobuf_g)(g)
255
256 // switch to g0
257 MOVD R5, g
258 BL runtime·save_g(SB)
259 MOVD (g_sched+gobuf_sp)(g), R3
260 // make it look like mstart called systemstack on g0, to stop traceback
261 SUB $FIXED_FRAME, R3
262 MOVD $runtime·mstart(SB), R4
263 MOVD R4, 0(R3)
264 MOVD R3, R1
265
266 // call target function
267 MOVD 0(R11), R12 // code pointer
268 MOVD R12, CTR
269 BL (CTR)
270
271 // restore TOC pointer. It seems unlikely that we will use systemstack
272 // to call a function defined in another module, but the results of
273 // doing so would be so confusing that it's worth doing this.
274 MOVD g_m(g), R3
275 MOVD m_curg(R3), g
276 MOVD (g_sched+gobuf_sp)(g), R3
277 #ifndef GOOS_aix
278 MOVD 24(R3), R2
279 #endif
280 // switch back to g
281 MOVD g_m(g), R3
282 MOVD m_curg(R3), g
283 BL runtime·save_g(SB)
284 MOVD (g_sched+gobuf_sp)(g), R1
285 MOVD R0, (g_sched+gobuf_sp)(g)
286 RET
287
288 noswitch:
289 // already on m stack, just call directly
290 // On other arches we do a tail call here, but it appears to be
291 // impossible to tail call a function pointer in shared mode on
292 // ppc64 because the caller is responsible for restoring the TOC.
293 MOVD 0(R11), R12 // code pointer
294 MOVD R12, CTR
295 BL (CTR)
296 #ifndef GOOS_aix
297 MOVD 24(R1), R2
298 #endif
299 RET
300
301 /*
302 * support for morestack
303 */
304
305 // Called during function prolog when more stack is needed.
306 // Caller has already loaded:
307 // R3: framesize, R4: argsize, R5: LR
308 //
309 // The traceback routines see morestack on a g0 as being
310 // the top of a stack (for example, morestack calling newstack
311 // calling the scheduler calling newm calling gc), so we must
312 // record an argument size. For that purpose, it has no arguments.
313 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
314 // Cannot grow scheduler stack (m->g0).
315 MOVD g_m(g), R7
316 MOVD m_g0(R7), R8
317 CMP g, R8
318 BNE 3(PC)
319 BL runtime·badmorestackg0(SB)
320 BL runtime·abort(SB)
321
322 // Cannot grow signal stack (m->gsignal).
323 MOVD m_gsignal(R7), R8
324 CMP g, R8
325 BNE 3(PC)
326 BL runtime·badmorestackgsignal(SB)
327 BL runtime·abort(SB)
328
329 // Called from f.
330 // Set g->sched to context in f.
331 MOVD R1, (g_sched+gobuf_sp)(g)
332 MOVD LR, R8
333 MOVD R8, (g_sched+gobuf_pc)(g)
334 MOVD R5, (g_sched+gobuf_lr)(g)
335 MOVD R11, (g_sched+gobuf_ctxt)(g)
336
337 // Called from f.
338 // Set m->morebuf to f's caller.
339 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
340 MOVD R1, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
341 MOVD g, (m_morebuf+gobuf_g)(R7)
342
343 // Call newstack on m->g0's stack.
344 MOVD m_g0(R7), g
345 BL runtime·save_g(SB)
346 MOVD (g_sched+gobuf_sp)(g), R1
347 MOVDU R0, -(FIXED_FRAME+0)(R1) // create a call frame on g0
348 BL runtime·newstack(SB)
349
350 // Not reached, but make sure the return PC from the call to newstack
351 // is still in this function, and not the beginning of the next.
352 UNDEF
353
354 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
355 MOVD R0, R11
356 BR runtime·morestack(SB)
357
358 // reflectcall: call a function with the given argument list
359 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
360 // we don't have variable-sized frames, so we use a small number
361 // of constant-sized-frame functions to encode a few bits of size in the pc.
362 // Caution: ugly multiline assembly macros in your future!
363
364 #define DISPATCH(NAME,MAXSIZE) \
365 MOVD $MAXSIZE, R31; \
366 CMP R3, R31; \
367 BGT 4(PC); \
368 MOVD $NAME(SB), R12; \
369 MOVD R12, CTR; \
370 BR (CTR)
371 // Note: can't just "BR NAME(SB)" - bad inlining results.
372
373 TEXT ·reflectcall(SB), NOSPLIT|NOFRAME, $0-32
374 MOVWZ argsize+24(FP), R3
375 DISPATCH(runtime·call32, 32)
376 DISPATCH(runtime·call64, 64)
377 DISPATCH(runtime·call128, 128)
378 DISPATCH(runtime·call256, 256)
379 DISPATCH(runtime·call512, 512)
380 DISPATCH(runtime·call1024, 1024)
381 DISPATCH(runtime·call2048, 2048)
382 DISPATCH(runtime·call4096, 4096)
383 DISPATCH(runtime·call8192, 8192)
384 DISPATCH(runtime·call16384, 16384)
385 DISPATCH(runtime·call32768, 32768)
386 DISPATCH(runtime·call65536, 65536)
387 DISPATCH(runtime·call131072, 131072)
388 DISPATCH(runtime·call262144, 262144)
389 DISPATCH(runtime·call524288, 524288)
390 DISPATCH(runtime·call1048576, 1048576)
391 DISPATCH(runtime·call2097152, 2097152)
392 DISPATCH(runtime·call4194304, 4194304)
393 DISPATCH(runtime·call8388608, 8388608)
394 DISPATCH(runtime·call16777216, 16777216)
395 DISPATCH(runtime·call33554432, 33554432)
396 DISPATCH(runtime·call67108864, 67108864)
397 DISPATCH(runtime·call134217728, 134217728)
398 DISPATCH(runtime·call268435456, 268435456)
399 DISPATCH(runtime·call536870912, 536870912)
400 DISPATCH(runtime·call1073741824, 1073741824)
401 MOVD $runtime·badreflectcall(SB), R12
402 MOVD R12, CTR
403 BR (CTR)
404
405 #define CALLFN(NAME,MAXSIZE) \
406 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
407 NO_LOCAL_POINTERS; \
408 /* copy arguments to stack */ \
409 MOVD arg+16(FP), R3; \
410 MOVWZ argsize+24(FP), R4; \
411 MOVD R1, R5; \
412 CMP R4, $8; \
413 BLT tailsetup; \
414 /* copy 8 at a time if possible */ \
415 ADD $(FIXED_FRAME-8), R5; \
416 SUB $8, R3; \
417 top: \
418 MOVDU 8(R3), R7; \
419 MOVDU R7, 8(R5); \
420 SUB $8, R4; \
421 CMP R4, $8; \
422 BGE top; \
423 /* handle remaining bytes */ \
424 CMP $0, R4; \
425 BEQ callfn; \
426 ADD $7, R3; \
427 ADD $7, R5; \
428 BR tail; \
429 tailsetup: \
430 CMP $0, R4; \
431 BEQ callfn; \
432 ADD $(FIXED_FRAME-1), R5; \
433 SUB $1, R3; \
434 tail: \
435 MOVBU 1(R3), R6; \
436 MOVBU R6, 1(R5); \
437 SUB $1, R4; \
438 CMP $0, R4; \
439 BGT tail; \
440 callfn: \
441 /* call function */ \
442 MOVD f+8(FP), R11; \
443 #ifdef GOOS_aix \
444 /* AIX won't trigger a SIGSEGV if R11 = nil */ \
445 /* So it manually triggers it */ \
446 CMP R0, R11 \
447 BNE 2(PC) \
448 MOVD R0, 0(R0) \
449 #endif \
450 MOVD (R11), R12; \
451 MOVD R12, CTR; \
452 PCDATA $PCDATA_StackMapIndex, $0; \
453 BL (CTR); \
454 #ifndef GOOS_aix \
455 MOVD 24(R1), R2; \
456 #endif \
457 /* copy return values back */ \
458 MOVD argtype+0(FP), R7; \
459 MOVD arg+16(FP), R3; \
460 MOVWZ n+24(FP), R4; \
461 MOVWZ retoffset+28(FP), R6; \
462 ADD $FIXED_FRAME, R1, R5; \
463 ADD R6, R5; \
464 ADD R6, R3; \
465 SUB R6, R4; \
466 BL callRet<>(SB); \
467 RET
468
469 // callRet copies return values back at the end of call*. This is a
470 // separate function so it can allocate stack space for the arguments
471 // to reflectcallmove. It does not follow the Go ABI; it expects its
472 // arguments in registers.
473 TEXT callRet<>(SB), NOSPLIT, $32-0
474 MOVD R7, FIXED_FRAME+0(R1)
475 MOVD R3, FIXED_FRAME+8(R1)
476 MOVD R5, FIXED_FRAME+16(R1)
477 MOVD R4, FIXED_FRAME+24(R1)
478 BL runtime·reflectcallmove(SB)
479 RET
480
481 CALLFN(·call32, 32)
482 CALLFN(·call64, 64)
483 CALLFN(·call128, 128)
484 CALLFN(·call256, 256)
485 CALLFN(·call512, 512)
486 CALLFN(·call1024, 1024)
487 CALLFN(·call2048, 2048)
488 CALLFN(·call4096, 4096)
489 CALLFN(·call8192, 8192)
490 CALLFN(·call16384, 16384)
491 CALLFN(·call32768, 32768)
492 CALLFN(·call65536, 65536)
493 CALLFN(·call131072, 131072)
494 CALLFN(·call262144, 262144)
495 CALLFN(·call524288, 524288)
496 CALLFN(·call1048576, 1048576)
497 CALLFN(·call2097152, 2097152)
498 CALLFN(·call4194304, 4194304)
499 CALLFN(·call8388608, 8388608)
500 CALLFN(·call16777216, 16777216)
501 CALLFN(·call33554432, 33554432)
502 CALLFN(·call67108864, 67108864)
503 CALLFN(·call134217728, 134217728)
504 CALLFN(·call268435456, 268435456)
505 CALLFN(·call536870912, 536870912)
506 CALLFN(·call1073741824, 1073741824)
507
508 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0-4
509 MOVW cycles+0(FP), R7
510 // POWER does not have a pause/yield instruction equivalent.
511 // Instead, we can lower the program priority by setting the
512 // Program Priority Register prior to the wait loop and set it
513 // back to default afterwards. On Linux, the default priority is
514 // medium-low. For details, see page 837 of the ISA 3.0.
515 OR R1, R1, R1 // Set PPR priority to low
516 again:
517 SUB $1, R7
518 CMP $0, R7
519 BNE again
520 OR R6, R6, R6 // Set PPR priority back to medium-low
521 RET
522
523 // void jmpdefer(fv, sp);
524 // called from deferreturn.
525 // 1. grab stored LR for caller
526 // 2. sub 8 bytes to get back to either nop or toc reload before deferreturn
527 // 3. BR to fn
528 // When dynamically linking Go, it is not sufficient to rewind to the BL
529 // deferreturn -- we might be jumping between modules and so we need to reset
530 // the TOC pointer in r2. To do this, codegen inserts MOVD 24(R1), R2 *before*
531 // the BL deferreturn and jmpdefer rewinds to that.
532 TEXT runtime·jmpdefer(SB), NOSPLIT|NOFRAME, $0-16
533 MOVD 0(R1), R31
534 SUB $8, R31
535 MOVD R31, LR
536
537 MOVD fv+0(FP), R11
538 MOVD argp+8(FP), R1
539 SUB $FIXED_FRAME, R1
540 #ifdef GOOS_aix
541 // AIX won't trigger a SIGSEGV if R11 = nil
542 // So it manually triggers it
543 CMP R0, R11
544 BNE 2(PC)
545 MOVD R0, 0(R0)
546 #endif
547 MOVD 0(R11), R12
548 MOVD R12, CTR
549 BR (CTR)
550
551 // Save state of caller into g->sched. Smashes R31.
552 TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0
553 MOVD LR, R31
554 MOVD R31, (g_sched+gobuf_pc)(g)
555 MOVD R1, (g_sched+gobuf_sp)(g)
556 MOVD R0, (g_sched+gobuf_lr)(g)
557 MOVD R0, (g_sched+gobuf_ret)(g)
558 // Assert ctxt is zero. See func save.
559 MOVD (g_sched+gobuf_ctxt)(g), R31
560 CMP R0, R31
561 BEQ 2(PC)
562 BL runtime·badctxt(SB)
563 RET
564
565 #ifdef GOOS_aix
566 #define asmcgocallSaveOffset cgoCalleeStackSize + 8
567 #else
568 #define asmcgocallSaveOffset cgoCalleeStackSize
569 #endif
570
571 // func asmcgocall(fn, arg unsafe.Pointer) int32
572 // Call fn(arg) on the scheduler stack,
573 // aligned appropriately for the gcc ABI.
574 // See cgocall.go for more details.
575 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
576 MOVD fn+0(FP), R3
577 MOVD arg+8(FP), R4
578
579 MOVD R1, R7 // save original stack pointer
580 MOVD g, R5
581
582 // Figure out if we need to switch to m->g0 stack.
583 // We get called to create new OS threads too, and those
584 // come in on the m->g0 stack already.
585 // Moreover, if it's called inside the signal handler, it must not switch
586 // to g0 as it can be in use by another syscall.
587 MOVD g_m(g), R8
588 MOVD m_gsignal(R8), R6
589 CMP R6, g
590 BEQ g0
591 MOVD m_g0(R8), R6
592 CMP R6, g
593 BEQ g0
594 BL gosave<>(SB)
595 MOVD R6, g
596 BL runtime·save_g(SB)
597 MOVD (g_sched+gobuf_sp)(g), R1
598
599 // Now on a scheduling stack (a pthread-created stack).
600 g0:
601 #ifdef GOOS_aix
602 // Create a fake LR to improve backtrace.
603 MOVD $runtime·asmcgocall(SB), R6
604 MOVD R6, 16(R1)
605 // AIX also save one argument on the stack.
606 SUB $8, R1
607 #endif
608 // Save room for two of our pointers, plus the callee
609 // save area that lives on the caller stack.
610 SUB $(asmcgocallSaveOffset+16), R1
611 RLDCR $0, R1, $~15, R1 // 16-byte alignment for gcc ABI
612 MOVD R5, (asmcgocallSaveOffset+8)(R1)// save old g on stack
613 MOVD (g_stack+stack_hi)(R5), R5
614 SUB R7, R5
615 MOVD R5, asmcgocallSaveOffset(R1) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
616 #ifdef GOOS_aix
617 MOVD R7, 0(R1) // Save frame pointer to allow manual backtrace with gdb
618 #else
619 MOVD R0, 0(R1) // clear back chain pointer (TODO can we give it real back trace information?)
620 #endif
621 // This is a "global call", so put the global entry point in r12
622 MOVD R3, R12
623
624 #ifdef GOARCH_ppc64
625 // ppc64 use elf ABI v1. we must get the real entry address from
626 // first slot of the function descriptor before call.
627 // Same for AIX.
628 MOVD 8(R12), R2
629 MOVD (R12), R12
630 #endif
631 MOVD R12, CTR
632 MOVD R4, R3 // arg in r3
633 BL (CTR)
634 // C code can clobber R0, so set it back to 0. F27-F31 are
635 // callee save, so we don't need to recover those.
636 XOR R0, R0
637 // Restore g, stack pointer, toc pointer.
638 // R3 is errno, so don't touch it
639 MOVD (asmcgocallSaveOffset+8)(R1), g
640 MOVD (g_stack+stack_hi)(g), R5
641 MOVD asmcgocallSaveOffset(R1), R6
642 SUB R6, R5
643 #ifndef GOOS_aix
644 MOVD 24(R5), R2
645 #endif
646 MOVD R5, R1
647 BL runtime·save_g(SB)
648
649 MOVW R3, ret+16(FP)
650 RET
651
652 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt)
653 // Turn the fn into a Go func (by taking its address) and call
654 // cgocallback_gofunc.
655 TEXT runtime·cgocallback(SB),NOSPLIT,$32-32
656 MOVD $fn+0(FP), R3
657 MOVD R3, FIXED_FRAME+0(R1)
658 MOVD frame+8(FP), R3
659 MOVD R3, FIXED_FRAME+8(R1)
660 MOVD framesize+16(FP), R3
661 MOVD R3, FIXED_FRAME+16(R1)
662 MOVD ctxt+24(FP), R3
663 MOVD R3, FIXED_FRAME+24(R1)
664 MOVD $runtime·cgocallback_gofunc(SB), R12
665 MOVD R12, CTR
666 BL (CTR)
667 RET
668
669 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt)
670 // See cgocall.go for more details.
671 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32
672 NO_LOCAL_POINTERS
673
674 // Load m and g from thread-local storage.
675 MOVBZ runtime·iscgo(SB), R3
676 CMP R3, $0
677 BEQ nocgo
678 BL runtime·load_g(SB)
679 nocgo:
680
681 // If g is nil, Go did not create the current thread.
682 // Call needm to obtain one for temporary use.
683 // In this case, we're running on the thread stack, so there's
684 // lots of space, but the linker doesn't know. Hide the call from
685 // the linker analysis by using an indirect call.
686 CMP g, $0
687 BEQ needm
688
689 MOVD g_m(g), R8
690 MOVD R8, savedm-8(SP)
691 BR havem
692
693 needm:
694 MOVD g, savedm-8(SP) // g is zero, so is m.
695 MOVD $runtime·needm(SB), R12
696 MOVD R12, CTR
697 BL (CTR)
698
699 // Set m->sched.sp = SP, so that if a panic happens
700 // during the function we are about to execute, it will
701 // have a valid SP to run on the g0 stack.
702 // The next few lines (after the havem label)
703 // will save this SP onto the stack and then write
704 // the same SP back to m->sched.sp. That seems redundant,
705 // but if an unrecovered panic happens, unwindm will
706 // restore the g->sched.sp from the stack location
707 // and then systemstack will try to use it. If we don't set it here,
708 // that restored SP will be uninitialized (typically 0) and
709 // will not be usable.
710 MOVD g_m(g), R8
711 MOVD m_g0(R8), R3
712 MOVD R1, (g_sched+gobuf_sp)(R3)
713
714 havem:
715 // Now there's a valid m, and we're running on its m->g0.
716 // Save current m->g0->sched.sp on stack and then set it to SP.
717 // Save current sp in m->g0->sched.sp in preparation for
718 // switch back to m->curg stack.
719 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
720 MOVD m_g0(R8), R3
721 MOVD (g_sched+gobuf_sp)(R3), R4
722 MOVD R4, savedsp-16(SP)
723 MOVD R1, (g_sched+gobuf_sp)(R3)
724
725 // Switch to m->curg stack and call runtime.cgocallbackg.
726 // Because we are taking over the execution of m->curg
727 // but *not* resuming what had been running, we need to
728 // save that information (m->curg->sched) so we can restore it.
729 // We can restore m->curg->sched.sp easily, because calling
730 // runtime.cgocallbackg leaves SP unchanged upon return.
731 // To save m->curg->sched.pc, we push it onto the stack.
732 // This has the added benefit that it looks to the traceback
733 // routine like cgocallbackg is going to return to that
734 // PC (because the frame we allocate below has the same
735 // size as cgocallback_gofunc's frame declared above)
736 // so that the traceback will seamlessly trace back into
737 // the earlier calls.
738 //
739 // In the new goroutine, -8(SP) is unused (where SP refers to
740 // m->curg's SP while we're setting it up, before we've adjusted it).
741 MOVD m_curg(R8), g
742 BL runtime·save_g(SB)
743 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
744 MOVD (g_sched+gobuf_pc)(g), R5
745 MOVD R5, -(FIXED_FRAME+16)(R4)
746 MOVD ctxt+24(FP), R3
747 MOVD R3, -16(R4)
748 MOVD $-(FIXED_FRAME+16)(R4), R1
749 BL runtime·cgocallbackg(SB)
750
751 // Restore g->sched (== m->curg->sched) from saved values.
752 MOVD 0(R1), R5
753 MOVD R5, (g_sched+gobuf_pc)(g)
754 MOVD $(FIXED_FRAME+16)(R1), R4
755 MOVD R4, (g_sched+gobuf_sp)(g)
756
757 // Switch back to m->g0's stack and restore m->g0->sched.sp.
758 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
759 // so we do not have to restore it.)
760 MOVD g_m(g), R8
761 MOVD m_g0(R8), g
762 BL runtime·save_g(SB)
763 MOVD (g_sched+gobuf_sp)(g), R1
764 MOVD savedsp-16(SP), R4
765 MOVD R4, (g_sched+gobuf_sp)(g)
766
767 // If the m on entry was nil, we called needm above to borrow an m
768 // for the duration of the call. Since the call is over, return it with dropm.
769 MOVD savedm-8(SP), R6
770 CMP R6, $0
771 BNE droppedm
772 MOVD $runtime·dropm(SB), R12
773 MOVD R12, CTR
774 BL (CTR)
775 droppedm:
776
777 // Done!
778 RET
779
780 // void setg(G*); set g. for use by needm.
781 TEXT runtime·setg(SB), NOSPLIT, $0-8
782 MOVD gg+0(FP), g
783 // This only happens if iscgo, so jump straight to save_g
784 BL runtime·save_g(SB)
785 RET
786
787 #ifdef GOARCH_ppc64
788 #ifdef GOOS_aix
789 DATA setg_gcc<>+0(SB)/8, $_setg_gcc<>(SB)
790 DATA setg_gcc<>+8(SB)/8, $TOC(SB)
791 DATA setg_gcc<>+16(SB)/8, $0
792 GLOBL setg_gcc<>(SB), NOPTR, $24
793 #else
794 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
795 DWORD $_setg_gcc<>(SB)
796 DWORD $0
797 DWORD $0
798 #endif
799 #endif
800
801 // void setg_gcc(G*); set g in C TLS.
802 // Must obey the gcc calling convention.
803 #ifdef GOARCH_ppc64le
804 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
805 #else
806 TEXT _setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
807 #endif
808 // The standard prologue clobbers R31, which is callee-save in
809 // the C ABI, so we have to use $-8-0 and save LR ourselves.
810 MOVD LR, R4
811 // Also save g and R31, since they're callee-save in C ABI
812 MOVD R31, R5
813 MOVD g, R6
814
815 MOVD R3, g
816 BL runtime·save_g(SB)
817
818 MOVD R6, g
819 MOVD R5, R31
820 MOVD R4, LR
821 RET
822
823 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
824 MOVW (R0), R0
825 UNDEF
826
827 #define TBR 268
828
829 // int64 runtime·cputicks(void)
830 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
831 MOVD SPR(TBR), R3
832 MOVD R3, ret+0(FP)
833 RET
834
835 // AES hashing not implemented for ppc64
836 TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-0
837 MOVW (R0), R1
838 TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-0
839 MOVW (R0), R1
840 TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-0
841 MOVW (R0), R1
842 TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-0
843 MOVW (R0), R1
844
845 TEXT runtime·return0(SB), NOSPLIT, $0
846 MOVW $0, R3
847 RET
848
849 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
850 // Must obey the gcc calling convention.
851 #ifdef GOOS_aix
852 // On AIX, _cgo_topofstack is defined in runtime/cgo, because it must
853 // be a longcall in order to prevent trampolines from ld.
854 TEXT __cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
855 #else
856 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
857 #endif
858 // g (R30) and R31 are callee-save in the C ABI, so save them
859 MOVD g, R4
860 MOVD R31, R5
861 MOVD LR, R6
862
863 BL runtime·load_g(SB) // clobbers g (R30), R31
864 MOVD g_m(g), R3
865 MOVD m_curg(R3), R3
866 MOVD (g_stack+stack_hi)(R3), R3
867
868 MOVD R4, g
869 MOVD R5, R31
870 MOVD R6, LR
871 RET
872
873 // The top-most function running on a goroutine
874 // returns to goexit+PCQuantum.
875 //
876 // When dynamically linking Go, it can be returned to from a function
877 // implemented in a different module and so needs to reload the TOC pointer
878 // from the stack (although this function declares that it does not set up x-a
879 // frame, newproc1 does in fact allocate one for goexit and saves the TOC
880 // pointer in the correct place).
881 // goexit+_PCQuantum is halfway through the usual global entry point prologue
882 // that derives r2 from r12 which is a bit silly, but not harmful.
883 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
884 MOVD 24(R1), R2
885 BL runtime·goexit1(SB) // does not return
886 // traceback from goexit1 must hit code range of goexit
887 MOVD R0, R0 // NOP
888
889 // prepGoExitFrame saves the current TOC pointer (i.e. the TOC pointer for the
890 // module containing runtime) to the frame that goexit will execute in when
891 // the goroutine exits. It's implemented in assembly mainly because that's the
892 // easiest way to get access to R2.
893 TEXT runtime·prepGoExitFrame(SB),NOSPLIT,$0-8
894 MOVD sp+0(FP), R3
895 MOVD R2, 24(R3)
896 RET
897
898 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
899 ADD $-8, R1
900 MOVD R31, 0(R1)
901 MOVD runtime·lastmoduledatap(SB), R4
902 MOVD R3, moduledata_next(R4)
903 MOVD R3, runtime·lastmoduledatap(SB)
904 MOVD 0(R1), R31
905 ADD $8, R1
906 RET
907
908 TEXT ·checkASM(SB),NOSPLIT,$0-1
909 MOVW $1, R3
910 MOVB R3, ret+0(FP)
911 RET
912
913 // gcWriteBarrier performs a heap pointer write and informs the GC.
914 //
915 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
916 // - R20 is the destination of the write
917 // - R21 is the value being written at R20.
918 // It clobbers condition codes.
919 // It does not clobber R0 through R15,
920 // but may clobber any other register, *including* R31.
921 TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$112
922 // The standard prologue clobbers R31.
923 // We use R16 and R17 as scratch registers.
924 MOVD g_m(g), R16
925 MOVD m_p(R16), R16
926 MOVD (p_wbBuf+wbBuf_next)(R16), R17
927 // Increment wbBuf.next position.
928 ADD $16, R17
929 MOVD R17, (p_wbBuf+wbBuf_next)(R16)
930 MOVD (p_wbBuf+wbBuf_end)(R16), R16
931 CMP R16, R17
932 // Record the write.
933 MOVD R21, -16(R17) // Record value
934 MOVD (R20), R16 // TODO: This turns bad writes into bad reads.
935 MOVD R16, -8(R17) // Record *slot
936 // Is the buffer full? (flags set in CMP above)
937 BEQ flush
938 ret:
939 // Do the write.
940 MOVD R21, (R20)
941 RET
942
943 flush:
944 // Save registers R0 through R15 since these were not saved by the caller.
945 // We don't save all registers on ppc64 because it takes too much space.
946 MOVD R20, (FIXED_FRAME+0)(R1) // Also first argument to wbBufFlush
947 MOVD R21, (FIXED_FRAME+8)(R1) // Also second argument to wbBufFlush
948 // R0 is always 0, so no need to spill.
949 // R1 is SP.
950 // R2 is SB.
951 MOVD R3, (FIXED_FRAME+16)(R1)
952 MOVD R4, (FIXED_FRAME+24)(R1)
953 MOVD R5, (FIXED_FRAME+32)(R1)
954 MOVD R6, (FIXED_FRAME+40)(R1)
955 MOVD R7, (FIXED_FRAME+48)(R1)
956 MOVD R8, (FIXED_FRAME+56)(R1)
957 MOVD R9, (FIXED_FRAME+64)(R1)
958 MOVD R10, (FIXED_FRAME+72)(R1)
959 MOVD R11, (FIXED_FRAME+80)(R1)
960 MOVD R12, (FIXED_FRAME+88)(R1)
961 // R13 is REGTLS
962 MOVD R14, (FIXED_FRAME+96)(R1)
963 MOVD R15, (FIXED_FRAME+104)(R1)
964
965 // This takes arguments R20 and R21.
966 CALL runtime·wbBufFlush(SB)
967
968 MOVD (FIXED_FRAME+0)(R1), R20
969 MOVD (FIXED_FRAME+8)(R1), R21
970 MOVD (FIXED_FRAME+16)(R1), R3
971 MOVD (FIXED_FRAME+24)(R1), R4
972 MOVD (FIXED_FRAME+32)(R1), R5
973 MOVD (FIXED_FRAME+40)(R1), R6
974 MOVD (FIXED_FRAME+48)(R1), R7
975 MOVD (FIXED_FRAME+56)(R1), R8
976 MOVD (FIXED_FRAME+64)(R1), R9
977 MOVD (FIXED_FRAME+72)(R1), R10
978 MOVD (FIXED_FRAME+80)(R1), R11
979 MOVD (FIXED_FRAME+88)(R1), R12
980 MOVD (FIXED_FRAME+96)(R1), R14
981 MOVD (FIXED_FRAME+104)(R1), R15
982 JMP ret
983
984 // Note: these functions use a special calling convention to save generated code space.
985 // Arguments are passed in registers, but the space for those arguments are allocated
986 // in the caller's stack frame. These stubs write the args into that stack space and
987 // then tail call to the corresponding runtime handler.
988 // The tail call makes these stubs disappear in backtraces.
989 TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
990 MOVD R3, x+0(FP)
991 MOVD R4, y+8(FP)
992 JMP runtime·goPanicIndex(SB)
993 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
994 MOVD R3, x+0(FP)
995 MOVD R4, y+8(FP)
996 JMP runtime·goPanicIndexU(SB)
997 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
998 MOVD R4, x+0(FP)
999 MOVD R5, y+8(FP)
1000 JMP runtime·goPanicSliceAlen(SB)
1001 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
1002 MOVD R4, x+0(FP)
1003 MOVD R5, y+8(FP)
1004 JMP runtime·goPanicSliceAlenU(SB)
1005 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
1006 MOVD R4, x+0(FP)
1007 MOVD R5, y+8(FP)
1008 JMP runtime·goPanicSliceAcap(SB)
1009 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
1010 MOVD R4, x+0(FP)
1011 MOVD R5, y+8(FP)
1012 JMP runtime·goPanicSliceAcapU(SB)
1013 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
1014 MOVD R3, x+0(FP)
1015 MOVD R4, y+8(FP)
1016 JMP runtime·goPanicSliceB(SB)
1017 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
1018 MOVD R3, x+0(FP)
1019 MOVD R4, y+8(FP)
1020 JMP runtime·goPanicSliceBU(SB)
1021 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
1022 MOVD R5, x+0(FP)
1023 MOVD R6, y+8(FP)
1024 JMP runtime·goPanicSlice3Alen(SB)
1025 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
1026 MOVD R5, x+0(FP)
1027 MOVD R6, y+8(FP)
1028 JMP runtime·goPanicSlice3AlenU(SB)
1029 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
1030 MOVD R5, x+0(FP)
1031 MOVD R6, y+8(FP)
1032 JMP runtime·goPanicSlice3Acap(SB)
1033 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
1034 MOVD R5, x+0(FP)
1035 MOVD R6, y+8(FP)
1036 JMP runtime·goPanicSlice3AcapU(SB)
1037 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
1038 MOVD R4, x+0(FP)
1039 MOVD R5, y+8(FP)
1040 JMP runtime·goPanicSlice3B(SB)
1041 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
1042 MOVD R4, x+0(FP)
1043 MOVD R5, y+8(FP)
1044 JMP runtime·goPanicSlice3BU(SB)
1045 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
1046 MOVD R3, x+0(FP)
1047 MOVD R4, y+8(FP)
1048 JMP runtime·goPanicSlice3C(SB)
1049 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
1050 MOVD R3, x+0(FP)
1051 MOVD R4, y+8(FP)
1052 JMP runtime·goPanicSlice3CU(SB)
View as plain text