Text file src/runtime/asm_s390x.s
1 // Copyright 2016 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 // _rt0_s390x_lib is common startup code for s390x systems when
11 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
12 // arrange to invoke this function as a global constructor (for
13 // c-archive) or when the shared library is loaded (for c-shared).
14 // We expect argc and argv to be passed in the usual C ABI registers
15 // R2 and R3.
16 TEXT _rt0_s390x_lib(SB), NOSPLIT|NOFRAME, $0
17 STMG R6, R15, 48(R15)
18 MOVD R2, _rt0_s390x_lib_argc<>(SB)
19 MOVD R3, _rt0_s390x_lib_argv<>(SB)
20
21 // Save R6-R15 in the register save area of the calling function.
22 STMG R6, R15, 48(R15)
23
24 // Allocate 80 bytes on the stack.
25 MOVD $-80(R15), R15
26
27 // Save F8-F15 in our stack frame.
28 FMOVD F8, 16(R15)
29 FMOVD F9, 24(R15)
30 FMOVD F10, 32(R15)
31 FMOVD F11, 40(R15)
32 FMOVD F12, 48(R15)
33 FMOVD F13, 56(R15)
34 FMOVD F14, 64(R15)
35 FMOVD F15, 72(R15)
36
37 // Synchronous initialization.
38 MOVD $runtime·libpreinit(SB), R1
39 BL R1
40
41 // Create a new thread to finish Go runtime initialization.
42 MOVD _cgo_sys_thread_create(SB), R1
43 CMP R1, $0
44 BEQ nocgo
45 MOVD $_rt0_s390x_lib_go(SB), R2
46 MOVD $0, R3
47 BL R1
48 BR restore
49
50 nocgo:
51 MOVD $0x800000, R1 // stacksize
52 MOVD R1, 0(R15)
53 MOVD $_rt0_s390x_lib_go(SB), R1
54 MOVD R1, 8(R15) // fn
55 MOVD $runtime·newosproc(SB), R1
56 BL R1
57
58 restore:
59 // Restore F8-F15 from our stack frame.
60 FMOVD 16(R15), F8
61 FMOVD 24(R15), F9
62 FMOVD 32(R15), F10
63 FMOVD 40(R15), F11
64 FMOVD 48(R15), F12
65 FMOVD 56(R15), F13
66 FMOVD 64(R15), F14
67 FMOVD 72(R15), F15
68 MOVD $80(R15), R15
69
70 // Restore R6-R15.
71 LMG 48(R15), R6, R15
72 RET
73
74 // _rt0_s390x_lib_go initializes the Go runtime.
75 // This is started in a separate thread by _rt0_s390x_lib.
76 TEXT _rt0_s390x_lib_go(SB), NOSPLIT|NOFRAME, $0
77 MOVD _rt0_s390x_lib_argc<>(SB), R2
78 MOVD _rt0_s390x_lib_argv<>(SB), R3
79 MOVD $runtime·rt0_go(SB), R1
80 BR R1
81
82 DATA _rt0_s390x_lib_argc<>(SB)/8, $0
83 GLOBL _rt0_s390x_lib_argc<>(SB), NOPTR, $8
84 DATA _rt0_s90x_lib_argv<>(SB)/8, $0
85 GLOBL _rt0_s390x_lib_argv<>(SB), NOPTR, $8
86
87 TEXT runtime·rt0_go(SB),NOSPLIT,$0
88 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
89 // C TLS base pointer in AR0:AR1
90
91 // initialize essential registers
92 XOR R0, R0
93
94 SUB $24, R15
95 MOVW R2, 8(R15) // argc
96 MOVD R3, 16(R15) // argv
97
98 // create istack out of the given (operating system) stack.
99 // _cgo_init may update stackguard.
100 MOVD $runtime·g0(SB), g
101 MOVD R15, R11
102 SUB $(64*1024), R11
103 MOVD R11, g_stackguard0(g)
104 MOVD R11, g_stackguard1(g)
105 MOVD R11, (g_stack+stack_lo)(g)
106 MOVD R15, (g_stack+stack_hi)(g)
107
108 // if there is a _cgo_init, call it using the gcc ABI.
109 MOVD _cgo_init(SB), R11
110 CMPBEQ R11, $0, nocgo
111 MOVW AR0, R4 // (AR0 << 32 | AR1) is the TLS base pointer; MOVD is translated to EAR
112 SLD $32, R4, R4
113 MOVW AR1, R4 // arg 2: TLS base pointer
114 MOVD $setg_gcc<>(SB), R3 // arg 1: setg
115 MOVD g, R2 // arg 0: G
116 // C functions expect 160 bytes of space on caller stack frame
117 // and an 8-byte aligned stack pointer
118 MOVD R15, R9 // save current stack (R9 is preserved in the Linux ABI)
119 SUB $160, R15 // reserve 160 bytes
120 MOVD $~7, R6
121 AND R6, R15 // 8-byte align
122 BL R11 // this call clobbers volatile registers according to Linux ABI (R0-R5, R14)
123 MOVD R9, R15 // restore stack
124 XOR R0, R0 // zero R0
125
126 nocgo:
127 // update stackguard after _cgo_init
128 MOVD (g_stack+stack_lo)(g), R2
129 ADD $const__StackGuard, R2
130 MOVD R2, g_stackguard0(g)
131 MOVD R2, g_stackguard1(g)
132
133 // set the per-goroutine and per-mach "registers"
134 MOVD $runtime·m0(SB), R2
135
136 // save m->g0 = g0
137 MOVD g, m_g0(R2)
138 // save m0 to g0->m
139 MOVD R2, g_m(g)
140
141 BL runtime·check(SB)
142
143 // argc/argv are already prepared on stack
144 BL runtime·args(SB)
145 BL runtime·osinit(SB)
146 BL runtime·schedinit(SB)
147
148 // create a new goroutine to start program
149 MOVD $runtime·mainPC(SB), R2 // entry
150 SUB $24, R15
151 MOVD R2, 16(R15)
152 MOVD $0, 8(R15)
153 MOVD $0, 0(R15)
154 BL runtime·newproc(SB)
155 ADD $24, R15
156
157 // start this M
158 BL runtime·mstart(SB)
159
160 MOVD $0, 1(R0)
161 RET
162
163 DATA runtime·mainPC+0(SB)/8,$runtime·main(SB)
164 GLOBL runtime·mainPC(SB),RODATA,$8
165
166 TEXT runtime·breakpoint(SB),NOSPLIT|NOFRAME,$0-0
167 MOVD $0, 2(R0)
168 RET
169
170 TEXT runtime·asminit(SB),NOSPLIT|NOFRAME,$0-0
171 RET
172
173 /*
174 * go-routine
175 */
176
177 // void gosave(Gobuf*)
178 // save state in Gobuf; setjmp
179 TEXT runtime·gosave(SB), NOSPLIT, $-8-8
180 MOVD buf+0(FP), R3
181 MOVD R15, gobuf_sp(R3)
182 MOVD LR, gobuf_pc(R3)
183 MOVD g, gobuf_g(R3)
184 MOVD $0, gobuf_lr(R3)
185 MOVD $0, gobuf_ret(R3)
186 // Assert ctxt is zero. See func save.
187 MOVD gobuf_ctxt(R3), R3
188 CMPBEQ R3, $0, 2(PC)
189 BL runtime·badctxt(SB)
190 RET
191
192 // void gogo(Gobuf*)
193 // restore state from Gobuf; longjmp
194 TEXT runtime·gogo(SB), NOSPLIT, $16-8
195 MOVD buf+0(FP), R5
196 MOVD gobuf_g(R5), g // make sure g is not nil
197 BL runtime·save_g(SB)
198
199 MOVD 0(g), R4
200 MOVD gobuf_sp(R5), R15
201 MOVD gobuf_lr(R5), LR
202 MOVD gobuf_ret(R5), R3
203 MOVD gobuf_ctxt(R5), R12
204 MOVD $0, gobuf_sp(R5)
205 MOVD $0, gobuf_ret(R5)
206 MOVD $0, gobuf_lr(R5)
207 MOVD $0, gobuf_ctxt(R5)
208 CMP R0, R0 // set condition codes for == test, needed by stack split
209 MOVD gobuf_pc(R5), R6
210 BR (R6)
211
212 // void mcall(fn func(*g))
213 // Switch to m->g0's stack, call fn(g).
214 // Fn must never return. It should gogo(&g->sched)
215 // to keep running g.
216 TEXT runtime·mcall(SB), NOSPLIT, $-8-8
217 // Save caller state in g->sched
218 MOVD R15, (g_sched+gobuf_sp)(g)
219 MOVD LR, (g_sched+gobuf_pc)(g)
220 MOVD $0, (g_sched+gobuf_lr)(g)
221 MOVD g, (g_sched+gobuf_g)(g)
222
223 // Switch to m->g0 & its stack, call fn.
224 MOVD g, R3
225 MOVD g_m(g), R8
226 MOVD m_g0(R8), g
227 BL runtime·save_g(SB)
228 CMP g, R3
229 BNE 2(PC)
230 BR runtime·badmcall(SB)
231 MOVD fn+0(FP), R12 // context
232 MOVD 0(R12), R4 // code pointer
233 MOVD (g_sched+gobuf_sp)(g), R15 // sp = m->g0->sched.sp
234 SUB $16, R15
235 MOVD R3, 8(R15)
236 MOVD $0, 0(R15)
237 BL (R4)
238 BR runtime·badmcall2(SB)
239
240 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
241 // of the G stack. We need to distinguish the routine that
242 // lives at the bottom of the G stack from the one that lives
243 // at the top of the system stack because the one at the top of
244 // the system stack terminates the stack walk (see topofstack()).
245 TEXT runtime·systemstack_switch(SB), NOSPLIT, $0-0
246 UNDEF
247 BL (LR) // make sure this function is not leaf
248 RET
249
250 // func systemstack(fn func())
251 TEXT runtime·systemstack(SB), NOSPLIT, $0-8
252 MOVD fn+0(FP), R3 // R3 = fn
253 MOVD R3, R12 // context
254 MOVD g_m(g), R4 // R4 = m
255
256 MOVD m_gsignal(R4), R5 // R5 = gsignal
257 CMPBEQ g, R5, noswitch
258
259 MOVD m_g0(R4), R5 // R5 = g0
260 CMPBEQ g, R5, noswitch
261
262 MOVD m_curg(R4), R6
263 CMPBEQ g, R6, switch
264
265 // Bad: g is not gsignal, not g0, not curg. What is it?
266 // Hide call from linker nosplit analysis.
267 MOVD $runtime·badsystemstack(SB), R3
268 BL (R3)
269 BL runtime·abort(SB)
270
271 switch:
272 // save our state in g->sched. Pretend to
273 // be systemstack_switch if the G stack is scanned.
274 MOVD $runtime·systemstack_switch(SB), R6
275 ADD $16, R6 // get past prologue
276 MOVD R6, (g_sched+gobuf_pc)(g)
277 MOVD R15, (g_sched+gobuf_sp)(g)
278 MOVD $0, (g_sched+gobuf_lr)(g)
279 MOVD g, (g_sched+gobuf_g)(g)
280
281 // switch to g0
282 MOVD R5, g
283 BL runtime·save_g(SB)
284 MOVD (g_sched+gobuf_sp)(g), R3
285 // make it look like mstart called systemstack on g0, to stop traceback
286 SUB $8, R3
287 MOVD $runtime·mstart(SB), R4
288 MOVD R4, 0(R3)
289 MOVD R3, R15
290
291 // call target function
292 MOVD 0(R12), R3 // code pointer
293 BL (R3)
294
295 // switch back to g
296 MOVD g_m(g), R3
297 MOVD m_curg(R3), g
298 BL runtime·save_g(SB)
299 MOVD (g_sched+gobuf_sp)(g), R15
300 MOVD $0, (g_sched+gobuf_sp)(g)
301 RET
302
303 noswitch:
304 // already on m stack, just call directly
305 // Using a tail call here cleans up tracebacks since we won't stop
306 // at an intermediate systemstack.
307 MOVD 0(R12), R3 // code pointer
308 MOVD 0(R15), LR // restore LR
309 ADD $8, R15
310 BR (R3)
311
312 /*
313 * support for morestack
314 */
315
316 // Called during function prolog when more stack is needed.
317 // Caller has already loaded:
318 // R3: framesize, R4: argsize, R5: LR
319 //
320 // The traceback routines see morestack on a g0 as being
321 // the top of a stack (for example, morestack calling newstack
322 // calling the scheduler calling newm calling gc), so we must
323 // record an argument size. For that purpose, it has no arguments.
324 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
325 // Cannot grow scheduler stack (m->g0).
326 MOVD g_m(g), R7
327 MOVD m_g0(R7), R8
328 CMPBNE g, R8, 3(PC)
329 BL runtime·badmorestackg0(SB)
330 BL runtime·abort(SB)
331
332 // Cannot grow signal stack (m->gsignal).
333 MOVD m_gsignal(R7), R8
334 CMP g, R8
335 BNE 3(PC)
336 BL runtime·badmorestackgsignal(SB)
337 BL runtime·abort(SB)
338
339 // Called from f.
340 // Set g->sched to context in f.
341 MOVD R15, (g_sched+gobuf_sp)(g)
342 MOVD LR, R8
343 MOVD R8, (g_sched+gobuf_pc)(g)
344 MOVD R5, (g_sched+gobuf_lr)(g)
345 MOVD R12, (g_sched+gobuf_ctxt)(g)
346
347 // Called from f.
348 // Set m->morebuf to f's caller.
349 MOVD R5, (m_morebuf+gobuf_pc)(R7) // f's caller's PC
350 MOVD R15, (m_morebuf+gobuf_sp)(R7) // f's caller's SP
351 MOVD g, (m_morebuf+gobuf_g)(R7)
352
353 // Call newstack on m->g0's stack.
354 MOVD m_g0(R7), g
355 BL runtime·save_g(SB)
356 MOVD (g_sched+gobuf_sp)(g), R15
357 // Create a stack frame on g0 to call newstack.
358 MOVD $0, -8(R15) // Zero saved LR in frame
359 SUB $8, R15
360 BL runtime·newstack(SB)
361
362 // Not reached, but make sure the return PC from the call to newstack
363 // is still in this function, and not the beginning of the next.
364 UNDEF
365
366 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
367 MOVD $0, R12
368 BR runtime·morestack(SB)
369
370 // reflectcall: call a function with the given argument list
371 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
372 // we don't have variable-sized frames, so we use a small number
373 // of constant-sized-frame functions to encode a few bits of size in the pc.
374 // Caution: ugly multiline assembly macros in your future!
375
376 #define DISPATCH(NAME,MAXSIZE) \
377 MOVD $MAXSIZE, R4; \
378 CMP R3, R4; \
379 BGT 3(PC); \
380 MOVD $NAME(SB), R5; \
381 BR (R5)
382 // Note: can't just "BR NAME(SB)" - bad inlining results.
383
384 TEXT ·reflectcall(SB), NOSPLIT, $-8-32
385 MOVWZ argsize+24(FP), R3
386 DISPATCH(runtime·call32, 32)
387 DISPATCH(runtime·call64, 64)
388 DISPATCH(runtime·call128, 128)
389 DISPATCH(runtime·call256, 256)
390 DISPATCH(runtime·call512, 512)
391 DISPATCH(runtime·call1024, 1024)
392 DISPATCH(runtime·call2048, 2048)
393 DISPATCH(runtime·call4096, 4096)
394 DISPATCH(runtime·call8192, 8192)
395 DISPATCH(runtime·call16384, 16384)
396 DISPATCH(runtime·call32768, 32768)
397 DISPATCH(runtime·call65536, 65536)
398 DISPATCH(runtime·call131072, 131072)
399 DISPATCH(runtime·call262144, 262144)
400 DISPATCH(runtime·call524288, 524288)
401 DISPATCH(runtime·call1048576, 1048576)
402 DISPATCH(runtime·call2097152, 2097152)
403 DISPATCH(runtime·call4194304, 4194304)
404 DISPATCH(runtime·call8388608, 8388608)
405 DISPATCH(runtime·call16777216, 16777216)
406 DISPATCH(runtime·call33554432, 33554432)
407 DISPATCH(runtime·call67108864, 67108864)
408 DISPATCH(runtime·call134217728, 134217728)
409 DISPATCH(runtime·call268435456, 268435456)
410 DISPATCH(runtime·call536870912, 536870912)
411 DISPATCH(runtime·call1073741824, 1073741824)
412 MOVD $runtime·badreflectcall(SB), R5
413 BR (R5)
414
415 #define CALLFN(NAME,MAXSIZE) \
416 TEXT NAME(SB), WRAPPER, $MAXSIZE-24; \
417 NO_LOCAL_POINTERS; \
418 /* copy arguments to stack */ \
419 MOVD arg+16(FP), R4; \
420 MOVWZ argsize+24(FP), R5; \
421 MOVD $stack-MAXSIZE(SP), R6; \
422 loopArgs: /* copy 256 bytes at a time */ \
423 CMP R5, $256; \
424 BLT tailArgs; \
425 SUB $256, R5; \
426 MVC $256, 0(R4), 0(R6); \
427 MOVD $256(R4), R4; \
428 MOVD $256(R6), R6; \
429 BR loopArgs; \
430 tailArgs: /* copy remaining bytes */ \
431 CMP R5, $0; \
432 BEQ callFunction; \
433 SUB $1, R5; \
434 EXRL $callfnMVC<>(SB), R5; \
435 callFunction: \
436 MOVD f+8(FP), R12; \
437 MOVD (R12), R8; \
438 PCDATA $PCDATA_StackMapIndex, $0; \
439 BL (R8); \
440 /* copy return values back */ \
441 MOVD argtype+0(FP), R7; \
442 MOVD arg+16(FP), R6; \
443 MOVWZ n+24(FP), R5; \
444 MOVD $stack-MAXSIZE(SP), R4; \
445 MOVWZ retoffset+28(FP), R1; \
446 ADD R1, R4; \
447 ADD R1, R6; \
448 SUB R1, R5; \
449 BL callRet<>(SB); \
450 RET
451
452 // callRet copies return values back at the end of call*. This is a
453 // separate function so it can allocate stack space for the arguments
454 // to reflectcallmove. It does not follow the Go ABI; it expects its
455 // arguments in registers.
456 TEXT callRet<>(SB), NOSPLIT, $32-0
457 MOVD R7, 8(R15)
458 MOVD R6, 16(R15)
459 MOVD R4, 24(R15)
460 MOVD R5, 32(R15)
461 BL runtime·reflectcallmove(SB)
462 RET
463
464 CALLFN(·call32, 32)
465 CALLFN(·call64, 64)
466 CALLFN(·call128, 128)
467 CALLFN(·call256, 256)
468 CALLFN(·call512, 512)
469 CALLFN(·call1024, 1024)
470 CALLFN(·call2048, 2048)
471 CALLFN(·call4096, 4096)
472 CALLFN(·call8192, 8192)
473 CALLFN(·call16384, 16384)
474 CALLFN(·call32768, 32768)
475 CALLFN(·call65536, 65536)
476 CALLFN(·call131072, 131072)
477 CALLFN(·call262144, 262144)
478 CALLFN(·call524288, 524288)
479 CALLFN(·call1048576, 1048576)
480 CALLFN(·call2097152, 2097152)
481 CALLFN(·call4194304, 4194304)
482 CALLFN(·call8388608, 8388608)
483 CALLFN(·call16777216, 16777216)
484 CALLFN(·call33554432, 33554432)
485 CALLFN(·call67108864, 67108864)
486 CALLFN(·call134217728, 134217728)
487 CALLFN(·call268435456, 268435456)
488 CALLFN(·call536870912, 536870912)
489 CALLFN(·call1073741824, 1073741824)
490
491 // Not a function: target for EXRL (execute relative long) instruction.
492 TEXT callfnMVC<>(SB),NOSPLIT|NOFRAME,$0-0
493 MVC $1, 0(R4), 0(R6)
494
495 TEXT runtime·procyield(SB),NOSPLIT,$0-0
496 RET
497
498 // void jmpdefer(fv, sp);
499 // called from deferreturn.
500 // 1. grab stored LR for caller
501 // 2. sub 6 bytes to get back to BL deferreturn (size of BRASL instruction)
502 // 3. BR to fn
503 TEXT runtime·jmpdefer(SB),NOSPLIT|NOFRAME,$0-16
504 MOVD 0(R15), R1
505 SUB $6, R1, LR
506
507 MOVD fv+0(FP), R12
508 MOVD argp+8(FP), R15
509 SUB $8, R15
510 MOVD 0(R12), R3
511 BR (R3)
512
513 // Save state of caller into g->sched. Smashes R1.
514 TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0
515 MOVD LR, (g_sched+gobuf_pc)(g)
516 MOVD R15, (g_sched+gobuf_sp)(g)
517 MOVD $0, (g_sched+gobuf_lr)(g)
518 MOVD $0, (g_sched+gobuf_ret)(g)
519 // Assert ctxt is zero. See func save.
520 MOVD (g_sched+gobuf_ctxt)(g), R1
521 CMPBEQ R1, $0, 2(PC)
522 BL runtime·badctxt(SB)
523 RET
524
525 // func asmcgocall(fn, arg unsafe.Pointer) int32
526 // Call fn(arg) on the scheduler stack,
527 // aligned appropriately for the gcc ABI.
528 // See cgocall.go for more details.
529 TEXT ·asmcgocall(SB),NOSPLIT,$0-20
530 // R2 = argc; R3 = argv; R11 = temp; R13 = g; R15 = stack pointer
531 // C TLS base pointer in AR0:AR1
532 MOVD fn+0(FP), R3
533 MOVD arg+8(FP), R4
534
535 MOVD R15, R2 // save original stack pointer
536 MOVD g, R5
537
538 // Figure out if we need to switch to m->g0 stack.
539 // We get called to create new OS threads too, and those
540 // come in on the m->g0 stack already.
541 MOVD g_m(g), R6
542 MOVD m_g0(R6), R6
543 CMPBEQ R6, g, g0
544 BL gosave<>(SB)
545 MOVD R6, g
546 BL runtime·save_g(SB)
547 MOVD (g_sched+gobuf_sp)(g), R15
548
549 // Now on a scheduling stack (a pthread-created stack).
550 g0:
551 // Save room for two of our pointers, plus 160 bytes of callee
552 // save area that lives on the caller stack.
553 SUB $176, R15
554 MOVD $~7, R6
555 AND R6, R15 // 8-byte alignment for gcc ABI
556 MOVD R5, 168(R15) // save old g on stack
557 MOVD (g_stack+stack_hi)(R5), R5
558 SUB R2, R5
559 MOVD R5, 160(R15) // save depth in old g stack (can't just save SP, as stack might be copied during a callback)
560 MOVD $0, 0(R15) // clear back chain pointer (TODO can we give it real back trace information?)
561 MOVD R4, R2 // arg in R2
562 BL R3 // can clobber: R0-R5, R14, F0-F3, F5, F7-F15
563
564 XOR R0, R0 // set R0 back to 0.
565 // Restore g, stack pointer.
566 MOVD 168(R15), g
567 BL runtime·save_g(SB)
568 MOVD (g_stack+stack_hi)(g), R5
569 MOVD 160(R15), R6
570 SUB R6, R5
571 MOVD R5, R15
572
573 MOVW R2, ret+16(FP)
574 RET
575
576 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt)
577 // Turn the fn into a Go func (by taking its address) and call
578 // cgocallback_gofunc.
579 TEXT runtime·cgocallback(SB),NOSPLIT,$32-32
580 MOVD $fn+0(FP), R3
581 MOVD R3, 8(R15)
582 MOVD frame+8(FP), R3
583 MOVD R3, 16(R15)
584 MOVD framesize+16(FP), R3
585 MOVD R3, 24(R15)
586 MOVD ctxt+24(FP), R3
587 MOVD R3, 32(R15)
588 MOVD $runtime·cgocallback_gofunc(SB), R3
589 BL (R3)
590 RET
591
592 // cgocallback_gofunc(FuncVal*, void *frame, uintptr framesize, uintptr ctxt)
593 // See cgocall.go for more details.
594 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$16-32
595 NO_LOCAL_POINTERS
596
597 // Load m and g from thread-local storage.
598 MOVB runtime·iscgo(SB), R3
599 CMPBEQ R3, $0, nocgo
600 BL runtime·load_g(SB)
601
602 nocgo:
603 // If g is nil, Go did not create the current thread.
604 // Call needm to obtain one for temporary use.
605 // In this case, we're running on the thread stack, so there's
606 // lots of space, but the linker doesn't know. Hide the call from
607 // the linker analysis by using an indirect call.
608 CMPBEQ g, $0, needm
609
610 MOVD g_m(g), R8
611 MOVD R8, savedm-8(SP)
612 BR havem
613
614 needm:
615 MOVD g, savedm-8(SP) // g is zero, so is m.
616 MOVD $runtime·needm(SB), R3
617 BL (R3)
618
619 // Set m->sched.sp = SP, so that if a panic happens
620 // during the function we are about to execute, it will
621 // have a valid SP to run on the g0 stack.
622 // The next few lines (after the havem label)
623 // will save this SP onto the stack and then write
624 // the same SP back to m->sched.sp. That seems redundant,
625 // but if an unrecovered panic happens, unwindm will
626 // restore the g->sched.sp from the stack location
627 // and then systemstack will try to use it. If we don't set it here,
628 // that restored SP will be uninitialized (typically 0) and
629 // will not be usable.
630 MOVD g_m(g), R8
631 MOVD m_g0(R8), R3
632 MOVD R15, (g_sched+gobuf_sp)(R3)
633
634 havem:
635 // Now there's a valid m, and we're running on its m->g0.
636 // Save current m->g0->sched.sp on stack and then set it to SP.
637 // Save current sp in m->g0->sched.sp in preparation for
638 // switch back to m->curg stack.
639 // NOTE: unwindm knows that the saved g->sched.sp is at 8(R1) aka savedsp-16(SP).
640 MOVD m_g0(R8), R3
641 MOVD (g_sched+gobuf_sp)(R3), R4
642 MOVD R4, savedsp-16(SP)
643 MOVD R15, (g_sched+gobuf_sp)(R3)
644
645 // Switch to m->curg stack and call runtime.cgocallbackg.
646 // Because we are taking over the execution of m->curg
647 // but *not* resuming what had been running, we need to
648 // save that information (m->curg->sched) so we can restore it.
649 // We can restore m->curg->sched.sp easily, because calling
650 // runtime.cgocallbackg leaves SP unchanged upon return.
651 // To save m->curg->sched.pc, we push it onto the stack.
652 // This has the added benefit that it looks to the traceback
653 // routine like cgocallbackg is going to return to that
654 // PC (because the frame we allocate below has the same
655 // size as cgocallback_gofunc's frame declared above)
656 // so that the traceback will seamlessly trace back into
657 // the earlier calls.
658 //
659 // In the new goroutine, -8(SP) is unused (where SP refers to
660 // m->curg's SP while we're setting it up, before we've adjusted it).
661 MOVD m_curg(R8), g
662 BL runtime·save_g(SB)
663 MOVD (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
664 MOVD (g_sched+gobuf_pc)(g), R5
665 MOVD R5, -24(R4)
666 MOVD ctxt+24(FP), R5
667 MOVD R5, -16(R4)
668 MOVD $-24(R4), R15
669 BL runtime·cgocallbackg(SB)
670
671 // Restore g->sched (== m->curg->sched) from saved values.
672 MOVD 0(R15), R5
673 MOVD R5, (g_sched+gobuf_pc)(g)
674 MOVD $24(R15), R4
675 MOVD R4, (g_sched+gobuf_sp)(g)
676
677 // Switch back to m->g0's stack and restore m->g0->sched.sp.
678 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
679 // so we do not have to restore it.)
680 MOVD g_m(g), R8
681 MOVD m_g0(R8), g
682 BL runtime·save_g(SB)
683 MOVD (g_sched+gobuf_sp)(g), R15
684 MOVD savedsp-16(SP), R4
685 MOVD R4, (g_sched+gobuf_sp)(g)
686
687 // If the m on entry was nil, we called needm above to borrow an m
688 // for the duration of the call. Since the call is over, return it with dropm.
689 MOVD savedm-8(SP), R6
690 CMPBNE R6, $0, droppedm
691 MOVD $runtime·dropm(SB), R3
692 BL (R3)
693 droppedm:
694
695 // Done!
696 RET
697
698 // void setg(G*); set g. for use by needm.
699 TEXT runtime·setg(SB), NOSPLIT, $0-8
700 MOVD gg+0(FP), g
701 // This only happens if iscgo, so jump straight to save_g
702 BL runtime·save_g(SB)
703 RET
704
705 // void setg_gcc(G*); set g in C TLS.
706 // Must obey the gcc calling convention.
707 TEXT setg_gcc<>(SB),NOSPLIT|NOFRAME,$0-0
708 // The standard prologue clobbers LR (R14), which is callee-save in
709 // the C ABI, so we have to use NOFRAME and save LR ourselves.
710 MOVD LR, R1
711 // Also save g, R10, and R11 since they're callee-save in C ABI
712 MOVD R10, R3
713 MOVD g, R4
714 MOVD R11, R5
715
716 MOVD R2, g
717 BL runtime·save_g(SB)
718
719 MOVD R5, R11
720 MOVD R4, g
721 MOVD R3, R10
722 MOVD R1, LR
723 RET
724
725 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
726 MOVW (R0), R0
727 UNDEF
728
729 // int64 runtime·cputicks(void)
730 TEXT runtime·cputicks(SB),NOSPLIT,$0-8
731 // The TOD clock on s390 counts from the year 1900 in ~250ps intervals.
732 // This means that since about 1972 the msb has been set, making the
733 // result of a call to STORE CLOCK (stck) a negative number.
734 // We clear the msb to make it positive.
735 STCK ret+0(FP) // serialises before and after call
736 MOVD ret+0(FP), R3 // R3 will wrap to 0 in the year 2043
737 SLD $1, R3
738 SRD $1, R3
739 MOVD R3, ret+0(FP)
740 RET
741
742 // AES hashing not implemented for s390x
743 TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-0
744 MOVW (R0), R15
745 TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-0
746 MOVW (R0), R15
747 TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-0
748 MOVW (R0), R15
749 TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-0
750 MOVW (R0), R15
751
752 TEXT runtime·return0(SB), NOSPLIT, $0
753 MOVW $0, R3
754 RET
755
756 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
757 // Must obey the gcc calling convention.
758 TEXT _cgo_topofstack(SB),NOSPLIT|NOFRAME,$0
759 // g (R13), R10, R11 and LR (R14) are callee-save in the C ABI, so save them
760 MOVD g, R1
761 MOVD R10, R3
762 MOVD LR, R4
763 MOVD R11, R5
764
765 BL runtime·load_g(SB) // clobbers g (R13), R10, R11
766 MOVD g_m(g), R2
767 MOVD m_curg(R2), R2
768 MOVD (g_stack+stack_hi)(R2), R2
769
770 MOVD R1, g
771 MOVD R3, R10
772 MOVD R4, LR
773 MOVD R5, R11
774 RET
775
776 // The top-most function running on a goroutine
777 // returns to goexit+PCQuantum.
778 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
779 BYTE $0x07; BYTE $0x00; // 2-byte nop
780 BL runtime·goexit1(SB) // does not return
781 // traceback from goexit1 must hit code range of goexit
782 BYTE $0x07; BYTE $0x00; // 2-byte nop
783
784 TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
785 // Stores are already ordered on s390x, so this is just a
786 // compile barrier.
787 RET
788
789 // This is called from .init_array and follows the platform, not Go, ABI.
790 // We are overly conservative. We could only save the registers we use.
791 // However, since this function is only called once per loaded module
792 // performance is unimportant.
793 TEXT runtime·addmoduledata(SB),NOSPLIT|NOFRAME,$0-0
794 // Save R6-R15 in the register save area of the calling function.
795 // Don't bother saving F8-F15 as we aren't doing any calls.
796 STMG R6, R15, 48(R15)
797
798 // append the argument (passed in R2, as per the ELF ABI) to the
799 // moduledata linked list.
800 MOVD runtime·lastmoduledatap(SB), R1
801 MOVD R2, moduledata_next(R1)
802 MOVD R2, runtime·lastmoduledatap(SB)
803
804 // Restore R6-R15.
805 LMG 48(R15), R6, R15
806 RET
807
808 TEXT ·checkASM(SB),NOSPLIT,$0-1
809 MOVB $1, ret+0(FP)
810 RET
811
812 // gcWriteBarrier performs a heap pointer write and informs the GC.
813 //
814 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
815 // - R2 is the destination of the write
816 // - R3 is the value being written at R2.
817 // It clobbers R10 (the temp register).
818 // It does not clobber any other general-purpose registers,
819 // but may clobber others (e.g., floating point registers).
820 TEXT runtime·gcWriteBarrier(SB),NOSPLIT,$104
821 // Save the registers clobbered by the fast path.
822 MOVD R1, 96(R15)
823 MOVD R4, 104(R15)
824 MOVD g_m(g), R1
825 MOVD m_p(R1), R1
826 // Increment wbBuf.next position.
827 MOVD $16, R4
828 ADD (p_wbBuf+wbBuf_next)(R1), R4
829 MOVD R4, (p_wbBuf+wbBuf_next)(R1)
830 MOVD (p_wbBuf+wbBuf_end)(R1), R1
831 // Record the write.
832 MOVD R3, -16(R4) // Record value
833 MOVD (R2), R10 // TODO: This turns bad writes into bad reads.
834 MOVD R10, -8(R4) // Record *slot
835 // Is the buffer full?
836 CMPBEQ R4, R1, flush
837 ret:
838 MOVD 96(R15), R1
839 MOVD 104(R15), R4
840 // Do the write.
841 MOVD R3, (R2)
842 RET
843
844 flush:
845 // Save all general purpose registers since these could be
846 // clobbered by wbBufFlush and were not saved by the caller.
847 STMG R2, R3, 8(R15) // set R2 and R3 as arguments for wbBufFlush
848 MOVD R0, 24(R15)
849 // R1 already saved.
850 // R4 already saved.
851 STMG R5, R12, 32(R15) // save R5 - R12
852 // R13 is g.
853 // R14 is LR.
854 // R15 is SP.
855
856 // This takes arguments R2 and R3.
857 CALL runtime·wbBufFlush(SB)
858
859 LMG 8(R15), R2, R3 // restore R2 - R3
860 MOVD 24(R15), R0 // restore R0
861 LMG 32(R15), R5, R12 // restore R5 - R12
862 JMP ret
863
864 // Note: these functions use a special calling convention to save generated code space.
865 // Arguments are passed in registers, but the space for those arguments are allocated
866 // in the caller's stack frame. These stubs write the args into that stack space and
867 // then tail call to the corresponding runtime handler.
868 // The tail call makes these stubs disappear in backtraces.
869 TEXT runtime·panicIndex(SB),NOSPLIT,$0-16
870 MOVD R0, x+0(FP)
871 MOVD R1, y+8(FP)
872 JMP runtime·goPanicIndex(SB)
873 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-16
874 MOVD R0, x+0(FP)
875 MOVD R1, y+8(FP)
876 JMP runtime·goPanicIndexU(SB)
877 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-16
878 MOVD R1, x+0(FP)
879 MOVD R2, y+8(FP)
880 JMP runtime·goPanicSliceAlen(SB)
881 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-16
882 MOVD R1, x+0(FP)
883 MOVD R2, y+8(FP)
884 JMP runtime·goPanicSliceAlenU(SB)
885 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-16
886 MOVD R1, x+0(FP)
887 MOVD R2, y+8(FP)
888 JMP runtime·goPanicSliceAcap(SB)
889 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-16
890 MOVD R1, x+0(FP)
891 MOVD R2, y+8(FP)
892 JMP runtime·goPanicSliceAcapU(SB)
893 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-16
894 MOVD R0, x+0(FP)
895 MOVD R1, y+8(FP)
896 JMP runtime·goPanicSliceB(SB)
897 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-16
898 MOVD R0, x+0(FP)
899 MOVD R1, y+8(FP)
900 JMP runtime·goPanicSliceBU(SB)
901 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-16
902 MOVD R2, x+0(FP)
903 MOVD R3, y+8(FP)
904 JMP runtime·goPanicSlice3Alen(SB)
905 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-16
906 MOVD R2, x+0(FP)
907 MOVD R3, y+8(FP)
908 JMP runtime·goPanicSlice3AlenU(SB)
909 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-16
910 MOVD R2, x+0(FP)
911 MOVD R3, y+8(FP)
912 JMP runtime·goPanicSlice3Acap(SB)
913 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-16
914 MOVD R2, x+0(FP)
915 MOVD R3, y+8(FP)
916 JMP runtime·goPanicSlice3AcapU(SB)
917 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-16
918 MOVD R1, x+0(FP)
919 MOVD R2, y+8(FP)
920 JMP runtime·goPanicSlice3B(SB)
921 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-16
922 MOVD R1, x+0(FP)
923 MOVD R2, y+8(FP)
924 JMP runtime·goPanicSlice3BU(SB)
925 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-16
926 MOVD R0, x+0(FP)
927 MOVD R1, y+8(FP)
928 JMP runtime·goPanicSlice3C(SB)
929 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-16
930 MOVD R0, x+0(FP)
931 MOVD R1, y+8(FP)
932 JMP runtime·goPanicSlice3CU(SB)
View as plain text