Text file src/pkg/runtime/asm_arm.s
1 // Copyright 2009 The Go Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style
3 // license that can be found in the LICENSE file.
4
5 #include "go_asm.h"
6 #include "go_tls.h"
7 #include "funcdata.h"
8 #include "textflag.h"
9
10 // _rt0_arm is common startup code for most ARM systems when using
11 // internal linking. This is the entry point for the program from the
12 // kernel for an ordinary -buildmode=exe program. The stack holds the
13 // number of arguments and the C-style argv.
14 TEXT _rt0_arm(SB),NOSPLIT|NOFRAME,$0
15 MOVW (R13), R0 // argc
16 MOVW $4(R13), R1 // argv
17 B runtime·rt0_go(SB)
18
19 // main is common startup code for most ARM systems when using
20 // external linking. The C startup code will call the symbol "main"
21 // passing argc and argv in the usual C ABI registers R0 and R1.
22 TEXT main(SB),NOSPLIT|NOFRAME,$0
23 B runtime·rt0_go(SB)
24
25 // _rt0_arm_lib is common startup code for most ARM systems when
26 // using -buildmode=c-archive or -buildmode=c-shared. The linker will
27 // arrange to invoke this function as a global constructor (for
28 // c-archive) or when the shared library is loaded (for c-shared).
29 // We expect argc and argv to be passed in the usual C ABI registers
30 // R0 and R1.
31 TEXT _rt0_arm_lib(SB),NOSPLIT,$104
32 // Preserve callee-save registers. Raspberry Pi's dlopen(), for example,
33 // actually cares that R11 is preserved.
34 MOVW R4, 12(R13)
35 MOVW R5, 16(R13)
36 MOVW R6, 20(R13)
37 MOVW R7, 24(R13)
38 MOVW R8, 28(R13)
39 MOVW g, 32(R13)
40 MOVW R11, 36(R13)
41
42 // Skip floating point registers on GOARM < 6.
43 MOVB runtime·goarm(SB), R11
44 CMP $6, R11
45 BLT skipfpsave
46 MOVD F8, (40+8*0)(R13)
47 MOVD F9, (40+8*1)(R13)
48 MOVD F10, (40+8*2)(R13)
49 MOVD F11, (40+8*3)(R13)
50 MOVD F12, (40+8*4)(R13)
51 MOVD F13, (40+8*5)(R13)
52 MOVD F14, (40+8*6)(R13)
53 MOVD F15, (40+8*7)(R13)
54 skipfpsave:
55 // Save argc/argv.
56 MOVW R0, _rt0_arm_lib_argc<>(SB)
57 MOVW R1, _rt0_arm_lib_argv<>(SB)
58
59 MOVW $0, g // Initialize g.
60
61 // Synchronous initialization.
62 CALL runtime·libpreinit(SB)
63
64 // Create a new thread to do the runtime initialization.
65 MOVW _cgo_sys_thread_create(SB), R2
66 CMP $0, R2
67 BEQ nocgo
68 MOVW $_rt0_arm_lib_go<>(SB), R0
69 MOVW $0, R1
70 BL (R2)
71 B rr
72 nocgo:
73 MOVW $0x800000, R0 // stacksize = 8192KB
74 MOVW $_rt0_arm_lib_go<>(SB), R1 // fn
75 MOVW R0, 4(R13)
76 MOVW R1, 8(R13)
77 BL runtime·newosproc0(SB)
78 rr:
79 // Restore callee-save registers and return.
80 MOVB runtime·goarm(SB), R11
81 CMP $6, R11
82 BLT skipfprest
83 MOVD (40+8*0)(R13), F8
84 MOVD (40+8*1)(R13), F9
85 MOVD (40+8*2)(R13), F10
86 MOVD (40+8*3)(R13), F11
87 MOVD (40+8*4)(R13), F12
88 MOVD (40+8*5)(R13), F13
89 MOVD (40+8*6)(R13), F14
90 MOVD (40+8*7)(R13), F15
91 skipfprest:
92 MOVW 12(R13), R4
93 MOVW 16(R13), R5
94 MOVW 20(R13), R6
95 MOVW 24(R13), R7
96 MOVW 28(R13), R8
97 MOVW 32(R13), g
98 MOVW 36(R13), R11
99 RET
100
101 // _rt0_arm_lib_go initializes the Go runtime.
102 // This is started in a separate thread by _rt0_arm_lib.
103 TEXT _rt0_arm_lib_go<>(SB),NOSPLIT,$8
104 MOVW _rt0_arm_lib_argc<>(SB), R0
105 MOVW _rt0_arm_lib_argv<>(SB), R1
106 B runtime·rt0_go(SB)
107
108 DATA _rt0_arm_lib_argc<>(SB)/4,$0
109 GLOBL _rt0_arm_lib_argc<>(SB),NOPTR,$4
110 DATA _rt0_arm_lib_argv<>(SB)/4,$0
111 GLOBL _rt0_arm_lib_argv<>(SB),NOPTR,$4
112
113 // using NOFRAME means do not save LR on stack.
114 // argc is in R0, argv is in R1.
115 TEXT runtime·rt0_go(SB),NOSPLIT|NOFRAME,$0
116 MOVW $0xcafebabe, R12
117
118 // copy arguments forward on an even stack
119 // use R13 instead of SP to avoid linker rewriting the offsets
120 SUB $64, R13 // plenty of scratch
121 AND $~7, R13
122 MOVW R0, 60(R13) // save argc, argv away
123 MOVW R1, 64(R13)
124
125 // set up g register
126 // g is R10
127 MOVW $runtime·g0(SB), g
128 MOVW $runtime·m0(SB), R8
129
130 // save m->g0 = g0
131 MOVW g, m_g0(R8)
132 // save g->m = m0
133 MOVW R8, g_m(g)
134
135 // create istack out of the OS stack
136 // (1MB of system stack is available on iOS and Android)
137 MOVW $(-64*1024+104)(R13), R0
138 MOVW R0, g_stackguard0(g)
139 MOVW R0, g_stackguard1(g)
140 MOVW R0, (g_stack+stack_lo)(g)
141 MOVW R13, (g_stack+stack_hi)(g)
142
143 BL runtime·emptyfunc(SB) // fault if stack check is wrong
144
145 BL runtime·_initcgo(SB) // will clobber R0-R3
146
147 // update stackguard after _cgo_init
148 MOVW (g_stack+stack_lo)(g), R0
149 ADD $const__StackGuard, R0
150 MOVW R0, g_stackguard0(g)
151 MOVW R0, g_stackguard1(g)
152
153 BL runtime·check(SB)
154
155 // saved argc, argv
156 MOVW 60(R13), R0
157 MOVW R0, 4(R13)
158 MOVW 64(R13), R1
159 MOVW R1, 8(R13)
160 BL runtime·args(SB)
161 BL runtime·checkgoarm(SB)
162 BL runtime·osinit(SB)
163 BL runtime·schedinit(SB)
164
165 // create a new goroutine to start program
166 MOVW $runtime·mainPC(SB), R0
167 MOVW.W R0, -4(R13)
168 MOVW $8, R0
169 MOVW.W R0, -4(R13)
170 MOVW $0, R0
171 MOVW.W R0, -4(R13) // push $0 as guard
172 BL runtime·newproc(SB)
173 MOVW $12(R13), R13 // pop args and LR
174
175 // start this M
176 BL runtime·mstart(SB)
177
178 MOVW $1234, R0
179 MOVW $1000, R1
180 MOVW R0, (R1) // fail hard
181
182 DATA runtime·mainPC+0(SB)/4,$runtime·main(SB)
183 GLOBL runtime·mainPC(SB),RODATA,$4
184
185 TEXT runtime·breakpoint(SB),NOSPLIT,$0-0
186 // gdb won't skip this breakpoint instruction automatically,
187 // so you must manually "set $pc+=4" to skip it and continue.
188 #ifdef GOOS_nacl
189 WORD $0xe125be7f // BKPT 0x5bef, NACL_INSTR_ARM_BREAKPOINT
190 #else
191 #ifdef GOOS_plan9
192 WORD $0xD1200070 // undefined instruction used as armv5 breakpoint in Plan 9
193 #else
194 WORD $0xe7f001f0 // undefined instruction that gdb understands is a software breakpoint
195 #endif
196 #endif
197 RET
198
199 TEXT runtime·asminit(SB),NOSPLIT,$0-0
200 // disable runfast (flush-to-zero) mode of vfp if runtime.goarm > 5
201 MOVB runtime·goarm(SB), R11
202 CMP $5, R11
203 BLE 4(PC)
204 WORD $0xeef1ba10 // vmrs r11, fpscr
205 BIC $(1<<24), R11
206 WORD $0xeee1ba10 // vmsr fpscr, r11
207 RET
208
209 /*
210 * go-routine
211 */
212
213 // void gosave(Gobuf*)
214 // save state in Gobuf; setjmp
215 TEXT runtime·gosave(SB),NOSPLIT|NOFRAME,$0-4
216 MOVW buf+0(FP), R0
217 MOVW R13, gobuf_sp(R0)
218 MOVW LR, gobuf_pc(R0)
219 MOVW g, gobuf_g(R0)
220 MOVW $0, R11
221 MOVW R11, gobuf_lr(R0)
222 MOVW R11, gobuf_ret(R0)
223 // Assert ctxt is zero. See func save.
224 MOVW gobuf_ctxt(R0), R0
225 CMP R0, R11
226 B.EQ 2(PC)
227 CALL runtime·badctxt(SB)
228 RET
229
230 // void gogo(Gobuf*)
231 // restore state from Gobuf; longjmp
232 TEXT runtime·gogo(SB),NOSPLIT,$8-4
233 MOVW buf+0(FP), R1
234 MOVW gobuf_g(R1), R0
235 BL setg<>(SB)
236
237 // NOTE: We updated g above, and we are about to update SP.
238 // Until LR and PC are also updated, the g/SP/LR/PC quadruple
239 // are out of sync and must not be used as the basis of a traceback.
240 // Sigprof skips the traceback when SP is not within g's bounds,
241 // and when the PC is inside this function, runtime.gogo.
242 // Since we are about to update SP, until we complete runtime.gogo
243 // we must not leave this function. In particular, no calls
244 // after this point: it must be straight-line code until the
245 // final B instruction.
246 // See large comment in sigprof for more details.
247 MOVW gobuf_sp(R1), R13 // restore SP==R13
248 MOVW gobuf_lr(R1), LR
249 MOVW gobuf_ret(R1), R0
250 MOVW gobuf_ctxt(R1), R7
251 MOVW $0, R11
252 MOVW R11, gobuf_sp(R1) // clear to help garbage collector
253 MOVW R11, gobuf_ret(R1)
254 MOVW R11, gobuf_lr(R1)
255 MOVW R11, gobuf_ctxt(R1)
256 MOVW gobuf_pc(R1), R11
257 CMP R11, R11 // set condition codes for == test, needed by stack split
258 B (R11)
259
260 // func mcall(fn func(*g))
261 // Switch to m->g0's stack, call fn(g).
262 // Fn must never return. It should gogo(&g->sched)
263 // to keep running g.
264 TEXT runtime·mcall(SB),NOSPLIT|NOFRAME,$0-4
265 // Save caller state in g->sched.
266 MOVW R13, (g_sched+gobuf_sp)(g)
267 MOVW LR, (g_sched+gobuf_pc)(g)
268 MOVW $0, R11
269 MOVW R11, (g_sched+gobuf_lr)(g)
270 MOVW g, (g_sched+gobuf_g)(g)
271
272 // Switch to m->g0 & its stack, call fn.
273 MOVW g, R1
274 MOVW g_m(g), R8
275 MOVW m_g0(R8), R0
276 BL setg<>(SB)
277 CMP g, R1
278 B.NE 2(PC)
279 B runtime·badmcall(SB)
280 MOVB runtime·iscgo(SB), R11
281 CMP $0, R11
282 BL.NE runtime·save_g(SB)
283 MOVW fn+0(FP), R0
284 MOVW (g_sched+gobuf_sp)(g), R13
285 SUB $8, R13
286 MOVW R1, 4(R13)
287 MOVW R0, R7
288 MOVW 0(R0), R0
289 BL (R0)
290 B runtime·badmcall2(SB)
291 RET
292
293 // systemstack_switch is a dummy routine that systemstack leaves at the bottom
294 // of the G stack. We need to distinguish the routine that
295 // lives at the bottom of the G stack from the one that lives
296 // at the top of the system stack because the one at the top of
297 // the system stack terminates the stack walk (see topofstack()).
298 TEXT runtime·systemstack_switch(SB),NOSPLIT,$0-0
299 MOVW $0, R0
300 BL (R0) // clobber lr to ensure push {lr} is kept
301 RET
302
303 // func systemstack(fn func())
304 TEXT runtime·systemstack(SB),NOSPLIT,$0-4
305 MOVW fn+0(FP), R0 // R0 = fn
306 MOVW g_m(g), R1 // R1 = m
307
308 MOVW m_gsignal(R1), R2 // R2 = gsignal
309 CMP g, R2
310 B.EQ noswitch
311
312 MOVW m_g0(R1), R2 // R2 = g0
313 CMP g, R2
314 B.EQ noswitch
315
316 MOVW m_curg(R1), R3
317 CMP g, R3
318 B.EQ switch
319
320 // Bad: g is not gsignal, not g0, not curg. What is it?
321 // Hide call from linker nosplit analysis.
322 MOVW $runtime·badsystemstack(SB), R0
323 BL (R0)
324 B runtime·abort(SB)
325
326 switch:
327 // save our state in g->sched. Pretend to
328 // be systemstack_switch if the G stack is scanned.
329 MOVW $runtime·systemstack_switch(SB), R3
330 #ifdef GOOS_nacl
331 ADD $4, R3, R3 // get past nacl-insert bic instruction
332 #endif
333 ADD $4, R3, R3 // get past push {lr}
334 MOVW R3, (g_sched+gobuf_pc)(g)
335 MOVW R13, (g_sched+gobuf_sp)(g)
336 MOVW LR, (g_sched+gobuf_lr)(g)
337 MOVW g, (g_sched+gobuf_g)(g)
338
339 // switch to g0
340 MOVW R0, R5
341 MOVW R2, R0
342 BL setg<>(SB)
343 MOVW R5, R0
344 MOVW (g_sched+gobuf_sp)(R2), R3
345 // make it look like mstart called systemstack on g0, to stop traceback
346 SUB $4, R3, R3
347 MOVW $runtime·mstart(SB), R4
348 MOVW R4, 0(R3)
349 MOVW R3, R13
350
351 // call target function
352 MOVW R0, R7
353 MOVW 0(R0), R0
354 BL (R0)
355
356 // switch back to g
357 MOVW g_m(g), R1
358 MOVW m_curg(R1), R0
359 BL setg<>(SB)
360 MOVW (g_sched+gobuf_sp)(g), R13
361 MOVW $0, R3
362 MOVW R3, (g_sched+gobuf_sp)(g)
363 RET
364
365 noswitch:
366 // Using a tail call here cleans up tracebacks since we won't stop
367 // at an intermediate systemstack.
368 MOVW R0, R7
369 MOVW 0(R0), R0
370 MOVW.P 4(R13), R14 // restore LR
371 B (R0)
372
373 /*
374 * support for morestack
375 */
376
377 // Called during function prolog when more stack is needed.
378 // R3 prolog's LR
379 // using NOFRAME means do not save LR on stack.
380 //
381 // The traceback routines see morestack on a g0 as being
382 // the top of a stack (for example, morestack calling newstack
383 // calling the scheduler calling newm calling gc), so we must
384 // record an argument size. For that purpose, it has no arguments.
385 TEXT runtime·morestack(SB),NOSPLIT|NOFRAME,$0-0
386 // Cannot grow scheduler stack (m->g0).
387 MOVW g_m(g), R8
388 MOVW m_g0(R8), R4
389 CMP g, R4
390 BNE 3(PC)
391 BL runtime·badmorestackg0(SB)
392 B runtime·abort(SB)
393
394 // Cannot grow signal stack (m->gsignal).
395 MOVW m_gsignal(R8), R4
396 CMP g, R4
397 BNE 3(PC)
398 BL runtime·badmorestackgsignal(SB)
399 B runtime·abort(SB)
400
401 // Called from f.
402 // Set g->sched to context in f.
403 MOVW R13, (g_sched+gobuf_sp)(g)
404 MOVW LR, (g_sched+gobuf_pc)(g)
405 MOVW R3, (g_sched+gobuf_lr)(g)
406 MOVW R7, (g_sched+gobuf_ctxt)(g)
407
408 // Called from f.
409 // Set m->morebuf to f's caller.
410 MOVW R3, (m_morebuf+gobuf_pc)(R8) // f's caller's PC
411 MOVW R13, (m_morebuf+gobuf_sp)(R8) // f's caller's SP
412 MOVW g, (m_morebuf+gobuf_g)(R8)
413
414 // Call newstack on m->g0's stack.
415 MOVW m_g0(R8), R0
416 BL setg<>(SB)
417 MOVW (g_sched+gobuf_sp)(g), R13
418 MOVW $0, R0
419 MOVW.W R0, -4(R13) // create a call frame on g0 (saved LR)
420 BL runtime·newstack(SB)
421
422 // Not reached, but make sure the return PC from the call to newstack
423 // is still in this function, and not the beginning of the next.
424 RET
425
426 TEXT runtime·morestack_noctxt(SB),NOSPLIT|NOFRAME,$0-0
427 MOVW $0, R7
428 B runtime·morestack(SB)
429
430 // reflectcall: call a function with the given argument list
431 // func call(argtype *_type, f *FuncVal, arg *byte, argsize, retoffset uint32).
432 // we don't have variable-sized frames, so we use a small number
433 // of constant-sized-frame functions to encode a few bits of size in the pc.
434 // Caution: ugly multiline assembly macros in your future!
435
436 #define DISPATCH(NAME,MAXSIZE) \
437 CMP $MAXSIZE, R0; \
438 B.HI 3(PC); \
439 MOVW $NAME(SB), R1; \
440 B (R1)
441
442 TEXT ·reflectcall(SB),NOSPLIT|NOFRAME,$0-20
443 MOVW argsize+12(FP), R0
444 DISPATCH(runtime·call16, 16)
445 DISPATCH(runtime·call32, 32)
446 DISPATCH(runtime·call64, 64)
447 DISPATCH(runtime·call128, 128)
448 DISPATCH(runtime·call256, 256)
449 DISPATCH(runtime·call512, 512)
450 DISPATCH(runtime·call1024, 1024)
451 DISPATCH(runtime·call2048, 2048)
452 DISPATCH(runtime·call4096, 4096)
453 DISPATCH(runtime·call8192, 8192)
454 DISPATCH(runtime·call16384, 16384)
455 DISPATCH(runtime·call32768, 32768)
456 DISPATCH(runtime·call65536, 65536)
457 DISPATCH(runtime·call131072, 131072)
458 DISPATCH(runtime·call262144, 262144)
459 DISPATCH(runtime·call524288, 524288)
460 DISPATCH(runtime·call1048576, 1048576)
461 DISPATCH(runtime·call2097152, 2097152)
462 DISPATCH(runtime·call4194304, 4194304)
463 DISPATCH(runtime·call8388608, 8388608)
464 DISPATCH(runtime·call16777216, 16777216)
465 DISPATCH(runtime·call33554432, 33554432)
466 DISPATCH(runtime·call67108864, 67108864)
467 DISPATCH(runtime·call134217728, 134217728)
468 DISPATCH(runtime·call268435456, 268435456)
469 DISPATCH(runtime·call536870912, 536870912)
470 DISPATCH(runtime·call1073741824, 1073741824)
471 MOVW $runtime·badreflectcall(SB), R1
472 B (R1)
473
474 #define CALLFN(NAME,MAXSIZE) \
475 TEXT NAME(SB), WRAPPER, $MAXSIZE-20; \
476 NO_LOCAL_POINTERS; \
477 /* copy arguments to stack */ \
478 MOVW argptr+8(FP), R0; \
479 MOVW argsize+12(FP), R2; \
480 ADD $4, R13, R1; \
481 CMP $0, R2; \
482 B.EQ 5(PC); \
483 MOVBU.P 1(R0), R5; \
484 MOVBU.P R5, 1(R1); \
485 SUB $1, R2, R2; \
486 B -5(PC); \
487 /* call function */ \
488 MOVW f+4(FP), R7; \
489 MOVW (R7), R0; \
490 PCDATA $PCDATA_StackMapIndex, $0; \
491 BL (R0); \
492 /* copy return values back */ \
493 MOVW argtype+0(FP), R4; \
494 MOVW argptr+8(FP), R0; \
495 MOVW argsize+12(FP), R2; \
496 MOVW retoffset+16(FP), R3; \
497 ADD $4, R13, R1; \
498 ADD R3, R1; \
499 ADD R3, R0; \
500 SUB R3, R2; \
501 BL callRet<>(SB); \
502 RET
503
504 // callRet copies return values back at the end of call*. This is a
505 // separate function so it can allocate stack space for the arguments
506 // to reflectcallmove. It does not follow the Go ABI; it expects its
507 // arguments in registers.
508 TEXT callRet<>(SB), NOSPLIT, $16-0
509 MOVW R4, 4(R13)
510 MOVW R0, 8(R13)
511 MOVW R1, 12(R13)
512 MOVW R2, 16(R13)
513 BL runtime·reflectcallmove(SB)
514 RET
515
516 CALLFN(·call16, 16)
517 CALLFN(·call32, 32)
518 CALLFN(·call64, 64)
519 CALLFN(·call128, 128)
520 CALLFN(·call256, 256)
521 CALLFN(·call512, 512)
522 CALLFN(·call1024, 1024)
523 CALLFN(·call2048, 2048)
524 CALLFN(·call4096, 4096)
525 CALLFN(·call8192, 8192)
526 CALLFN(·call16384, 16384)
527 CALLFN(·call32768, 32768)
528 CALLFN(·call65536, 65536)
529 CALLFN(·call131072, 131072)
530 CALLFN(·call262144, 262144)
531 CALLFN(·call524288, 524288)
532 CALLFN(·call1048576, 1048576)
533 CALLFN(·call2097152, 2097152)
534 CALLFN(·call4194304, 4194304)
535 CALLFN(·call8388608, 8388608)
536 CALLFN(·call16777216, 16777216)
537 CALLFN(·call33554432, 33554432)
538 CALLFN(·call67108864, 67108864)
539 CALLFN(·call134217728, 134217728)
540 CALLFN(·call268435456, 268435456)
541 CALLFN(·call536870912, 536870912)
542 CALLFN(·call1073741824, 1073741824)
543
544 // void jmpdefer(fn, sp);
545 // called from deferreturn.
546 // 1. grab stored LR for caller
547 // 2. sub 4 bytes to get back to BL deferreturn
548 // 3. B to fn
549 // TODO(rsc): Push things on stack and then use pop
550 // to load all registers simultaneously, so that a profiling
551 // interrupt can never see mismatched SP/LR/PC.
552 // (And double-check that pop is atomic in that way.)
553 TEXT runtime·jmpdefer(SB),NOSPLIT,$0-8
554 MOVW 0(R13), LR
555 MOVW $-4(LR), LR // BL deferreturn
556 MOVW fv+0(FP), R7
557 MOVW argp+4(FP), R13
558 MOVW $-4(R13), R13 // SP is 4 below argp, due to saved LR
559 MOVW 0(R7), R1
560 B (R1)
561
562 // Save state of caller into g->sched. Smashes R11.
563 TEXT gosave<>(SB),NOSPLIT|NOFRAME,$0
564 MOVW LR, (g_sched+gobuf_pc)(g)
565 MOVW R13, (g_sched+gobuf_sp)(g)
566 MOVW $0, R11
567 MOVW R11, (g_sched+gobuf_lr)(g)
568 MOVW R11, (g_sched+gobuf_ret)(g)
569 MOVW R11, (g_sched+gobuf_ctxt)(g)
570 // Assert ctxt is zero. See func save.
571 MOVW (g_sched+gobuf_ctxt)(g), R11
572 CMP $0, R11
573 B.EQ 2(PC)
574 CALL runtime·badctxt(SB)
575 RET
576
577 // func asmcgocall(fn, arg unsafe.Pointer) int32
578 // Call fn(arg) on the scheduler stack,
579 // aligned appropriately for the gcc ABI.
580 // See cgocall.go for more details.
581 TEXT ·asmcgocall(SB),NOSPLIT,$0-12
582 MOVW fn+0(FP), R1
583 MOVW arg+4(FP), R0
584
585 MOVW R13, R2
586 CMP $0, g
587 BEQ nosave
588 MOVW g, R4
589
590 // Figure out if we need to switch to m->g0 stack.
591 // We get called to create new OS threads too, and those
592 // come in on the m->g0 stack already.
593 MOVW g_m(g), R8
594 MOVW m_gsignal(R8), R3
595 CMP R3, g
596 BEQ nosave
597 MOVW m_g0(R8), R3
598 CMP R3, g
599 BEQ nosave
600 BL gosave<>(SB)
601 MOVW R0, R5
602 MOVW R3, R0
603 BL setg<>(SB)
604 MOVW R5, R0
605 MOVW (g_sched+gobuf_sp)(g), R13
606
607 // Now on a scheduling stack (a pthread-created stack).
608 SUB $24, R13
609 BIC $0x7, R13 // alignment for gcc ABI
610 MOVW R4, 20(R13) // save old g
611 MOVW (g_stack+stack_hi)(R4), R4
612 SUB R2, R4
613 MOVW R4, 16(R13) // save depth in stack (can't just save SP, as stack might be copied during a callback)
614 BL (R1)
615
616 // Restore registers, g, stack pointer.
617 MOVW R0, R5
618 MOVW 20(R13), R0
619 BL setg<>(SB)
620 MOVW (g_stack+stack_hi)(g), R1
621 MOVW 16(R13), R2
622 SUB R2, R1
623 MOVW R5, R0
624 MOVW R1, R13
625
626 MOVW R0, ret+8(FP)
627 RET
628
629 nosave:
630 // Running on a system stack, perhaps even without a g.
631 // Having no g can happen during thread creation or thread teardown
632 // (see needm/dropm on Solaris, for example).
633 // This code is like the above sequence but without saving/restoring g
634 // and without worrying about the stack moving out from under us
635 // (because we're on a system stack, not a goroutine stack).
636 // The above code could be used directly if already on a system stack,
637 // but then the only path through this code would be a rare case on Solaris.
638 // Using this code for all "already on system stack" calls exercises it more,
639 // which should help keep it correct.
640 SUB $24, R13
641 BIC $0x7, R13 // alignment for gcc ABI
642 // save null g in case someone looks during debugging.
643 MOVW $0, R4
644 MOVW R4, 20(R13)
645 MOVW R2, 16(R13) // Save old stack pointer.
646 BL (R1)
647 // Restore stack pointer.
648 MOVW 16(R13), R2
649 MOVW R2, R13
650 MOVW R0, ret+8(FP)
651 RET
652
653 // cgocallback(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt)
654 // Turn the fn into a Go func (by taking its address) and call
655 // cgocallback_gofunc.
656 TEXT runtime·cgocallback(SB),NOSPLIT,$16-16
657 MOVW $fn+0(FP), R0
658 MOVW R0, 4(R13)
659 MOVW frame+4(FP), R0
660 MOVW R0, 8(R13)
661 MOVW framesize+8(FP), R0
662 MOVW R0, 12(R13)
663 MOVW ctxt+12(FP), R0
664 MOVW R0, 16(R13)
665 MOVW $runtime·cgocallback_gofunc(SB), R0
666 BL (R0)
667 RET
668
669 // cgocallback_gofunc(void (*fn)(void*), void *frame, uintptr framesize, uintptr ctxt)
670 // See cgocall.go for more details.
671 TEXT ·cgocallback_gofunc(SB),NOSPLIT,$8-16
672 NO_LOCAL_POINTERS
673
674 // Load m and g from thread-local storage.
675 MOVB runtime·iscgo(SB), R0
676 CMP $0, R0
677 BL.NE runtime·load_g(SB)
678
679 // If g is nil, Go did not create the current thread.
680 // Call needm to obtain one for temporary use.
681 // In this case, we're running on the thread stack, so there's
682 // lots of space, but the linker doesn't know. Hide the call from
683 // the linker analysis by using an indirect call.
684 CMP $0, g
685 B.EQ needm
686
687 MOVW g_m(g), R8
688 MOVW R8, savedm-4(SP)
689 B havem
690
691 needm:
692 MOVW g, savedm-4(SP) // g is zero, so is m.
693 MOVW $runtime·needm(SB), R0
694 BL (R0)
695
696 // Set m->sched.sp = SP, so that if a panic happens
697 // during the function we are about to execute, it will
698 // have a valid SP to run on the g0 stack.
699 // The next few lines (after the havem label)
700 // will save this SP onto the stack and then write
701 // the same SP back to m->sched.sp. That seems redundant,
702 // but if an unrecovered panic happens, unwindm will
703 // restore the g->sched.sp from the stack location
704 // and then systemstack will try to use it. If we don't set it here,
705 // that restored SP will be uninitialized (typically 0) and
706 // will not be usable.
707 MOVW g_m(g), R8
708 MOVW m_g0(R8), R3
709 MOVW R13, (g_sched+gobuf_sp)(R3)
710
711 havem:
712 // Now there's a valid m, and we're running on its m->g0.
713 // Save current m->g0->sched.sp on stack and then set it to SP.
714 // Save current sp in m->g0->sched.sp in preparation for
715 // switch back to m->curg stack.
716 // NOTE: unwindm knows that the saved g->sched.sp is at 4(R13) aka savedsp-8(SP).
717 MOVW m_g0(R8), R3
718 MOVW (g_sched+gobuf_sp)(R3), R4
719 MOVW R4, savedsp-8(SP)
720 MOVW R13, (g_sched+gobuf_sp)(R3)
721
722 // Switch to m->curg stack and call runtime.cgocallbackg.
723 // Because we are taking over the execution of m->curg
724 // but *not* resuming what had been running, we need to
725 // save that information (m->curg->sched) so we can restore it.
726 // We can restore m->curg->sched.sp easily, because calling
727 // runtime.cgocallbackg leaves SP unchanged upon return.
728 // To save m->curg->sched.pc, we push it onto the stack.
729 // This has the added benefit that it looks to the traceback
730 // routine like cgocallbackg is going to return to that
731 // PC (because the frame we allocate below has the same
732 // size as cgocallback_gofunc's frame declared above)
733 // so that the traceback will seamlessly trace back into
734 // the earlier calls.
735 //
736 // In the new goroutine, -4(SP) is unused (where SP refers to
737 // m->curg's SP while we're setting it up, before we've adjusted it).
738 MOVW m_curg(R8), R0
739 BL setg<>(SB)
740 MOVW (g_sched+gobuf_sp)(g), R4 // prepare stack as R4
741 MOVW (g_sched+gobuf_pc)(g), R5
742 MOVW R5, -12(R4)
743 MOVW ctxt+12(FP), R0
744 MOVW R0, -8(R4)
745 MOVW $-12(R4), R13
746 BL runtime·cgocallbackg(SB)
747
748 // Restore g->sched (== m->curg->sched) from saved values.
749 MOVW 0(R13), R5
750 MOVW R5, (g_sched+gobuf_pc)(g)
751 MOVW $12(R13), R4
752 MOVW R4, (g_sched+gobuf_sp)(g)
753
754 // Switch back to m->g0's stack and restore m->g0->sched.sp.
755 // (Unlike m->curg, the g0 goroutine never uses sched.pc,
756 // so we do not have to restore it.)
757 MOVW g_m(g), R8
758 MOVW m_g0(R8), R0
759 BL setg<>(SB)
760 MOVW (g_sched+gobuf_sp)(g), R13
761 MOVW savedsp-8(SP), R4
762 MOVW R4, (g_sched+gobuf_sp)(g)
763
764 // If the m on entry was nil, we called needm above to borrow an m
765 // for the duration of the call. Since the call is over, return it with dropm.
766 MOVW savedm-4(SP), R6
767 CMP $0, R6
768 B.NE 3(PC)
769 MOVW $runtime·dropm(SB), R0
770 BL (R0)
771
772 // Done!
773 RET
774
775 // void setg(G*); set g. for use by needm.
776 TEXT runtime·setg(SB),NOSPLIT|NOFRAME,$0-4
777 MOVW gg+0(FP), R0
778 B setg<>(SB)
779
780 TEXT setg<>(SB),NOSPLIT|NOFRAME,$0-0
781 MOVW R0, g
782
783 // Save g to thread-local storage.
784 #ifdef GOOS_windows
785 B runtime·save_g(SB)
786 #else
787 MOVB runtime·iscgo(SB), R0
788 CMP $0, R0
789 B.EQ 2(PC)
790 B runtime·save_g(SB)
791
792 MOVW g, R0
793 RET
794 #endif
795
796 TEXT runtime·emptyfunc(SB),0,$0-0
797 RET
798
799 TEXT runtime·abort(SB),NOSPLIT|NOFRAME,$0-0
800 MOVW $0, R0
801 MOVW (R0), R1
802
803 // armPublicationBarrier is a native store/store barrier for ARMv7+.
804 // On earlier ARM revisions, armPublicationBarrier is a no-op.
805 // This will not work on SMP ARMv6 machines, if any are in use.
806 // To implement publicationBarrier in sys_$GOOS_arm.s using the native
807 // instructions, use:
808 //
809 // TEXT ·publicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
810 // B runtime·armPublicationBarrier(SB)
811 //
812 TEXT runtime·armPublicationBarrier(SB),NOSPLIT|NOFRAME,$0-0
813 MOVB runtime·goarm(SB), R11
814 CMP $7, R11
815 BLT 2(PC)
816 DMB MB_ST
817 RET
818
819 // AES hashing not implemented for ARM
820 TEXT runtime·aeshash(SB),NOSPLIT|NOFRAME,$0-0
821 MOVW $0, R0
822 MOVW (R0), R1
823 TEXT runtime·aeshash32(SB),NOSPLIT|NOFRAME,$0-0
824 MOVW $0, R0
825 MOVW (R0), R1
826 TEXT runtime·aeshash64(SB),NOSPLIT|NOFRAME,$0-0
827 MOVW $0, R0
828 MOVW (R0), R1
829 TEXT runtime·aeshashstr(SB),NOSPLIT|NOFRAME,$0-0
830 MOVW $0, R0
831 MOVW (R0), R1
832
833 TEXT runtime·return0(SB),NOSPLIT,$0
834 MOVW $0, R0
835 RET
836
837 TEXT runtime·procyield(SB),NOSPLIT|NOFRAME,$0
838 MOVW cycles+0(FP), R1
839 MOVW $0, R0
840 yieldloop:
841 WORD $0xe320f001 // YIELD (NOP pre-ARMv6K)
842 CMP R0, R1
843 B.NE 2(PC)
844 RET
845 SUB $1, R1
846 B yieldloop
847
848 // Called from cgo wrappers, this function returns g->m->curg.stack.hi.
849 // Must obey the gcc calling convention.
850 TEXT _cgo_topofstack(SB),NOSPLIT,$8
851 // R11 and g register are clobbered by load_g. They are
852 // callee-save in the gcc calling convention, so save them here.
853 MOVW R11, saveR11-4(SP)
854 MOVW g, saveG-8(SP)
855
856 BL runtime·load_g(SB)
857 MOVW g_m(g), R0
858 MOVW m_curg(R0), R0
859 MOVW (g_stack+stack_hi)(R0), R0
860
861 MOVW saveG-8(SP), g
862 MOVW saveR11-4(SP), R11
863 RET
864
865 // The top-most function running on a goroutine
866 // returns to goexit+PCQuantum.
867 TEXT runtime·goexit(SB),NOSPLIT|NOFRAME|TOPFRAME,$0-0
868 MOVW R0, R0 // NOP
869 BL runtime·goexit1(SB) // does not return
870 // traceback from goexit1 must hit code range of goexit
871 MOVW R0, R0 // NOP
872
873 // x -> x/1000000, x%1000000, called from Go with args, results on stack.
874 TEXT runtime·usplit(SB),NOSPLIT,$0-12
875 MOVW x+0(FP), R0
876 CALL runtime·usplitR0(SB)
877 MOVW R0, q+4(FP)
878 MOVW R1, r+8(FP)
879 RET
880
881 // R0, R1 = R0/1000000, R0%1000000
882 TEXT runtime·usplitR0(SB),NOSPLIT,$0
883 // magic multiply to avoid software divide without available m.
884 // see output of go tool compile -S for x/1000000.
885 MOVW R0, R3
886 MOVW $1125899907, R1
887 MULLU R1, R0, (R0, R1)
888 MOVW R0>>18, R0
889 MOVW $1000000, R1
890 MULU R0, R1
891 SUB R1, R3, R1
892 RET
893
894 #ifndef GOOS_nacl
895 // This is called from .init_array and follows the platform, not Go, ABI.
896 TEXT runtime·addmoduledata(SB),NOSPLIT,$0-0
897 MOVW R9, saver9-4(SP) // The access to global variables below implicitly uses R9, which is callee-save
898 MOVW R11, saver11-8(SP) // Likewise, R11 is the temp register, but callee-save in C ABI
899 MOVW runtime·lastmoduledatap(SB), R1
900 MOVW R0, moduledata_next(R1)
901 MOVW R0, runtime·lastmoduledatap(SB)
902 MOVW saver11-8(SP), R11
903 MOVW saver9-4(SP), R9
904 RET
905 #endif
906
907 TEXT ·checkASM(SB),NOSPLIT,$0-1
908 MOVW $1, R3
909 MOVB R3, ret+0(FP)
910 RET
911
912 // gcWriteBarrier performs a heap pointer write and informs the GC.
913 //
914 // gcWriteBarrier does NOT follow the Go ABI. It takes two arguments:
915 // - R2 is the destination of the write
916 // - R3 is the value being written at R2
917 // It clobbers condition codes.
918 // It does not clobber any other general-purpose registers,
919 // but may clobber others (e.g., floating point registers).
920 // The act of CALLing gcWriteBarrier will clobber R14 (LR).
921 TEXT runtime·gcWriteBarrier(SB),NOSPLIT|NOFRAME,$0
922 // Save the registers clobbered by the fast path.
923 MOVM.DB.W [R0,R1], (R13)
924 MOVW g_m(g), R0
925 MOVW m_p(R0), R0
926 MOVW (p_wbBuf+wbBuf_next)(R0), R1
927 // Increment wbBuf.next position.
928 ADD $8, R1
929 MOVW R1, (p_wbBuf+wbBuf_next)(R0)
930 MOVW (p_wbBuf+wbBuf_end)(R0), R0
931 CMP R1, R0
932 // Record the write.
933 MOVW R3, -8(R1) // Record value
934 MOVW (R2), R0 // TODO: This turns bad writes into bad reads.
935 MOVW R0, -4(R1) // Record *slot
936 // Is the buffer full? (flags set in CMP above)
937 B.EQ flush
938 ret:
939 MOVM.IA.W (R13), [R0,R1]
940 // Do the write.
941 MOVW R3, (R2)
942 // Normally RET on nacl clobbers R12, but because this
943 // function has no frame it doesn't have to usual epilogue.
944 RET
945
946 flush:
947 // Save all general purpose registers since these could be
948 // clobbered by wbBufFlush and were not saved by the caller.
949 //
950 // R0 and R1 were saved at entry.
951 // R10 is g, so preserved.
952 // R11 is linker temp, so no need to save.
953 // R13 is stack pointer.
954 // R15 is PC.
955 //
956 // This also sets up R2 and R3 as the arguments to wbBufFlush.
957 MOVM.DB.W [R2-R9,R12], (R13)
958 // Save R14 (LR) because the fast path above doesn't save it,
959 // but needs it to RET. This is after the MOVM so it appears below
960 // the arguments in the stack frame.
961 MOVM.DB.W [R14], (R13)
962
963 // This takes arguments R2 and R3.
964 CALL runtime·wbBufFlush(SB)
965
966 MOVM.IA.W (R13), [R14]
967 MOVM.IA.W (R13), [R2-R9,R12]
968 JMP ret
969
970 // Note: these functions use a special calling convention to save generated code space.
971 // Arguments are passed in registers, but the space for those arguments are allocated
972 // in the caller's stack frame. These stubs write the args into that stack space and
973 // then tail call to the corresponding runtime handler.
974 // The tail call makes these stubs disappear in backtraces.
975 TEXT runtime·panicIndex(SB),NOSPLIT,$0-8
976 MOVW R0, x+0(FP)
977 MOVW R1, y+4(FP)
978 JMP runtime·goPanicIndex(SB)
979 TEXT runtime·panicIndexU(SB),NOSPLIT,$0-8
980 MOVW R0, x+0(FP)
981 MOVW R1, y+4(FP)
982 JMP runtime·goPanicIndexU(SB)
983 TEXT runtime·panicSliceAlen(SB),NOSPLIT,$0-8
984 MOVW R1, x+0(FP)
985 MOVW R2, y+4(FP)
986 JMP runtime·goPanicSliceAlen(SB)
987 TEXT runtime·panicSliceAlenU(SB),NOSPLIT,$0-8
988 MOVW R1, x+0(FP)
989 MOVW R2, y+4(FP)
990 JMP runtime·goPanicSliceAlenU(SB)
991 TEXT runtime·panicSliceAcap(SB),NOSPLIT,$0-8
992 MOVW R1, x+0(FP)
993 MOVW R2, y+4(FP)
994 JMP runtime·goPanicSliceAcap(SB)
995 TEXT runtime·panicSliceAcapU(SB),NOSPLIT,$0-8
996 MOVW R1, x+0(FP)
997 MOVW R2, y+4(FP)
998 JMP runtime·goPanicSliceAcapU(SB)
999 TEXT runtime·panicSliceB(SB),NOSPLIT,$0-8
1000 MOVW R0, x+0(FP)
1001 MOVW R1, y+4(FP)
1002 JMP runtime·goPanicSliceB(SB)
1003 TEXT runtime·panicSliceBU(SB),NOSPLIT,$0-8
1004 MOVW R0, x+0(FP)
1005 MOVW R1, y+4(FP)
1006 JMP runtime·goPanicSliceBU(SB)
1007 TEXT runtime·panicSlice3Alen(SB),NOSPLIT,$0-8
1008 MOVW R2, x+0(FP)
1009 MOVW R3, y+4(FP)
1010 JMP runtime·goPanicSlice3Alen(SB)
1011 TEXT runtime·panicSlice3AlenU(SB),NOSPLIT,$0-8
1012 MOVW R2, x+0(FP)
1013 MOVW R3, y+4(FP)
1014 JMP runtime·goPanicSlice3AlenU(SB)
1015 TEXT runtime·panicSlice3Acap(SB),NOSPLIT,$0-8
1016 MOVW R2, x+0(FP)
1017 MOVW R3, y+4(FP)
1018 JMP runtime·goPanicSlice3Acap(SB)
1019 TEXT runtime·panicSlice3AcapU(SB),NOSPLIT,$0-8
1020 MOVW R2, x+0(FP)
1021 MOVW R3, y+4(FP)
1022 JMP runtime·goPanicSlice3AcapU(SB)
1023 TEXT runtime·panicSlice3B(SB),NOSPLIT,$0-8
1024 MOVW R1, x+0(FP)
1025 MOVW R2, y+4(FP)
1026 JMP runtime·goPanicSlice3B(SB)
1027 TEXT runtime·panicSlice3BU(SB),NOSPLIT,$0-8
1028 MOVW R1, x+0(FP)
1029 MOVW R2, y+4(FP)
1030 JMP runtime·goPanicSlice3BU(SB)
1031 TEXT runtime·panicSlice3C(SB),NOSPLIT,$0-8
1032 MOVW R0, x+0(FP)
1033 MOVW R1, y+4(FP)
1034 JMP runtime·goPanicSlice3C(SB)
1035 TEXT runtime·panicSlice3CU(SB),NOSPLIT,$0-8
1036 MOVW R0, x+0(FP)
1037 MOVW R1, y+4(FP)
1038 JMP runtime·goPanicSlice3CU(SB)
1039
1040 // Extended versions for 64-bit indexes.
1041 TEXT runtime·panicExtendIndex(SB),NOSPLIT,$0-12
1042 MOVW R4, hi+0(FP)
1043 MOVW R0, lo+4(FP)
1044 MOVW R1, y+8(FP)
1045 JMP runtime·goPanicExtendIndex(SB)
1046 TEXT runtime·panicExtendIndexU(SB),NOSPLIT,$0-12
1047 MOVW R4, hi+0(FP)
1048 MOVW R0, lo+4(FP)
1049 MOVW R1, y+8(FP)
1050 JMP runtime·goPanicExtendIndexU(SB)
1051 TEXT runtime·panicExtendSliceAlen(SB),NOSPLIT,$0-12
1052 MOVW R4, hi+0(FP)
1053 MOVW R1, lo+4(FP)
1054 MOVW R2, y+8(FP)
1055 JMP runtime·goPanicExtendSliceAlen(SB)
1056 TEXT runtime·panicExtendSliceAlenU(SB),NOSPLIT,$0-12
1057 MOVW R4, hi+0(FP)
1058 MOVW R1, lo+4(FP)
1059 MOVW R2, y+8(FP)
1060 JMP runtime·goPanicExtendSliceAlenU(SB)
1061 TEXT runtime·panicExtendSliceAcap(SB),NOSPLIT,$0-12
1062 MOVW R4, hi+0(FP)
1063 MOVW R1, lo+4(FP)
1064 MOVW R2, y+8(FP)
1065 JMP runtime·goPanicExtendSliceAcap(SB)
1066 TEXT runtime·panicExtendSliceAcapU(SB),NOSPLIT,$0-12
1067 MOVW R4, hi+0(FP)
1068 MOVW R1, lo+4(FP)
1069 MOVW R2, y+8(FP)
1070 JMP runtime·goPanicExtendSliceAcapU(SB)
1071 TEXT runtime·panicExtendSliceB(SB),NOSPLIT,$0-12
1072 MOVW R4, hi+0(FP)
1073 MOVW R0, lo+4(FP)
1074 MOVW R1, y+8(FP)
1075 JMP runtime·goPanicExtendSliceB(SB)
1076 TEXT runtime·panicExtendSliceBU(SB),NOSPLIT,$0-12
1077 MOVW R4, hi+0(FP)
1078 MOVW R0, lo+4(FP)
1079 MOVW R1, y+8(FP)
1080 JMP runtime·goPanicExtendSliceBU(SB)
1081 TEXT runtime·panicExtendSlice3Alen(SB),NOSPLIT,$0-12
1082 MOVW R4, hi+0(FP)
1083 MOVW R2, lo+4(FP)
1084 MOVW R3, y+8(FP)
1085 JMP runtime·goPanicExtendSlice3Alen(SB)
1086 TEXT runtime·panicExtendSlice3AlenU(SB),NOSPLIT,$0-12
1087 MOVW R4, hi+0(FP)
1088 MOVW R2, lo+4(FP)
1089 MOVW R3, y+8(FP)
1090 JMP runtime·goPanicExtendSlice3AlenU(SB)
1091 TEXT runtime·panicExtendSlice3Acap(SB),NOSPLIT,$0-12
1092 MOVW R4, hi+0(FP)
1093 MOVW R2, lo+4(FP)
1094 MOVW R3, y+8(FP)
1095 JMP runtime·goPanicExtendSlice3Acap(SB)
1096 TEXT runtime·panicExtendSlice3AcapU(SB),NOSPLIT,$0-12
1097 MOVW R4, hi+0(FP)
1098 MOVW R2, lo+4(FP)
1099 MOVW R3, y+8(FP)
1100 JMP runtime·goPanicExtendSlice3AcapU(SB)
1101 TEXT runtime·panicExtendSlice3B(SB),NOSPLIT,$0-12
1102 MOVW R4, hi+0(FP)
1103 MOVW R1, lo+4(FP)
1104 MOVW R2, y+8(FP)
1105 JMP runtime·goPanicExtendSlice3B(SB)
1106 TEXT runtime·panicExtendSlice3BU(SB),NOSPLIT,$0-12
1107 MOVW R4, hi+0(FP)
1108 MOVW R1, lo+4(FP)
1109 MOVW R2, y+8(FP)
1110 JMP runtime·goPanicExtendSlice3BU(SB)
1111 TEXT runtime·panicExtendSlice3C(SB),NOSPLIT,$0-12
1112 MOVW R4, hi+0(FP)
1113 MOVW R0, lo+4(FP)
1114 MOVW R1, y+8(FP)
1115 JMP runtime·goPanicExtendSlice3C(SB)
1116 TEXT runtime·panicExtendSlice3CU(SB),NOSPLIT,$0-12
1117 MOVW R4, hi+0(FP)
1118 MOVW R0, lo+4(FP)
1119 MOVW R1, y+8(FP)
1120 JMP runtime·goPanicExtendSlice3CU(SB)
View as plain text