1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import "unsafe" 8 9 // Should be a built-in for unsafe.Pointer? 10 //go:nosplit 11 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 12 return unsafe.Pointer(uintptr(p) + x) 13 } 14 15 // getg returns the pointer to the current g. 16 // The compiler rewrites calls to this function into instructions 17 // that fetch the g directly (from TLS or from the dedicated register). 18 func getg() *g 19 20 // mcall switches from the g to the g0 stack and invokes fn(g), 21 // where g is the goroutine that made the call. 22 // mcall saves g's current PC/SP in g->sched so that it can be restored later. 23 // It is up to fn to arrange for that later execution, typically by recording 24 // g in a data structure, causing something to call ready(g) later. 25 // mcall returns to the original goroutine g later, when g has been rescheduled. 26 // fn must not return at all; typically it ends by calling schedule, to let the m 27 // run other goroutines. 28 // 29 // mcall can only be called from g stacks (not g0, not gsignal). 30 // 31 // This must NOT be go:noescape: if fn is a stack-allocated closure, 32 // fn puts g on a run queue, and g executes before fn returns, the 33 // closure will be invalidated while it is still executing. 34 func mcall(fn func(*g)) 35 36 // systemstack runs fn on a system stack. 37 // If systemstack is called from the per-OS-thread (g0) stack, or 38 // if systemstack is called from the signal handling (gsignal) stack, 39 // systemstack calls fn directly and returns. 40 // Otherwise, systemstack is being called from the limited stack 41 // of an ordinary goroutine. In this case, systemstack switches 42 // to the per-OS-thread stack, calls fn, and switches back. 43 // It is common to use a func literal as the argument, in order 44 // to share inputs and outputs with the code around the call 45 // to system stack: 46 // 47 // ... set up y ... 48 // systemstack(func() { 49 // x = bigcall(y) 50 // }) 51 // ... use x ... 52 // 53 //go:noescape 54 func systemstack(fn func()) 55 56 var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine" 57 58 //go:nosplit 59 //go:nowritebarrierrec 60 func badsystemstack() { 61 sp := stringStructOf(&badsystemstackMsg) 62 write(2, sp.str, int32(sp.len)) 63 } 64 65 // memclrNoHeapPointers clears n bytes starting at ptr. 66 // 67 // Usually you should use typedmemclr. memclrNoHeapPointers should be 68 // used only when the caller knows that *ptr contains no heap pointers 69 // because either: 70 // 71 // *ptr is initialized memory and its type is pointer-free, or 72 // 73 // *ptr is uninitialized memory (e.g., memory that's being reused 74 // for a new allocation) and hence contains only "junk". 75 // 76 // The (CPU-specific) implementations of this function are in memclr_*.s. 77 //go:noescape 78 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 79 80 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers 81 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { 82 memclrNoHeapPointers(ptr, n) 83 } 84 85 // memmove copies n bytes from "from" to "to". 86 // in memmove_*.s 87 //go:noescape 88 func memmove(to, from unsafe.Pointer, n uintptr) 89 90 //go:linkname reflect_memmove reflect.memmove 91 func reflect_memmove(to, from unsafe.Pointer, n uintptr) { 92 memmove(to, from, n) 93 } 94 95 // exported value for testing 96 var hashLoad = float32(loadFactorNum) / float32(loadFactorDen) 97 98 //go:nosplit 99 func fastrand() uint32 { 100 mp := getg().m 101 // Implement xorshift64+: 2 32-bit xorshift sequences added together. 102 // Shift triplet [17,7,16] was calculated as indicated in Marsaglia's 103 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf 104 // This generator passes the SmallCrush suite, part of TestU01 framework: 105 // http://simul.iro.umontreal.ca/testu01/tu01.html 106 s1, s0 := mp.fastrand[0], mp.fastrand[1] 107 s1 ^= s1 << 17 108 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 109 mp.fastrand[0], mp.fastrand[1] = s0, s1 110 return s0 + s1 111 } 112 113 //go:nosplit 114 func fastrandn(n uint32) uint32 { 115 // This is similar to fastrand() % n, but faster. 116 // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ 117 return uint32(uint64(fastrand()) * uint64(n) >> 32) 118 } 119 120 //go:linkname sync_fastrand sync.fastrand 121 func sync_fastrand() uint32 { return fastrand() } 122 123 // in asm_*.s 124 //go:noescape 125 func memequal(a, b unsafe.Pointer, size uintptr) bool 126 127 // noescape hides a pointer from escape analysis. noescape is 128 // the identity function but escape analysis doesn't think the 129 // output depends on the input. noescape is inlined and currently 130 // compiles down to zero instructions. 131 // USE CAREFULLY! 132 //go:nosplit 133 func noescape(p unsafe.Pointer) unsafe.Pointer { 134 x := uintptr(p) 135 return unsafe.Pointer(x ^ 0) 136 } 137 138 func cgocallback(fn, frame unsafe.Pointer, framesize, ctxt uintptr) 139 func gogo(buf *gobuf) 140 func gosave(buf *gobuf) 141 142 //go:noescape 143 func jmpdefer(fv *funcval, argp uintptr) 144 func asminit() 145 func setg(gg *g) 146 func breakpoint() 147 148 // reflectcall calls fn with a copy of the n argument bytes pointed at by arg. 149 // After fn returns, reflectcall copies n-retoffset result bytes 150 // back into arg+retoffset before returning. If copying result bytes back, 151 // the caller should pass the argument frame type as argtype, so that 152 // call can execute appropriate write barriers during the copy. 153 // Package reflect passes a frame type. In package runtime, there is only 154 // one call that copies results back, in cgocallbackg1, and it does NOT pass a 155 // frame type, meaning there are no write barriers invoked. See that call 156 // site for justification. 157 // 158 // Package reflect accesses this symbol through a linkname. 159 func reflectcall(argtype *_type, fn, arg unsafe.Pointer, argsize uint32, retoffset uint32) 160 161 func procyield(cycles uint32) 162 163 type neverCallThisFunction struct{} 164 165 // goexit is the return stub at the top of every goroutine call stack. 166 // Each goroutine stack is constructed as if goexit called the 167 // goroutine's entry point function, so that when the entry point 168 // function returns, it will return to goexit, which will call goexit1 169 // to perform the actual exit. 170 // 171 // This function must never be called directly. Call goexit1 instead. 172 // gentraceback assumes that goexit terminates the stack. A direct 173 // call on the stack will cause gentraceback to stop walking the stack 174 // prematurely and if there is leftover state it may panic. 175 func goexit(neverCallThisFunction) 176 177 // Not all cgocallback_gofunc frames are actually cgocallback_gofunc, 178 // so not all have these arguments. Mark them uintptr so that the GC 179 // does not misinterpret memory when the arguments are not present. 180 // cgocallback_gofunc is not called from go, only from cgocallback, 181 // so the arguments will be found via cgocallback's pointer-declared arguments. 182 // See the assembly implementations for more details. 183 func cgocallback_gofunc(fv, frame, framesize, ctxt uintptr) 184 185 // publicationBarrier performs a store/store barrier (a "publication" 186 // or "export" barrier). Some form of synchronization is required 187 // between initializing an object and making that object accessible to 188 // another processor. Without synchronization, the initialization 189 // writes and the "publication" write may be reordered, allowing the 190 // other processor to follow the pointer and observe an uninitialized 191 // object. In general, higher-level synchronization should be used, 192 // such as locking or an atomic pointer write. publicationBarrier is 193 // for when those aren't an option, such as in the implementation of 194 // the memory manager. 195 // 196 // There's no corresponding barrier for the read side because the read 197 // side naturally has a data dependency order. All architectures that 198 // Go supports or seems likely to ever support automatically enforce 199 // data dependency ordering. 200 func publicationBarrier() 201 202 // getcallerpc returns the program counter (PC) of its caller's caller. 203 // getcallersp returns the stack pointer (SP) of its caller's caller. 204 // The implementation may be a compiler intrinsic; there is not 205 // necessarily code implementing this on every platform. 206 // 207 // For example: 208 // 209 // func f(arg1, arg2, arg3 int) { 210 // pc := getcallerpc() 211 // sp := getcallersp() 212 // } 213 // 214 // These two lines find the PC and SP immediately following 215 // the call to f (where f will return). 216 // 217 // The call to getcallerpc and getcallersp must be done in the 218 // frame being asked about. 219 // 220 // The result of getcallersp is correct at the time of the return, 221 // but it may be invalidated by any subsequent call to a function 222 // that might relocate the stack in order to grow or shrink it. 223 // A general rule is that the result of getcallersp should be used 224 // immediately and can only be passed to nosplit functions. 225 226 //go:noescape 227 func getcallerpc() uintptr 228 229 //go:noescape 230 func getcallersp() uintptr // implemented as an intrinsic on all platforms 231 232 // getclosureptr returns the pointer to the current closure. 233 // getclosureptr can only be used in an assignment statement 234 // at the entry of a function. Moreover, go:nosplit directive 235 // must be specified at the declaration of caller function, 236 // so that the function prolog does not clobber the closure register. 237 // for example: 238 // 239 // //go:nosplit 240 // func f(arg1, arg2, arg3 int) { 241 // dx := getclosureptr() 242 // } 243 // 244 // The compiler rewrites calls to this function into instructions that fetch the 245 // pointer from a well-known register (DX on x86 architecture, etc.) directly. 246 func getclosureptr() uintptr 247 248 //go:noescape 249 func asmcgocall(fn, arg unsafe.Pointer) int32 250 251 func morestack() 252 func morestack_noctxt() 253 func rt0_go() 254 255 // return0 is a stub used to return 0 from deferproc. 256 // It is called at the very end of deferproc to signal 257 // the calling Go function that it should not jump 258 // to deferreturn. 259 // in asm_*.s 260 func return0() 261 262 // in asm_*.s 263 // not called directly; definitions here supply type information for traceback. 264 func call32(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 265 func call64(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 266 func call128(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 267 func call256(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 268 func call512(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 269 func call1024(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 270 func call2048(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 271 func call4096(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 272 func call8192(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 273 func call16384(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 274 func call32768(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 275 func call65536(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 276 func call131072(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 277 func call262144(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 278 func call524288(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 279 func call1048576(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 280 func call2097152(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 281 func call4194304(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 282 func call8388608(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 283 func call16777216(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 284 func call33554432(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 285 func call67108864(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 286 func call134217728(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 287 func call268435456(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 288 func call536870912(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 289 func call1073741824(typ, fn, arg unsafe.Pointer, n, retoffset uint32) 290 291 func systemstack_switch() 292 293 // round n up to a multiple of a. a must be a power of 2. 294 func round(n, a uintptr) uintptr { 295 return (n + a - 1) &^ (a - 1) 296 } 297 298 // checkASM reports whether assembly runtime checks have passed. 299 func checkASM() bool 300 301 func memequal_varlen(a, b unsafe.Pointer) bool 302 303 // bool2int returns 0 if x is false or 1 if x is true. 304 func bool2int(x bool) int { 305 // Avoid branches. In the SSA compiler, this compiles to 306 // exactly what you would want it to. 307 return int(uint8(*(*uint8)(unsafe.Pointer(&x)))) 308 } 309 310 // abort crashes the runtime in situations where even throw might not 311 // work. In general it should do something a debugger will recognize 312 // (e.g., an INT3 on x86). A crash in abort is recognized by the 313 // signal handler, which will attempt to tear down the runtime 314 // immediately. 315 func abort() 316 317 // Called from compiled code; declared for vet; do NOT call from Go. 318 func gcWriteBarrier() 319 func duffzero() 320 func duffcopy() 321 322 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. 323 func addmoduledata() 324