1 // Copyright 2014 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 package runtime 6 7 import ( 8 "internal/abi" 9 "internal/goexperiment" 10 "unsafe" 11 ) 12 13 // Should be a built-in for unsafe.Pointer? 14 //go:nosplit 15 func add(p unsafe.Pointer, x uintptr) unsafe.Pointer { 16 return unsafe.Pointer(uintptr(p) + x) 17 } 18 19 // getg returns the pointer to the current g. 20 // The compiler rewrites calls to this function into instructions 21 // that fetch the g directly (from TLS or from the dedicated register). 22 func getg() *g 23 24 // mcall switches from the g to the g0 stack and invokes fn(g), 25 // where g is the goroutine that made the call. 26 // mcall saves g's current PC/SP in g->sched so that it can be restored later. 27 // It is up to fn to arrange for that later execution, typically by recording 28 // g in a data structure, causing something to call ready(g) later. 29 // mcall returns to the original goroutine g later, when g has been rescheduled. 30 // fn must not return at all; typically it ends by calling schedule, to let the m 31 // run other goroutines. 32 // 33 // mcall can only be called from g stacks (not g0, not gsignal). 34 // 35 // This must NOT be go:noescape: if fn is a stack-allocated closure, 36 // fn puts g on a run queue, and g executes before fn returns, the 37 // closure will be invalidated while it is still executing. 38 func mcall(fn func(*g)) 39 40 // systemstack runs fn on a system stack. 41 // If systemstack is called from the per-OS-thread (g0) stack, or 42 // if systemstack is called from the signal handling (gsignal) stack, 43 // systemstack calls fn directly and returns. 44 // Otherwise, systemstack is being called from the limited stack 45 // of an ordinary goroutine. In this case, systemstack switches 46 // to the per-OS-thread stack, calls fn, and switches back. 47 // It is common to use a func literal as the argument, in order 48 // to share inputs and outputs with the code around the call 49 // to system stack: 50 // 51 // ... set up y ... 52 // systemstack(func() { 53 // x = bigcall(y) 54 // }) 55 // ... use x ... 56 // 57 //go:noescape 58 func systemstack(fn func()) 59 60 var badsystemstackMsg = "fatal: systemstack called from unexpected goroutine" 61 62 //go:nosplit 63 //go:nowritebarrierrec 64 func badsystemstack() { 65 sp := stringStructOf(&badsystemstackMsg) 66 write(2, sp.str, int32(sp.len)) 67 } 68 69 // memclrNoHeapPointers clears n bytes starting at ptr. 70 // 71 // Usually you should use typedmemclr. memclrNoHeapPointers should be 72 // used only when the caller knows that *ptr contains no heap pointers 73 // because either: 74 // 75 // *ptr is initialized memory and its type is pointer-free, or 76 // 77 // *ptr is uninitialized memory (e.g., memory that's being reused 78 // for a new allocation) and hence contains only "junk". 79 // 80 // memclrNoHeapPointers ensures that if ptr is pointer-aligned, and n 81 // is a multiple of the pointer size, then any pointer-aligned, 82 // pointer-sized portion is cleared atomically. Despite the function 83 // name, this is necessary because this function is the underlying 84 // implementation of typedmemclr and memclrHasPointers. See the doc of 85 // memmove for more details. 86 // 87 // The (CPU-specific) implementations of this function are in memclr_*.s. 88 // 89 //go:noescape 90 func memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) 91 92 //go:linkname reflect_memclrNoHeapPointers reflect.memclrNoHeapPointers 93 func reflect_memclrNoHeapPointers(ptr unsafe.Pointer, n uintptr) { 94 memclrNoHeapPointers(ptr, n) 95 } 96 97 // memmove copies n bytes from "from" to "to". 98 // 99 // memmove ensures that any pointer in "from" is written to "to" with 100 // an indivisible write, so that racy reads cannot observe a 101 // half-written pointer. This is necessary to prevent the garbage 102 // collector from observing invalid pointers, and differs from memmove 103 // in unmanaged languages. However, memmove is only required to do 104 // this if "from" and "to" may contain pointers, which can only be the 105 // case if "from", "to", and "n" are all be word-aligned. 106 // 107 // Implementations are in memmove_*.s. 108 // 109 //go:noescape 110 func memmove(to, from unsafe.Pointer, n uintptr) 111 112 // Outside assembly calls memmove. Make sure it has ABI wrappers. 113 //go:linkname memmove 114 115 //go:linkname reflect_memmove reflect.memmove 116 func reflect_memmove(to, from unsafe.Pointer, n uintptr) { 117 memmove(to, from, n) 118 } 119 120 // exported value for testing 121 var hashLoad = float32(loadFactorNum) / float32(loadFactorDen) 122 123 //go:nosplit 124 func fastrand() uint32 { 125 mp := getg().m 126 // Implement xorshift64+: 2 32-bit xorshift sequences added together. 127 // Shift triplet [17,7,16] was calculated as indicated in Marsaglia's 128 // Xorshift paper: https://www.jstatsoft.org/article/view/v008i14/xorshift.pdf 129 // This generator passes the SmallCrush suite, part of TestU01 framework: 130 // http://simul.iro.umontreal.ca/testu01/tu01.html 131 s1, s0 := mp.fastrand[0], mp.fastrand[1] 132 s1 ^= s1 << 17 133 s1 = s1 ^ s0 ^ s1>>7 ^ s0>>16 134 mp.fastrand[0], mp.fastrand[1] = s0, s1 135 return s0 + s1 136 } 137 138 //go:nosplit 139 func fastrandn(n uint32) uint32 { 140 // This is similar to fastrand() % n, but faster. 141 // See https://lemire.me/blog/2016/06/27/a-fast-alternative-to-the-modulo-reduction/ 142 return uint32(uint64(fastrand()) * uint64(n) >> 32) 143 } 144 145 //go:linkname sync_fastrand sync.fastrand 146 func sync_fastrand() uint32 { return fastrand() } 147 148 //go:linkname net_fastrand net.fastrand 149 func net_fastrand() uint32 { return fastrand() } 150 151 //go:linkname os_fastrand os.fastrand 152 func os_fastrand() uint32 { return fastrand() } 153 154 // in internal/bytealg/equal_*.s 155 //go:noescape 156 func memequal(a, b unsafe.Pointer, size uintptr) bool 157 158 // noescape hides a pointer from escape analysis. noescape is 159 // the identity function but escape analysis doesn't think the 160 // output depends on the input. noescape is inlined and currently 161 // compiles down to zero instructions. 162 // USE CAREFULLY! 163 //go:nosplit 164 func noescape(p unsafe.Pointer) unsafe.Pointer { 165 x := uintptr(p) 166 return unsafe.Pointer(x ^ 0) 167 } 168 169 // Not all cgocallback frames are actually cgocallback, 170 // so not all have these arguments. Mark them uintptr so that the GC 171 // does not misinterpret memory when the arguments are not present. 172 // cgocallback is not called from Go, only from crosscall2. 173 // This in turn calls cgocallbackg, which is where we'll find 174 // pointer-declared arguments. 175 func cgocallback(fn, frame, ctxt uintptr) 176 177 func gogo(buf *gobuf) 178 179 //go:noescape 180 func jmpdefer(fv *funcval, argp uintptr) 181 func asminit() 182 func setg(gg *g) 183 func breakpoint() 184 185 // reflectcall calls fn with arguments described by stackArgs, stackArgsSize, 186 // frameSize, and regArgs. 187 // 188 // Arguments passed on the stack and space for return values passed on the stack 189 // must be laid out at the space pointed to by stackArgs (with total length 190 // stackArgsSize) according to the ABI. 191 // 192 // stackRetOffset must be some value <= stackArgsSize that indicates the 193 // offset within stackArgs where the return value space begins. 194 // 195 // frameSize is the total size of the argument frame at stackArgs and must 196 // therefore be >= stackArgsSize. It must include additional space for spilling 197 // register arguments for stack growth and preemption. 198 // 199 // TODO(mknyszek): Once we don't need the additional spill space, remove frameSize, 200 // since frameSize will be redundant with stackArgsSize. 201 // 202 // Arguments passed in registers must be laid out in regArgs according to the ABI. 203 // regArgs will hold any return values passed in registers after the call. 204 // 205 // reflectcall copies stack arguments from stackArgs to the goroutine stack, and 206 // then copies back stackArgsSize-stackRetOffset bytes back to the return space 207 // in stackArgs once fn has completed. It also "unspills" argument registers from 208 // regArgs before calling fn, and spills them back into regArgs immediately 209 // following the call to fn. If there are results being returned on the stack, 210 // the caller should pass the argument frame type as stackArgsType so that 211 // reflectcall can execute appropriate write barriers during the copy. 212 // 213 // reflectcall expects regArgs.ReturnIsPtr to be populated indicating which 214 // registers on the return path will contain Go pointers. It will then store 215 // these pointers in regArgs.Ptrs such that they are visible to the GC. 216 // 217 // Package reflect passes a frame type. In package runtime, there is only 218 // one call that copies results back, in callbackWrap in syscall_windows.go, and it 219 // does NOT pass a frame type, meaning there are no write barriers invoked. See that 220 // call site for justification. 221 // 222 // Package reflect accesses this symbol through a linkname. 223 // 224 // Arguments passed through to reflectcall do not escape. The type is used 225 // only in a very limited callee of reflectcall, the stackArgs are copied, and 226 // regArgs is only used in the reflectcall frame. 227 //go:noescape 228 func reflectcall(stackArgsType *_type, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 229 230 func procyield(cycles uint32) 231 232 type neverCallThisFunction struct{} 233 234 // goexit is the return stub at the top of every goroutine call stack. 235 // Each goroutine stack is constructed as if goexit called the 236 // goroutine's entry point function, so that when the entry point 237 // function returns, it will return to goexit, which will call goexit1 238 // to perform the actual exit. 239 // 240 // This function must never be called directly. Call goexit1 instead. 241 // gentraceback assumes that goexit terminates the stack. A direct 242 // call on the stack will cause gentraceback to stop walking the stack 243 // prematurely and if there is leftover state it may panic. 244 func goexit(neverCallThisFunction) 245 246 // publicationBarrier performs a store/store barrier (a "publication" 247 // or "export" barrier). Some form of synchronization is required 248 // between initializing an object and making that object accessible to 249 // another processor. Without synchronization, the initialization 250 // writes and the "publication" write may be reordered, allowing the 251 // other processor to follow the pointer and observe an uninitialized 252 // object. In general, higher-level synchronization should be used, 253 // such as locking or an atomic pointer write. publicationBarrier is 254 // for when those aren't an option, such as in the implementation of 255 // the memory manager. 256 // 257 // There's no corresponding barrier for the read side because the read 258 // side naturally has a data dependency order. All architectures that 259 // Go supports or seems likely to ever support automatically enforce 260 // data dependency ordering. 261 func publicationBarrier() 262 263 // getcallerpc returns the program counter (PC) of its caller's caller. 264 // getcallersp returns the stack pointer (SP) of its caller's caller. 265 // The implementation may be a compiler intrinsic; there is not 266 // necessarily code implementing this on every platform. 267 // 268 // For example: 269 // 270 // func f(arg1, arg2, arg3 int) { 271 // pc := getcallerpc() 272 // sp := getcallersp() 273 // } 274 // 275 // These two lines find the PC and SP immediately following 276 // the call to f (where f will return). 277 // 278 // The call to getcallerpc and getcallersp must be done in the 279 // frame being asked about. 280 // 281 // The result of getcallersp is correct at the time of the return, 282 // but it may be invalidated by any subsequent call to a function 283 // that might relocate the stack in order to grow or shrink it. 284 // A general rule is that the result of getcallersp should be used 285 // immediately and can only be passed to nosplit functions. 286 287 //go:noescape 288 func getcallerpc() uintptr 289 290 //go:noescape 291 func getcallersp() uintptr // implemented as an intrinsic on all platforms 292 293 // getclosureptr returns the pointer to the current closure. 294 // getclosureptr can only be used in an assignment statement 295 // at the entry of a function. Moreover, go:nosplit directive 296 // must be specified at the declaration of caller function, 297 // so that the function prolog does not clobber the closure register. 298 // for example: 299 // 300 // //go:nosplit 301 // func f(arg1, arg2, arg3 int) { 302 // dx := getclosureptr() 303 // } 304 // 305 // The compiler rewrites calls to this function into instructions that fetch the 306 // pointer from a well-known register (DX on x86 architecture, etc.) directly. 307 func getclosureptr() uintptr 308 309 //go:noescape 310 func asmcgocall(fn, arg unsafe.Pointer) int32 311 312 func morestack() 313 func morestack_noctxt() 314 func rt0_go() 315 316 // return0 is a stub used to return 0 from deferproc. 317 // It is called at the very end of deferproc to signal 318 // the calling Go function that it should not jump 319 // to deferreturn. 320 // in asm_*.s 321 func return0() 322 323 // in asm_*.s 324 // not called directly; definitions here supply type information for traceback. 325 // These must have the same signature (arg pointer map) as reflectcall. 326 func call16(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 327 func call32(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 328 func call64(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 329 func call128(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 330 func call256(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 331 func call512(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 332 func call1024(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 333 func call2048(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 334 func call4096(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 335 func call8192(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 336 func call16384(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 337 func call32768(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 338 func call65536(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 339 func call131072(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 340 func call262144(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 341 func call524288(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 342 func call1048576(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 343 func call2097152(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 344 func call4194304(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 345 func call8388608(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 346 func call16777216(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 347 func call33554432(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 348 func call67108864(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 349 func call134217728(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 350 func call268435456(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 351 func call536870912(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 352 func call1073741824(typ, fn, stackArgs unsafe.Pointer, stackArgsSize, stackRetOffset, frameSize uint32, regArgs *abi.RegArgs) 353 354 func systemstack_switch() 355 356 // alignUp rounds n up to a multiple of a. a must be a power of 2. 357 func alignUp(n, a uintptr) uintptr { 358 return (n + a - 1) &^ (a - 1) 359 } 360 361 // alignDown rounds n down to a multiple of a. a must be a power of 2. 362 func alignDown(n, a uintptr) uintptr { 363 return n &^ (a - 1) 364 } 365 366 // divRoundUp returns ceil(n / a). 367 func divRoundUp(n, a uintptr) uintptr { 368 // a is generally a power of two. This will get inlined and 369 // the compiler will optimize the division. 370 return (n + a - 1) / a 371 } 372 373 // checkASM reports whether assembly runtime checks have passed. 374 func checkASM() bool 375 376 func memequal_varlen(a, b unsafe.Pointer) bool 377 378 // bool2int returns 0 if x is false or 1 if x is true. 379 func bool2int(x bool) int { 380 // Avoid branches. In the SSA compiler, this compiles to 381 // exactly what you would want it to. 382 return int(uint8(*(*uint8)(unsafe.Pointer(&x)))) 383 } 384 385 // abort crashes the runtime in situations where even throw might not 386 // work. In general it should do something a debugger will recognize 387 // (e.g., an INT3 on x86). A crash in abort is recognized by the 388 // signal handler, which will attempt to tear down the runtime 389 // immediately. 390 func abort() 391 392 // Called from compiled code; declared for vet; do NOT call from Go. 393 func gcWriteBarrier() 394 func duffzero() 395 func duffcopy() 396 397 // Called from linker-generated .initarray; declared for go vet; do NOT call from Go. 398 func addmoduledata() 399 400 // Injected by the signal handler for panicking signals. 401 // Initializes any registers that have fixed meaning at calls but 402 // are scratch in bodies and calls sigpanic. 403 // On many platforms it just jumps to sigpanic. 404 func sigpanic0() 405 406 // intArgRegs is used by the various register assignment 407 // algorithm implementations in the runtime. These include:. 408 // - Finalizers (mfinal.go) 409 // - Windows callbacks (syscall_windows.go) 410 // 411 // Both are stripped-down versions of the algorithm since they 412 // only have to deal with a subset of cases (finalizers only 413 // take a pointer or interface argument, Go Windows callbacks 414 // don't support floating point). 415 // 416 // It should be modified with care and are generally only 417 // modified when testing this package. 418 // 419 // It should never be set higher than its internal/abi 420 // constant counterparts, because the system relies on a 421 // structure that is at least large enough to hold the 422 // registers the system supports. 423 // 424 // Currently it's set to zero because using the actual 425 // constant will break every part of the toolchain that 426 // uses finalizers or Windows callbacks to call functions 427 // The value that is currently commented out there should be 428 // the actual value once we're ready to use the register ABI 429 // everywhere. 430 // 431 // Protected by finlock. 432 var intArgRegs = abi.IntArgRegs * goexperiment.RegabiArgsInt 433