1 // Copyright 2009 The Go Authors. All rights reserved. 2 // Use of this source code is governed by a BSD-style 3 // license that can be found in the LICENSE file. 4 5 // Cgo call and callback support. 6 // 7 // To call into the C function f from Go, the cgo-generated code calls 8 // runtime.cgocall(_cgo_Cfunc_f, frame), where _cgo_Cfunc_f is a 9 // gcc-compiled function written by cgo. 10 // 11 // runtime.cgocall (below) calls entersyscall so as not to block 12 // other goroutines or the garbage collector, and then calls 13 // runtime.asmcgocall(_cgo_Cfunc_f, frame). 14 // 15 // runtime.asmcgocall (in asm_$GOARCH.s) switches to the m->g0 stack 16 // (assumed to be an operating system-allocated stack, so safe to run 17 // gcc-compiled code on) and calls _cgo_Cfunc_f(frame). 18 // 19 // _cgo_Cfunc_f invokes the actual C function f with arguments 20 // taken from the frame structure, records the results in the frame, 21 // and returns to runtime.asmcgocall. 22 // 23 // After it regains control, runtime.asmcgocall switches back to the 24 // original g (m->curg)'s stack and returns to runtime.cgocall. 25 // 26 // After it regains control, runtime.cgocall calls exitsyscall, which blocks 27 // until this m can run Go code without violating the $GOMAXPROCS limit, 28 // and then unlocks g from m. 29 // 30 // The above description skipped over the possibility of the gcc-compiled 31 // function f calling back into Go. If that happens, we continue down 32 // the rabbit hole during the execution of f. 33 // 34 // To make it possible for gcc-compiled C code to call a Go function p.GoF, 35 // cgo writes a gcc-compiled function named GoF (not p.GoF, since gcc doesn't 36 // know about packages). The gcc-compiled C function f calls GoF. 37 // 38 // GoF initializes "frame", a structure containing all of its 39 // arguments and slots for p.GoF's results. It calls 40 // crosscall2(_cgoexp_GoF, frame, framesize, ctxt) using the gcc ABI. 41 // 42 // crosscall2 (in cgo/asm_$GOARCH.s) is a four-argument adapter from 43 // the gcc function call ABI to the gc function call ABI. At this 44 // point we're in the Go runtime, but we're still running on m.g0's 45 // stack and outside the $GOMAXPROCS limit. crosscall2 calls 46 // runtime.cgocallback(_cgoexp_GoF, frame, ctxt) using the gc ABI. 47 // (crosscall2's framesize argument is no longer used, but there's one 48 // case where SWIG calls crosscall2 directly and expects to pass this 49 // argument. See _cgo_panic.) 50 // 51 // runtime.cgocallback (in asm_$GOARCH.s) switches from m.g0's stack 52 // to the original g (m.curg)'s stack, on which it calls 53 // runtime.cgocallbackg(_cgoexp_GoF, frame, ctxt). As part of the 54 // stack switch, runtime.cgocallback saves the current SP as 55 // m.g0.sched.sp, so that any use of m.g0's stack during the execution 56 // of the callback will be done below the existing stack frames. 57 // Before overwriting m.g0.sched.sp, it pushes the old value on the 58 // m.g0 stack, so that it can be restored later. 59 // 60 // runtime.cgocallbackg (below) is now running on a real goroutine 61 // stack (not an m.g0 stack). First it calls runtime.exitsyscall, which will 62 // block until the $GOMAXPROCS limit allows running this goroutine. 63 // Once exitsyscall has returned, it is safe to do things like call the memory 64 // allocator or invoke the Go callback function. runtime.cgocallbackg 65 // first defers a function to unwind m.g0.sched.sp, so that if p.GoF 66 // panics, m.g0.sched.sp will be restored to its old value: the m.g0 stack 67 // and the m.curg stack will be unwound in lock step. 68 // Then it calls _cgoexp_GoF(frame). 69 // 70 // _cgoexp_GoF, which was generated by cmd/cgo, unpacks the arguments 71 // from frame, calls p.GoF, writes the results back to frame, and 72 // returns. Now we start unwinding this whole process. 73 // 74 // runtime.cgocallbackg pops but does not execute the deferred 75 // function to unwind m.g0.sched.sp, calls runtime.entersyscall, and 76 // returns to runtime.cgocallback. 77 // 78 // After it regains control, runtime.cgocallback switches back to 79 // m.g0's stack (the pointer is still in m.g0.sched.sp), restores the old 80 // m.g0.sched.sp value from the stack, and returns to crosscall2. 81 // 82 // crosscall2 restores the callee-save registers for gcc and returns 83 // to GoF, which unpacks any result values and returns to f. 84 85 package runtime 86 87 import ( 88 "runtime/internal/atomic" 89 "runtime/internal/sys" 90 "unsafe" 91 ) 92 93 // Addresses collected in a cgo backtrace when crashing. 94 // Length must match arg.Max in x_cgo_callers in runtime/cgo/gcc_traceback.c. 95 type cgoCallers [32]uintptr 96 97 // argset matches runtime/cgo/linux_syscall.c:argset_t 98 type argset struct { 99 args unsafe.Pointer 100 retval uintptr 101 } 102 103 // wrapper for syscall package to call cgocall for libc (cgo) calls. 104 //go:linkname syscall_cgocaller syscall.cgocaller 105 //go:nosplit 106 //go:uintptrescapes 107 func syscall_cgocaller(fn unsafe.Pointer, args ...uintptr) uintptr { 108 as := argset{args: unsafe.Pointer(&args[0])} 109 cgocall(fn, unsafe.Pointer(&as)) 110 return as.retval 111 } 112 113 var ncgocall uint64 // number of cgo calls in total for dead m 114 115 // Call from Go to C. 116 // 117 // This must be nosplit because it's used for syscalls on some 118 // platforms. Syscalls may have untyped arguments on the stack, so 119 // it's not safe to grow or scan the stack. 120 // 121 //go:nosplit 122 func cgocall(fn, arg unsafe.Pointer) int32 { 123 if !iscgo && GOOS != "solaris" && GOOS != "illumos" && GOOS != "windows" { 124 throw("cgocall unavailable") 125 } 126 127 if fn == nil { 128 throw("cgocall nil") 129 } 130 131 if raceenabled { 132 racereleasemerge(unsafe.Pointer(&racecgosync)) 133 } 134 135 mp := getg().m 136 mp.ncgocall++ 137 mp.ncgo++ 138 139 // Reset traceback. 140 mp.cgoCallers[0] = 0 141 142 // Announce we are entering a system call 143 // so that the scheduler knows to create another 144 // M to run goroutines while we are in the 145 // foreign code. 146 // 147 // The call to asmcgocall is guaranteed not to 148 // grow the stack and does not allocate memory, 149 // so it is safe to call while "in a system call", outside 150 // the $GOMAXPROCS accounting. 151 // 152 // fn may call back into Go code, in which case we'll exit the 153 // "system call", run the Go code (which may grow the stack), 154 // and then re-enter the "system call" reusing the PC and SP 155 // saved by entersyscall here. 156 entersyscall() 157 158 // Tell asynchronous preemption that we're entering external 159 // code. We do this after entersyscall because this may block 160 // and cause an async preemption to fail, but at this point a 161 // sync preemption will succeed (though this is not a matter 162 // of correctness). 163 osPreemptExtEnter(mp) 164 165 mp.incgo = true 166 errno := asmcgocall(fn, arg) 167 168 // Update accounting before exitsyscall because exitsyscall may 169 // reschedule us on to a different M. 170 mp.incgo = false 171 mp.ncgo-- 172 173 osPreemptExtExit(mp) 174 175 exitsyscall() 176 177 // Note that raceacquire must be called only after exitsyscall has 178 // wired this M to a P. 179 if raceenabled { 180 raceacquire(unsafe.Pointer(&racecgosync)) 181 } 182 183 // From the garbage collector's perspective, time can move 184 // backwards in the sequence above. If there's a callback into 185 // Go code, GC will see this function at the call to 186 // asmcgocall. When the Go call later returns to C, the 187 // syscall PC/SP is rolled back and the GC sees this function 188 // back at the call to entersyscall. Normally, fn and arg 189 // would be live at entersyscall and dead at asmcgocall, so if 190 // time moved backwards, GC would see these arguments as dead 191 // and then live. Prevent these undead arguments from crashing 192 // GC by forcing them to stay live across this time warp. 193 KeepAlive(fn) 194 KeepAlive(arg) 195 KeepAlive(mp) 196 197 return errno 198 } 199 200 // Call from C back to Go. fn must point to an ABIInternal Go entry-point. 201 //go:nosplit 202 func cgocallbackg(fn, frame unsafe.Pointer, ctxt uintptr) { 203 gp := getg() 204 if gp != gp.m.curg { 205 println("runtime: bad g in cgocallback") 206 exit(2) 207 } 208 209 // The call from C is on gp.m's g0 stack, so we must ensure 210 // that we stay on that M. We have to do this before calling 211 // exitsyscall, since it would otherwise be free to move us to 212 // a different M. The call to unlockOSThread is in unwindm. 213 lockOSThread() 214 215 checkm := gp.m 216 217 // Save current syscall parameters, so m.syscall can be 218 // used again if callback decide to make syscall. 219 syscall := gp.m.syscall 220 221 // entersyscall saves the caller's SP to allow the GC to trace the Go 222 // stack. However, since we're returning to an earlier stack frame and 223 // need to pair with the entersyscall() call made by cgocall, we must 224 // save syscall* and let reentersyscall restore them. 225 savedsp := unsafe.Pointer(gp.syscallsp) 226 savedpc := gp.syscallpc 227 exitsyscall() // coming out of cgo call 228 gp.m.incgo = false 229 230 osPreemptExtExit(gp.m) 231 232 cgocallbackg1(fn, frame, ctxt) // will call unlockOSThread 233 234 // At this point unlockOSThread has been called. 235 // The following code must not change to a different m. 236 // This is enforced by checking incgo in the schedule function. 237 238 gp.m.incgo = true 239 240 if gp.m != checkm { 241 throw("m changed unexpectedly in cgocallbackg") 242 } 243 244 osPreemptExtEnter(gp.m) 245 246 // going back to cgo call 247 reentersyscall(savedpc, uintptr(savedsp)) 248 249 gp.m.syscall = syscall 250 } 251 252 func cgocallbackg1(fn, frame unsafe.Pointer, ctxt uintptr) { 253 gp := getg() 254 255 // When we return, undo the call to lockOSThread in cgocallbackg. 256 // We must still stay on the same m. 257 defer unlockOSThread() 258 259 if gp.m.needextram || atomic.Load(&extraMWaiters) > 0 { 260 gp.m.needextram = false 261 systemstack(newextram) 262 } 263 264 if ctxt != 0 { 265 s := append(gp.cgoCtxt, ctxt) 266 267 // Now we need to set gp.cgoCtxt = s, but we could get 268 // a SIGPROF signal while manipulating the slice, and 269 // the SIGPROF handler could pick up gp.cgoCtxt while 270 // tracing up the stack. We need to ensure that the 271 // handler always sees a valid slice, so set the 272 // values in an order such that it always does. 273 p := (*slice)(unsafe.Pointer(&gp.cgoCtxt)) 274 atomicstorep(unsafe.Pointer(&p.array), unsafe.Pointer(&s[0])) 275 p.cap = cap(s) 276 p.len = len(s) 277 278 defer func(gp *g) { 279 // Decrease the length of the slice by one, safely. 280 p := (*slice)(unsafe.Pointer(&gp.cgoCtxt)) 281 p.len-- 282 }(gp) 283 } 284 285 if gp.m.ncgo == 0 { 286 // The C call to Go came from a thread not currently running 287 // any Go. In the case of -buildmode=c-archive or c-shared, 288 // this call may be coming in before package initialization 289 // is complete. Wait until it is. 290 <-main_init_done 291 } 292 293 // Add entry to defer stack in case of panic. 294 restore := true 295 defer unwindm(&restore) 296 297 if raceenabled { 298 raceacquire(unsafe.Pointer(&racecgosync)) 299 } 300 301 // Invoke callback. This function is generated by cmd/cgo and 302 // will unpack the argument frame and call the Go function. 303 var cb func(frame unsafe.Pointer) 304 cbFV := funcval{uintptr(fn)} 305 *(*unsafe.Pointer)(unsafe.Pointer(&cb)) = noescape(unsafe.Pointer(&cbFV)) 306 cb(frame) 307 308 if raceenabled { 309 racereleasemerge(unsafe.Pointer(&racecgosync)) 310 } 311 312 // Do not unwind m->g0->sched.sp. 313 // Our caller, cgocallback, will do that. 314 restore = false 315 } 316 317 func unwindm(restore *bool) { 318 if *restore { 319 // Restore sp saved by cgocallback during 320 // unwind of g's stack (see comment at top of file). 321 mp := acquirem() 322 sched := &mp.g0.sched 323 sched.sp = *(*uintptr)(unsafe.Pointer(sched.sp + alignUp(sys.MinFrameSize, sys.StackAlign))) 324 325 // Do the accounting that cgocall will not have a chance to do 326 // during an unwind. 327 // 328 // In the case where a Go call originates from C, ncgo is 0 329 // and there is no matching cgocall to end. 330 if mp.ncgo > 0 { 331 mp.incgo = false 332 mp.ncgo-- 333 osPreemptExtExit(mp) 334 } 335 336 releasem(mp) 337 } 338 } 339 340 // called from assembly 341 func badcgocallback() { 342 throw("misaligned stack in cgocallback") 343 } 344 345 // called from (incomplete) assembly 346 func cgounimpl() { 347 throw("cgo not implemented") 348 } 349 350 var racecgosync uint64 // represents possible synchronization in C code 351 352 // Pointer checking for cgo code. 353 354 // We want to detect all cases where a program that does not use 355 // unsafe makes a cgo call passing a Go pointer to memory that 356 // contains a Go pointer. Here a Go pointer is defined as a pointer 357 // to memory allocated by the Go runtime. Programs that use unsafe 358 // can evade this restriction easily, so we don't try to catch them. 359 // The cgo program will rewrite all possibly bad pointer arguments to 360 // call cgoCheckPointer, where we can catch cases of a Go pointer 361 // pointing to a Go pointer. 362 363 // Complicating matters, taking the address of a slice or array 364 // element permits the C program to access all elements of the slice 365 // or array. In that case we will see a pointer to a single element, 366 // but we need to check the entire data structure. 367 368 // The cgoCheckPointer call takes additional arguments indicating that 369 // it was called on an address expression. An additional argument of 370 // true means that it only needs to check a single element. An 371 // additional argument of a slice or array means that it needs to 372 // check the entire slice/array, but nothing else. Otherwise, the 373 // pointer could be anything, and we check the entire heap object, 374 // which is conservative but safe. 375 376 // When and if we implement a moving garbage collector, 377 // cgoCheckPointer will pin the pointer for the duration of the cgo 378 // call. (This is necessary but not sufficient; the cgo program will 379 // also have to change to pin Go pointers that cannot point to Go 380 // pointers.) 381 382 // cgoCheckPointer checks if the argument contains a Go pointer that 383 // points to a Go pointer, and panics if it does. 384 func cgoCheckPointer(ptr interface{}, arg interface{}) { 385 if debug.cgocheck == 0 { 386 return 387 } 388 389 ep := efaceOf(&ptr) 390 t := ep._type 391 392 top := true 393 if arg != nil && (t.kind&kindMask == kindPtr || t.kind&kindMask == kindUnsafePointer) { 394 p := ep.data 395 if t.kind&kindDirectIface == 0 { 396 p = *(*unsafe.Pointer)(p) 397 } 398 if p == nil || !cgoIsGoPointer(p) { 399 return 400 } 401 aep := efaceOf(&arg) 402 switch aep._type.kind & kindMask { 403 case kindBool: 404 if t.kind&kindMask == kindUnsafePointer { 405 // We don't know the type of the element. 406 break 407 } 408 pt := (*ptrtype)(unsafe.Pointer(t)) 409 cgoCheckArg(pt.elem, p, true, false, cgoCheckPointerFail) 410 return 411 case kindSlice: 412 // Check the slice rather than the pointer. 413 ep = aep 414 t = ep._type 415 case kindArray: 416 // Check the array rather than the pointer. 417 // Pass top as false since we have a pointer 418 // to the array. 419 ep = aep 420 t = ep._type 421 top = false 422 default: 423 throw("can't happen") 424 } 425 } 426 427 cgoCheckArg(t, ep.data, t.kind&kindDirectIface == 0, top, cgoCheckPointerFail) 428 } 429 430 const cgoCheckPointerFail = "cgo argument has Go pointer to Go pointer" 431 const cgoResultFail = "cgo result has Go pointer" 432 433 // cgoCheckArg is the real work of cgoCheckPointer. The argument p 434 // is either a pointer to the value (of type t), or the value itself, 435 // depending on indir. The top parameter is whether we are at the top 436 // level, where Go pointers are allowed. 437 func cgoCheckArg(t *_type, p unsafe.Pointer, indir, top bool, msg string) { 438 if t.ptrdata == 0 || p == nil { 439 // If the type has no pointers there is nothing to do. 440 return 441 } 442 443 switch t.kind & kindMask { 444 default: 445 throw("can't happen") 446 case kindArray: 447 at := (*arraytype)(unsafe.Pointer(t)) 448 if !indir { 449 if at.len != 1 { 450 throw("can't happen") 451 } 452 cgoCheckArg(at.elem, p, at.elem.kind&kindDirectIface == 0, top, msg) 453 return 454 } 455 for i := uintptr(0); i < at.len; i++ { 456 cgoCheckArg(at.elem, p, true, top, msg) 457 p = add(p, at.elem.size) 458 } 459 case kindChan, kindMap: 460 // These types contain internal pointers that will 461 // always be allocated in the Go heap. It's never OK 462 // to pass them to C. 463 panic(errorString(msg)) 464 case kindFunc: 465 if indir { 466 p = *(*unsafe.Pointer)(p) 467 } 468 if !cgoIsGoPointer(p) { 469 return 470 } 471 panic(errorString(msg)) 472 case kindInterface: 473 it := *(**_type)(p) 474 if it == nil { 475 return 476 } 477 // A type known at compile time is OK since it's 478 // constant. A type not known at compile time will be 479 // in the heap and will not be OK. 480 if inheap(uintptr(unsafe.Pointer(it))) { 481 panic(errorString(msg)) 482 } 483 p = *(*unsafe.Pointer)(add(p, sys.PtrSize)) 484 if !cgoIsGoPointer(p) { 485 return 486 } 487 if !top { 488 panic(errorString(msg)) 489 } 490 cgoCheckArg(it, p, it.kind&kindDirectIface == 0, false, msg) 491 case kindSlice: 492 st := (*slicetype)(unsafe.Pointer(t)) 493 s := (*slice)(p) 494 p = s.array 495 if p == nil || !cgoIsGoPointer(p) { 496 return 497 } 498 if !top { 499 panic(errorString(msg)) 500 } 501 if st.elem.ptrdata == 0 { 502 return 503 } 504 for i := 0; i < s.cap; i++ { 505 cgoCheckArg(st.elem, p, true, false, msg) 506 p = add(p, st.elem.size) 507 } 508 case kindString: 509 ss := (*stringStruct)(p) 510 if !cgoIsGoPointer(ss.str) { 511 return 512 } 513 if !top { 514 panic(errorString(msg)) 515 } 516 case kindStruct: 517 st := (*structtype)(unsafe.Pointer(t)) 518 if !indir { 519 if len(st.fields) != 1 { 520 throw("can't happen") 521 } 522 cgoCheckArg(st.fields[0].typ, p, st.fields[0].typ.kind&kindDirectIface == 0, top, msg) 523 return 524 } 525 for _, f := range st.fields { 526 if f.typ.ptrdata == 0 { 527 continue 528 } 529 cgoCheckArg(f.typ, add(p, f.offset()), true, top, msg) 530 } 531 case kindPtr, kindUnsafePointer: 532 if indir { 533 p = *(*unsafe.Pointer)(p) 534 if p == nil { 535 return 536 } 537 } 538 539 if !cgoIsGoPointer(p) { 540 return 541 } 542 if !top { 543 panic(errorString(msg)) 544 } 545 546 cgoCheckUnknownPointer(p, msg) 547 } 548 } 549 550 // cgoCheckUnknownPointer is called for an arbitrary pointer into Go 551 // memory. It checks whether that Go memory contains any other 552 // pointer into Go memory. If it does, we panic. 553 // The return values are unused but useful to see in panic tracebacks. 554 func cgoCheckUnknownPointer(p unsafe.Pointer, msg string) (base, i uintptr) { 555 if inheap(uintptr(p)) { 556 b, span, _ := findObject(uintptr(p), 0, 0) 557 base = b 558 if base == 0 { 559 return 560 } 561 hbits := heapBitsForAddr(base) 562 n := span.elemsize 563 for i = uintptr(0); i < n; i += sys.PtrSize { 564 if !hbits.morePointers() { 565 // No more possible pointers. 566 break 567 } 568 if hbits.isPointer() && cgoIsGoPointer(*(*unsafe.Pointer)(unsafe.Pointer(base + i))) { 569 panic(errorString(msg)) 570 } 571 hbits = hbits.next() 572 } 573 574 return 575 } 576 577 for _, datap := range activeModules() { 578 if cgoInRange(p, datap.data, datap.edata) || cgoInRange(p, datap.bss, datap.ebss) { 579 // We have no way to know the size of the object. 580 // We have to assume that it might contain a pointer. 581 panic(errorString(msg)) 582 } 583 // In the text or noptr sections, we know that the 584 // pointer does not point to a Go pointer. 585 } 586 587 return 588 } 589 590 // cgoIsGoPointer reports whether the pointer is a Go pointer--a 591 // pointer to Go memory. We only care about Go memory that might 592 // contain pointers. 593 //go:nosplit 594 //go:nowritebarrierrec 595 func cgoIsGoPointer(p unsafe.Pointer) bool { 596 if p == nil { 597 return false 598 } 599 600 if inHeapOrStack(uintptr(p)) { 601 return true 602 } 603 604 for _, datap := range activeModules() { 605 if cgoInRange(p, datap.data, datap.edata) || cgoInRange(p, datap.bss, datap.ebss) { 606 return true 607 } 608 } 609 610 return false 611 } 612 613 // cgoInRange reports whether p is between start and end. 614 //go:nosplit 615 //go:nowritebarrierrec 616 func cgoInRange(p unsafe.Pointer, start, end uintptr) bool { 617 return start <= uintptr(p) && uintptr(p) < end 618 } 619 620 // cgoCheckResult is called to check the result parameter of an 621 // exported Go function. It panics if the result is or contains a Go 622 // pointer. 623 func cgoCheckResult(val interface{}) { 624 if debug.cgocheck == 0 { 625 return 626 } 627 628 ep := efaceOf(&val) 629 t := ep._type 630 cgoCheckArg(t, ep.data, t.kind&kindDirectIface == 0, false, cgoResultFail) 631 } 632