atomic_pointer.go

Documentation: runtime

		 1  // Copyright 2009 The Go Authors. All rights reserved.
		 2  // Use of this source code is governed by a BSD-style
		 3  // license that can be found in the LICENSE file.
		 4  
		 5  package runtime
		 6  
		 7  import (
		 8  	"runtime/internal/atomic"
		 9  	"unsafe"
		10  )
		11  
		12  // These functions cannot have go:noescape annotations,
		13  // because while ptr does not escape, new does.
		14  // If new is marked as not escaping, the compiler will make incorrect
		15  // escape analysis decisions about the pointer value being stored.
		16  
		17  // atomicwb performs a write barrier before an atomic pointer write.
		18  // The caller should guard the call with "if writeBarrier.enabled".
		19  //
		20  //go:nosplit
		21  func atomicwb(ptr *unsafe.Pointer, new unsafe.Pointer) {
		22  	slot := (*uintptr)(unsafe.Pointer(ptr))
		23  	if !getg().m.p.ptr().wbBuf.putFast(*slot, uintptr(new)) {
		24  		wbBufFlush(slot, uintptr(new))
		25  	}
		26  }
		27  
		28  // atomicstorep performs *ptr = new atomically and invokes a write barrier.
		29  //
		30  //go:nosplit
		31  func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) {
		32  	if writeBarrier.enabled {
		33  		atomicwb((*unsafe.Pointer)(ptr), new)
		34  	}
		35  	atomic.StorepNoWB(noescape(ptr), new)
		36  }
		37  
		38  // Like above, but implement in terms of sync/atomic's uintptr operations.
		39  // We cannot just call the runtime routines, because the race detector expects
		40  // to be able to intercept the sync/atomic forms but not the runtime forms.
		41  
		42  //go:linkname sync_atomic_StoreUintptr sync/atomic.StoreUintptr
		43  func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr)
		44  
		45  //go:linkname sync_atomic_StorePointer sync/atomic.StorePointer
		46  //go:nosplit
		47  func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
		48  	if writeBarrier.enabled {
		49  		atomicwb(ptr, new)
		50  	}
		51  	sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
		52  }
		53  
		54  //go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr
		55  func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr
		56  
		57  //go:linkname sync_atomic_SwapPointer sync/atomic.SwapPointer
		58  //go:nosplit
		59  func sync_atomic_SwapPointer(ptr *unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
		60  	if writeBarrier.enabled {
		61  		atomicwb(ptr, new)
		62  	}
		63  	old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(new)))
		64  	return old
		65  }
		66  
		67  //go:linkname sync_atomic_CompareAndSwapUintptr sync/atomic.CompareAndSwapUintptr
		68  func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
		69  
		70  //go:linkname sync_atomic_CompareAndSwapPointer sync/atomic.CompareAndSwapPointer
		71  //go:nosplit
		72  func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
		73  	if writeBarrier.enabled {
		74  		atomicwb(ptr, new)
		75  	}
		76  	return sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new))
		77  }
		78
View as plain text