Source file src/runtime/atomic_pointer.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package runtime
     6	
     7	import (
     8		"runtime/internal/atomic"
     9		"unsafe"
    10	)
    11	
    12	// These functions cannot have go:noescape annotations,
    13	// because while ptr does not escape, new does.
    14	// If new is marked as not escaping, the compiler will make incorrect
    15	// escape analysis decisions about the pointer value being stored.
    16	
    17	// atomicwb performs a write barrier before an atomic pointer write.
    18	// The caller should guard the call with "if writeBarrier.enabled".
    19	//
    20	//go:nosplit
    21	func atomicwb(ptr *unsafe.Pointer, new unsafe.Pointer) {
    22		slot := (*uintptr)(unsafe.Pointer(ptr))
    23		if !getg().m.p.ptr().wbBuf.putFast(*slot, uintptr(new)) {
    24			wbBufFlush(slot, uintptr(new))
    25		}
    26	}
    27	
    28	// atomicstorep performs *ptr = new atomically and invokes a write barrier.
    29	//
    30	//go:nosplit
    31	func atomicstorep(ptr unsafe.Pointer, new unsafe.Pointer) {
    32		if writeBarrier.enabled {
    33			atomicwb((*unsafe.Pointer)(ptr), new)
    34		}
    35		atomic.StorepNoWB(noescape(ptr), new)
    36	}
    37	
    38	// Like above, but implement in terms of sync/atomic's uintptr operations.
    39	// We cannot just call the runtime routines, because the race detector expects
    40	// to be able to intercept the sync/atomic forms but not the runtime forms.
    41	
    42	//go:linkname sync_atomic_StoreUintptr sync/atomic.StoreUintptr
    43	func sync_atomic_StoreUintptr(ptr *uintptr, new uintptr)
    44	
    45	//go:linkname sync_atomic_StorePointer sync/atomic.StorePointer
    46	//go:nosplit
    47	func sync_atomic_StorePointer(ptr *unsafe.Pointer, new unsafe.Pointer) {
    48		if writeBarrier.enabled {
    49			atomicwb(ptr, new)
    50		}
    51		sync_atomic_StoreUintptr((*uintptr)(unsafe.Pointer(ptr)), uintptr(new))
    52	}
    53	
    54	//go:linkname sync_atomic_SwapUintptr sync/atomic.SwapUintptr
    55	func sync_atomic_SwapUintptr(ptr *uintptr, new uintptr) uintptr
    56	
    57	//go:linkname sync_atomic_SwapPointer sync/atomic.SwapPointer
    58	//go:nosplit
    59	func sync_atomic_SwapPointer(ptr *unsafe.Pointer, new unsafe.Pointer) unsafe.Pointer {
    60		if writeBarrier.enabled {
    61			atomicwb(ptr, new)
    62		}
    63		old := unsafe.Pointer(sync_atomic_SwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(new)))
    64		return old
    65	}
    66	
    67	//go:linkname sync_atomic_CompareAndSwapUintptr sync/atomic.CompareAndSwapUintptr
    68	func sync_atomic_CompareAndSwapUintptr(ptr *uintptr, old, new uintptr) bool
    69	
    70	//go:linkname sync_atomic_CompareAndSwapPointer sync/atomic.CompareAndSwapPointer
    71	//go:nosplit
    72	func sync_atomic_CompareAndSwapPointer(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool {
    73		if writeBarrier.enabled {
    74			atomicwb(ptr, new)
    75		}
    76		return sync_atomic_CompareAndSwapUintptr((*uintptr)(noescape(unsafe.Pointer(ptr))), uintptr(old), uintptr(new))
    77	}
    78	

View as plain text