Source file src/pkg/runtime/race.go

     1	// Copyright 2012 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// +build race
     6	
     7	package runtime
     8	
     9	import (
    10		"unsafe"
    11	)
    12	
    13	// Public race detection API, present iff build with -race.
    14	
    15	func RaceRead(addr unsafe.Pointer)
    16	func RaceWrite(addr unsafe.Pointer)
    17	func RaceReadRange(addr unsafe.Pointer, len int)
    18	func RaceWriteRange(addr unsafe.Pointer, len int)
    19	
    20	func RaceErrors() int {
    21		var n uint64
    22		racecall(&__tsan_report_count, uintptr(unsafe.Pointer(&n)), 0, 0, 0)
    23		return int(n)
    24	}
    25	
    26	//go:nosplit
    27	
    28	// RaceAcquire/RaceRelease/RaceReleaseMerge establish happens-before relations
    29	// between goroutines. These inform the race detector about actual synchronization
    30	// that it can't see for some reason (e.g. synchronization within RaceDisable/RaceEnable
    31	// sections of code).
    32	// RaceAcquire establishes a happens-before relation with the preceding
    33	// RaceReleaseMerge on addr up to and including the last RaceRelease on addr.
    34	// In terms of the C memory model (C11 §5.1.2.4, §7.17.3),
    35	// RaceAcquire is equivalent to atomic_load(memory_order_acquire).
    36	func RaceAcquire(addr unsafe.Pointer) {
    37		raceacquire(addr)
    38	}
    39	
    40	//go:nosplit
    41	
    42	// RaceRelease performs a release operation on addr that
    43	// can synchronize with a later RaceAcquire on addr.
    44	//
    45	// In terms of the C memory model, RaceRelease is equivalent to
    46	// atomic_store(memory_order_release).
    47	func RaceRelease(addr unsafe.Pointer) {
    48		racerelease(addr)
    49	}
    50	
    51	//go:nosplit
    52	
    53	// RaceReleaseMerge is like RaceRelease, but also establishes a happens-before
    54	// relation with the preceding RaceRelease or RaceReleaseMerge on addr.
    55	//
    56	// In terms of the C memory model, RaceReleaseMerge is equivalent to
    57	// atomic_exchange(memory_order_release).
    58	func RaceReleaseMerge(addr unsafe.Pointer) {
    59		racereleasemerge(addr)
    60	}
    61	
    62	//go:nosplit
    63	
    64	// RaceDisable disables handling of race synchronization events in the current goroutine.
    65	// Handling is re-enabled with RaceEnable. RaceDisable/RaceEnable can be nested.
    66	// Non-synchronization events (memory accesses, function entry/exit) still affect
    67	// the race detector.
    68	func RaceDisable() {
    69		_g_ := getg()
    70		if _g_.raceignore == 0 {
    71			racecall(&__tsan_go_ignore_sync_begin, _g_.racectx, 0, 0, 0)
    72		}
    73		_g_.raceignore++
    74	}
    75	
    76	//go:nosplit
    77	
    78	// RaceEnable re-enables handling of race events in the current goroutine.
    79	func RaceEnable() {
    80		_g_ := getg()
    81		_g_.raceignore--
    82		if _g_.raceignore == 0 {
    83			racecall(&__tsan_go_ignore_sync_end, _g_.racectx, 0, 0, 0)
    84		}
    85	}
    86	
    87	// Private interface for the runtime.
    88	
    89	const raceenabled = true
    90	
    91	// For all functions accepting callerpc and pc,
    92	// callerpc is a return PC of the function that calls this function,
    93	// pc is start PC of the function that calls this function.
    94	func raceReadObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
    95		kind := t.kind & kindMask
    96		if kind == kindArray || kind == kindStruct {
    97			// for composite objects we have to read every address
    98			// because a write might happen to any subobject.
    99			racereadrangepc(addr, t.size, callerpc, pc)
   100		} else {
   101			// for non-composite objects we can read just the start
   102			// address, as any write must write the first byte.
   103			racereadpc(addr, callerpc, pc)
   104		}
   105	}
   106	
   107	func raceWriteObjectPC(t *_type, addr unsafe.Pointer, callerpc, pc uintptr) {
   108		kind := t.kind & kindMask
   109		if kind == kindArray || kind == kindStruct {
   110			// for composite objects we have to write every address
   111			// because a write might happen to any subobject.
   112			racewriterangepc(addr, t.size, callerpc, pc)
   113		} else {
   114			// for non-composite objects we can write just the start
   115			// address, as any write must write the first byte.
   116			racewritepc(addr, callerpc, pc)
   117		}
   118	}
   119	
   120	//go:noescape
   121	func racereadpc(addr unsafe.Pointer, callpc, pc uintptr)
   122	
   123	//go:noescape
   124	func racewritepc(addr unsafe.Pointer, callpc, pc uintptr)
   125	
   126	type symbolizeCodeContext struct {
   127		pc   uintptr
   128		fn   *byte
   129		file *byte
   130		line uintptr
   131		off  uintptr
   132		res  uintptr
   133	}
   134	
   135	var qq = [...]byte{'?', '?', 0}
   136	var dash = [...]byte{'-', 0}
   137	
   138	const (
   139		raceGetProcCmd = iota
   140		raceSymbolizeCodeCmd
   141		raceSymbolizeDataCmd
   142	)
   143	
   144	// Callback from C into Go, runs on g0.
   145	func racecallback(cmd uintptr, ctx unsafe.Pointer) {
   146		switch cmd {
   147		case raceGetProcCmd:
   148			throw("should have been handled by racecallbackthunk")
   149		case raceSymbolizeCodeCmd:
   150			raceSymbolizeCode((*symbolizeCodeContext)(ctx))
   151		case raceSymbolizeDataCmd:
   152			raceSymbolizeData((*symbolizeDataContext)(ctx))
   153		default:
   154			throw("unknown command")
   155		}
   156	}
   157	
   158	func raceSymbolizeCode(ctx *symbolizeCodeContext) {
   159		f := findfunc(ctx.pc)._Func()
   160		if f != nil {
   161			file, line := f.FileLine(ctx.pc)
   162			if line != 0 {
   163				ctx.fn = cfuncname(f.funcInfo())
   164				ctx.line = uintptr(line)
   165				ctx.file = &bytes(file)[0] // assume NUL-terminated
   166				ctx.off = ctx.pc - f.Entry()
   167				ctx.res = 1
   168				return
   169			}
   170		}
   171		ctx.fn = &qq[0]
   172		ctx.file = &dash[0]
   173		ctx.line = 0
   174		ctx.off = ctx.pc
   175		ctx.res = 1
   176	}
   177	
   178	type symbolizeDataContext struct {
   179		addr  uintptr
   180		heap  uintptr
   181		start uintptr
   182		size  uintptr
   183		name  *byte
   184		file  *byte
   185		line  uintptr
   186		res   uintptr
   187	}
   188	
   189	func raceSymbolizeData(ctx *symbolizeDataContext) {
   190		if base, span, _ := findObject(ctx.addr, 0, 0); base != 0 {
   191			ctx.heap = 1
   192			ctx.start = base
   193			ctx.size = span.elemsize
   194			ctx.res = 1
   195		}
   196	}
   197	
   198	// Race runtime functions called via runtime·racecall.
   199	//go:linkname __tsan_init __tsan_init
   200	var __tsan_init byte
   201	
   202	//go:linkname __tsan_fini __tsan_fini
   203	var __tsan_fini byte
   204	
   205	//go:linkname __tsan_proc_create __tsan_proc_create
   206	var __tsan_proc_create byte
   207	
   208	//go:linkname __tsan_proc_destroy __tsan_proc_destroy
   209	var __tsan_proc_destroy byte
   210	
   211	//go:linkname __tsan_map_shadow __tsan_map_shadow
   212	var __tsan_map_shadow byte
   213	
   214	//go:linkname __tsan_finalizer_goroutine __tsan_finalizer_goroutine
   215	var __tsan_finalizer_goroutine byte
   216	
   217	//go:linkname __tsan_go_start __tsan_go_start
   218	var __tsan_go_start byte
   219	
   220	//go:linkname __tsan_go_end __tsan_go_end
   221	var __tsan_go_end byte
   222	
   223	//go:linkname __tsan_malloc __tsan_malloc
   224	var __tsan_malloc byte
   225	
   226	//go:linkname __tsan_free __tsan_free
   227	var __tsan_free byte
   228	
   229	//go:linkname __tsan_acquire __tsan_acquire
   230	var __tsan_acquire byte
   231	
   232	//go:linkname __tsan_release __tsan_release
   233	var __tsan_release byte
   234	
   235	//go:linkname __tsan_release_merge __tsan_release_merge
   236	var __tsan_release_merge byte
   237	
   238	//go:linkname __tsan_go_ignore_sync_begin __tsan_go_ignore_sync_begin
   239	var __tsan_go_ignore_sync_begin byte
   240	
   241	//go:linkname __tsan_go_ignore_sync_end __tsan_go_ignore_sync_end
   242	var __tsan_go_ignore_sync_end byte
   243	
   244	//go:linkname __tsan_report_count __tsan_report_count
   245	var __tsan_report_count byte
   246	
   247	// Mimic what cmd/cgo would do.
   248	//go:cgo_import_static __tsan_init
   249	//go:cgo_import_static __tsan_fini
   250	//go:cgo_import_static __tsan_proc_create
   251	//go:cgo_import_static __tsan_proc_destroy
   252	//go:cgo_import_static __tsan_map_shadow
   253	//go:cgo_import_static __tsan_finalizer_goroutine
   254	//go:cgo_import_static __tsan_go_start
   255	//go:cgo_import_static __tsan_go_end
   256	//go:cgo_import_static __tsan_malloc
   257	//go:cgo_import_static __tsan_free
   258	//go:cgo_import_static __tsan_acquire
   259	//go:cgo_import_static __tsan_release
   260	//go:cgo_import_static __tsan_release_merge
   261	//go:cgo_import_static __tsan_go_ignore_sync_begin
   262	//go:cgo_import_static __tsan_go_ignore_sync_end
   263	//go:cgo_import_static __tsan_report_count
   264	
   265	// These are called from race_amd64.s.
   266	//go:cgo_import_static __tsan_read
   267	//go:cgo_import_static __tsan_read_pc
   268	//go:cgo_import_static __tsan_read_range
   269	//go:cgo_import_static __tsan_write
   270	//go:cgo_import_static __tsan_write_pc
   271	//go:cgo_import_static __tsan_write_range
   272	//go:cgo_import_static __tsan_func_enter
   273	//go:cgo_import_static __tsan_func_exit
   274	
   275	//go:cgo_import_static __tsan_go_atomic32_load
   276	//go:cgo_import_static __tsan_go_atomic64_load
   277	//go:cgo_import_static __tsan_go_atomic32_store
   278	//go:cgo_import_static __tsan_go_atomic64_store
   279	//go:cgo_import_static __tsan_go_atomic32_exchange
   280	//go:cgo_import_static __tsan_go_atomic64_exchange
   281	//go:cgo_import_static __tsan_go_atomic32_fetch_add
   282	//go:cgo_import_static __tsan_go_atomic64_fetch_add
   283	//go:cgo_import_static __tsan_go_atomic32_compare_exchange
   284	//go:cgo_import_static __tsan_go_atomic64_compare_exchange
   285	
   286	// start/end of global data (data+bss).
   287	var racedatastart uintptr
   288	var racedataend uintptr
   289	
   290	// start/end of heap for race_amd64.s
   291	var racearenastart uintptr
   292	var racearenaend uintptr
   293	
   294	func racefuncenter(callpc uintptr)
   295	func racefuncenterfp(fp uintptr)
   296	func racefuncexit()
   297	func raceread(addr uintptr)
   298	func racewrite(addr uintptr)
   299	func racereadrange(addr, size uintptr)
   300	func racewriterange(addr, size uintptr)
   301	func racereadrangepc1(addr, size, pc uintptr)
   302	func racewriterangepc1(addr, size, pc uintptr)
   303	func racecallbackthunk(uintptr)
   304	
   305	// racecall allows calling an arbitrary function f from C race runtime
   306	// with up to 4 uintptr arguments.
   307	func racecall(fn *byte, arg0, arg1, arg2, arg3 uintptr)
   308	
   309	// checks if the address has shadow (i.e. heap or data/bss)
   310	//go:nosplit
   311	func isvalidaddr(addr unsafe.Pointer) bool {
   312		return racearenastart <= uintptr(addr) && uintptr(addr) < racearenaend ||
   313			racedatastart <= uintptr(addr) && uintptr(addr) < racedataend
   314	}
   315	
   316	//go:nosplit
   317	func raceinit() (gctx, pctx uintptr) {
   318		// cgo is required to initialize libc, which is used by race runtime
   319		if !iscgo {
   320			throw("raceinit: race build must use cgo")
   321		}
   322	
   323		racecall(&__tsan_init, uintptr(unsafe.Pointer(&gctx)), uintptr(unsafe.Pointer(&pctx)), funcPC(racecallbackthunk), 0)
   324	
   325		// Round data segment to page boundaries, because it's used in mmap().
   326		start := ^uintptr(0)
   327		end := uintptr(0)
   328		if start > firstmoduledata.noptrdata {
   329			start = firstmoduledata.noptrdata
   330		}
   331		if start > firstmoduledata.data {
   332			start = firstmoduledata.data
   333		}
   334		if start > firstmoduledata.noptrbss {
   335			start = firstmoduledata.noptrbss
   336		}
   337		if start > firstmoduledata.bss {
   338			start = firstmoduledata.bss
   339		}
   340		if end < firstmoduledata.enoptrdata {
   341			end = firstmoduledata.enoptrdata
   342		}
   343		if end < firstmoduledata.edata {
   344			end = firstmoduledata.edata
   345		}
   346		if end < firstmoduledata.enoptrbss {
   347			end = firstmoduledata.enoptrbss
   348		}
   349		if end < firstmoduledata.ebss {
   350			end = firstmoduledata.ebss
   351		}
   352		size := round(end-start, _PageSize)
   353		racecall(&__tsan_map_shadow, start, size, 0, 0)
   354		racedatastart = start
   355		racedataend = start + size
   356	
   357		return
   358	}
   359	
   360	var raceFiniLock mutex
   361	
   362	//go:nosplit
   363	func racefini() {
   364		// racefini() can only be called once to avoid races.
   365		// This eventually (via __tsan_fini) calls C.exit which has
   366		// undefined behavior if called more than once. If the lock is
   367		// already held it's assumed that the first caller exits the program
   368		// so other calls can hang forever without an issue.
   369		lock(&raceFiniLock)
   370		racecall(&__tsan_fini, 0, 0, 0, 0)
   371	}
   372	
   373	//go:nosplit
   374	func raceproccreate() uintptr {
   375		var ctx uintptr
   376		racecall(&__tsan_proc_create, uintptr(unsafe.Pointer(&ctx)), 0, 0, 0)
   377		return ctx
   378	}
   379	
   380	//go:nosplit
   381	func raceprocdestroy(ctx uintptr) {
   382		racecall(&__tsan_proc_destroy, ctx, 0, 0, 0)
   383	}
   384	
   385	//go:nosplit
   386	func racemapshadow(addr unsafe.Pointer, size uintptr) {
   387		if racearenastart == 0 {
   388			racearenastart = uintptr(addr)
   389		}
   390		if racearenaend < uintptr(addr)+size {
   391			racearenaend = uintptr(addr) + size
   392		}
   393		racecall(&__tsan_map_shadow, uintptr(addr), size, 0, 0)
   394	}
   395	
   396	//go:nosplit
   397	func racemalloc(p unsafe.Pointer, sz uintptr) {
   398		racecall(&__tsan_malloc, 0, 0, uintptr(p), sz)
   399	}
   400	
   401	//go:nosplit
   402	func racefree(p unsafe.Pointer, sz uintptr) {
   403		racecall(&__tsan_free, uintptr(p), sz, 0, 0)
   404	}
   405	
   406	//go:nosplit
   407	func racegostart(pc uintptr) uintptr {
   408		_g_ := getg()
   409		var spawng *g
   410		if _g_.m.curg != nil {
   411			spawng = _g_.m.curg
   412		} else {
   413			spawng = _g_
   414		}
   415	
   416		var racectx uintptr
   417		racecall(&__tsan_go_start, spawng.racectx, uintptr(unsafe.Pointer(&racectx)), pc, 0)
   418		return racectx
   419	}
   420	
   421	//go:nosplit
   422	func racegoend() {
   423		racecall(&__tsan_go_end, getg().racectx, 0, 0, 0)
   424	}
   425	
   426	//go:nosplit
   427	func racewriterangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
   428		_g_ := getg()
   429		if _g_ != _g_.m.curg {
   430			// The call is coming from manual instrumentation of Go code running on g0/gsignal.
   431			// Not interesting.
   432			return
   433		}
   434		if callpc != 0 {
   435			racefuncenter(callpc)
   436		}
   437		racewriterangepc1(uintptr(addr), sz, pc)
   438		if callpc != 0 {
   439			racefuncexit()
   440		}
   441	}
   442	
   443	//go:nosplit
   444	func racereadrangepc(addr unsafe.Pointer, sz, callpc, pc uintptr) {
   445		_g_ := getg()
   446		if _g_ != _g_.m.curg {
   447			// The call is coming from manual instrumentation of Go code running on g0/gsignal.
   448			// Not interesting.
   449			return
   450		}
   451		if callpc != 0 {
   452			racefuncenter(callpc)
   453		}
   454		racereadrangepc1(uintptr(addr), sz, pc)
   455		if callpc != 0 {
   456			racefuncexit()
   457		}
   458	}
   459	
   460	//go:nosplit
   461	func raceacquire(addr unsafe.Pointer) {
   462		raceacquireg(getg(), addr)
   463	}
   464	
   465	//go:nosplit
   466	func raceacquireg(gp *g, addr unsafe.Pointer) {
   467		if getg().raceignore != 0 || !isvalidaddr(addr) {
   468			return
   469		}
   470		racecall(&__tsan_acquire, gp.racectx, uintptr(addr), 0, 0)
   471	}
   472	
   473	//go:nosplit
   474	func racerelease(addr unsafe.Pointer) {
   475		racereleaseg(getg(), addr)
   476	}
   477	
   478	//go:nosplit
   479	func racereleaseg(gp *g, addr unsafe.Pointer) {
   480		if getg().raceignore != 0 || !isvalidaddr(addr) {
   481			return
   482		}
   483		racecall(&__tsan_release, gp.racectx, uintptr(addr), 0, 0)
   484	}
   485	
   486	//go:nosplit
   487	func racereleasemerge(addr unsafe.Pointer) {
   488		racereleasemergeg(getg(), addr)
   489	}
   490	
   491	//go:nosplit
   492	func racereleasemergeg(gp *g, addr unsafe.Pointer) {
   493		if getg().raceignore != 0 || !isvalidaddr(addr) {
   494			return
   495		}
   496		racecall(&__tsan_release_merge, gp.racectx, uintptr(addr), 0, 0)
   497	}
   498	
   499	//go:nosplit
   500	func racefingo() {
   501		racecall(&__tsan_finalizer_goroutine, getg().racectx, 0, 0, 0)
   502	}
   503	
   504	// The declarations below generate ABI wrappers for functions
   505	// implemented in assembly in this package but declared in another
   506	// package.
   507	
   508	//go:linkname abigen_sync_atomic_LoadInt32 sync/atomic.LoadInt32
   509	func abigen_sync_atomic_LoadInt32(addr *int32) (val int32)
   510	
   511	//go:linkname abigen_sync_atomic_LoadInt64 sync/atomic.LoadInt64
   512	func abigen_sync_atomic_LoadInt64(addr *int64) (val int64)
   513	
   514	//go:linkname abigen_sync_atomic_LoadUint32 sync/atomic.LoadUint32
   515	func abigen_sync_atomic_LoadUint32(addr *uint32) (val uint32)
   516	
   517	//go:linkname abigen_sync_atomic_LoadUint64 sync/atomic.LoadUint64
   518	func abigen_sync_atomic_LoadUint64(addr *uint64) (val uint64)
   519	
   520	//go:linkname abigen_sync_atomic_LoadUintptr sync/atomic.LoadUintptr
   521	func abigen_sync_atomic_LoadUintptr(addr *uintptr) (val uintptr)
   522	
   523	//go:linkname abigen_sync_atomic_LoadPointer sync/atomic.LoadPointer
   524	func abigen_sync_atomic_LoadPointer(addr *unsafe.Pointer) (val unsafe.Pointer)
   525	
   526	//go:linkname abigen_sync_atomic_StoreInt32 sync/atomic.StoreInt32
   527	func abigen_sync_atomic_StoreInt32(addr *int32, val int32)
   528	
   529	//go:linkname abigen_sync_atomic_StoreInt64 sync/atomic.StoreInt64
   530	func abigen_sync_atomic_StoreInt64(addr *int64, val int64)
   531	
   532	//go:linkname abigen_sync_atomic_StoreUint32 sync/atomic.StoreUint32
   533	func abigen_sync_atomic_StoreUint32(addr *uint32, val uint32)
   534	
   535	//go:linkname abigen_sync_atomic_StoreUint64 sync/atomic.StoreUint64
   536	func abigen_sync_atomic_StoreUint64(addr *uint64, val uint64)
   537	
   538	//go:linkname abigen_sync_atomic_SwapInt32 sync/atomic.SwapInt32
   539	func abigen_sync_atomic_SwapInt32(addr *int32, new int32) (old int32)
   540	
   541	//go:linkname abigen_sync_atomic_SwapInt64 sync/atomic.SwapInt64
   542	func abigen_sync_atomic_SwapInt64(addr *int64, new int64) (old int64)
   543	
   544	//go:linkname abigen_sync_atomic_SwapUint32 sync/atomic.SwapUint32
   545	func abigen_sync_atomic_SwapUint32(addr *uint32, new uint32) (old uint32)
   546	
   547	//go:linkname abigen_sync_atomic_SwapUint64 sync/atomic.SwapUint64
   548	func abigen_sync_atomic_SwapUint64(addr *uint64, new uint64) (old uint64)
   549	
   550	//go:linkname abigen_sync_atomic_AddInt32 sync/atomic.AddInt32
   551	func abigen_sync_atomic_AddInt32(addr *int32, delta int32) (new int32)
   552	
   553	//go:linkname abigen_sync_atomic_AddUint32 sync/atomic.AddUint32
   554	func abigen_sync_atomic_AddUint32(addr *uint32, delta uint32) (new uint32)
   555	
   556	//go:linkname abigen_sync_atomic_AddInt64 sync/atomic.AddInt64
   557	func abigen_sync_atomic_AddInt64(addr *int64, delta int64) (new int64)
   558	
   559	//go:linkname abigen_sync_atomic_AddUint64 sync/atomic.AddUint64
   560	func abigen_sync_atomic_AddUint64(addr *uint64, delta uint64) (new uint64)
   561	
   562	//go:linkname abigen_sync_atomic_AddUintptr sync/atomic.AddUintptr
   563	func abigen_sync_atomic_AddUintptr(addr *uintptr, delta uintptr) (new uintptr)
   564	
   565	//go:linkname abigen_sync_atomic_CompareAndSwapInt32 sync/atomic.CompareAndSwapInt32
   566	func abigen_sync_atomic_CompareAndSwapInt32(addr *int32, old, new int32) (swapped bool)
   567	
   568	//go:linkname abigen_sync_atomic_CompareAndSwapInt64 sync/atomic.CompareAndSwapInt64
   569	func abigen_sync_atomic_CompareAndSwapInt64(addr *int64, old, new int64) (swapped bool)
   570	
   571	//go:linkname abigen_sync_atomic_CompareAndSwapUint32 sync/atomic.CompareAndSwapUint32
   572	func abigen_sync_atomic_CompareAndSwapUint32(addr *uint32, old, new uint32) (swapped bool)
   573	
   574	//go:linkname abigen_sync_atomic_CompareAndSwapUint64 sync/atomic.CompareAndSwapUint64
   575	func abigen_sync_atomic_CompareAndSwapUint64(addr *uint64, old, new uint64) (swapped bool)
   576	

View as plain text