Source file src/pkg/runtime/mgcmark.go

     1	// Copyright 2009 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Garbage collector: marking and scanning
     6	
     7	package runtime
     8	
     9	import (
    10		"runtime/internal/atomic"
    11		"runtime/internal/sys"
    12		"unsafe"
    13	)
    14	
    15	const (
    16		fixedRootFinalizers = iota
    17		fixedRootFreeGStacks
    18		fixedRootCount
    19	
    20		// rootBlockBytes is the number of bytes to scan per data or
    21		// BSS root.
    22		rootBlockBytes = 256 << 10
    23	
    24		// rootBlockSpans is the number of spans to scan per span
    25		// root.
    26		rootBlockSpans = 8 * 1024 // 64MB worth of spans
    27	
    28		// maxObletBytes is the maximum bytes of an object to scan at
    29		// once. Larger objects will be split up into "oblets" of at
    30		// most this size. Since we can scan 1–2 MB/ms, 128 KB bounds
    31		// scan preemption at ~100 µs.
    32		//
    33		// This must be > _MaxSmallSize so that the object base is the
    34		// span base.
    35		maxObletBytes = 128 << 10
    36	
    37		// drainCheckThreshold specifies how many units of work to do
    38		// between self-preemption checks in gcDrain. Assuming a scan
    39		// rate of 1 MB/ms, this is ~100 µs. Lower values have higher
    40		// overhead in the scan loop (the scheduler check may perform
    41		// a syscall, so its overhead is nontrivial). Higher values
    42		// make the system less responsive to incoming work.
    43		drainCheckThreshold = 100000
    44	)
    45	
    46	// gcMarkRootPrepare queues root scanning jobs (stacks, globals, and
    47	// some miscellany) and initializes scanning-related state.
    48	//
    49	// The caller must have call gcCopySpans().
    50	//
    51	// The world must be stopped.
    52	//
    53	//go:nowritebarrier
    54	func gcMarkRootPrepare() {
    55		work.nFlushCacheRoots = 0
    56	
    57		// Compute how many data and BSS root blocks there are.
    58		nBlocks := func(bytes uintptr) int {
    59			return int((bytes + rootBlockBytes - 1) / rootBlockBytes)
    60		}
    61	
    62		work.nDataRoots = 0
    63		work.nBSSRoots = 0
    64	
    65		// Scan globals.
    66		for _, datap := range activeModules() {
    67			nDataRoots := nBlocks(datap.edata - datap.data)
    68			if nDataRoots > work.nDataRoots {
    69				work.nDataRoots = nDataRoots
    70			}
    71		}
    72	
    73		for _, datap := range activeModules() {
    74			nBSSRoots := nBlocks(datap.ebss - datap.bss)
    75			if nBSSRoots > work.nBSSRoots {
    76				work.nBSSRoots = nBSSRoots
    77			}
    78		}
    79	
    80		// Scan span roots for finalizer specials.
    81		//
    82		// We depend on addfinalizer to mark objects that get
    83		// finalizers after root marking.
    84		//
    85		// We're only interested in scanning the in-use spans,
    86		// which will all be swept at this point. More spans
    87		// may be added to this list during concurrent GC, but
    88		// we only care about spans that were allocated before
    89		// this mark phase.
    90		work.nSpanRoots = mheap_.sweepSpans[mheap_.sweepgen/2%2].numBlocks()
    91	
    92		// Scan stacks.
    93		//
    94		// Gs may be created after this point, but it's okay that we
    95		// ignore them because they begin life without any roots, so
    96		// there's nothing to scan, and any roots they create during
    97		// the concurrent phase will be scanned during mark
    98		// termination.
    99		work.nStackRoots = int(atomic.Loaduintptr(&allglen))
   100	
   101		work.markrootNext = 0
   102		work.markrootJobs = uint32(fixedRootCount + work.nFlushCacheRoots + work.nDataRoots + work.nBSSRoots + work.nSpanRoots + work.nStackRoots)
   103	}
   104	
   105	// gcMarkRootCheck checks that all roots have been scanned. It is
   106	// purely for debugging.
   107	func gcMarkRootCheck() {
   108		if work.markrootNext < work.markrootJobs {
   109			print(work.markrootNext, " of ", work.markrootJobs, " markroot jobs done\n")
   110			throw("left over markroot jobs")
   111		}
   112	
   113		lock(&allglock)
   114		// Check that stacks have been scanned.
   115		var gp *g
   116		for i := 0; i < work.nStackRoots; i++ {
   117			gp = allgs[i]
   118			if !gp.gcscandone {
   119				goto fail
   120			}
   121		}
   122		unlock(&allglock)
   123		return
   124	
   125	fail:
   126		println("gp", gp, "goid", gp.goid,
   127			"status", readgstatus(gp),
   128			"gcscandone", gp.gcscandone,
   129			"gcscanvalid", gp.gcscanvalid)
   130		unlock(&allglock) // Avoid self-deadlock with traceback.
   131		throw("scan missed a g")
   132	}
   133	
   134	// ptrmask for an allocation containing a single pointer.
   135	var oneptrmask = [...]uint8{1}
   136	
   137	// markroot scans the i'th root.
   138	//
   139	// Preemption must be disabled (because this uses a gcWork).
   140	//
   141	// nowritebarrier is only advisory here.
   142	//
   143	//go:nowritebarrier
   144	func markroot(gcw *gcWork, i uint32) {
   145		// TODO(austin): This is a bit ridiculous. Compute and store
   146		// the bases in gcMarkRootPrepare instead of the counts.
   147		baseFlushCache := uint32(fixedRootCount)
   148		baseData := baseFlushCache + uint32(work.nFlushCacheRoots)
   149		baseBSS := baseData + uint32(work.nDataRoots)
   150		baseSpans := baseBSS + uint32(work.nBSSRoots)
   151		baseStacks := baseSpans + uint32(work.nSpanRoots)
   152		end := baseStacks + uint32(work.nStackRoots)
   153	
   154		// Note: if you add a case here, please also update heapdump.go:dumproots.
   155		switch {
   156		case baseFlushCache <= i && i < baseData:
   157			flushmcache(int(i - baseFlushCache))
   158	
   159		case baseData <= i && i < baseBSS:
   160			for _, datap := range activeModules() {
   161				markrootBlock(datap.data, datap.edata-datap.data, datap.gcdatamask.bytedata, gcw, int(i-baseData))
   162			}
   163	
   164		case baseBSS <= i && i < baseSpans:
   165			for _, datap := range activeModules() {
   166				markrootBlock(datap.bss, datap.ebss-datap.bss, datap.gcbssmask.bytedata, gcw, int(i-baseBSS))
   167			}
   168	
   169		case i == fixedRootFinalizers:
   170			for fb := allfin; fb != nil; fb = fb.alllink {
   171				cnt := uintptr(atomic.Load(&fb.cnt))
   172				scanblock(uintptr(unsafe.Pointer(&fb.fin[0])), cnt*unsafe.Sizeof(fb.fin[0]), &finptrmask[0], gcw, nil)
   173			}
   174	
   175		case i == fixedRootFreeGStacks:
   176			// Switch to the system stack so we can call
   177			// stackfree.
   178			systemstack(markrootFreeGStacks)
   179	
   180		case baseSpans <= i && i < baseStacks:
   181			// mark mspan.specials
   182			markrootSpans(gcw, int(i-baseSpans))
   183	
   184		default:
   185			// the rest is scanning goroutine stacks
   186			var gp *g
   187			if baseStacks <= i && i < end {
   188				gp = allgs[i-baseStacks]
   189			} else {
   190				throw("markroot: bad index")
   191			}
   192	
   193			// remember when we've first observed the G blocked
   194			// needed only to output in traceback
   195			status := readgstatus(gp) // We are not in a scan state
   196			if (status == _Gwaiting || status == _Gsyscall) && gp.waitsince == 0 {
   197				gp.waitsince = work.tstart
   198			}
   199	
   200			// scang must be done on the system stack in case
   201			// we're trying to scan our own stack.
   202			systemstack(func() {
   203				// If this is a self-scan, put the user G in
   204				// _Gwaiting to prevent self-deadlock. It may
   205				// already be in _Gwaiting if this is a mark
   206				// worker or we're in mark termination.
   207				userG := getg().m.curg
   208				selfScan := gp == userG && readgstatus(userG) == _Grunning
   209				if selfScan {
   210					casgstatus(userG, _Grunning, _Gwaiting)
   211					userG.waitreason = waitReasonGarbageCollectionScan
   212				}
   213	
   214				// TODO: scang blocks until gp's stack has
   215				// been scanned, which may take a while for
   216				// running goroutines. Consider doing this in
   217				// two phases where the first is non-blocking:
   218				// we scan the stacks we can and ask running
   219				// goroutines to scan themselves; and the
   220				// second blocks.
   221				scang(gp, gcw)
   222	
   223				if selfScan {
   224					casgstatus(userG, _Gwaiting, _Grunning)
   225				}
   226			})
   227		}
   228	}
   229	
   230	// markrootBlock scans the shard'th shard of the block of memory [b0,
   231	// b0+n0), with the given pointer mask.
   232	//
   233	//go:nowritebarrier
   234	func markrootBlock(b0, n0 uintptr, ptrmask0 *uint8, gcw *gcWork, shard int) {
   235		if rootBlockBytes%(8*sys.PtrSize) != 0 {
   236			// This is necessary to pick byte offsets in ptrmask0.
   237			throw("rootBlockBytes must be a multiple of 8*ptrSize")
   238		}
   239	
   240		b := b0 + uintptr(shard)*rootBlockBytes
   241		if b >= b0+n0 {
   242			return
   243		}
   244		ptrmask := (*uint8)(add(unsafe.Pointer(ptrmask0), uintptr(shard)*(rootBlockBytes/(8*sys.PtrSize))))
   245		n := uintptr(rootBlockBytes)
   246		if b+n > b0+n0 {
   247			n = b0 + n0 - b
   248		}
   249	
   250		// Scan this shard.
   251		scanblock(b, n, ptrmask, gcw, nil)
   252	}
   253	
   254	// markrootFreeGStacks frees stacks of dead Gs.
   255	//
   256	// This does not free stacks of dead Gs cached on Ps, but having a few
   257	// cached stacks around isn't a problem.
   258	func markrootFreeGStacks() {
   259		// Take list of dead Gs with stacks.
   260		lock(&sched.gFree.lock)
   261		list := sched.gFree.stack
   262		sched.gFree.stack = gList{}
   263		unlock(&sched.gFree.lock)
   264		if list.empty() {
   265			return
   266		}
   267	
   268		// Free stacks.
   269		q := gQueue{list.head, list.head}
   270		for gp := list.head.ptr(); gp != nil; gp = gp.schedlink.ptr() {
   271			stackfree(gp.stack)
   272			gp.stack.lo = 0
   273			gp.stack.hi = 0
   274			// Manipulate the queue directly since the Gs are
   275			// already all linked the right way.
   276			q.tail.set(gp)
   277		}
   278	
   279		// Put Gs back on the free list.
   280		lock(&sched.gFree.lock)
   281		sched.gFree.noStack.pushAll(q)
   282		unlock(&sched.gFree.lock)
   283	}
   284	
   285	// markrootSpans marks roots for one shard of work.spans.
   286	//
   287	//go:nowritebarrier
   288	func markrootSpans(gcw *gcWork, shard int) {
   289		// Objects with finalizers have two GC-related invariants:
   290		//
   291		// 1) Everything reachable from the object must be marked.
   292		// This ensures that when we pass the object to its finalizer,
   293		// everything the finalizer can reach will be retained.
   294		//
   295		// 2) Finalizer specials (which are not in the garbage
   296		// collected heap) are roots. In practice, this means the fn
   297		// field must be scanned.
   298		//
   299		// TODO(austin): There are several ideas for making this more
   300		// efficient in issue #11485.
   301	
   302		sg := mheap_.sweepgen
   303		spans := mheap_.sweepSpans[mheap_.sweepgen/2%2].block(shard)
   304		// Note that work.spans may not include spans that were
   305		// allocated between entering the scan phase and now. This is
   306		// okay because any objects with finalizers in those spans
   307		// must have been allocated and given finalizers after we
   308		// entered the scan phase, so addfinalizer will have ensured
   309		// the above invariants for them.
   310		for _, s := range spans {
   311			if s.state != mSpanInUse {
   312				continue
   313			}
   314			// Check that this span was swept (it may be cached or uncached).
   315			if !useCheckmark && !(s.sweepgen == sg || s.sweepgen == sg+3) {
   316				// sweepgen was updated (+2) during non-checkmark GC pass
   317				print("sweep ", s.sweepgen, " ", sg, "\n")
   318				throw("gc: unswept span")
   319			}
   320	
   321			// Speculatively check if there are any specials
   322			// without acquiring the span lock. This may race with
   323			// adding the first special to a span, but in that
   324			// case addfinalizer will observe that the GC is
   325			// active (which is globally synchronized) and ensure
   326			// the above invariants. We may also ensure the
   327			// invariants, but it's okay to scan an object twice.
   328			if s.specials == nil {
   329				continue
   330			}
   331	
   332			// Lock the specials to prevent a special from being
   333			// removed from the list while we're traversing it.
   334			lock(&s.speciallock)
   335	
   336			for sp := s.specials; sp != nil; sp = sp.next {
   337				if sp.kind != _KindSpecialFinalizer {
   338					continue
   339				}
   340				// don't mark finalized object, but scan it so we
   341				// retain everything it points to.
   342				spf := (*specialfinalizer)(unsafe.Pointer(sp))
   343				// A finalizer can be set for an inner byte of an object, find object beginning.
   344				p := s.base() + uintptr(spf.special.offset)/s.elemsize*s.elemsize
   345	
   346				// Mark everything that can be reached from
   347				// the object (but *not* the object itself or
   348				// we'll never collect it).
   349				scanobject(p, gcw)
   350	
   351				// The special itself is a root.
   352				scanblock(uintptr(unsafe.Pointer(&spf.fn)), sys.PtrSize, &oneptrmask[0], gcw, nil)
   353			}
   354	
   355			unlock(&s.speciallock)
   356		}
   357	}
   358	
   359	// gcAssistAlloc performs GC work to make gp's assist debt positive.
   360	// gp must be the calling user gorountine.
   361	//
   362	// This must be called with preemption enabled.
   363	func gcAssistAlloc(gp *g) {
   364		// Don't assist in non-preemptible contexts. These are
   365		// generally fragile and won't allow the assist to block.
   366		if getg() == gp.m.g0 {
   367			return
   368		}
   369		if mp := getg().m; mp.locks > 0 || mp.preemptoff != "" {
   370			return
   371		}
   372	
   373		traced := false
   374	retry:
   375		// Compute the amount of scan work we need to do to make the
   376		// balance positive. When the required amount of work is low,
   377		// we over-assist to build up credit for future allocations
   378		// and amortize the cost of assisting.
   379		debtBytes := -gp.gcAssistBytes
   380		scanWork := int64(gcController.assistWorkPerByte * float64(debtBytes))
   381		if scanWork < gcOverAssistWork {
   382			scanWork = gcOverAssistWork
   383			debtBytes = int64(gcController.assistBytesPerWork * float64(scanWork))
   384		}
   385	
   386		// Steal as much credit as we can from the background GC's
   387		// scan credit. This is racy and may drop the background
   388		// credit below 0 if two mutators steal at the same time. This
   389		// will just cause steals to fail until credit is accumulated
   390		// again, so in the long run it doesn't really matter, but we
   391		// do have to handle the negative credit case.
   392		bgScanCredit := atomic.Loadint64(&gcController.bgScanCredit)
   393		stolen := int64(0)
   394		if bgScanCredit > 0 {
   395			if bgScanCredit < scanWork {
   396				stolen = bgScanCredit
   397				gp.gcAssistBytes += 1 + int64(gcController.assistBytesPerWork*float64(stolen))
   398			} else {
   399				stolen = scanWork
   400				gp.gcAssistBytes += debtBytes
   401			}
   402			atomic.Xaddint64(&gcController.bgScanCredit, -stolen)
   403	
   404			scanWork -= stolen
   405	
   406			if scanWork == 0 {
   407				// We were able to steal all of the credit we
   408				// needed.
   409				if traced {
   410					traceGCMarkAssistDone()
   411				}
   412				return
   413			}
   414		}
   415	
   416		if trace.enabled && !traced {
   417			traced = true
   418			traceGCMarkAssistStart()
   419		}
   420	
   421		// Perform assist work
   422		systemstack(func() {
   423			gcAssistAlloc1(gp, scanWork)
   424			// The user stack may have moved, so this can't touch
   425			// anything on it until it returns from systemstack.
   426		})
   427	
   428		completed := gp.param != nil
   429		gp.param = nil
   430		if completed {
   431			gcMarkDone()
   432		}
   433	
   434		if gp.gcAssistBytes < 0 {
   435			// We were unable steal enough credit or perform
   436			// enough work to pay off the assist debt. We need to
   437			// do one of these before letting the mutator allocate
   438			// more to prevent over-allocation.
   439			//
   440			// If this is because we were preempted, reschedule
   441			// and try some more.
   442			if gp.preempt {
   443				Gosched()
   444				goto retry
   445			}
   446	
   447			// Add this G to an assist queue and park. When the GC
   448			// has more background credit, it will satisfy queued
   449			// assists before flushing to the global credit pool.
   450			//
   451			// Note that this does *not* get woken up when more
   452			// work is added to the work list. The theory is that
   453			// there wasn't enough work to do anyway, so we might
   454			// as well let background marking take care of the
   455			// work that is available.
   456			if !gcParkAssist() {
   457				goto retry
   458			}
   459	
   460			// At this point either background GC has satisfied
   461			// this G's assist debt, or the GC cycle is over.
   462		}
   463		if traced {
   464			traceGCMarkAssistDone()
   465		}
   466	}
   467	
   468	// gcAssistAlloc1 is the part of gcAssistAlloc that runs on the system
   469	// stack. This is a separate function to make it easier to see that
   470	// we're not capturing anything from the user stack, since the user
   471	// stack may move while we're in this function.
   472	//
   473	// gcAssistAlloc1 indicates whether this assist completed the mark
   474	// phase by setting gp.param to non-nil. This can't be communicated on
   475	// the stack since it may move.
   476	//
   477	//go:systemstack
   478	func gcAssistAlloc1(gp *g, scanWork int64) {
   479		// Clear the flag indicating that this assist completed the
   480		// mark phase.
   481		gp.param = nil
   482	
   483		if atomic.Load(&gcBlackenEnabled) == 0 {
   484			// The gcBlackenEnabled check in malloc races with the
   485			// store that clears it but an atomic check in every malloc
   486			// would be a performance hit.
   487			// Instead we recheck it here on the non-preemptable system
   488			// stack to determine if we should perform an assist.
   489	
   490			// GC is done, so ignore any remaining debt.
   491			gp.gcAssistBytes = 0
   492			return
   493		}
   494		// Track time spent in this assist. Since we're on the
   495		// system stack, this is non-preemptible, so we can
   496		// just measure start and end time.
   497		startTime := nanotime()
   498	
   499		decnwait := atomic.Xadd(&work.nwait, -1)
   500		if decnwait == work.nproc {
   501			println("runtime: work.nwait =", decnwait, "work.nproc=", work.nproc)
   502			throw("nwait > work.nprocs")
   503		}
   504	
   505		// gcDrainN requires the caller to be preemptible.
   506		casgstatus(gp, _Grunning, _Gwaiting)
   507		gp.waitreason = waitReasonGCAssistMarking
   508	
   509		// drain own cached work first in the hopes that it
   510		// will be more cache friendly.
   511		gcw := &getg().m.p.ptr().gcw
   512		workDone := gcDrainN(gcw, scanWork)
   513	
   514		casgstatus(gp, _Gwaiting, _Grunning)
   515	
   516		// Record that we did this much scan work.
   517		//
   518		// Back out the number of bytes of assist credit that
   519		// this scan work counts for. The "1+" is a poor man's
   520		// round-up, to ensure this adds credit even if
   521		// assistBytesPerWork is very low.
   522		gp.gcAssistBytes += 1 + int64(gcController.assistBytesPerWork*float64(workDone))
   523	
   524		// If this is the last worker and we ran out of work,
   525		// signal a completion point.
   526		incnwait := atomic.Xadd(&work.nwait, +1)
   527		if incnwait > work.nproc {
   528			println("runtime: work.nwait=", incnwait,
   529				"work.nproc=", work.nproc)
   530			throw("work.nwait > work.nproc")
   531		}
   532	
   533		if incnwait == work.nproc && !gcMarkWorkAvailable(nil) {
   534			// This has reached a background completion point. Set
   535			// gp.param to a non-nil value to indicate this. It
   536			// doesn't matter what we set it to (it just has to be
   537			// a valid pointer).
   538			gp.param = unsafe.Pointer(gp)
   539		}
   540		duration := nanotime() - startTime
   541		_p_ := gp.m.p.ptr()
   542		_p_.gcAssistTime += duration
   543		if _p_.gcAssistTime > gcAssistTimeSlack {
   544			atomic.Xaddint64(&gcController.assistTime, _p_.gcAssistTime)
   545			_p_.gcAssistTime = 0
   546		}
   547	}
   548	
   549	// gcWakeAllAssists wakes all currently blocked assists. This is used
   550	// at the end of a GC cycle. gcBlackenEnabled must be false to prevent
   551	// new assists from going to sleep after this point.
   552	func gcWakeAllAssists() {
   553		lock(&work.assistQueue.lock)
   554		list := work.assistQueue.q.popList()
   555		injectglist(&list)
   556		unlock(&work.assistQueue.lock)
   557	}
   558	
   559	// gcParkAssist puts the current goroutine on the assist queue and parks.
   560	//
   561	// gcParkAssist reports whether the assist is now satisfied. If it
   562	// returns false, the caller must retry the assist.
   563	//
   564	//go:nowritebarrier
   565	func gcParkAssist() bool {
   566		lock(&work.assistQueue.lock)
   567		// If the GC cycle finished while we were getting the lock,
   568		// exit the assist. The cycle can't finish while we hold the
   569		// lock.
   570		if atomic.Load(&gcBlackenEnabled) == 0 {
   571			unlock(&work.assistQueue.lock)
   572			return true
   573		}
   574	
   575		gp := getg()
   576		oldList := work.assistQueue.q
   577		work.assistQueue.q.pushBack(gp)
   578	
   579		// Recheck for background credit now that this G is in
   580		// the queue, but can still back out. This avoids a
   581		// race in case background marking has flushed more
   582		// credit since we checked above.
   583		if atomic.Loadint64(&gcController.bgScanCredit) > 0 {
   584			work.assistQueue.q = oldList
   585			if oldList.tail != 0 {
   586				oldList.tail.ptr().schedlink.set(nil)
   587			}
   588			unlock(&work.assistQueue.lock)
   589			return false
   590		}
   591		// Park.
   592		goparkunlock(&work.assistQueue.lock, waitReasonGCAssistWait, traceEvGoBlockGC, 2)
   593		return true
   594	}
   595	
   596	// gcFlushBgCredit flushes scanWork units of background scan work
   597	// credit. This first satisfies blocked assists on the
   598	// work.assistQueue and then flushes any remaining credit to
   599	// gcController.bgScanCredit.
   600	//
   601	// Write barriers are disallowed because this is used by gcDrain after
   602	// it has ensured that all work is drained and this must preserve that
   603	// condition.
   604	//
   605	//go:nowritebarrierrec
   606	func gcFlushBgCredit(scanWork int64) {
   607		if work.assistQueue.q.empty() {
   608			// Fast path; there are no blocked assists. There's a
   609			// small window here where an assist may add itself to
   610			// the blocked queue and park. If that happens, we'll
   611			// just get it on the next flush.
   612			atomic.Xaddint64(&gcController.bgScanCredit, scanWork)
   613			return
   614		}
   615	
   616		scanBytes := int64(float64(scanWork) * gcController.assistBytesPerWork)
   617	
   618		lock(&work.assistQueue.lock)
   619		for !work.assistQueue.q.empty() && scanBytes > 0 {
   620			gp := work.assistQueue.q.pop()
   621			// Note that gp.gcAssistBytes is negative because gp
   622			// is in debt. Think carefully about the signs below.
   623			if scanBytes+gp.gcAssistBytes >= 0 {
   624				// Satisfy this entire assist debt.
   625				scanBytes += gp.gcAssistBytes
   626				gp.gcAssistBytes = 0
   627				// It's important that we *not* put gp in
   628				// runnext. Otherwise, it's possible for user
   629				// code to exploit the GC worker's high
   630				// scheduler priority to get itself always run
   631				// before other goroutines and always in the
   632				// fresh quantum started by GC.
   633				ready(gp, 0, false)
   634			} else {
   635				// Partially satisfy this assist.
   636				gp.gcAssistBytes += scanBytes
   637				scanBytes = 0
   638				// As a heuristic, we move this assist to the
   639				// back of the queue so that large assists
   640				// can't clog up the assist queue and
   641				// substantially delay small assists.
   642				work.assistQueue.q.pushBack(gp)
   643				break
   644			}
   645		}
   646	
   647		if scanBytes > 0 {
   648			// Convert from scan bytes back to work.
   649			scanWork = int64(float64(scanBytes) * gcController.assistWorkPerByte)
   650			atomic.Xaddint64(&gcController.bgScanCredit, scanWork)
   651		}
   652		unlock(&work.assistQueue.lock)
   653	}
   654	
   655	// scanstack scans gp's stack, greying all pointers found on the stack.
   656	//
   657	// scanstack is marked go:systemstack because it must not be preempted
   658	// while using a workbuf.
   659	//
   660	//go:nowritebarrier
   661	//go:systemstack
   662	func scanstack(gp *g, gcw *gcWork) {
   663		if gp.gcscanvalid {
   664			return
   665		}
   666	
   667		if readgstatus(gp)&_Gscan == 0 {
   668			print("runtime:scanstack: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", hex(readgstatus(gp)), "\n")
   669			throw("scanstack - bad status")
   670		}
   671	
   672		switch readgstatus(gp) &^ _Gscan {
   673		default:
   674			print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
   675			throw("mark - bad status")
   676		case _Gdead:
   677			return
   678		case _Grunning:
   679			print("runtime: gp=", gp, ", goid=", gp.goid, ", gp->atomicstatus=", readgstatus(gp), "\n")
   680			throw("scanstack: goroutine not stopped")
   681		case _Grunnable, _Gsyscall, _Gwaiting:
   682			// ok
   683		}
   684	
   685		if gp == getg() {
   686			throw("can't scan our own stack")
   687		}
   688	
   689		// Shrink the stack if not much of it is being used.
   690		shrinkstack(gp)
   691	
   692		var state stackScanState
   693		state.stack = gp.stack
   694	
   695		if stackTraceDebug {
   696			println("stack trace goroutine", gp.goid)
   697		}
   698	
   699		// Scan the saved context register. This is effectively a live
   700		// register that gets moved back and forth between the
   701		// register and sched.ctxt without a write barrier.
   702		if gp.sched.ctxt != nil {
   703			scanblock(uintptr(unsafe.Pointer(&gp.sched.ctxt)), sys.PtrSize, &oneptrmask[0], gcw, &state)
   704		}
   705	
   706		// Scan the stack. Accumulate a list of stack objects.
   707		scanframe := func(frame *stkframe, unused unsafe.Pointer) bool {
   708			scanframeworker(frame, &state, gcw)
   709			return true
   710		}
   711		gentraceback(^uintptr(0), ^uintptr(0), 0, gp, 0, nil, 0x7fffffff, scanframe, nil, 0)
   712	
   713		// Find additional pointers that point into the stack from the heap.
   714		// Currently this includes defers and panics. See also function copystack.
   715	
   716		// Find and trace all defer arguments.
   717		tracebackdefers(gp, scanframe, nil)
   718	
   719		// Find and trace other pointers in defer records.
   720		for d := gp._defer; d != nil; d = d.link {
   721			if d.fn != nil {
   722				// tracebackdefers above does not scan the func value, which could
   723				// be a stack allocated closure. See issue 30453.
   724				scanblock(uintptr(unsafe.Pointer(&d.fn)), sys.PtrSize, &oneptrmask[0], gcw, &state)
   725			}
   726			if d.link != nil {
   727				// The link field of a stack-allocated defer record might point
   728				// to a heap-allocated defer record. Keep that heap record live.
   729				scanblock(uintptr(unsafe.Pointer(&d.link)), sys.PtrSize, &oneptrmask[0], gcw, &state)
   730			}
   731			// Retain defers records themselves.
   732			// Defer records might not be reachable from the G through regular heap
   733			// tracing because the defer linked list might weave between the stack and the heap.
   734			if d.heap {
   735				scanblock(uintptr(unsafe.Pointer(&d)), sys.PtrSize, &oneptrmask[0], gcw, &state)
   736			}
   737		}
   738		if gp._panic != nil {
   739			// Panics are always stack allocated.
   740			state.putPtr(uintptr(unsafe.Pointer(gp._panic)))
   741		}
   742	
   743		// Find and scan all reachable stack objects.
   744		state.buildIndex()
   745		for {
   746			p := state.getPtr()
   747			if p == 0 {
   748				break
   749			}
   750			obj := state.findObject(p)
   751			if obj == nil {
   752				continue
   753			}
   754			t := obj.typ
   755			if t == nil {
   756				// We've already scanned this object.
   757				continue
   758			}
   759			obj.setType(nil) // Don't scan it again.
   760			if stackTraceDebug {
   761				println("  live stkobj at", hex(state.stack.lo+uintptr(obj.off)), "of type", t.string())
   762			}
   763			gcdata := t.gcdata
   764			var s *mspan
   765			if t.kind&kindGCProg != 0 {
   766				// This path is pretty unlikely, an object large enough
   767				// to have a GC program allocated on the stack.
   768				// We need some space to unpack the program into a straight
   769				// bitmask, which we allocate/free here.
   770				// TODO: it would be nice if there were a way to run a GC
   771				// program without having to store all its bits. We'd have
   772				// to change from a Lempel-Ziv style program to something else.
   773				// Or we can forbid putting objects on stacks if they require
   774				// a gc program (see issue 27447).
   775				s = materializeGCProg(t.ptrdata, gcdata)
   776				gcdata = (*byte)(unsafe.Pointer(s.startAddr))
   777			}
   778	
   779			scanblock(state.stack.lo+uintptr(obj.off), t.ptrdata, gcdata, gcw, &state)
   780	
   781			if s != nil {
   782				dematerializeGCProg(s)
   783			}
   784		}
   785	
   786		// Deallocate object buffers.
   787		// (Pointer buffers were all deallocated in the loop above.)
   788		for state.head != nil {
   789			x := state.head
   790			state.head = x.next
   791			if stackTraceDebug {
   792				for _, obj := range x.obj[:x.nobj] {
   793					if obj.typ == nil { // reachable
   794						continue
   795					}
   796					println("  dead stkobj at", hex(gp.stack.lo+uintptr(obj.off)), "of type", obj.typ.string())
   797					// Note: not necessarily really dead - only reachable-from-ptr dead.
   798				}
   799			}
   800			x.nobj = 0
   801			putempty((*workbuf)(unsafe.Pointer(x)))
   802		}
   803		if state.buf != nil || state.freeBuf != nil {
   804			throw("remaining pointer buffers")
   805		}
   806	
   807		gp.gcscanvalid = true
   808	}
   809	
   810	// Scan a stack frame: local variables and function arguments/results.
   811	//go:nowritebarrier
   812	func scanframeworker(frame *stkframe, state *stackScanState, gcw *gcWork) {
   813		if _DebugGC > 1 && frame.continpc != 0 {
   814			print("scanframe ", funcname(frame.fn), "\n")
   815		}
   816	
   817		locals, args, objs := getStackMap(frame, &state.cache, false)
   818	
   819		// Scan local variables if stack frame has been allocated.
   820		if locals.n > 0 {
   821			size := uintptr(locals.n) * sys.PtrSize
   822			scanblock(frame.varp-size, size, locals.bytedata, gcw, state)
   823		}
   824	
   825		// Scan arguments.
   826		if args.n > 0 {
   827			scanblock(frame.argp, uintptr(args.n)*sys.PtrSize, args.bytedata, gcw, state)
   828		}
   829	
   830		// Add all stack objects to the stack object list.
   831		if frame.varp != 0 {
   832			// varp is 0 for defers, where there are no locals.
   833			// In that case, there can't be a pointer to its args, either.
   834			// (And all args would be scanned above anyway.)
   835			for _, obj := range objs {
   836				off := obj.off
   837				base := frame.varp // locals base pointer
   838				if off >= 0 {
   839					base = frame.argp // arguments and return values base pointer
   840				}
   841				ptr := base + uintptr(off)
   842				if ptr < frame.sp {
   843					// object hasn't been allocated in the frame yet.
   844					continue
   845				}
   846				if stackTraceDebug {
   847					println("stkobj at", hex(ptr), "of type", obj.typ.string())
   848				}
   849				state.addObject(ptr, obj.typ)
   850			}
   851		}
   852	}
   853	
   854	type gcDrainFlags int
   855	
   856	const (
   857		gcDrainUntilPreempt gcDrainFlags = 1 << iota
   858		gcDrainFlushBgCredit
   859		gcDrainIdle
   860		gcDrainFractional
   861	)
   862	
   863	// gcDrain scans roots and objects in work buffers, blackening grey
   864	// objects until it is unable to get more work. It may return before
   865	// GC is done; it's the caller's responsibility to balance work from
   866	// other Ps.
   867	//
   868	// If flags&gcDrainUntilPreempt != 0, gcDrain returns when g.preempt
   869	// is set.
   870	//
   871	// If flags&gcDrainIdle != 0, gcDrain returns when there is other work
   872	// to do.
   873	//
   874	// If flags&gcDrainFractional != 0, gcDrain self-preempts when
   875	// pollFractionalWorkerExit() returns true. This implies
   876	// gcDrainNoBlock.
   877	//
   878	// If flags&gcDrainFlushBgCredit != 0, gcDrain flushes scan work
   879	// credit to gcController.bgScanCredit every gcCreditSlack units of
   880	// scan work.
   881	//
   882	//go:nowritebarrier
   883	func gcDrain(gcw *gcWork, flags gcDrainFlags) {
   884		if !writeBarrier.needed {
   885			throw("gcDrain phase incorrect")
   886		}
   887	
   888		gp := getg().m.curg
   889		preemptible := flags&gcDrainUntilPreempt != 0
   890		flushBgCredit := flags&gcDrainFlushBgCredit != 0
   891		idle := flags&gcDrainIdle != 0
   892	
   893		initScanWork := gcw.scanWork
   894	
   895		// checkWork is the scan work before performing the next
   896		// self-preempt check.
   897		checkWork := int64(1<<63 - 1)
   898		var check func() bool
   899		if flags&(gcDrainIdle|gcDrainFractional) != 0 {
   900			checkWork = initScanWork + drainCheckThreshold
   901			if idle {
   902				check = pollWork
   903			} else if flags&gcDrainFractional != 0 {
   904				check = pollFractionalWorkerExit
   905			}
   906		}
   907	
   908		// Drain root marking jobs.
   909		if work.markrootNext < work.markrootJobs {
   910			for !(preemptible && gp.preempt) {
   911				job := atomic.Xadd(&work.markrootNext, +1) - 1
   912				if job >= work.markrootJobs {
   913					break
   914				}
   915				markroot(gcw, job)
   916				if check != nil && check() {
   917					goto done
   918				}
   919			}
   920		}
   921	
   922		// Drain heap marking jobs.
   923		for !(preemptible && gp.preempt) {
   924			// Try to keep work available on the global queue. We used to
   925			// check if there were waiting workers, but it's better to
   926			// just keep work available than to make workers wait. In the
   927			// worst case, we'll do O(log(_WorkbufSize)) unnecessary
   928			// balances.
   929			if work.full == 0 {
   930				gcw.balance()
   931			}
   932	
   933			b := gcw.tryGetFast()
   934			if b == 0 {
   935				b = gcw.tryGet()
   936				if b == 0 {
   937					// Flush the write barrier
   938					// buffer; this may create
   939					// more work.
   940					wbBufFlush(nil, 0)
   941					b = gcw.tryGet()
   942				}
   943			}
   944			if b == 0 {
   945				// Unable to get work.
   946				break
   947			}
   948			scanobject(b, gcw)
   949	
   950			// Flush background scan work credit to the global
   951			// account if we've accumulated enough locally so
   952			// mutator assists can draw on it.
   953			if gcw.scanWork >= gcCreditSlack {
   954				atomic.Xaddint64(&gcController.scanWork, gcw.scanWork)
   955				if flushBgCredit {
   956					gcFlushBgCredit(gcw.scanWork - initScanWork)
   957					initScanWork = 0
   958				}
   959				checkWork -= gcw.scanWork
   960				gcw.scanWork = 0
   961	
   962				if checkWork <= 0 {
   963					checkWork += drainCheckThreshold
   964					if check != nil && check() {
   965						break
   966					}
   967				}
   968			}
   969		}
   970	
   971	done:
   972		// Flush remaining scan work credit.
   973		if gcw.scanWork > 0 {
   974			atomic.Xaddint64(&gcController.scanWork, gcw.scanWork)
   975			if flushBgCredit {
   976				gcFlushBgCredit(gcw.scanWork - initScanWork)
   977			}
   978			gcw.scanWork = 0
   979		}
   980	}
   981	
   982	// gcDrainN blackens grey objects until it has performed roughly
   983	// scanWork units of scan work or the G is preempted. This is
   984	// best-effort, so it may perform less work if it fails to get a work
   985	// buffer. Otherwise, it will perform at least n units of work, but
   986	// may perform more because scanning is always done in whole object
   987	// increments. It returns the amount of scan work performed.
   988	//
   989	// The caller goroutine must be in a preemptible state (e.g.,
   990	// _Gwaiting) to prevent deadlocks during stack scanning. As a
   991	// consequence, this must be called on the system stack.
   992	//
   993	//go:nowritebarrier
   994	//go:systemstack
   995	func gcDrainN(gcw *gcWork, scanWork int64) int64 {
   996		if !writeBarrier.needed {
   997			throw("gcDrainN phase incorrect")
   998		}
   999	
  1000		// There may already be scan work on the gcw, which we don't
  1001		// want to claim was done by this call.
  1002		workFlushed := -gcw.scanWork
  1003	
  1004		gp := getg().m.curg
  1005		for !gp.preempt && workFlushed+gcw.scanWork < scanWork {
  1006			// See gcDrain comment.
  1007			if work.full == 0 {
  1008				gcw.balance()
  1009			}
  1010	
  1011			// This might be a good place to add prefetch code...
  1012			// if(wbuf.nobj > 4) {
  1013			//         PREFETCH(wbuf->obj[wbuf.nobj - 3];
  1014			//  }
  1015			//
  1016			b := gcw.tryGetFast()
  1017			if b == 0 {
  1018				b = gcw.tryGet()
  1019				if b == 0 {
  1020					// Flush the write barrier buffer;
  1021					// this may create more work.
  1022					wbBufFlush(nil, 0)
  1023					b = gcw.tryGet()
  1024				}
  1025			}
  1026	
  1027			if b == 0 {
  1028				// Try to do a root job.
  1029				//
  1030				// TODO: Assists should get credit for this
  1031				// work.
  1032				if work.markrootNext < work.markrootJobs {
  1033					job := atomic.Xadd(&work.markrootNext, +1) - 1
  1034					if job < work.markrootJobs {
  1035						markroot(gcw, job)
  1036						continue
  1037					}
  1038				}
  1039				// No heap or root jobs.
  1040				break
  1041			}
  1042			scanobject(b, gcw)
  1043	
  1044			// Flush background scan work credit.
  1045			if gcw.scanWork >= gcCreditSlack {
  1046				atomic.Xaddint64(&gcController.scanWork, gcw.scanWork)
  1047				workFlushed += gcw.scanWork
  1048				gcw.scanWork = 0
  1049			}
  1050		}
  1051	
  1052		// Unlike gcDrain, there's no need to flush remaining work
  1053		// here because this never flushes to bgScanCredit and
  1054		// gcw.dispose will flush any remaining work to scanWork.
  1055	
  1056		return workFlushed + gcw.scanWork
  1057	}
  1058	
  1059	// scanblock scans b as scanobject would, but using an explicit
  1060	// pointer bitmap instead of the heap bitmap.
  1061	//
  1062	// This is used to scan non-heap roots, so it does not update
  1063	// gcw.bytesMarked or gcw.scanWork.
  1064	//
  1065	// If stk != nil, possible stack pointers are also reported to stk.putPtr.
  1066	//go:nowritebarrier
  1067	func scanblock(b0, n0 uintptr, ptrmask *uint8, gcw *gcWork, stk *stackScanState) {
  1068		// Use local copies of original parameters, so that a stack trace
  1069		// due to one of the throws below shows the original block
  1070		// base and extent.
  1071		b := b0
  1072		n := n0
  1073	
  1074		for i := uintptr(0); i < n; {
  1075			// Find bits for the next word.
  1076			bits := uint32(*addb(ptrmask, i/(sys.PtrSize*8)))
  1077			if bits == 0 {
  1078				i += sys.PtrSize * 8
  1079				continue
  1080			}
  1081			for j := 0; j < 8 && i < n; j++ {
  1082				if bits&1 != 0 {
  1083					// Same work as in scanobject; see comments there.
  1084					p := *(*uintptr)(unsafe.Pointer(b + i))
  1085					if p != 0 {
  1086						if obj, span, objIndex := findObject(p, b, i); obj != 0 {
  1087							greyobject(obj, b, i, span, gcw, objIndex)
  1088						} else if stk != nil && p >= stk.stack.lo && p < stk.stack.hi {
  1089							stk.putPtr(p)
  1090						}
  1091					}
  1092				}
  1093				bits >>= 1
  1094				i += sys.PtrSize
  1095			}
  1096		}
  1097	}
  1098	
  1099	// scanobject scans the object starting at b, adding pointers to gcw.
  1100	// b must point to the beginning of a heap object or an oblet.
  1101	// scanobject consults the GC bitmap for the pointer mask and the
  1102	// spans for the size of the object.
  1103	//
  1104	//go:nowritebarrier
  1105	func scanobject(b uintptr, gcw *gcWork) {
  1106		// Find the bits for b and the size of the object at b.
  1107		//
  1108		// b is either the beginning of an object, in which case this
  1109		// is the size of the object to scan, or it points to an
  1110		// oblet, in which case we compute the size to scan below.
  1111		hbits := heapBitsForAddr(b)
  1112		s := spanOfUnchecked(b)
  1113		n := s.elemsize
  1114		if n == 0 {
  1115			throw("scanobject n == 0")
  1116		}
  1117	
  1118		if n > maxObletBytes {
  1119			// Large object. Break into oblets for better
  1120			// parallelism and lower latency.
  1121			if b == s.base() {
  1122				// It's possible this is a noscan object (not
  1123				// from greyobject, but from other code
  1124				// paths), in which case we must *not* enqueue
  1125				// oblets since their bitmaps will be
  1126				// uninitialized.
  1127				if s.spanclass.noscan() {
  1128					// Bypass the whole scan.
  1129					gcw.bytesMarked += uint64(n)
  1130					return
  1131				}
  1132	
  1133				// Enqueue the other oblets to scan later.
  1134				// Some oblets may be in b's scalar tail, but
  1135				// these will be marked as "no more pointers",
  1136				// so we'll drop out immediately when we go to
  1137				// scan those.
  1138				for oblet := b + maxObletBytes; oblet < s.base()+s.elemsize; oblet += maxObletBytes {
  1139					if !gcw.putFast(oblet) {
  1140						gcw.put(oblet)
  1141					}
  1142				}
  1143			}
  1144	
  1145			// Compute the size of the oblet. Since this object
  1146			// must be a large object, s.base() is the beginning
  1147			// of the object.
  1148			n = s.base() + s.elemsize - b
  1149			if n > maxObletBytes {
  1150				n = maxObletBytes
  1151			}
  1152		}
  1153	
  1154		var i uintptr
  1155		for i = 0; i < n; i += sys.PtrSize {
  1156			// Find bits for this word.
  1157			if i != 0 {
  1158				// Avoid needless hbits.next() on last iteration.
  1159				hbits = hbits.next()
  1160			}
  1161			// Load bits once. See CL 22712 and issue 16973 for discussion.
  1162			bits := hbits.bits()
  1163			// During checkmarking, 1-word objects store the checkmark
  1164			// in the type bit for the one word. The only one-word objects
  1165			// are pointers, or else they'd be merged with other non-pointer
  1166			// data into larger allocations.
  1167			if i != 1*sys.PtrSize && bits&bitScan == 0 {
  1168				break // no more pointers in this object
  1169			}
  1170			if bits&bitPointer == 0 {
  1171				continue // not a pointer
  1172			}
  1173	
  1174			// Work here is duplicated in scanblock and above.
  1175			// If you make changes here, make changes there too.
  1176			obj := *(*uintptr)(unsafe.Pointer(b + i))
  1177	
  1178			// At this point we have extracted the next potential pointer.
  1179			// Quickly filter out nil and pointers back to the current object.
  1180			if obj != 0 && obj-b >= n {
  1181				// Test if obj points into the Go heap and, if so,
  1182				// mark the object.
  1183				//
  1184				// Note that it's possible for findObject to
  1185				// fail if obj points to a just-allocated heap
  1186				// object because of a race with growing the
  1187				// heap. In this case, we know the object was
  1188				// just allocated and hence will be marked by
  1189				// allocation itself.
  1190				if obj, span, objIndex := findObject(obj, b, i); obj != 0 {
  1191					greyobject(obj, b, i, span, gcw, objIndex)
  1192				}
  1193			}
  1194		}
  1195		gcw.bytesMarked += uint64(n)
  1196		gcw.scanWork += int64(i)
  1197	}
  1198	
  1199	// Shade the object if it isn't already.
  1200	// The object is not nil and known to be in the heap.
  1201	// Preemption must be disabled.
  1202	//go:nowritebarrier
  1203	func shade(b uintptr) {
  1204		if obj, span, objIndex := findObject(b, 0, 0); obj != 0 {
  1205			gcw := &getg().m.p.ptr().gcw
  1206			greyobject(obj, 0, 0, span, gcw, objIndex)
  1207		}
  1208	}
  1209	
  1210	// obj is the start of an object with mark mbits.
  1211	// If it isn't already marked, mark it and enqueue into gcw.
  1212	// base and off are for debugging only and could be removed.
  1213	//
  1214	// See also wbBufFlush1, which partially duplicates this logic.
  1215	//
  1216	//go:nowritebarrierrec
  1217	func greyobject(obj, base, off uintptr, span *mspan, gcw *gcWork, objIndex uintptr) {
  1218		// obj should be start of allocation, and so must be at least pointer-aligned.
  1219		if obj&(sys.PtrSize-1) != 0 {
  1220			throw("greyobject: obj not pointer-aligned")
  1221		}
  1222		mbits := span.markBitsForIndex(objIndex)
  1223	
  1224		if useCheckmark {
  1225			if !mbits.isMarked() {
  1226				printlock()
  1227				print("runtime:greyobject: checkmarks finds unexpected unmarked object obj=", hex(obj), "\n")
  1228				print("runtime: found obj at *(", hex(base), "+", hex(off), ")\n")
  1229	
  1230				// Dump the source (base) object
  1231				gcDumpObject("base", base, off)
  1232	
  1233				// Dump the object
  1234				gcDumpObject("obj", obj, ^uintptr(0))
  1235	
  1236				getg().m.traceback = 2
  1237				throw("checkmark found unmarked object")
  1238			}
  1239			hbits := heapBitsForAddr(obj)
  1240			if hbits.isCheckmarked(span.elemsize) {
  1241				return
  1242			}
  1243			hbits.setCheckmarked(span.elemsize)
  1244			if !hbits.isCheckmarked(span.elemsize) {
  1245				throw("setCheckmarked and isCheckmarked disagree")
  1246			}
  1247		} else {
  1248			if debug.gccheckmark > 0 && span.isFree(objIndex) {
  1249				print("runtime: marking free object ", hex(obj), " found at *(", hex(base), "+", hex(off), ")\n")
  1250				gcDumpObject("base", base, off)
  1251				gcDumpObject("obj", obj, ^uintptr(0))
  1252				getg().m.traceback = 2
  1253				throw("marking free object")
  1254			}
  1255	
  1256			// If marked we have nothing to do.
  1257			if mbits.isMarked() {
  1258				return
  1259			}
  1260			mbits.setMarked()
  1261	
  1262			// Mark span.
  1263			arena, pageIdx, pageMask := pageIndexOf(span.base())
  1264			if arena.pageMarks[pageIdx]&pageMask == 0 {
  1265				atomic.Or8(&arena.pageMarks[pageIdx], pageMask)
  1266			}
  1267	
  1268			// If this is a noscan object, fast-track it to black
  1269			// instead of greying it.
  1270			if span.spanclass.noscan() {
  1271				gcw.bytesMarked += uint64(span.elemsize)
  1272				return
  1273			}
  1274		}
  1275	
  1276		// Queue the obj for scanning. The PREFETCH(obj) logic has been removed but
  1277		// seems like a nice optimization that can be added back in.
  1278		// There needs to be time between the PREFETCH and the use.
  1279		// Previously we put the obj in an 8 element buffer that is drained at a rate
  1280		// to give the PREFETCH time to do its work.
  1281		// Use of PREFETCHNTA might be more appropriate than PREFETCH
  1282		if !gcw.putFast(obj) {
  1283			gcw.put(obj)
  1284		}
  1285	}
  1286	
  1287	// gcDumpObject dumps the contents of obj for debugging and marks the
  1288	// field at byte offset off in obj.
  1289	func gcDumpObject(label string, obj, off uintptr) {
  1290		s := spanOf(obj)
  1291		print(label, "=", hex(obj))
  1292		if s == nil {
  1293			print(" s=nil\n")
  1294			return
  1295		}
  1296		print(" s.base()=", hex(s.base()), " s.limit=", hex(s.limit), " s.spanclass=", s.spanclass, " s.elemsize=", s.elemsize, " s.state=")
  1297		if 0 <= s.state && int(s.state) < len(mSpanStateNames) {
  1298			print(mSpanStateNames[s.state], "\n")
  1299		} else {
  1300			print("unknown(", s.state, ")\n")
  1301		}
  1302	
  1303		skipped := false
  1304		size := s.elemsize
  1305		if s.state == mSpanManual && size == 0 {
  1306			// We're printing something from a stack frame. We
  1307			// don't know how big it is, so just show up to an
  1308			// including off.
  1309			size = off + sys.PtrSize
  1310		}
  1311		for i := uintptr(0); i < size; i += sys.PtrSize {
  1312			// For big objects, just print the beginning (because
  1313			// that usually hints at the object's type) and the
  1314			// fields around off.
  1315			if !(i < 128*sys.PtrSize || off-16*sys.PtrSize < i && i < off+16*sys.PtrSize) {
  1316				skipped = true
  1317				continue
  1318			}
  1319			if skipped {
  1320				print(" ...\n")
  1321				skipped = false
  1322			}
  1323			print(" *(", label, "+", i, ") = ", hex(*(*uintptr)(unsafe.Pointer(obj + i))))
  1324			if i == off {
  1325				print(" <==")
  1326			}
  1327			print("\n")
  1328		}
  1329		if skipped {
  1330			print(" ...\n")
  1331		}
  1332	}
  1333	
  1334	// gcmarknewobject marks a newly allocated object black. obj must
  1335	// not contain any non-nil pointers.
  1336	//
  1337	// This is nosplit so it can manipulate a gcWork without preemption.
  1338	//
  1339	//go:nowritebarrier
  1340	//go:nosplit
  1341	func gcmarknewobject(obj, size, scanSize uintptr) {
  1342		if useCheckmark { // The world should be stopped so this should not happen.
  1343			throw("gcmarknewobject called while doing checkmark")
  1344		}
  1345		markBitsForAddr(obj).setMarked()
  1346		gcw := &getg().m.p.ptr().gcw
  1347		gcw.bytesMarked += uint64(size)
  1348		gcw.scanWork += int64(scanSize)
  1349	}
  1350	
  1351	// gcMarkTinyAllocs greys all active tiny alloc blocks.
  1352	//
  1353	// The world must be stopped.
  1354	func gcMarkTinyAllocs() {
  1355		for _, p := range allp {
  1356			c := p.mcache
  1357			if c == nil || c.tiny == 0 {
  1358				continue
  1359			}
  1360			_, span, objIndex := findObject(c.tiny, 0, 0)
  1361			gcw := &p.gcw
  1362			greyobject(c.tiny, 0, 0, span, gcw, objIndex)
  1363		}
  1364	}
  1365	
  1366	// Checkmarking
  1367	
  1368	// To help debug the concurrent GC we remark with the world
  1369	// stopped ensuring that any object encountered has their normal
  1370	// mark bit set. To do this we use an orthogonal bit
  1371	// pattern to indicate the object is marked. The following pattern
  1372	// uses the upper two bits in the object's boundary nibble.
  1373	// 01: scalar  not marked
  1374	// 10: pointer not marked
  1375	// 11: pointer     marked
  1376	// 00: scalar      marked
  1377	// Xoring with 01 will flip the pattern from marked to unmarked and vica versa.
  1378	// The higher bit is 1 for pointers and 0 for scalars, whether the object
  1379	// is marked or not.
  1380	// The first nibble no longer holds the typeDead pattern indicating that the
  1381	// there are no more pointers in the object. This information is held
  1382	// in the second nibble.
  1383	
  1384	// If useCheckmark is true, marking of an object uses the
  1385	// checkmark bits (encoding above) instead of the standard
  1386	// mark bits.
  1387	var useCheckmark = false
  1388	
  1389	//go:nowritebarrier
  1390	func initCheckmarks() {
  1391		useCheckmark = true
  1392		for _, s := range mheap_.allspans {
  1393			if s.state == mSpanInUse {
  1394				heapBitsForAddr(s.base()).initCheckmarkSpan(s.layout())
  1395			}
  1396		}
  1397	}
  1398	
  1399	func clearCheckmarks() {
  1400		useCheckmark = false
  1401		for _, s := range mheap_.allspans {
  1402			if s.state == mSpanInUse {
  1403				heapBitsForAddr(s.base()).clearCheckmarkSpan(s.layout())
  1404			}
  1405		}
  1406	}
  1407	

View as plain text