Source file src/pkg/cmd/go/internal/modload/load.go

     1	// Copyright 2018 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package modload
     6	
     7	import (
     8		"bytes"
     9		"errors"
    10		"fmt"
    11		"go/build"
    12		"io/ioutil"
    13		"os"
    14		"path"
    15		pathpkg "path"
    16		"path/filepath"
    17		"sort"
    18		"strings"
    19		"sync"
    20	
    21		"cmd/go/internal/base"
    22		"cmd/go/internal/cfg"
    23		"cmd/go/internal/imports"
    24		"cmd/go/internal/modfetch"
    25		"cmd/go/internal/modfile"
    26		"cmd/go/internal/module"
    27		"cmd/go/internal/mvs"
    28		"cmd/go/internal/par"
    29		"cmd/go/internal/search"
    30		"cmd/go/internal/semver"
    31		"cmd/go/internal/str"
    32	)
    33	
    34	// buildList is the list of modules to use for building packages.
    35	// It is initialized by calling ImportPaths, ImportFromFiles,
    36	// LoadALL, or LoadBuildList, each of which uses loaded.load.
    37	//
    38	// Ideally, exactly ONE of those functions would be called,
    39	// and exactly once. Most of the time, that's true.
    40	// During "go get" it may not be. TODO(rsc): Figure out if
    41	// that restriction can be established, or else document why not.
    42	//
    43	var buildList []module.Version
    44	
    45	// loaded is the most recently-used package loader.
    46	// It holds details about individual packages.
    47	//
    48	// Note that loaded.buildList is only valid during a load operation;
    49	// afterward, it is copied back into the global buildList,
    50	// which should be used instead.
    51	var loaded *loader
    52	
    53	// ImportPaths returns the set of packages matching the args (patterns),
    54	// on the target platform. Modules may be added to the build list
    55	// to satisfy new imports.
    56	func ImportPaths(patterns []string) []*search.Match {
    57		matches := ImportPathsQuiet(patterns, imports.Tags())
    58		search.WarnUnmatched(matches)
    59		return matches
    60	}
    61	
    62	// ImportPathsQuiet is like ImportPaths but does not warn about patterns with
    63	// no matches. It also lets the caller specify a set of build tags to match
    64	// packages. The build tags should typically be imports.Tags() or
    65	// imports.AnyTags(); a nil map has no special meaning.
    66	func ImportPathsQuiet(patterns []string, tags map[string]bool) []*search.Match {
    67		var fsDirs [][]string
    68		updateMatches := func(matches []*search.Match, iterating bool) {
    69			for i, m := range matches {
    70				switch {
    71				case build.IsLocalImport(m.Pattern) || filepath.IsAbs(m.Pattern):
    72					// Evaluate list of file system directories on first iteration.
    73					if fsDirs == nil {
    74						fsDirs = make([][]string, len(matches))
    75					}
    76					if fsDirs[i] == nil {
    77						var dirs []string
    78						if m.Literal {
    79							dirs = []string{m.Pattern}
    80						} else {
    81							dirs = search.MatchPackagesInFS(m.Pattern).Pkgs
    82						}
    83						fsDirs[i] = dirs
    84					}
    85	
    86					// Make a copy of the directory list and translate to import paths.
    87					// Note that whether a directory corresponds to an import path
    88					// changes as the build list is updated, and a directory can change
    89					// from not being in the build list to being in it and back as
    90					// the exact version of a particular module increases during
    91					// the loader iterations.
    92					m.Pkgs = str.StringList(fsDirs[i])
    93					pkgs := m.Pkgs
    94					m.Pkgs = m.Pkgs[:0]
    95					for _, pkg := range pkgs {
    96						dir := pkg
    97						if !filepath.IsAbs(dir) {
    98							dir = filepath.Join(cwd, pkg)
    99						} else {
   100							dir = filepath.Clean(dir)
   101						}
   102	
   103						// golang.org/issue/32917: We should resolve a relative path to a
   104						// package path only if the relative path actually contains the code
   105						// for that package.
   106						if !dirContainsPackage(dir) {
   107							// If we're outside of a module, ensure that the failure mode
   108							// indicates that.
   109							ModRoot()
   110	
   111							// If the directory is local but does not exist, don't return it
   112							// while loader is iterating, since this might trigger a fetch.
   113							// After loader is done iterating, we still need to return the
   114							// path, so that "go list -e" produces valid output.
   115							if !iterating {
   116								// We don't have a valid path to resolve to, so report the
   117								// unresolved path.
   118								m.Pkgs = append(m.Pkgs, pkg)
   119							}
   120							continue
   121						}
   122	
   123						// Note: The checks for @ here are just to avoid misinterpreting
   124						// the module cache directories (formerly GOPATH/src/mod/foo@v1.5.2/bar).
   125						// It's not strictly necessary but helpful to keep the checks.
   126						if modRoot != "" && dir == modRoot {
   127							pkg = targetPrefix
   128						} else if modRoot != "" && strings.HasPrefix(dir, modRoot+string(filepath.Separator)) && !strings.Contains(dir[len(modRoot):], "@") {
   129							suffix := filepath.ToSlash(dir[len(modRoot):])
   130							if strings.HasPrefix(suffix, "/vendor/") {
   131								// TODO getmode vendor check
   132								pkg = strings.TrimPrefix(suffix, "/vendor/")
   133							} else if targetInGorootSrc && Target.Path == "std" {
   134								// Don't add the prefix "std/" to packages in the "std" module.
   135								// It's the one module path that isn't a prefix of its packages.
   136								pkg = strings.TrimPrefix(suffix, "/")
   137								if pkg == "builtin" {
   138									// "builtin" is a pseudo-package with a real source file.
   139									// It's not included in "std", so it shouldn't be included in
   140									// "./..." within module "std" either.
   141									continue
   142								}
   143							} else {
   144								modPkg := targetPrefix + suffix
   145								if _, ok := dirInModule(modPkg, targetPrefix, modRoot, true); ok {
   146									pkg = modPkg
   147								} else if !iterating {
   148									ModRoot()
   149									base.Errorf("go: directory %s is outside main module", base.ShortPath(dir))
   150								}
   151							}
   152						} else if sub := search.InDir(dir, cfg.GOROOTsrc); sub != "" && sub != "." && !strings.Contains(sub, "@") {
   153							pkg = filepath.ToSlash(sub)
   154						} else if path := pathInModuleCache(dir); path != "" {
   155							pkg = path
   156						} else {
   157							pkg = ""
   158							if !iterating {
   159								ModRoot()
   160								base.Errorf("go: directory %s outside available modules", base.ShortPath(dir))
   161							}
   162						}
   163						m.Pkgs = append(m.Pkgs, pkg)
   164					}
   165	
   166				case strings.Contains(m.Pattern, "..."):
   167					m.Pkgs = matchPackages(m.Pattern, loaded.tags, true, buildList)
   168	
   169				case m.Pattern == "all":
   170					loaded.testAll = true
   171					if iterating {
   172						// Enumerate the packages in the main module.
   173						// We'll load the dependencies as we find them.
   174						m.Pkgs = matchPackages("...", loaded.tags, false, []module.Version{Target})
   175					} else {
   176						// Starting with the packages in the main module,
   177						// enumerate the full list of "all".
   178						m.Pkgs = loaded.computePatternAll(m.Pkgs)
   179					}
   180	
   181				case search.IsMetaPackage(m.Pattern): // std, cmd
   182					if len(m.Pkgs) == 0 {
   183						m.Pkgs = search.MatchPackages(m.Pattern).Pkgs
   184					}
   185	
   186				default:
   187					m.Pkgs = []string{m.Pattern}
   188				}
   189			}
   190		}
   191	
   192		InitMod()
   193	
   194		var matches []*search.Match
   195		for _, pattern := range search.CleanPatterns(patterns) {
   196			matches = append(matches, &search.Match{
   197				Pattern: pattern,
   198				Literal: !strings.Contains(pattern, "...") && !search.IsMetaPackage(pattern),
   199			})
   200		}
   201	
   202		loaded = newLoader(tags)
   203		loaded.load(func() []string {
   204			var roots []string
   205			updateMatches(matches, true)
   206			for _, m := range matches {
   207				roots = append(roots, m.Pkgs...)
   208			}
   209			return roots
   210		})
   211	
   212		// One last pass to finalize wildcards.
   213		updateMatches(matches, false)
   214	
   215		// A given module path may be used as itself or as a replacement for another
   216		// module, but not both at the same time. Otherwise, the aliasing behavior is
   217		// too subtle (see https://golang.org/issue/26607), and we don't want to
   218		// commit to a specific behavior at this point.
   219		firstPath := make(map[module.Version]string, len(buildList))
   220		for _, mod := range buildList {
   221			src := mod
   222			if rep := Replacement(mod); rep.Path != "" {
   223				src = rep
   224			}
   225			if prev, ok := firstPath[src]; !ok {
   226				firstPath[src] = mod.Path
   227			} else if prev != mod.Path {
   228				base.Errorf("go: %s@%s used for two different module paths (%s and %s)", src.Path, src.Version, prev, mod.Path)
   229			}
   230		}
   231		base.ExitIfErrors()
   232		WriteGoMod()
   233	
   234		return matches
   235	}
   236	
   237	// pathInModuleCache returns the import path of the directory dir,
   238	// if dir is in the module cache copy of a module in our build list.
   239	func pathInModuleCache(dir string) string {
   240		for _, m := range buildList[1:] {
   241			var root string
   242			var err error
   243			if repl := Replacement(m); repl.Path != "" && repl.Version == "" {
   244				root = repl.Path
   245				if !filepath.IsAbs(root) {
   246					root = filepath.Join(ModRoot(), root)
   247				}
   248			} else if repl.Path != "" {
   249				root, err = modfetch.DownloadDir(repl)
   250			} else {
   251				root, err = modfetch.DownloadDir(m)
   252			}
   253			if err != nil {
   254				continue
   255			}
   256			if sub := search.InDir(dir, root); sub != "" {
   257				sub = filepath.ToSlash(sub)
   258				if !strings.Contains(sub, "/vendor/") && !strings.HasPrefix(sub, "vendor/") && !strings.Contains(sub, "@") {
   259					return path.Join(m.Path, filepath.ToSlash(sub))
   260				}
   261			}
   262		}
   263		return ""
   264	}
   265	
   266	var dirContainsPackageCache sync.Map // absolute dir → bool
   267	
   268	func dirContainsPackage(dir string) bool {
   269		isPkg, ok := dirContainsPackageCache.Load(dir)
   270		if !ok {
   271			_, err := cfg.BuildContext.ImportDir(dir, 0)
   272			if err == nil {
   273				isPkg = true
   274			} else {
   275				if fi, statErr := os.Stat(dir); statErr != nil || !fi.IsDir() {
   276					// A non-directory or inaccessible directory is not a Go package.
   277					isPkg = false
   278				} else if _, noGo := err.(*build.NoGoError); noGo {
   279					// A directory containing no Go source files is not a Go package.
   280					isPkg = false
   281				} else {
   282					// An error other than *build.NoGoError indicates that the package exists
   283					// but has some other problem (such as a syntax error).
   284					isPkg = true
   285				}
   286			}
   287			isPkg, _ = dirContainsPackageCache.LoadOrStore(dir, isPkg)
   288		}
   289		return isPkg.(bool)
   290	}
   291	
   292	// ImportFromFiles adds modules to the build list as needed
   293	// to satisfy the imports in the named Go source files.
   294	func ImportFromFiles(gofiles []string) {
   295		InitMod()
   296	
   297		tags := imports.Tags()
   298		imports, testImports, err := imports.ScanFiles(gofiles, tags)
   299		if err != nil {
   300			base.Fatalf("go: %v", err)
   301		}
   302	
   303		loaded = newLoader(tags)
   304		loaded.load(func() []string {
   305			var roots []string
   306			roots = append(roots, imports...)
   307			roots = append(roots, testImports...)
   308			return roots
   309		})
   310		WriteGoMod()
   311	}
   312	
   313	// DirImportPath returns the effective import path for dir,
   314	// provided it is within the main module, or else returns ".".
   315	func DirImportPath(dir string) string {
   316		if modRoot == "" {
   317			return "."
   318		}
   319	
   320		if !filepath.IsAbs(dir) {
   321			dir = filepath.Join(cwd, dir)
   322		} else {
   323			dir = filepath.Clean(dir)
   324		}
   325	
   326		if dir == modRoot {
   327			return targetPrefix
   328		}
   329		if strings.HasPrefix(dir, modRoot+string(filepath.Separator)) {
   330			suffix := filepath.ToSlash(dir[len(modRoot):])
   331			if strings.HasPrefix(suffix, "/vendor/") {
   332				return strings.TrimPrefix(suffix, "/vendor/")
   333			}
   334			return targetPrefix + suffix
   335		}
   336		return "."
   337	}
   338	
   339	// LoadBuildList loads and returns the build list from go.mod.
   340	// The loading of the build list happens automatically in ImportPaths:
   341	// LoadBuildList need only be called if ImportPaths is not
   342	// (typically in commands that care about the module but
   343	// no particular package).
   344	func LoadBuildList() []module.Version {
   345		InitMod()
   346		ReloadBuildList()
   347		WriteGoMod()
   348		return buildList
   349	}
   350	
   351	func ReloadBuildList() []module.Version {
   352		loaded = newLoader(imports.Tags())
   353		loaded.load(func() []string { return nil })
   354		return buildList
   355	}
   356	
   357	// LoadALL returns the set of all packages in the current module
   358	// and their dependencies in any other modules, without filtering
   359	// due to build tags, except "+build ignore".
   360	// It adds modules to the build list as needed to satisfy new imports.
   361	// This set is useful for deciding whether a particular import is needed
   362	// anywhere in a module.
   363	func LoadALL() []string {
   364		return loadAll(true)
   365	}
   366	
   367	// LoadVendor is like LoadALL but only follows test dependencies
   368	// for tests in the main module. Tests in dependency modules are
   369	// ignored completely.
   370	// This set is useful for identifying the which packages to include in a vendor directory.
   371	func LoadVendor() []string {
   372		return loadAll(false)
   373	}
   374	
   375	func loadAll(testAll bool) []string {
   376		InitMod()
   377	
   378		loaded = newLoader(imports.AnyTags())
   379		loaded.isALL = true
   380		loaded.testAll = testAll
   381		if !testAll {
   382			loaded.testRoots = true
   383		}
   384		all := TargetPackages("...")
   385		loaded.load(func() []string { return all })
   386		WriteGoMod()
   387	
   388		var paths []string
   389		for _, pkg := range loaded.pkgs {
   390			if pkg.err != nil {
   391				base.Errorf("%s: %v", pkg.stackText(), pkg.err)
   392				continue
   393			}
   394			paths = append(paths, pkg.path)
   395		}
   396		base.ExitIfErrors()
   397		return paths
   398	}
   399	
   400	// TargetPackages returns the list of packages in the target (top-level) module
   401	// matching pattern, which may be relative to the working directory, under all
   402	// build tag settings.
   403	func TargetPackages(pattern string) []string {
   404		return matchPackages(pattern, imports.AnyTags(), false, []module.Version{Target})
   405	}
   406	
   407	// BuildList returns the module build list,
   408	// typically constructed by a previous call to
   409	// LoadBuildList or ImportPaths.
   410	// The caller must not modify the returned list.
   411	func BuildList() []module.Version {
   412		return buildList
   413	}
   414	
   415	// SetBuildList sets the module build list.
   416	// The caller is responsible for ensuring that the list is valid.
   417	// SetBuildList does not retain a reference to the original list.
   418	func SetBuildList(list []module.Version) {
   419		buildList = append([]module.Version{}, list...)
   420	}
   421	
   422	// ImportMap returns the actual package import path
   423	// for an import path found in source code.
   424	// If the given import path does not appear in the source code
   425	// for the packages that have been loaded, ImportMap returns the empty string.
   426	func ImportMap(path string) string {
   427		pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   428		if !ok {
   429			return ""
   430		}
   431		return pkg.path
   432	}
   433	
   434	// PackageDir returns the directory containing the source code
   435	// for the package named by the import path.
   436	func PackageDir(path string) string {
   437		pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   438		if !ok {
   439			return ""
   440		}
   441		return pkg.dir
   442	}
   443	
   444	// PackageModule returns the module providing the package named by the import path.
   445	func PackageModule(path string) module.Version {
   446		pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   447		if !ok {
   448			return module.Version{}
   449		}
   450		return pkg.mod
   451	}
   452	
   453	// PackageImports returns the imports for the package named by the import path.
   454	// Test imports will be returned as well if tests were loaded for the package
   455	// (i.e., if "all" was loaded or if LoadTests was set and the path was matched
   456	// by a command line argument). PackageImports will return nil for
   457	// unknown package paths.
   458	func PackageImports(path string) (imports, testImports []string) {
   459		pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   460		if !ok {
   461			return nil, nil
   462		}
   463		imports = make([]string, len(pkg.imports))
   464		for i, p := range pkg.imports {
   465			imports[i] = p.path
   466		}
   467		if pkg.test != nil {
   468			testImports = make([]string, len(pkg.test.imports))
   469			for i, p := range pkg.test.imports {
   470				testImports[i] = p.path
   471			}
   472		}
   473		return imports, testImports
   474	}
   475	
   476	// ModuleUsedDirectly reports whether the main module directly imports
   477	// some package in the module with the given path.
   478	func ModuleUsedDirectly(path string) bool {
   479		return loaded.direct[path]
   480	}
   481	
   482	// Lookup returns the source directory, import path, and any loading error for
   483	// the package at path as imported from the package in parentDir.
   484	// Lookup requires that one of the Load functions in this package has already
   485	// been called.
   486	func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) {
   487		if path == "" {
   488			panic("Lookup called with empty package path")
   489		}
   490	
   491		if parentIsStd {
   492			path = loaded.stdVendor(parentPath, path)
   493		}
   494		pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   495		if !ok {
   496			// The loader should have found all the relevant paths.
   497			// There are a few exceptions, though:
   498			//	- during go list without -test, the p.Resolve calls to process p.TestImports and p.XTestImports
   499			//	  end up here to canonicalize the import paths.
   500			//	- during any load, non-loaded packages like "unsafe" end up here.
   501			//	- during any load, build-injected dependencies like "runtime/cgo" end up here.
   502			//	- because we ignore appengine/* in the module loader,
   503			//	  the dependencies of any actual appengine/* library end up here.
   504			dir := findStandardImportPath(path)
   505			if dir != "" {
   506				return dir, path, nil
   507			}
   508			return "", "", errMissing
   509		}
   510		return pkg.dir, pkg.path, pkg.err
   511	}
   512	
   513	// A loader manages the process of loading information about
   514	// the required packages for a particular build,
   515	// checking that the packages are available in the module set,
   516	// and updating the module set if needed.
   517	// Loading is an iterative process: try to load all the needed packages,
   518	// but if imports are missing, try to resolve those imports, and repeat.
   519	//
   520	// Although most of the loading state is maintained in the loader struct,
   521	// one key piece - the build list - is a global, so that it can be modified
   522	// separate from the loading operation, such as during "go get"
   523	// upgrades/downgrades or in "go mod" operations.
   524	// TODO(rsc): It might be nice to make the loader take and return
   525	// a buildList rather than hard-coding use of the global.
   526	type loader struct {
   527		tags           map[string]bool // tags for scanDir
   528		testRoots      bool            // include tests for roots
   529		isALL          bool            // created with LoadALL
   530		testAll        bool            // include tests for all packages
   531		forceStdVendor bool            // if true, load standard-library dependencies from the vendor subtree
   532	
   533		// reset on each iteration
   534		roots    []*loadPkg
   535		pkgs     []*loadPkg
   536		work     *par.Work  // current work queue
   537		pkgCache *par.Cache // map from string to *loadPkg
   538	
   539		// computed at end of iterations
   540		direct    map[string]bool   // imported directly by main module
   541		goVersion map[string]string // go version recorded in each module
   542	}
   543	
   544	// LoadTests controls whether the loaders load tests of the root packages.
   545	var LoadTests bool
   546	
   547	func newLoader(tags map[string]bool) *loader {
   548		ld := new(loader)
   549		ld.tags = tags
   550		ld.testRoots = LoadTests
   551	
   552		// Inside the "std" and "cmd" modules, we prefer to use the vendor directory
   553		// unless the command explicitly changes the module graph.
   554		if !targetInGorootSrc || (cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ")) {
   555			ld.forceStdVendor = true
   556		}
   557	
   558		return ld
   559	}
   560	
   561	func (ld *loader) reset() {
   562		ld.roots = nil
   563		ld.pkgs = nil
   564		ld.work = new(par.Work)
   565		ld.pkgCache = new(par.Cache)
   566	}
   567	
   568	// A loadPkg records information about a single loaded package.
   569	type loadPkg struct {
   570		path        string         // import path
   571		mod         module.Version // module providing package
   572		dir         string         // directory containing source code
   573		imports     []*loadPkg     // packages imported by this one
   574		err         error          // error loading package
   575		stack       *loadPkg       // package importing this one in minimal import stack for this pkg
   576		test        *loadPkg       // package with test imports, if we need test
   577		testOf      *loadPkg
   578		testImports []string // test-only imports, saved for use by pkg.test.
   579	}
   580	
   581	var errMissing = errors.New("cannot find package")
   582	
   583	// load attempts to load the build graph needed to process a set of root packages.
   584	// The set of root packages is defined by the addRoots function,
   585	// which must call add(path) with the import path of each root package.
   586	func (ld *loader) load(roots func() []string) {
   587		var err error
   588		reqs := Reqs()
   589		buildList, err = mvs.BuildList(Target, reqs)
   590		if err != nil {
   591			base.Fatalf("go: %v", err)
   592		}
   593	
   594		added := make(map[string]bool)
   595		for {
   596			ld.reset()
   597			if roots != nil {
   598				// Note: the returned roots can change on each iteration,
   599				// since the expansion of package patterns depends on the
   600				// build list we're using.
   601				for _, path := range roots() {
   602					ld.work.Add(ld.pkg(path, true))
   603				}
   604			}
   605			ld.work.Do(10, ld.doPkg)
   606			ld.buildStacks()
   607			numAdded := 0
   608			haveMod := make(map[module.Version]bool)
   609			for _, m := range buildList {
   610				haveMod[m] = true
   611			}
   612			modAddedBy := make(map[module.Version]*loadPkg)
   613			for _, pkg := range ld.pkgs {
   614				if err, ok := pkg.err.(*ImportMissingError); ok && err.Module.Path != "" {
   615					if err.newMissingVersion != "" {
   616						base.Fatalf("go: %s: package provided by %s at latest version %s but not at required version %s", pkg.stackText(), err.Module.Path, err.Module.Version, err.newMissingVersion)
   617					}
   618					if added[pkg.path] {
   619						base.Fatalf("go: %s: looping trying to add package", pkg.stackText())
   620					}
   621					added[pkg.path] = true
   622					numAdded++
   623					if !haveMod[err.Module] {
   624						haveMod[err.Module] = true
   625						modAddedBy[err.Module] = pkg
   626						buildList = append(buildList, err.Module)
   627					}
   628					continue
   629				}
   630				// Leave other errors for Import or load.Packages to report.
   631			}
   632			base.ExitIfErrors()
   633			if numAdded == 0 {
   634				break
   635			}
   636	
   637			// Recompute buildList with all our additions.
   638			reqs = Reqs()
   639			buildList, err = mvs.BuildList(Target, reqs)
   640			if err != nil {
   641				// If an error was found in a newly added module, report the package
   642				// import stack instead of the module requirement stack. Packages
   643				// are more descriptive.
   644				if err, ok := err.(*mvs.BuildListError); ok {
   645					if pkg := modAddedBy[err.Module()]; pkg != nil {
   646						base.Fatalf("go: %s: %v", pkg.stackText(), err.Err)
   647					}
   648				}
   649				base.Fatalf("go: %v", err)
   650			}
   651		}
   652		base.ExitIfErrors()
   653	
   654		// Compute directly referenced dependency modules.
   655		ld.direct = make(map[string]bool)
   656		for _, pkg := range ld.pkgs {
   657			if pkg.mod == Target {
   658				for _, dep := range pkg.imports {
   659					if dep.mod.Path != "" {
   660						ld.direct[dep.mod.Path] = true
   661					}
   662				}
   663			}
   664		}
   665	
   666		// Add Go versions, computed during walk.
   667		ld.goVersion = make(map[string]string)
   668		for _, m := range buildList {
   669			v, _ := reqs.(*mvsReqs).versions.Load(m)
   670			ld.goVersion[m.Path], _ = v.(string)
   671		}
   672	
   673		// Mix in direct markings (really, lack of indirect markings)
   674		// from go.mod, unless we scanned the whole module
   675		// and can therefore be sure we know better than go.mod.
   676		if !ld.isALL && modFile != nil {
   677			for _, r := range modFile.Require {
   678				if !r.Indirect {
   679					ld.direct[r.Mod.Path] = true
   680				}
   681			}
   682		}
   683	}
   684	
   685	// pkg returns the *loadPkg for path, creating and queuing it if needed.
   686	// If the package should be tested, its test is created but not queued
   687	// (the test is queued after processing pkg).
   688	// If isRoot is true, the pkg is being queued as one of the roots of the work graph.
   689	func (ld *loader) pkg(path string, isRoot bool) *loadPkg {
   690		return ld.pkgCache.Do(path, func() interface{} {
   691			pkg := &loadPkg{
   692				path: path,
   693			}
   694			if ld.testRoots && isRoot || ld.testAll {
   695				test := &loadPkg{
   696					path:   path,
   697					testOf: pkg,
   698				}
   699				pkg.test = test
   700			}
   701			if isRoot {
   702				ld.roots = append(ld.roots, pkg)
   703			}
   704			ld.work.Add(pkg)
   705			return pkg
   706		}).(*loadPkg)
   707	}
   708	
   709	// doPkg processes a package on the work queue.
   710	func (ld *loader) doPkg(item interface{}) {
   711		// TODO: what about replacements?
   712		pkg := item.(*loadPkg)
   713		var imports []string
   714		if pkg.testOf != nil {
   715			pkg.dir = pkg.testOf.dir
   716			pkg.mod = pkg.testOf.mod
   717			imports = pkg.testOf.testImports
   718		} else {
   719			if strings.Contains(pkg.path, "@") {
   720				// Leave for error during load.
   721				return
   722			}
   723			if build.IsLocalImport(pkg.path) {
   724				// Leave for error during load.
   725				// (Module mode does not allow local imports.)
   726				return
   727			}
   728	
   729			pkg.mod, pkg.dir, pkg.err = Import(pkg.path)
   730			if pkg.dir == "" {
   731				return
   732			}
   733			var testImports []string
   734			var err error
   735			imports, testImports, err = scanDir(pkg.dir, ld.tags)
   736			if err != nil {
   737				pkg.err = err
   738				return
   739			}
   740			if pkg.test != nil {
   741				pkg.testImports = testImports
   742			}
   743		}
   744	
   745		inStd := (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "")
   746		for _, path := range imports {
   747			if inStd {
   748				path = ld.stdVendor(pkg.path, path)
   749			}
   750			pkg.imports = append(pkg.imports, ld.pkg(path, false))
   751		}
   752	
   753		// Now that pkg.dir, pkg.mod, pkg.testImports are set, we can queue pkg.test.
   754		// TODO: All that's left is creating new imports. Why not just do it now?
   755		if pkg.test != nil {
   756			ld.work.Add(pkg.test)
   757		}
   758	}
   759	
   760	// stdVendor returns the canonical import path for the package with the given
   761	// path when imported from the standard-library package at parentPath.
   762	func (ld *loader) stdVendor(parentPath, path string) string {
   763		if search.IsStandardImportPath(path) {
   764			return path
   765		}
   766	
   767		if str.HasPathPrefix(parentPath, "cmd") {
   768			if ld.forceStdVendor || Target.Path != "cmd" {
   769				vendorPath := pathpkg.Join("cmd", "vendor", path)
   770				if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil {
   771					return vendorPath
   772				}
   773			}
   774		} else if ld.forceStdVendor || Target.Path != "std" {
   775			vendorPath := pathpkg.Join("vendor", path)
   776			if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil {
   777				return vendorPath
   778			}
   779		}
   780	
   781		// Not vendored: resolve from modules.
   782		return path
   783	}
   784	
   785	// computePatternAll returns the list of packages matching pattern "all",
   786	// starting with a list of the import paths for the packages in the main module.
   787	func (ld *loader) computePatternAll(paths []string) []string {
   788		seen := make(map[*loadPkg]bool)
   789		var all []string
   790		var walk func(*loadPkg)
   791		walk = func(pkg *loadPkg) {
   792			if seen[pkg] {
   793				return
   794			}
   795			seen[pkg] = true
   796			if pkg.testOf == nil {
   797				all = append(all, pkg.path)
   798			}
   799			for _, p := range pkg.imports {
   800				walk(p)
   801			}
   802			if p := pkg.test; p != nil {
   803				walk(p)
   804			}
   805		}
   806		for _, path := range paths {
   807			walk(ld.pkg(path, false))
   808		}
   809		sort.Strings(all)
   810	
   811		return all
   812	}
   813	
   814	// scanDir is like imports.ScanDir but elides known magic imports from the list,
   815	// so that we do not go looking for packages that don't really exist.
   816	//
   817	// The standard magic import is "C", for cgo.
   818	//
   819	// The only other known magic imports are appengine and appengine/*.
   820	// These are so old that they predate "go get" and did not use URL-like paths.
   821	// Most code today now uses google.golang.org/appengine instead,
   822	// but not all code has been so updated. When we mostly ignore build tags
   823	// during "go vendor", we look into "// +build appengine" files and
   824	// may see these legacy imports. We drop them so that the module
   825	// search does not look for modules to try to satisfy them.
   826	func scanDir(dir string, tags map[string]bool) (imports_, testImports []string, err error) {
   827		imports_, testImports, err = imports.ScanDir(dir, tags)
   828	
   829		filter := func(x []string) []string {
   830			w := 0
   831			for _, pkg := range x {
   832				if pkg != "C" && pkg != "appengine" && !strings.HasPrefix(pkg, "appengine/") &&
   833					pkg != "appengine_internal" && !strings.HasPrefix(pkg, "appengine_internal/") {
   834					x[w] = pkg
   835					w++
   836				}
   837			}
   838			return x[:w]
   839		}
   840	
   841		return filter(imports_), filter(testImports), err
   842	}
   843	
   844	// buildStacks computes minimal import stacks for each package,
   845	// for use in error messages. When it completes, packages that
   846	// are part of the original root set have pkg.stack == nil,
   847	// and other packages have pkg.stack pointing at the next
   848	// package up the import stack in their minimal chain.
   849	// As a side effect, buildStacks also constructs ld.pkgs,
   850	// the list of all packages loaded.
   851	func (ld *loader) buildStacks() {
   852		if len(ld.pkgs) > 0 {
   853			panic("buildStacks")
   854		}
   855		for _, pkg := range ld.roots {
   856			pkg.stack = pkg // sentinel to avoid processing in next loop
   857			ld.pkgs = append(ld.pkgs, pkg)
   858		}
   859		for i := 0; i < len(ld.pkgs); i++ { // not range: appending to ld.pkgs in loop
   860			pkg := ld.pkgs[i]
   861			for _, next := range pkg.imports {
   862				if next.stack == nil {
   863					next.stack = pkg
   864					ld.pkgs = append(ld.pkgs, next)
   865				}
   866			}
   867			if next := pkg.test; next != nil && next.stack == nil {
   868				next.stack = pkg
   869				ld.pkgs = append(ld.pkgs, next)
   870			}
   871		}
   872		for _, pkg := range ld.roots {
   873			pkg.stack = nil
   874		}
   875	}
   876	
   877	// stackText builds the import stack text to use when
   878	// reporting an error in pkg. It has the general form
   879	//
   880	//	root imports
   881	//		other imports
   882	//		other2 tested by
   883	//		other2.test imports
   884	//		pkg
   885	//
   886	func (pkg *loadPkg) stackText() string {
   887		var stack []*loadPkg
   888		for p := pkg; p != nil; p = p.stack {
   889			stack = append(stack, p)
   890		}
   891	
   892		var buf bytes.Buffer
   893		for i := len(stack) - 1; i >= 0; i-- {
   894			p := stack[i]
   895			fmt.Fprint(&buf, p.path)
   896			if p.testOf != nil {
   897				fmt.Fprint(&buf, ".test")
   898			}
   899			if i > 0 {
   900				if stack[i-1].testOf == p {
   901					fmt.Fprint(&buf, " tested by\n\t")
   902				} else {
   903					fmt.Fprint(&buf, " imports\n\t")
   904				}
   905			}
   906		}
   907		return buf.String()
   908	}
   909	
   910	// why returns the text to use in "go mod why" output about the given package.
   911	// It is less ornate than the stackText but contains the same information.
   912	func (pkg *loadPkg) why() string {
   913		var buf strings.Builder
   914		var stack []*loadPkg
   915		for p := pkg; p != nil; p = p.stack {
   916			stack = append(stack, p)
   917		}
   918	
   919		for i := len(stack) - 1; i >= 0; i-- {
   920			p := stack[i]
   921			if p.testOf != nil {
   922				fmt.Fprintf(&buf, "%s.test\n", p.testOf.path)
   923			} else {
   924				fmt.Fprintf(&buf, "%s\n", p.path)
   925			}
   926		}
   927		return buf.String()
   928	}
   929	
   930	// Why returns the "go mod why" output stanza for the given package,
   931	// without the leading # comment.
   932	// The package graph must have been loaded already, usually by LoadALL.
   933	// If there is no reason for the package to be in the current build,
   934	// Why returns an empty string.
   935	func Why(path string) string {
   936		pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
   937		if !ok {
   938			return ""
   939		}
   940		return pkg.why()
   941	}
   942	
   943	// WhyDepth returns the number of steps in the Why listing.
   944	// If there is no reason for the package to be in the current build,
   945	// WhyDepth returns 0.
   946	func WhyDepth(path string) int {
   947		n := 0
   948		pkg, _ := loaded.pkgCache.Get(path).(*loadPkg)
   949		for p := pkg; p != nil; p = p.stack {
   950			n++
   951		}
   952		return n
   953	}
   954	
   955	// Replacement returns the replacement for mod, if any, from go.mod.
   956	// If there is no replacement for mod, Replacement returns
   957	// a module.Version with Path == "".
   958	func Replacement(mod module.Version) module.Version {
   959		if modFile == nil {
   960			// Happens during testing and if invoking 'go get' or 'go list' outside a module.
   961			return module.Version{}
   962		}
   963	
   964		var found *modfile.Replace
   965		for _, r := range modFile.Replace {
   966			if r.Old.Path == mod.Path && (r.Old.Version == "" || r.Old.Version == mod.Version) {
   967				found = r // keep going
   968			}
   969		}
   970		if found == nil {
   971			return module.Version{}
   972		}
   973		return found.New
   974	}
   975	
   976	// mvsReqs implements mvs.Reqs for module semantic versions,
   977	// with any exclusions or replacements applied internally.
   978	type mvsReqs struct {
   979		buildList []module.Version
   980		cache     par.Cache
   981		versions  sync.Map
   982	}
   983	
   984	// Reqs returns the current module requirement graph.
   985	// Future calls to SetBuildList do not affect the operation
   986	// of the returned Reqs.
   987	func Reqs() mvs.Reqs {
   988		r := &mvsReqs{
   989			buildList: buildList,
   990		}
   991		return r
   992	}
   993	
   994	func (r *mvsReqs) Required(mod module.Version) ([]module.Version, error) {
   995		type cached struct {
   996			list []module.Version
   997			err  error
   998		}
   999	
  1000		c := r.cache.Do(mod, func() interface{} {
  1001			list, err := r.required(mod)
  1002			if err != nil {
  1003				return cached{nil, err}
  1004			}
  1005			for i, mv := range list {
  1006				for excluded[mv] {
  1007					mv1, err := r.next(mv)
  1008					if err != nil {
  1009						return cached{nil, err}
  1010					}
  1011					if mv1.Version == "none" {
  1012						return cached{nil, fmt.Errorf("%s(%s) depends on excluded %s(%s) with no newer version available", mod.Path, mod.Version, mv.Path, mv.Version)}
  1013					}
  1014					mv = mv1
  1015				}
  1016				list[i] = mv
  1017			}
  1018	
  1019			return cached{list, nil}
  1020		}).(cached)
  1021	
  1022		return c.list, c.err
  1023	}
  1024	
  1025	var vendorOnce sync.Once
  1026	
  1027	var (
  1028		vendorList []module.Version
  1029		vendorMap  map[string]module.Version
  1030	)
  1031	
  1032	// readVendorList reads the list of vendored modules from vendor/modules.txt.
  1033	func readVendorList() {
  1034		vendorOnce.Do(func() {
  1035			vendorList = nil
  1036			vendorMap = make(map[string]module.Version)
  1037			data, _ := ioutil.ReadFile(filepath.Join(ModRoot(), "vendor/modules.txt"))
  1038			var m module.Version
  1039			for _, line := range strings.Split(string(data), "\n") {
  1040				if strings.HasPrefix(line, "# ") {
  1041					f := strings.Fields(line)
  1042					m = module.Version{}
  1043					if len(f) == 3 && semver.IsValid(f[2]) {
  1044						m = module.Version{Path: f[1], Version: f[2]}
  1045						vendorList = append(vendorList, m)
  1046					}
  1047				} else if m.Path != "" {
  1048					f := strings.Fields(line)
  1049					if len(f) == 1 {
  1050						vendorMap[f[0]] = m
  1051					}
  1052				}
  1053			}
  1054		})
  1055	}
  1056	
  1057	func (r *mvsReqs) modFileToList(f *modfile.File) []module.Version {
  1058		list := make([]module.Version, 0, len(f.Require))
  1059		for _, r := range f.Require {
  1060			list = append(list, r.Mod)
  1061		}
  1062		return list
  1063	}
  1064	
  1065	// required returns a unique copy of the requirements of mod.
  1066	func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) {
  1067		if mod == Target {
  1068			if modFile != nil && modFile.Go != nil {
  1069				r.versions.LoadOrStore(mod, modFile.Go.Version)
  1070			}
  1071			return append([]module.Version(nil), r.buildList[1:]...), nil
  1072		}
  1073	
  1074		if cfg.BuildMod == "vendor" {
  1075			// For every module other than the target,
  1076			// return the full list of modules from modules.txt.
  1077			readVendorList()
  1078			return append([]module.Version(nil), vendorList...), nil
  1079		}
  1080	
  1081		if targetInGorootSrc {
  1082			// When inside "std" or "cmd", only fetch and read go.mod files if we're
  1083			// explicitly running a command that can change the module graph. If we have
  1084			// to resolve a new dependency, we might pick the wrong version, but 'go mod
  1085			// tidy' will fix it — and new standard-library dependencies should be rare
  1086			// anyway.
  1087			//
  1088			// TODO(golang.org/issue/30240): Drop this special-case.
  1089			if cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ") {
  1090				return nil, nil
  1091			}
  1092		}
  1093	
  1094		origPath := mod.Path
  1095		if repl := Replacement(mod); repl.Path != "" {
  1096			if repl.Version == "" {
  1097				// TODO: need to slip the new version into the tags list etc.
  1098				dir := repl.Path
  1099				if !filepath.IsAbs(dir) {
  1100					dir = filepath.Join(ModRoot(), dir)
  1101				}
  1102				gomod := filepath.Join(dir, "go.mod")
  1103				data, err := ioutil.ReadFile(gomod)
  1104				if err != nil {
  1105					return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err)
  1106				}
  1107				f, err := modfile.ParseLax(gomod, data, nil)
  1108				if err != nil {
  1109					return nil, fmt.Errorf("parsing %s: %v", base.ShortPath(gomod), err)
  1110				}
  1111				if f.Go != nil {
  1112					r.versions.LoadOrStore(mod, f.Go.Version)
  1113				}
  1114				return r.modFileToList(f), nil
  1115			}
  1116			mod = repl
  1117		}
  1118	
  1119		if mod.Version == "none" {
  1120			return nil, nil
  1121		}
  1122	
  1123		if !semver.IsValid(mod.Version) {
  1124			// Disallow the broader queries supported by fetch.Lookup.
  1125			base.Fatalf("go: internal error: %s@%s: unexpected invalid semantic version", mod.Path, mod.Version)
  1126		}
  1127	
  1128		data, err := modfetch.GoMod(mod.Path, mod.Version)
  1129		if err != nil {
  1130			return nil, err
  1131		}
  1132		f, err := modfile.ParseLax("go.mod", data, nil)
  1133		if err != nil {
  1134			return nil, module.VersionError(mod, fmt.Errorf("parsing go.mod: %v", err))
  1135		}
  1136	
  1137		if f.Module == nil {
  1138			return nil, module.VersionError(mod, errors.New("parsing go.mod: missing module line"))
  1139		}
  1140		if mpath := f.Module.Mod.Path; mpath != origPath && mpath != mod.Path {
  1141			return nil, module.VersionError(mod, fmt.Errorf(`parsing go.mod:
  1142		module declares its path as: %s
  1143		        but was required as: %s`, mpath, mod.Path))
  1144		}
  1145		if f.Go != nil {
  1146			r.versions.LoadOrStore(mod, f.Go.Version)
  1147		}
  1148	
  1149		return r.modFileToList(f), nil
  1150	}
  1151	
  1152	func (*mvsReqs) Max(v1, v2 string) string {
  1153		if v1 != "" && semver.Compare(v1, v2) == -1 {
  1154			return v2
  1155		}
  1156		return v1
  1157	}
  1158	
  1159	// Upgrade is a no-op, here to implement mvs.Reqs.
  1160	// The upgrade logic for go get -u is in ../modget/get.go.
  1161	func (*mvsReqs) Upgrade(m module.Version) (module.Version, error) {
  1162		return m, nil
  1163	}
  1164	
  1165	func versions(path string) ([]string, error) {
  1166		// Note: modfetch.Lookup and repo.Versions are cached,
  1167		// so there's no need for us to add extra caching here.
  1168		var versions []string
  1169		err := modfetch.TryProxies(func(proxy string) error {
  1170			repo, err := modfetch.Lookup(proxy, path)
  1171			if err == nil {
  1172				versions, err = repo.Versions("")
  1173			}
  1174			return err
  1175		})
  1176		return versions, err
  1177	}
  1178	
  1179	// Previous returns the tagged version of m.Path immediately prior to
  1180	// m.Version, or version "none" if no prior version is tagged.
  1181	func (*mvsReqs) Previous(m module.Version) (module.Version, error) {
  1182		list, err := versions(m.Path)
  1183		if err != nil {
  1184			return module.Version{}, err
  1185		}
  1186		i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) >= 0 })
  1187		if i > 0 {
  1188			return module.Version{Path: m.Path, Version: list[i-1]}, nil
  1189		}
  1190		return module.Version{Path: m.Path, Version: "none"}, nil
  1191	}
  1192	
  1193	// next returns the next version of m.Path after m.Version.
  1194	// It is only used by the exclusion processing in the Required method,
  1195	// not called directly by MVS.
  1196	func (*mvsReqs) next(m module.Version) (module.Version, error) {
  1197		list, err := versions(m.Path)
  1198		if err != nil {
  1199			return module.Version{}, err
  1200		}
  1201		i := sort.Search(len(list), func(i int) bool { return semver.Compare(list[i], m.Version) > 0 })
  1202		if i < len(list) {
  1203			return module.Version{Path: m.Path, Version: list[i]}, nil
  1204		}
  1205		return module.Version{Path: m.Path, Version: "none"}, nil
  1206	}
  1207	
  1208	// fetch downloads the given module (or its replacement)
  1209	// and returns its location.
  1210	//
  1211	// The isLocal return value reports whether the replacement,
  1212	// if any, is local to the filesystem.
  1213	func fetch(mod module.Version) (dir string, isLocal bool, err error) {
  1214		if mod == Target {
  1215			return ModRoot(), true, nil
  1216		}
  1217		if r := Replacement(mod); r.Path != "" {
  1218			if r.Version == "" {
  1219				dir = r.Path
  1220				if !filepath.IsAbs(dir) {
  1221					dir = filepath.Join(ModRoot(), dir)
  1222				}
  1223				return dir, true, nil
  1224			}
  1225			mod = r
  1226		}
  1227	
  1228		dir, err = modfetch.Download(mod)
  1229		return dir, false, err
  1230	}
  1231	

View as plain text