Source file src/pkg/cmd/go/internal/get/get.go

     1	// Copyright 2011 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Package get implements the ``go get'' command.
     6	package get
     7	
     8	import (
     9		"fmt"
    10		"go/build"
    11		"os"
    12		"path/filepath"
    13		"runtime"
    14		"strings"
    15	
    16		"cmd/go/internal/base"
    17		"cmd/go/internal/cfg"
    18		"cmd/go/internal/load"
    19		"cmd/go/internal/search"
    20		"cmd/go/internal/str"
    21		"cmd/go/internal/web"
    22		"cmd/go/internal/work"
    23	)
    24	
    25	var CmdGet = &base.Command{
    26		UsageLine: "go get [-d] [-f] [-t] [-u] [-v] [-fix] [-insecure] [build flags] [packages]",
    27		Short:     "download and install packages and dependencies",
    28		Long: `
    29	Get downloads the packages named by the import paths, along with their
    30	dependencies. It then installs the named packages, like 'go install'.
    31	
    32	The -d flag instructs get to stop after downloading the packages; that is,
    33	it instructs get not to install the packages.
    34	
    35	The -f flag, valid only when -u is set, forces get -u not to verify that
    36	each package has been checked out from the source control repository
    37	implied by its import path. This can be useful if the source is a local fork
    38	of the original.
    39	
    40	The -fix flag instructs get to run the fix tool on the downloaded packages
    41	before resolving dependencies or building the code.
    42	
    43	The -insecure flag permits fetching from repositories and resolving
    44	custom domains using insecure schemes such as HTTP. Use with caution.
    45	
    46	The -t flag instructs get to also download the packages required to build
    47	the tests for the specified packages.
    48	
    49	The -u flag instructs get to use the network to update the named packages
    50	and their dependencies. By default, get uses the network to check out
    51	missing packages but does not use it to look for updates to existing packages.
    52	
    53	The -v flag enables verbose progress and debug output.
    54	
    55	Get also accepts build flags to control the installation. See 'go help build'.
    56	
    57	When checking out a new package, get creates the target directory
    58	GOPATH/src/<import-path>. If the GOPATH contains multiple entries,
    59	get uses the first one. For more details see: 'go help gopath'.
    60	
    61	When checking out or updating a package, get looks for a branch or tag
    62	that matches the locally installed version of Go. The most important
    63	rule is that if the local installation is running version "go1", get
    64	searches for a branch or tag named "go1". If no such version exists
    65	it retrieves the default branch of the package.
    66	
    67	When go get checks out or updates a Git repository,
    68	it also updates any git submodules referenced by the repository.
    69	
    70	Get never checks out or updates code stored in vendor directories.
    71	
    72	For more about specifying packages, see 'go help packages'.
    73	
    74	For more about how 'go get' finds source code to
    75	download, see 'go help importpath'.
    76	
    77	This text describes the behavior of get when using GOPATH
    78	to manage source code and dependencies.
    79	If instead the go command is running in module-aware mode,
    80	the details of get's flags and effects change, as does 'go help get'.
    81	See 'go help modules' and 'go help module-get'.
    82	
    83	See also: go build, go install, go clean.
    84		`,
    85	}
    86	
    87	var HelpGopathGet = &base.Command{
    88		UsageLine: "gopath-get",
    89		Short:     "legacy GOPATH go get",
    90		Long: `
    91	The 'go get' command changes behavior depending on whether the
    92	go command is running in module-aware mode or legacy GOPATH mode.
    93	This help text, accessible as 'go help gopath-get' even in module-aware mode,
    94	describes 'go get' as it operates in legacy GOPATH mode.
    95	
    96	Usage: ` + CmdGet.UsageLine + `
    97	` + CmdGet.Long,
    98	}
    99	
   100	var (
   101		getD   = CmdGet.Flag.Bool("d", false, "")
   102		getF   = CmdGet.Flag.Bool("f", false, "")
   103		getT   = CmdGet.Flag.Bool("t", false, "")
   104		getU   = CmdGet.Flag.Bool("u", false, "")
   105		getFix = CmdGet.Flag.Bool("fix", false, "")
   106	
   107		Insecure bool
   108	)
   109	
   110	func init() {
   111		work.AddBuildFlags(CmdGet)
   112		CmdGet.Run = runGet // break init loop
   113		CmdGet.Flag.BoolVar(&Insecure, "insecure", Insecure, "")
   114	}
   115	
   116	func runGet(cmd *base.Command, args []string) {
   117		if cfg.ModulesEnabled {
   118			// Should not happen: main.go should install the separate module-enabled get code.
   119			base.Fatalf("go get: modules not implemented")
   120		}
   121	
   122		work.BuildInit()
   123	
   124		if *getF && !*getU {
   125			base.Fatalf("go get: cannot use -f flag without -u")
   126		}
   127	
   128		// Disable any prompting for passwords by Git.
   129		// Only has an effect for 2.3.0 or later, but avoiding
   130		// the prompt in earlier versions is just too hard.
   131		// If user has explicitly set GIT_TERMINAL_PROMPT=1, keep
   132		// prompting.
   133		// See golang.org/issue/9341 and golang.org/issue/12706.
   134		if os.Getenv("GIT_TERMINAL_PROMPT") == "" {
   135			os.Setenv("GIT_TERMINAL_PROMPT", "0")
   136		}
   137	
   138		// Disable any ssh connection pooling by Git.
   139		// If a Git subprocess forks a child into the background to cache a new connection,
   140		// that child keeps stdout/stderr open. After the Git subprocess exits,
   141		// os /exec expects to be able to read from the stdout/stderr pipe
   142		// until EOF to get all the data that the Git subprocess wrote before exiting.
   143		// The EOF doesn't come until the child exits too, because the child
   144		// is holding the write end of the pipe.
   145		// This is unfortunate, but it has come up at least twice
   146		// (see golang.org/issue/13453 and golang.org/issue/16104)
   147		// and confuses users when it does.
   148		// If the user has explicitly set GIT_SSH or GIT_SSH_COMMAND,
   149		// assume they know what they are doing and don't step on it.
   150		// But default to turning off ControlMaster.
   151		if os.Getenv("GIT_SSH") == "" && os.Getenv("GIT_SSH_COMMAND") == "" {
   152			os.Setenv("GIT_SSH_COMMAND", "ssh -o ControlMaster=no")
   153		}
   154	
   155		// Phase 1. Download/update.
   156		var stk load.ImportStack
   157		mode := 0
   158		if *getT {
   159			mode |= load.GetTestDeps
   160		}
   161		for _, pkg := range downloadPaths(args) {
   162			download(pkg, nil, &stk, mode)
   163		}
   164		base.ExitIfErrors()
   165	
   166		// Phase 2. Rescan packages and re-evaluate args list.
   167	
   168		// Code we downloaded and all code that depends on it
   169		// needs to be evicted from the package cache so that
   170		// the information will be recomputed. Instead of keeping
   171		// track of the reverse dependency information, evict
   172		// everything.
   173		load.ClearPackageCache()
   174	
   175		pkgs := load.PackagesForBuild(args)
   176	
   177		// Phase 3. Install.
   178		if *getD {
   179			// Download only.
   180			// Check delayed until now so that importPaths
   181			// and packagesForBuild have a chance to print errors.
   182			return
   183		}
   184	
   185		work.InstallPackages(args, pkgs)
   186	}
   187	
   188	// downloadPaths prepares the list of paths to pass to download.
   189	// It expands ... patterns that can be expanded. If there is no match
   190	// for a particular pattern, downloadPaths leaves it in the result list,
   191	// in the hope that we can figure out the repository from the
   192	// initial ...-free prefix.
   193	func downloadPaths(patterns []string) []string {
   194		for _, arg := range patterns {
   195			if strings.Contains(arg, "@") {
   196				base.Fatalf("go: cannot use path@version syntax in GOPATH mode")
   197			}
   198		}
   199		var pkgs []string
   200		for _, m := range search.ImportPathsQuiet(patterns) {
   201			if len(m.Pkgs) == 0 && strings.Contains(m.Pattern, "...") {
   202				pkgs = append(pkgs, m.Pattern)
   203			} else {
   204				pkgs = append(pkgs, m.Pkgs...)
   205			}
   206		}
   207		return pkgs
   208	}
   209	
   210	// downloadCache records the import paths we have already
   211	// considered during the download, to avoid duplicate work when
   212	// there is more than one dependency sequence leading to
   213	// a particular package.
   214	var downloadCache = map[string]bool{}
   215	
   216	// downloadRootCache records the version control repository
   217	// root directories we have already considered during the download.
   218	// For example, all the packages in the github.com/google/codesearch repo
   219	// share the same root (the directory for that path), and we only need
   220	// to run the hg commands to consider each repository once.
   221	var downloadRootCache = map[string]bool{}
   222	
   223	// download runs the download half of the get command
   224	// for the package or pattern named by the argument.
   225	func download(arg string, parent *load.Package, stk *load.ImportStack, mode int) {
   226		if mode&load.ResolveImport != 0 {
   227			// Caller is responsible for expanding vendor paths.
   228			panic("internal error: download mode has useVendor set")
   229		}
   230		load1 := func(path string, mode int) *load.Package {
   231			if parent == nil {
   232				mode := 0 // don't do module or vendor resolution
   233				return load.LoadImport(path, base.Cwd, nil, stk, nil, mode)
   234			}
   235			return load.LoadImport(path, parent.Dir, parent, stk, nil, mode|load.ResolveModule)
   236		}
   237	
   238		p := load1(arg, mode)
   239		if p.Error != nil && p.Error.Hard {
   240			base.Errorf("%s", p.Error)
   241			return
   242		}
   243	
   244		// loadPackage inferred the canonical ImportPath from arg.
   245		// Use that in the following to prevent hysteresis effects
   246		// in e.g. downloadCache and packageCache.
   247		// This allows invocations such as:
   248		//   mkdir -p $GOPATH/src/github.com/user
   249		//   cd $GOPATH/src/github.com/user
   250		//   go get ./foo
   251		// see: golang.org/issue/9767
   252		arg = p.ImportPath
   253	
   254		// There's nothing to do if this is a package in the standard library.
   255		if p.Standard {
   256			return
   257		}
   258	
   259		// Only process each package once.
   260		// (Unless we're fetching test dependencies for this package,
   261		// in which case we want to process it again.)
   262		if downloadCache[arg] && mode&load.GetTestDeps == 0 {
   263			return
   264		}
   265		downloadCache[arg] = true
   266	
   267		pkgs := []*load.Package{p}
   268		wildcardOkay := len(*stk) == 0
   269		isWildcard := false
   270	
   271		// Download if the package is missing, or update if we're using -u.
   272		if p.Dir == "" || *getU {
   273			// The actual download.
   274			stk.Push(arg)
   275			err := downloadPackage(p)
   276			if err != nil {
   277				base.Errorf("%s", &load.PackageError{ImportStack: stk.Copy(), Err: err.Error()})
   278				stk.Pop()
   279				return
   280			}
   281			stk.Pop()
   282	
   283			args := []string{arg}
   284			// If the argument has a wildcard in it, re-evaluate the wildcard.
   285			// We delay this until after reloadPackage so that the old entry
   286			// for p has been replaced in the package cache.
   287			if wildcardOkay && strings.Contains(arg, "...") {
   288				if build.IsLocalImport(arg) {
   289					args = search.MatchPackagesInFS(arg).Pkgs
   290				} else {
   291					args = search.MatchPackages(arg).Pkgs
   292				}
   293				isWildcard = true
   294			}
   295	
   296			// Clear all relevant package cache entries before
   297			// doing any new loads.
   298			load.ClearPackageCachePartial(args)
   299	
   300			pkgs = pkgs[:0]
   301			for _, arg := range args {
   302				// Note: load calls loadPackage or loadImport,
   303				// which push arg onto stk already.
   304				// Do not push here too, or else stk will say arg imports arg.
   305				p := load1(arg, mode)
   306				if p.Error != nil {
   307					base.Errorf("%s", p.Error)
   308					continue
   309				}
   310				pkgs = append(pkgs, p)
   311			}
   312		}
   313	
   314		// Process package, which might now be multiple packages
   315		// due to wildcard expansion.
   316		for _, p := range pkgs {
   317			if *getFix {
   318				files := base.RelPaths(p.InternalAllGoFiles())
   319				base.Run(cfg.BuildToolexec, str.StringList(base.Tool("fix"), files))
   320	
   321				// The imports might have changed, so reload again.
   322				p = load.ReloadPackageNoFlags(arg, stk)
   323				if p.Error != nil {
   324					base.Errorf("%s", p.Error)
   325					return
   326				}
   327			}
   328	
   329			if isWildcard {
   330				// Report both the real package and the
   331				// wildcard in any error message.
   332				stk.Push(p.ImportPath)
   333			}
   334	
   335			// Process dependencies, now that we know what they are.
   336			imports := p.Imports
   337			if mode&load.GetTestDeps != 0 {
   338				// Process test dependencies when -t is specified.
   339				// (But don't get test dependencies for test dependencies:
   340				// we always pass mode 0 to the recursive calls below.)
   341				imports = str.StringList(imports, p.TestImports, p.XTestImports)
   342			}
   343			for i, path := range imports {
   344				if path == "C" {
   345					continue
   346				}
   347				// Fail fast on import naming full vendor path.
   348				// Otherwise expand path as needed for test imports.
   349				// Note that p.Imports can have additional entries beyond p.Internal.Build.Imports.
   350				orig := path
   351				if i < len(p.Internal.Build.Imports) {
   352					orig = p.Internal.Build.Imports[i]
   353				}
   354				if j, ok := load.FindVendor(orig); ok {
   355					stk.Push(path)
   356					err := &load.PackageError{
   357						ImportStack: stk.Copy(),
   358						Err:         "must be imported as " + path[j+len("vendor/"):],
   359					}
   360					stk.Pop()
   361					base.Errorf("%s", err)
   362					continue
   363				}
   364				// If this is a test import, apply module and vendor lookup now.
   365				// We cannot pass ResolveImport to download, because
   366				// download does caching based on the value of path,
   367				// so it must be the fully qualified path already.
   368				if i >= len(p.Imports) {
   369					path = load.ResolveImportPath(p, path)
   370				}
   371				download(path, p, stk, 0)
   372			}
   373	
   374			if isWildcard {
   375				stk.Pop()
   376			}
   377		}
   378	}
   379	
   380	// downloadPackage runs the create or download command
   381	// to make the first copy of or update a copy of the given package.
   382	func downloadPackage(p *load.Package) error {
   383		var (
   384			vcs            *vcsCmd
   385			repo, rootPath string
   386			err            error
   387			blindRepo      bool // set if the repo has unusual configuration
   388		)
   389	
   390		security := web.SecureOnly
   391		if Insecure {
   392			security = web.Insecure
   393		}
   394	
   395		// p can be either a real package, or a pseudo-package whose “import path” is
   396		// actually a wildcard pattern.
   397		// Trim the path at the element containing the first wildcard,
   398		// and hope that it applies to the wildcarded parts too.
   399		// This makes 'go get rsc.io/pdf/...' work in a fresh GOPATH.
   400		importPrefix := p.ImportPath
   401		if i := strings.Index(importPrefix, "..."); i >= 0 {
   402			slash := strings.LastIndexByte(importPrefix[:i], '/')
   403			if slash < 0 {
   404				return fmt.Errorf("cannot expand ... in %q", p.ImportPath)
   405			}
   406			importPrefix = importPrefix[:slash]
   407		}
   408		if err := CheckImportPath(importPrefix); err != nil {
   409			return fmt.Errorf("%s: invalid import path: %v", p.ImportPath, err)
   410		}
   411	
   412		if p.Internal.Build.SrcRoot != "" {
   413			// Directory exists. Look for checkout along path to src.
   414			vcs, rootPath, err = vcsFromDir(p.Dir, p.Internal.Build.SrcRoot)
   415			if err != nil {
   416				return err
   417			}
   418			repo = "<local>" // should be unused; make distinctive
   419	
   420			// Double-check where it came from.
   421			if *getU && vcs.remoteRepo != nil {
   422				dir := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath))
   423				remote, err := vcs.remoteRepo(vcs, dir)
   424				if err != nil {
   425					// Proceed anyway. The package is present; we likely just don't understand
   426					// the repo configuration (e.g. unusual remote protocol).
   427					blindRepo = true
   428				}
   429				repo = remote
   430				if !*getF && err == nil {
   431					if rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security); err == nil {
   432						repo := rr.Repo
   433						if rr.vcs.resolveRepo != nil {
   434							resolved, err := rr.vcs.resolveRepo(rr.vcs, dir, repo)
   435							if err == nil {
   436								repo = resolved
   437							}
   438						}
   439						if remote != repo && rr.IsCustom {
   440							return fmt.Errorf("%s is a custom import path for %s, but %s is checked out from %s", rr.Root, repo, dir, remote)
   441						}
   442					}
   443				}
   444			}
   445		} else {
   446			// Analyze the import path to determine the version control system,
   447			// repository, and the import path for the root of the repository.
   448			rr, err := RepoRootForImportPath(importPrefix, IgnoreMod, security)
   449			if err != nil {
   450				return err
   451			}
   452			vcs, repo, rootPath = rr.vcs, rr.Repo, rr.Root
   453		}
   454		if !blindRepo && !vcs.isSecure(repo) && !Insecure {
   455			return fmt.Errorf("cannot download, %v uses insecure protocol", repo)
   456		}
   457	
   458		if p.Internal.Build.SrcRoot == "" {
   459			// Package not found. Put in first directory of $GOPATH.
   460			list := filepath.SplitList(cfg.BuildContext.GOPATH)
   461			if len(list) == 0 {
   462				return fmt.Errorf("cannot download, $GOPATH not set. For more details see: 'go help gopath'")
   463			}
   464			// Guard against people setting GOPATH=$GOROOT.
   465			if filepath.Clean(list[0]) == filepath.Clean(cfg.GOROOT) {
   466				return fmt.Errorf("cannot download, $GOPATH must not be set to $GOROOT. For more details see: 'go help gopath'")
   467			}
   468			if _, err := os.Stat(filepath.Join(list[0], "src/cmd/go/alldocs.go")); err == nil {
   469				return fmt.Errorf("cannot download, %s is a GOROOT, not a GOPATH. For more details see: 'go help gopath'", list[0])
   470			}
   471			p.Internal.Build.Root = list[0]
   472			p.Internal.Build.SrcRoot = filepath.Join(list[0], "src")
   473			p.Internal.Build.PkgRoot = filepath.Join(list[0], "pkg")
   474		}
   475		root := filepath.Join(p.Internal.Build.SrcRoot, filepath.FromSlash(rootPath))
   476	
   477		if err := checkNestedVCS(vcs, root, p.Internal.Build.SrcRoot); err != nil {
   478			return err
   479		}
   480	
   481		// If we've considered this repository already, don't do it again.
   482		if downloadRootCache[root] {
   483			return nil
   484		}
   485		downloadRootCache[root] = true
   486	
   487		if cfg.BuildV {
   488			fmt.Fprintf(os.Stderr, "%s (download)\n", rootPath)
   489		}
   490	
   491		// Check that this is an appropriate place for the repo to be checked out.
   492		// The target directory must either not exist or have a repo checked out already.
   493		meta := filepath.Join(root, "."+vcs.cmd)
   494		if _, err := os.Stat(meta); err != nil {
   495			// Metadata file or directory does not exist. Prepare to checkout new copy.
   496			// Some version control tools require the target directory not to exist.
   497			// We require that too, just to avoid stepping on existing work.
   498			if _, err := os.Stat(root); err == nil {
   499				return fmt.Errorf("%s exists but %s does not - stale checkout?", root, meta)
   500			}
   501	
   502			_, err := os.Stat(p.Internal.Build.Root)
   503			gopathExisted := err == nil
   504	
   505			// Some version control tools require the parent of the target to exist.
   506			parent, _ := filepath.Split(root)
   507			if err = os.MkdirAll(parent, 0777); err != nil {
   508				return err
   509			}
   510			if cfg.BuildV && !gopathExisted && p.Internal.Build.Root == cfg.BuildContext.GOPATH {
   511				fmt.Fprintf(os.Stderr, "created GOPATH=%s; see 'go help gopath'\n", p.Internal.Build.Root)
   512			}
   513	
   514			if err = vcs.create(root, repo); err != nil {
   515				return err
   516			}
   517		} else {
   518			// Metadata directory does exist; download incremental updates.
   519			if err = vcs.download(root); err != nil {
   520				return err
   521			}
   522		}
   523	
   524		if cfg.BuildN {
   525			// Do not show tag sync in -n; it's noise more than anything,
   526			// and since we're not running commands, no tag will be found.
   527			// But avoid printing nothing.
   528			fmt.Fprintf(os.Stderr, "# cd %s; %s sync/update\n", root, vcs.cmd)
   529			return nil
   530		}
   531	
   532		// Select and sync to appropriate version of the repository.
   533		tags, err := vcs.tags(root)
   534		if err != nil {
   535			return err
   536		}
   537		vers := runtime.Version()
   538		if i := strings.Index(vers, " "); i >= 0 {
   539			vers = vers[:i]
   540		}
   541		if err := vcs.tagSync(root, selectTag(vers, tags)); err != nil {
   542			return err
   543		}
   544	
   545		return nil
   546	}
   547	
   548	// selectTag returns the closest matching tag for a given version.
   549	// Closest means the latest one that is not after the current release.
   550	// Version "goX" (or "goX.Y" or "goX.Y.Z") matches tags of the same form.
   551	// Version "release.rN" matches tags of the form "go.rN" (N being a floating-point number).
   552	// Version "weekly.YYYY-MM-DD" matches tags like "go.weekly.YYYY-MM-DD".
   553	//
   554	// NOTE(rsc): Eventually we will need to decide on some logic here.
   555	// For now, there is only "go1". This matches the docs in go help get.
   556	func selectTag(goVersion string, tags []string) (match string) {
   557		for _, t := range tags {
   558			if t == "go1" {
   559				return "go1"
   560			}
   561		}
   562		return ""
   563	}
   564	

View as plain text