Source file src/pkg/cmd/go/internal/modfetch/cache.go

     1	// Copyright 2018 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package modfetch
     6	
     7	import (
     8		"bytes"
     9		"encoding/json"
    10		"fmt"
    11		"io"
    12		"io/ioutil"
    13		"os"
    14		"path/filepath"
    15		"strings"
    16	
    17		"cmd/go/internal/base"
    18		"cmd/go/internal/cfg"
    19		"cmd/go/internal/lockedfile"
    20		"cmd/go/internal/modfetch/codehost"
    21		"cmd/go/internal/module"
    22		"cmd/go/internal/par"
    23		"cmd/go/internal/renameio"
    24		"cmd/go/internal/semver"
    25	)
    26	
    27	var QuietLookup bool // do not print about lookups
    28	
    29	var PkgMod string // $GOPATH/pkg/mod; set by package modload
    30	
    31	func cacheDir(path string) (string, error) {
    32		if PkgMod == "" {
    33			return "", fmt.Errorf("internal error: modfetch.PkgMod not set")
    34		}
    35		enc, err := module.EncodePath(path)
    36		if err != nil {
    37			return "", err
    38		}
    39		return filepath.Join(PkgMod, "cache/download", enc, "/@v"), nil
    40	}
    41	
    42	func CachePath(m module.Version, suffix string) (string, error) {
    43		dir, err := cacheDir(m.Path)
    44		if err != nil {
    45			return "", err
    46		}
    47		if !semver.IsValid(m.Version) {
    48			return "", fmt.Errorf("non-semver module version %q", m.Version)
    49		}
    50		if module.CanonicalVersion(m.Version) != m.Version {
    51			return "", fmt.Errorf("non-canonical module version %q", m.Version)
    52		}
    53		encVer, err := module.EncodeVersion(m.Version)
    54		if err != nil {
    55			return "", err
    56		}
    57		return filepath.Join(dir, encVer+"."+suffix), nil
    58	}
    59	
    60	// DownloadDir returns the directory to which m should be downloaded.
    61	// Note that the directory may not yet exist.
    62	func DownloadDir(m module.Version) (string, error) {
    63		if PkgMod == "" {
    64			return "", fmt.Errorf("internal error: modfetch.PkgMod not set")
    65		}
    66		enc, err := module.EncodePath(m.Path)
    67		if err != nil {
    68			return "", err
    69		}
    70		if !semver.IsValid(m.Version) {
    71			return "", fmt.Errorf("non-semver module version %q", m.Version)
    72		}
    73		if module.CanonicalVersion(m.Version) != m.Version {
    74			return "", fmt.Errorf("non-canonical module version %q", m.Version)
    75		}
    76		encVer, err := module.EncodeVersion(m.Version)
    77		if err != nil {
    78			return "", err
    79		}
    80		return filepath.Join(PkgMod, enc+"@"+encVer), nil
    81	}
    82	
    83	// lockVersion locks a file within the module cache that guards the downloading
    84	// and extraction of the zipfile for the given module version.
    85	func lockVersion(mod module.Version) (unlock func(), err error) {
    86		path, err := CachePath(mod, "lock")
    87		if err != nil {
    88			return nil, err
    89		}
    90		if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
    91			return nil, err
    92		}
    93		return lockedfile.MutexAt(path).Lock()
    94	}
    95	
    96	// SideLock locks a file within the module cache that that guards edits to files
    97	// outside the cache, such as go.sum and go.mod files in the user's working
    98	// directory. It returns a function that must be called to unlock the file.
    99	func SideLock() (unlock func()) {
   100		if PkgMod == "" {
   101			base.Fatalf("go: internal error: modfetch.PkgMod not set")
   102		}
   103		path := filepath.Join(PkgMod, "cache", "lock")
   104		if err := os.MkdirAll(filepath.Dir(path), 0777); err != nil {
   105			base.Fatalf("go: failed to create cache directory %s: %v", filepath.Dir(path), err)
   106		}
   107		unlock, err := lockedfile.MutexAt(path).Lock()
   108		if err != nil {
   109			base.Fatalf("go: failed to lock file at %v", path)
   110		}
   111		return unlock
   112	}
   113	
   114	// A cachingRepo is a cache around an underlying Repo,
   115	// avoiding redundant calls to ModulePath, Versions, Stat, Latest, and GoMod (but not Zip).
   116	// It is also safe for simultaneous use by multiple goroutines
   117	// (so that it can be returned from Lookup multiple times).
   118	// It serializes calls to the underlying Repo.
   119	type cachingRepo struct {
   120		path  string
   121		cache par.Cache // cache for all operations
   122		r     Repo
   123	}
   124	
   125	func newCachingRepo(r Repo) *cachingRepo {
   126		return &cachingRepo{
   127			r:    r,
   128			path: r.ModulePath(),
   129		}
   130	}
   131	
   132	func (r *cachingRepo) ModulePath() string {
   133		return r.path
   134	}
   135	
   136	func (r *cachingRepo) Versions(prefix string) ([]string, error) {
   137		type cached struct {
   138			list []string
   139			err  error
   140		}
   141		c := r.cache.Do("versions:"+prefix, func() interface{} {
   142			list, err := r.r.Versions(prefix)
   143			return cached{list, err}
   144		}).(cached)
   145	
   146		if c.err != nil {
   147			return nil, c.err
   148		}
   149		return append([]string(nil), c.list...), nil
   150	}
   151	
   152	type cachedInfo struct {
   153		info *RevInfo
   154		err  error
   155	}
   156	
   157	func (r *cachingRepo) Stat(rev string) (*RevInfo, error) {
   158		c := r.cache.Do("stat:"+rev, func() interface{} {
   159			file, info, err := readDiskStat(r.path, rev)
   160			if err == nil {
   161				return cachedInfo{info, nil}
   162			}
   163	
   164			if !QuietLookup {
   165				fmt.Fprintf(os.Stderr, "go: finding %s %s\n", r.path, rev)
   166			}
   167			info, err = r.r.Stat(rev)
   168			if err == nil {
   169				// If we resolved, say, 1234abcde to v0.0.0-20180604122334-1234abcdef78,
   170				// then save the information under the proper version, for future use.
   171				if info.Version != rev {
   172					file, _ = CachePath(module.Version{Path: r.path, Version: info.Version}, "info")
   173					r.cache.Do("stat:"+info.Version, func() interface{} {
   174						return cachedInfo{info, err}
   175					})
   176				}
   177	
   178				if err := writeDiskStat(file, info); err != nil {
   179					fmt.Fprintf(os.Stderr, "go: writing stat cache: %v\n", err)
   180				}
   181			}
   182			return cachedInfo{info, err}
   183		}).(cachedInfo)
   184	
   185		if c.err != nil {
   186			return nil, c.err
   187		}
   188		info := *c.info
   189		return &info, nil
   190	}
   191	
   192	func (r *cachingRepo) Latest() (*RevInfo, error) {
   193		c := r.cache.Do("latest:", func() interface{} {
   194			if !QuietLookup {
   195				fmt.Fprintf(os.Stderr, "go: finding %s latest\n", r.path)
   196			}
   197			info, err := r.r.Latest()
   198	
   199			// Save info for likely future Stat call.
   200			if err == nil {
   201				r.cache.Do("stat:"+info.Version, func() interface{} {
   202					return cachedInfo{info, err}
   203				})
   204				if file, _, err := readDiskStat(r.path, info.Version); err != nil {
   205					writeDiskStat(file, info)
   206				}
   207			}
   208	
   209			return cachedInfo{info, err}
   210		}).(cachedInfo)
   211	
   212		if c.err != nil {
   213			return nil, c.err
   214		}
   215		info := *c.info
   216		return &info, nil
   217	}
   218	
   219	func (r *cachingRepo) GoMod(version string) ([]byte, error) {
   220		type cached struct {
   221			text []byte
   222			err  error
   223		}
   224		c := r.cache.Do("gomod:"+version, func() interface{} {
   225			file, text, err := readDiskGoMod(r.path, version)
   226			if err == nil {
   227				// Note: readDiskGoMod already called checkGoMod.
   228				return cached{text, nil}
   229			}
   230	
   231			text, err = r.r.GoMod(version)
   232			if err == nil {
   233				checkGoMod(r.path, version, text)
   234				if err := writeDiskGoMod(file, text); err != nil {
   235					fmt.Fprintf(os.Stderr, "go: writing go.mod cache: %v\n", err)
   236				}
   237			}
   238			return cached{text, err}
   239		}).(cached)
   240	
   241		if c.err != nil {
   242			return nil, c.err
   243		}
   244		return append([]byte(nil), c.text...), nil
   245	}
   246	
   247	func (r *cachingRepo) Zip(dst io.Writer, version string) error {
   248		return r.r.Zip(dst, version)
   249	}
   250	
   251	// Stat is like Lookup(path).Stat(rev) but avoids the
   252	// repository path resolution in Lookup if the result is
   253	// already cached on local disk.
   254	func Stat(proxy, path, rev string) (*RevInfo, error) {
   255		_, info, err := readDiskStat(path, rev)
   256		if err == nil {
   257			return info, nil
   258		}
   259		repo, err := Lookup(proxy, path)
   260		if err != nil {
   261			return nil, err
   262		}
   263		return repo.Stat(rev)
   264	}
   265	
   266	// InfoFile is like Stat but returns the name of the file containing
   267	// the cached information.
   268	func InfoFile(path, version string) (string, error) {
   269		if !semver.IsValid(version) {
   270			return "", fmt.Errorf("invalid version %q", version)
   271		}
   272	
   273		if file, _, err := readDiskStat(path, version); err == nil {
   274			return file, nil
   275		}
   276	
   277		err := TryProxies(func(proxy string) error {
   278			repo, err := Lookup(proxy, path)
   279			if err == nil {
   280				_, err = repo.Stat(version)
   281			}
   282			return err
   283		})
   284		if err != nil {
   285			return "", err
   286		}
   287	
   288		// Stat should have populated the disk cache for us.
   289		file, _, err := readDiskStat(path, version)
   290		if err != nil {
   291			return "", err
   292		}
   293		return file, nil
   294	}
   295	
   296	// GoMod is like Lookup(path).GoMod(rev) but avoids the
   297	// repository path resolution in Lookup if the result is
   298	// already cached on local disk.
   299	func GoMod(path, rev string) ([]byte, error) {
   300		// Convert commit hash to pseudo-version
   301		// to increase cache hit rate.
   302		if !semver.IsValid(rev) {
   303			if _, info, err := readDiskStat(path, rev); err == nil {
   304				rev = info.Version
   305			} else {
   306				err := TryProxies(func(proxy string) error {
   307					repo, err := Lookup(proxy, path)
   308					if err != nil {
   309						return err
   310					}
   311					info, err := repo.Stat(rev)
   312					if err == nil {
   313						rev = info.Version
   314					}
   315					return err
   316				})
   317				if err != nil {
   318					return nil, err
   319				}
   320			}
   321		}
   322	
   323		_, data, err := readDiskGoMod(path, rev)
   324		if err == nil {
   325			return data, nil
   326		}
   327	
   328		err = TryProxies(func(proxy string) error {
   329			repo, err := Lookup(proxy, path)
   330			if err == nil {
   331				data, err = repo.GoMod(rev)
   332			}
   333			return err
   334		})
   335		return data, err
   336	}
   337	
   338	// GoModFile is like GoMod but returns the name of the file containing
   339	// the cached information.
   340	func GoModFile(path, version string) (string, error) {
   341		if !semver.IsValid(version) {
   342			return "", fmt.Errorf("invalid version %q", version)
   343		}
   344		if _, err := GoMod(path, version); err != nil {
   345			return "", err
   346		}
   347		// GoMod should have populated the disk cache for us.
   348		file, _, err := readDiskGoMod(path, version)
   349		if err != nil {
   350			return "", err
   351		}
   352		return file, nil
   353	}
   354	
   355	// GoModSum returns the go.sum entry for the module version's go.mod file.
   356	// (That is, it returns the entry listed in go.sum as "path version/go.mod".)
   357	func GoModSum(path, version string) (string, error) {
   358		if !semver.IsValid(version) {
   359			return "", fmt.Errorf("invalid version %q", version)
   360		}
   361		data, err := GoMod(path, version)
   362		if err != nil {
   363			return "", err
   364		}
   365		sum, err := goModSum(data)
   366		if err != nil {
   367			return "", err
   368		}
   369		return sum, nil
   370	}
   371	
   372	var errNotCached = fmt.Errorf("not in cache")
   373	
   374	// readDiskStat reads a cached stat result from disk,
   375	// returning the name of the cache file and the result.
   376	// If the read fails, the caller can use
   377	// writeDiskStat(file, info) to write a new cache entry.
   378	func readDiskStat(path, rev string) (file string, info *RevInfo, err error) {
   379		file, data, err := readDiskCache(path, rev, "info")
   380		if err != nil {
   381			// If the cache already contains a pseudo-version with the given hash, we
   382			// would previously return that pseudo-version without checking upstream.
   383			// However, that produced an unfortunate side-effect: if the author added a
   384			// tag to the repository, 'go get' would not pick up the effect of that new
   385			// tag on the existing commits, and 'go' commands that referred to those
   386			// commits would use the previous name instead of the new one.
   387			//
   388			// That's especially problematic if the original pseudo-version starts with
   389			// v0.0.0-, as was the case for all pseudo-versions during vgo development,
   390			// since a v0.0.0- pseudo-version has lower precedence than pretty much any
   391			// tagged version.
   392			//
   393			// In practice, we're only looking up by hash during initial conversion of a
   394			// legacy config and during an explicit 'go get', and a little extra latency
   395			// for those operations seems worth the benefit of picking up more accurate
   396			// versions.
   397			//
   398			// Fall back to this resolution scheme only if the GOPROXY setting prohibits
   399			// us from resolving upstream tags.
   400			if cfg.GOPROXY == "off" {
   401				if file, info, err := readDiskStatByHash(path, rev); err == nil {
   402					return file, info, nil
   403				}
   404			}
   405			return file, nil, err
   406		}
   407		info = new(RevInfo)
   408		if err := json.Unmarshal(data, info); err != nil {
   409			return file, nil, errNotCached
   410		}
   411		// The disk might have stale .info files that have Name and Short fields set.
   412		// We want to canonicalize to .info files with those fields omitted.
   413		// Remarshal and update the cache file if needed.
   414		data2, err := json.Marshal(info)
   415		if err == nil && !bytes.Equal(data2, data) {
   416			writeDiskCache(file, data)
   417		}
   418		return file, info, nil
   419	}
   420	
   421	// readDiskStatByHash is a fallback for readDiskStat for the case
   422	// where rev is a commit hash instead of a proper semantic version.
   423	// In that case, we look for a cached pseudo-version that matches
   424	// the commit hash. If we find one, we use it.
   425	// This matters most for converting legacy package management
   426	// configs, when we are often looking up commits by full hash.
   427	// Without this check we'd be doing network I/O to the remote repo
   428	// just to find out about a commit we already know about
   429	// (and have cached under its pseudo-version).
   430	func readDiskStatByHash(path, rev string) (file string, info *RevInfo, err error) {
   431		if PkgMod == "" {
   432			// Do not download to current directory.
   433			return "", nil, errNotCached
   434		}
   435	
   436		if !codehost.AllHex(rev) || len(rev) < 12 {
   437			return "", nil, errNotCached
   438		}
   439		rev = rev[:12]
   440		cdir, err := cacheDir(path)
   441		if err != nil {
   442			return "", nil, errNotCached
   443		}
   444		dir, err := os.Open(cdir)
   445		if err != nil {
   446			return "", nil, errNotCached
   447		}
   448		names, err := dir.Readdirnames(-1)
   449		dir.Close()
   450		if err != nil {
   451			return "", nil, errNotCached
   452		}
   453	
   454		// A given commit hash may map to more than one pseudo-version,
   455		// depending on which tags are present on the repository.
   456		// Take the highest such version.
   457		var maxVersion string
   458		suffix := "-" + rev + ".info"
   459		err = errNotCached
   460		for _, name := range names {
   461			if strings.HasSuffix(name, suffix) {
   462				v := strings.TrimSuffix(name, ".info")
   463				if IsPseudoVersion(v) && semver.Max(maxVersion, v) == v {
   464					maxVersion = v
   465					file, info, err = readDiskStat(path, strings.TrimSuffix(name, ".info"))
   466				}
   467			}
   468		}
   469		return file, info, err
   470	}
   471	
   472	// oldVgoPrefix is the prefix in the old auto-generated cached go.mod files.
   473	// We stopped trying to auto-generate the go.mod files. Now we use a trivial
   474	// go.mod with only a module line, and we've dropped the version prefix
   475	// entirely. If we see a version prefix, that means we're looking at an old copy
   476	// and should ignore it.
   477	var oldVgoPrefix = []byte("//vgo 0.0.")
   478	
   479	// readDiskGoMod reads a cached go.mod file from disk,
   480	// returning the name of the cache file and the result.
   481	// If the read fails, the caller can use
   482	// writeDiskGoMod(file, data) to write a new cache entry.
   483	func readDiskGoMod(path, rev string) (file string, data []byte, err error) {
   484		file, data, err = readDiskCache(path, rev, "mod")
   485	
   486		// If the file has an old auto-conversion prefix, pretend it's not there.
   487		if bytes.HasPrefix(data, oldVgoPrefix) {
   488			err = errNotCached
   489			data = nil
   490		}
   491	
   492		if err == nil {
   493			checkGoMod(path, rev, data)
   494		}
   495	
   496		return file, data, err
   497	}
   498	
   499	// readDiskCache is the generic "read from a cache file" implementation.
   500	// It takes the revision and an identifying suffix for the kind of data being cached.
   501	// It returns the name of the cache file and the content of the file.
   502	// If the read fails, the caller can use
   503	// writeDiskCache(file, data) to write a new cache entry.
   504	func readDiskCache(path, rev, suffix string) (file string, data []byte, err error) {
   505		file, err = CachePath(module.Version{Path: path, Version: rev}, suffix)
   506		if err != nil {
   507			return "", nil, errNotCached
   508		}
   509		data, err = renameio.ReadFile(file)
   510		if err != nil {
   511			return file, nil, errNotCached
   512		}
   513		return file, data, nil
   514	}
   515	
   516	// writeDiskStat writes a stat result cache entry.
   517	// The file name must have been returned by a previous call to readDiskStat.
   518	func writeDiskStat(file string, info *RevInfo) error {
   519		if file == "" {
   520			return nil
   521		}
   522		js, err := json.Marshal(info)
   523		if err != nil {
   524			return err
   525		}
   526		return writeDiskCache(file, js)
   527	}
   528	
   529	// writeDiskGoMod writes a go.mod cache entry.
   530	// The file name must have been returned by a previous call to readDiskGoMod.
   531	func writeDiskGoMod(file string, text []byte) error {
   532		return writeDiskCache(file, text)
   533	}
   534	
   535	// writeDiskCache is the generic "write to a cache file" implementation.
   536	// The file must have been returned by a previous call to readDiskCache.
   537	func writeDiskCache(file string, data []byte) error {
   538		if file == "" {
   539			return nil
   540		}
   541		// Make sure directory for file exists.
   542		if err := os.MkdirAll(filepath.Dir(file), 0777); err != nil {
   543			return err
   544		}
   545	
   546		if err := renameio.WriteFile(file, data, 0666); err != nil {
   547			return err
   548		}
   549	
   550		if strings.HasSuffix(file, ".mod") {
   551			rewriteVersionList(filepath.Dir(file))
   552		}
   553		return nil
   554	}
   555	
   556	// rewriteVersionList rewrites the version list in dir
   557	// after a new *.mod file has been written.
   558	func rewriteVersionList(dir string) {
   559		if filepath.Base(dir) != "@v" {
   560			base.Fatalf("go: internal error: misuse of rewriteVersionList")
   561		}
   562	
   563		listFile := filepath.Join(dir, "list")
   564	
   565		// We use a separate lockfile here instead of locking listFile itself because
   566		// we want to use Rename to write the file atomically. The list may be read by
   567		// a GOPROXY HTTP server, and if we crash midway through a rewrite (or if the
   568		// HTTP server ignores our locking and serves the file midway through a
   569		// rewrite) it's better to serve a stale list than a truncated one.
   570		unlock, err := lockedfile.MutexAt(listFile + ".lock").Lock()
   571		if err != nil {
   572			base.Fatalf("go: can't lock version list lockfile: %v", err)
   573		}
   574		defer unlock()
   575	
   576		infos, err := ioutil.ReadDir(dir)
   577		if err != nil {
   578			return
   579		}
   580		var list []string
   581		for _, info := range infos {
   582			// We look for *.mod files on the theory that if we can't supply
   583			// the .mod file then there's no point in listing that version,
   584			// since it's unusable. (We can have *.info without *.mod.)
   585			// We don't require *.zip files on the theory that for code only
   586			// involved in module graph construction, many *.zip files
   587			// will never be requested.
   588			name := info.Name()
   589			if strings.HasSuffix(name, ".mod") {
   590				v := strings.TrimSuffix(name, ".mod")
   591				if v != "" && module.CanonicalVersion(v) == v {
   592					list = append(list, v)
   593				}
   594			}
   595		}
   596		SortVersions(list)
   597	
   598		var buf bytes.Buffer
   599		for _, v := range list {
   600			buf.WriteString(v)
   601			buf.WriteString("\n")
   602		}
   603		old, _ := renameio.ReadFile(listFile)
   604		if bytes.Equal(buf.Bytes(), old) {
   605			return
   606		}
   607	
   608		if err := renameio.WriteFile(listFile, buf.Bytes(), 0666); err != nil {
   609			base.Fatalf("go: failed to write version list: %v", err)
   610		}
   611	}
   612	

View as plain text