Source file src/pkg/cmd/go/internal/modfetch/codehost/codehost.go

     1	// Copyright 2018 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Package codehost defines the interface implemented by a code hosting source,
     6	// along with support code for use by implementations.
     7	package codehost
     8	
     9	import (
    10		"bytes"
    11		"crypto/sha256"
    12		"fmt"
    13		"io"
    14		"io/ioutil"
    15		"os"
    16		"os/exec"
    17		"path/filepath"
    18		"strings"
    19		"sync"
    20		"time"
    21	
    22		"cmd/go/internal/cfg"
    23		"cmd/go/internal/lockedfile"
    24		"cmd/go/internal/str"
    25	)
    26	
    27	// Downloaded size limits.
    28	const (
    29		MaxGoMod   = 16 << 20  // maximum size of go.mod file
    30		MaxLICENSE = 16 << 20  // maximum size of LICENSE file
    31		MaxZipFile = 500 << 20 // maximum size of downloaded zip file
    32	)
    33	
    34	// A Repo represents a code hosting source.
    35	// Typical implementations include local version control repositories,
    36	// remote version control servers, and code hosting sites.
    37	// A Repo must be safe for simultaneous use by multiple goroutines.
    38	type Repo interface {
    39		// List lists all tags with the given prefix.
    40		Tags(prefix string) (tags []string, err error)
    41	
    42		// Stat returns information about the revision rev.
    43		// A revision can be any identifier known to the underlying service:
    44		// commit hash, branch, tag, and so on.
    45		Stat(rev string) (*RevInfo, error)
    46	
    47		// Latest returns the latest revision on the default branch,
    48		// whatever that means in the underlying implementation.
    49		Latest() (*RevInfo, error)
    50	
    51		// ReadFile reads the given file in the file tree corresponding to revision rev.
    52		// It should refuse to read more than maxSize bytes.
    53		//
    54		// If the requested file does not exist it should return an error for which
    55		// os.IsNotExist(err) returns true.
    56		ReadFile(rev, file string, maxSize int64) (data []byte, err error)
    57	
    58		// ReadFileRevs reads a single file at multiple versions.
    59		// It should refuse to read more than maxSize bytes.
    60		// The result is a map from each requested rev strings
    61		// to the associated FileRev. The map must have a non-nil
    62		// entry for every requested rev (unless ReadFileRevs returned an error).
    63		// A file simply being missing or even corrupted in revs[i]
    64		// should be reported only in files[revs[i]].Err, not in the error result
    65		// from ReadFileRevs.
    66		// The overall call should return an error (and no map) only
    67		// in the case of a problem with obtaining the data, such as
    68		// a network failure.
    69		// Implementations may assume that revs only contain tags,
    70		// not direct commit hashes.
    71		ReadFileRevs(revs []string, file string, maxSize int64) (files map[string]*FileRev, err error)
    72	
    73		// ReadZip downloads a zip file for the subdir subdirectory
    74		// of the given revision to a new file in a given temporary directory.
    75		// It should refuse to read more than maxSize bytes.
    76		// It returns a ReadCloser for a streamed copy of the zip file,
    77		// along with the actual subdirectory (possibly shorter than subdir)
    78		// contained in the zip file. All files in the zip file are expected to be
    79		// nested in a single top-level directory, whose name is not specified.
    80		ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error)
    81	
    82		// RecentTag returns the most recent tag on rev or one of its predecessors
    83		// with the given prefix and major version.
    84		// An empty major string matches any major version.
    85		RecentTag(rev, prefix, major string) (tag string, err error)
    86	
    87		// DescendsFrom reports whether rev or any of its ancestors has the given tag.
    88		//
    89		// DescendsFrom must return true for any tag returned by RecentTag for the
    90		// same revision.
    91		DescendsFrom(rev, tag string) (bool, error)
    92	}
    93	
    94	// A Rev describes a single revision in a source code repository.
    95	type RevInfo struct {
    96		Name    string    // complete ID in underlying repository
    97		Short   string    // shortened ID, for use in pseudo-version
    98		Version string    // version used in lookup
    99		Time    time.Time // commit time
   100		Tags    []string  // known tags for commit
   101	}
   102	
   103	// A FileRev describes the result of reading a file at a given revision.
   104	type FileRev struct {
   105		Rev  string // requested revision
   106		Data []byte // file data
   107		Err  error  // error if any; os.IsNotExist(Err)==true if rev exists but file does not exist in that rev
   108	}
   109	
   110	// UnknownRevisionError is an error equivalent to os.ErrNotExist, but for a
   111	// revision rather than a file.
   112	type UnknownRevisionError struct {
   113		Rev string
   114	}
   115	
   116	func (e *UnknownRevisionError) Error() string {
   117		return "unknown revision " + e.Rev
   118	}
   119	func (UnknownRevisionError) Is(err error) bool {
   120		return err == os.ErrNotExist
   121	}
   122	
   123	// ErrNoCommits is an error equivalent to os.ErrNotExist indicating that a given
   124	// repository or module contains no commits.
   125	var ErrNoCommits error = noCommitsError{}
   126	
   127	type noCommitsError struct{}
   128	
   129	func (noCommitsError) Error() string {
   130		return "no commits"
   131	}
   132	func (noCommitsError) Is(err error) bool {
   133		return err == os.ErrNotExist
   134	}
   135	
   136	// AllHex reports whether the revision rev is entirely lower-case hexadecimal digits.
   137	func AllHex(rev string) bool {
   138		for i := 0; i < len(rev); i++ {
   139			c := rev[i]
   140			if '0' <= c && c <= '9' || 'a' <= c && c <= 'f' {
   141				continue
   142			}
   143			return false
   144		}
   145		return true
   146	}
   147	
   148	// ShortenSHA1 shortens a SHA1 hash (40 hex digits) to the canonical length
   149	// used in pseudo-versions (12 hex digits).
   150	func ShortenSHA1(rev string) string {
   151		if AllHex(rev) && len(rev) == 40 {
   152			return rev[:12]
   153		}
   154		return rev
   155	}
   156	
   157	// WorkRoot is the root of the cached work directory.
   158	// It is set by cmd/go/internal/modload.InitMod.
   159	var WorkRoot string
   160	
   161	// WorkDir returns the name of the cached work directory to use for the
   162	// given repository type and name.
   163	func WorkDir(typ, name string) (dir, lockfile string, err error) {
   164		if WorkRoot == "" {
   165			return "", "", fmt.Errorf("codehost.WorkRoot not set")
   166		}
   167	
   168		// We name the work directory for the SHA256 hash of the type and name.
   169		// We intentionally avoid the actual name both because of possible
   170		// conflicts with valid file system paths and because we want to ensure
   171		// that one checkout is never nested inside another. That nesting has
   172		// led to security problems in the past.
   173		if strings.Contains(typ, ":") {
   174			return "", "", fmt.Errorf("codehost.WorkDir: type cannot contain colon")
   175		}
   176		key := typ + ":" + name
   177		dir = filepath.Join(WorkRoot, fmt.Sprintf("%x", sha256.Sum256([]byte(key))))
   178	
   179		if cfg.BuildX {
   180			fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", filepath.Dir(dir), typ, name)
   181		}
   182		if err := os.MkdirAll(filepath.Dir(dir), 0777); err != nil {
   183			return "", "", err
   184		}
   185	
   186		lockfile = dir + ".lock"
   187		if cfg.BuildX {
   188			fmt.Fprintf(os.Stderr, "# lock %s", lockfile)
   189		}
   190	
   191		unlock, err := lockedfile.MutexAt(lockfile).Lock()
   192		if err != nil {
   193			return "", "", fmt.Errorf("codehost.WorkDir: can't find or create lock file: %v", err)
   194		}
   195		defer unlock()
   196	
   197		data, err := ioutil.ReadFile(dir + ".info")
   198		info, err2 := os.Stat(dir)
   199		if err == nil && err2 == nil && info.IsDir() {
   200			// Info file and directory both already exist: reuse.
   201			have := strings.TrimSuffix(string(data), "\n")
   202			if have != key {
   203				return "", "", fmt.Errorf("%s exists with wrong content (have %q want %q)", dir+".info", have, key)
   204			}
   205			if cfg.BuildX {
   206				fmt.Fprintf(os.Stderr, "# %s for %s %s\n", dir, typ, name)
   207			}
   208			return dir, lockfile, nil
   209		}
   210	
   211		// Info file or directory missing. Start from scratch.
   212		if cfg.BuildX {
   213			fmt.Fprintf(os.Stderr, "mkdir -p %s # %s %s\n", dir, typ, name)
   214		}
   215		os.RemoveAll(dir)
   216		if err := os.MkdirAll(dir, 0777); err != nil {
   217			return "", "", err
   218		}
   219		if err := ioutil.WriteFile(dir+".info", []byte(key), 0666); err != nil {
   220			os.RemoveAll(dir)
   221			return "", "", err
   222		}
   223		return dir, lockfile, nil
   224	}
   225	
   226	type RunError struct {
   227		Cmd      string
   228		Err      error
   229		Stderr   []byte
   230		HelpText string
   231	}
   232	
   233	func (e *RunError) Error() string {
   234		text := e.Cmd + ": " + e.Err.Error()
   235		stderr := bytes.TrimRight(e.Stderr, "\n")
   236		if len(stderr) > 0 {
   237			text += ":\n\t" + strings.ReplaceAll(string(stderr), "\n", "\n\t")
   238		}
   239		if len(e.HelpText) > 0 {
   240			text += "\n" + e.HelpText
   241		}
   242		return text
   243	}
   244	
   245	var dirLock sync.Map
   246	
   247	// Run runs the command line in the given directory
   248	// (an empty dir means the current directory).
   249	// It returns the standard output and, for a non-zero exit,
   250	// a *RunError indicating the command, exit status, and standard error.
   251	// Standard error is unavailable for commands that exit successfully.
   252	func Run(dir string, cmdline ...interface{}) ([]byte, error) {
   253		return RunWithStdin(dir, nil, cmdline...)
   254	}
   255	
   256	// bashQuoter escapes characters that have special meaning in double-quoted strings in the bash shell.
   257	// See https://www.gnu.org/software/bash/manual/html_node/Double-Quotes.html.
   258	var bashQuoter = strings.NewReplacer(`"`, `\"`, `$`, `\$`, "`", "\\`", `\`, `\\`)
   259	
   260	func RunWithStdin(dir string, stdin io.Reader, cmdline ...interface{}) ([]byte, error) {
   261		if dir != "" {
   262			muIface, ok := dirLock.Load(dir)
   263			if !ok {
   264				muIface, _ = dirLock.LoadOrStore(dir, new(sync.Mutex))
   265			}
   266			mu := muIface.(*sync.Mutex)
   267			mu.Lock()
   268			defer mu.Unlock()
   269		}
   270	
   271		cmd := str.StringList(cmdline...)
   272		if cfg.BuildX {
   273			text := new(strings.Builder)
   274			if dir != "" {
   275				text.WriteString("cd ")
   276				text.WriteString(dir)
   277				text.WriteString("; ")
   278			}
   279			for i, arg := range cmd {
   280				if i > 0 {
   281					text.WriteByte(' ')
   282				}
   283				switch {
   284				case strings.ContainsAny(arg, "'"):
   285					// Quote args that could be mistaken for quoted args.
   286					text.WriteByte('"')
   287					text.WriteString(bashQuoter.Replace(arg))
   288					text.WriteByte('"')
   289				case strings.ContainsAny(arg, "$`\\*?[\"\t\n\v\f\r \u0085\u00a0"):
   290					// Quote args that contain special characters, glob patterns, or spaces.
   291					text.WriteByte('\'')
   292					text.WriteString(arg)
   293					text.WriteByte('\'')
   294				default:
   295					text.WriteString(arg)
   296				}
   297			}
   298			fmt.Fprintf(os.Stderr, "%s\n", text)
   299			start := time.Now()
   300			defer func() {
   301				fmt.Fprintf(os.Stderr, "%.3fs # %s\n", time.Since(start).Seconds(), text)
   302			}()
   303		}
   304		// TODO: Impose limits on command output size.
   305		// TODO: Set environment to get English error messages.
   306		var stderr bytes.Buffer
   307		var stdout bytes.Buffer
   308		c := exec.Command(cmd[0], cmd[1:]...)
   309		c.Dir = dir
   310		c.Stdin = stdin
   311		c.Stderr = &stderr
   312		c.Stdout = &stdout
   313		err := c.Run()
   314		if err != nil {
   315			err = &RunError{Cmd: strings.Join(cmd, " ") + " in " + dir, Stderr: stderr.Bytes(), Err: err}
   316		}
   317		return stdout.Bytes(), err
   318	}
   319	

View as plain text