Source file src/pkg/cmd/go/internal/modfetch/codehost/vcs.go

     1	// Copyright 2018 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	package codehost
     6	
     7	import (
     8		"encoding/xml"
     9		"fmt"
    10		"internal/lazyregexp"
    11		"io"
    12		"io/ioutil"
    13		"os"
    14		"path/filepath"
    15		"sort"
    16		"strconv"
    17		"strings"
    18		"sync"
    19		"time"
    20	
    21		"cmd/go/internal/lockedfile"
    22		"cmd/go/internal/par"
    23		"cmd/go/internal/str"
    24	)
    25	
    26	// A VCSError indicates an error using a version control system.
    27	// The implication of a VCSError is that we know definitively where
    28	// to get the code, but we can't access it due to the error.
    29	// The caller should report this error instead of continuing to probe
    30	// other possible module paths.
    31	//
    32	// TODO(golang.org/issue/31730): See if we can invert this. (Return a
    33	// distinguished error for “repo not found” and treat everything else
    34	// as terminal.)
    35	type VCSError struct {
    36		Err error
    37	}
    38	
    39	func (e *VCSError) Error() string { return e.Err.Error() }
    40	
    41	func vcsErrorf(format string, a ...interface{}) error {
    42		return &VCSError{Err: fmt.Errorf(format, a...)}
    43	}
    44	
    45	func NewRepo(vcs, remote string) (Repo, error) {
    46		type key struct {
    47			vcs    string
    48			remote string
    49		}
    50		type cached struct {
    51			repo Repo
    52			err  error
    53		}
    54		c := vcsRepoCache.Do(key{vcs, remote}, func() interface{} {
    55			repo, err := newVCSRepo(vcs, remote)
    56			if err != nil {
    57				err = &VCSError{err}
    58			}
    59			return cached{repo, err}
    60		}).(cached)
    61	
    62		return c.repo, c.err
    63	}
    64	
    65	var vcsRepoCache par.Cache
    66	
    67	type vcsRepo struct {
    68		mu lockedfile.Mutex // protects all commands, so we don't have to decide which are safe on a per-VCS basis
    69	
    70		remote string
    71		cmd    *vcsCmd
    72		dir    string
    73	
    74		tagsOnce sync.Once
    75		tags     map[string]bool
    76	
    77		branchesOnce sync.Once
    78		branches     map[string]bool
    79	
    80		fetchOnce sync.Once
    81		fetchErr  error
    82	}
    83	
    84	func newVCSRepo(vcs, remote string) (Repo, error) {
    85		if vcs == "git" {
    86			return newGitRepo(remote, false)
    87		}
    88		cmd := vcsCmds[vcs]
    89		if cmd == nil {
    90			return nil, fmt.Errorf("unknown vcs: %s %s", vcs, remote)
    91		}
    92		if !strings.Contains(remote, "://") {
    93			return nil, fmt.Errorf("invalid vcs remote: %s %s", vcs, remote)
    94		}
    95	
    96		r := &vcsRepo{remote: remote, cmd: cmd}
    97		var err error
    98		r.dir, r.mu.Path, err = WorkDir(vcsWorkDirType+vcs, r.remote)
    99		if err != nil {
   100			return nil, err
   101		}
   102	
   103		if cmd.init == nil {
   104			return r, nil
   105		}
   106	
   107		unlock, err := r.mu.Lock()
   108		if err != nil {
   109			return nil, err
   110		}
   111		defer unlock()
   112	
   113		if _, err := os.Stat(filepath.Join(r.dir, "."+vcs)); err != nil {
   114			if _, err := Run(r.dir, cmd.init(r.remote)); err != nil {
   115				os.RemoveAll(r.dir)
   116				return nil, err
   117			}
   118		}
   119		return r, nil
   120	}
   121	
   122	const vcsWorkDirType = "vcs1."
   123	
   124	type vcsCmd struct {
   125		vcs           string                                            // vcs name "hg"
   126		init          func(remote string) []string                      // cmd to init repo to track remote
   127		tags          func(remote string) []string                      // cmd to list local tags
   128		tagRE         *lazyregexp.Regexp                                // regexp to extract tag names from output of tags cmd
   129		branches      func(remote string) []string                      // cmd to list local branches
   130		branchRE      *lazyregexp.Regexp                                // regexp to extract branch names from output of tags cmd
   131		badLocalRevRE *lazyregexp.Regexp                                // regexp of names that must not be served out of local cache without doing fetch first
   132		statLocal     func(rev, remote string) []string                 // cmd to stat local rev
   133		parseStat     func(rev, out string) (*RevInfo, error)           // cmd to parse output of statLocal
   134		fetch         []string                                          // cmd to fetch everything from remote
   135		latest        string                                            // name of latest commit on remote (tip, HEAD, etc)
   136		readFile      func(rev, file, remote string) []string           // cmd to read rev's file
   137		readZip       func(rev, subdir, remote, target string) []string // cmd to read rev's subdir as zip file
   138	}
   139	
   140	var re = lazyregexp.New
   141	
   142	var vcsCmds = map[string]*vcsCmd{
   143		"hg": {
   144			vcs: "hg",
   145			init: func(remote string) []string {
   146				return []string{"hg", "clone", "-U", "--", remote, "."}
   147			},
   148			tags: func(remote string) []string {
   149				return []string{"hg", "tags", "-q"}
   150			},
   151			tagRE: re(`(?m)^[^\n]+$`),
   152			branches: func(remote string) []string {
   153				return []string{"hg", "branches", "-c", "-q"}
   154			},
   155			branchRE:      re(`(?m)^[^\n]+$`),
   156			badLocalRevRE: re(`(?m)^(tip)$`),
   157			statLocal: func(rev, remote string) []string {
   158				return []string{"hg", "log", "-l1", "-r", rev, "--template", "{node} {date|hgdate} {tags}"}
   159			},
   160			parseStat: hgParseStat,
   161			fetch:     []string{"hg", "pull", "-f"},
   162			latest:    "tip",
   163			readFile: func(rev, file, remote string) []string {
   164				return []string{"hg", "cat", "-r", rev, file}
   165			},
   166			readZip: func(rev, subdir, remote, target string) []string {
   167				pattern := []string{}
   168				if subdir != "" {
   169					pattern = []string{"-I", subdir + "/**"}
   170				}
   171				return str.StringList("hg", "archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/", pattern, "--", target)
   172			},
   173		},
   174	
   175		"svn": {
   176			vcs:  "svn",
   177			init: nil, // no local checkout
   178			tags: func(remote string) []string {
   179				return []string{"svn", "list", "--", strings.TrimSuffix(remote, "/trunk") + "/tags"}
   180			},
   181			tagRE: re(`(?m)^(.*?)/?$`),
   182			statLocal: func(rev, remote string) []string {
   183				suffix := "@" + rev
   184				if rev == "latest" {
   185					suffix = ""
   186				}
   187				return []string{"svn", "log", "-l1", "--xml", "--", remote + suffix}
   188			},
   189			parseStat: svnParseStat,
   190			latest:    "latest",
   191			readFile: func(rev, file, remote string) []string {
   192				return []string{"svn", "cat", "--", remote + "/" + file + "@" + rev}
   193			},
   194			// TODO: zip
   195		},
   196	
   197		"bzr": {
   198			vcs: "bzr",
   199			init: func(remote string) []string {
   200				return []string{"bzr", "branch", "--use-existing-dir", "--", remote, "."}
   201			},
   202			fetch: []string{
   203				"bzr", "pull", "--overwrite-tags",
   204			},
   205			tags: func(remote string) []string {
   206				return []string{"bzr", "tags"}
   207			},
   208			tagRE:         re(`(?m)^\S+`),
   209			badLocalRevRE: re(`^revno:-`),
   210			statLocal: func(rev, remote string) []string {
   211				return []string{"bzr", "log", "-l1", "--long", "--show-ids", "-r", rev}
   212			},
   213			parseStat: bzrParseStat,
   214			latest:    "revno:-1",
   215			readFile: func(rev, file, remote string) []string {
   216				return []string{"bzr", "cat", "-r", rev, file}
   217			},
   218			readZip: func(rev, subdir, remote, target string) []string {
   219				extra := []string{}
   220				if subdir != "" {
   221					extra = []string{"./" + subdir}
   222				}
   223				return str.StringList("bzr", "export", "--format=zip", "-r", rev, "--root=prefix/", "--", target, extra)
   224			},
   225		},
   226	
   227		"fossil": {
   228			vcs: "fossil",
   229			init: func(remote string) []string {
   230				return []string{"fossil", "clone", "--", remote, ".fossil"}
   231			},
   232			fetch: []string{"fossil", "pull", "-R", ".fossil"},
   233			tags: func(remote string) []string {
   234				return []string{"fossil", "tag", "-R", ".fossil", "list"}
   235			},
   236			tagRE: re(`XXXTODO`),
   237			statLocal: func(rev, remote string) []string {
   238				return []string{"fossil", "info", "-R", ".fossil", rev}
   239			},
   240			parseStat: fossilParseStat,
   241			latest:    "trunk",
   242			readFile: func(rev, file, remote string) []string {
   243				return []string{"fossil", "cat", "-R", ".fossil", "-r", rev, file}
   244			},
   245			readZip: func(rev, subdir, remote, target string) []string {
   246				extra := []string{}
   247				if subdir != "" && !strings.ContainsAny(subdir, "*?[],") {
   248					extra = []string{"--include", subdir}
   249				}
   250				// Note that vcsRepo.ReadZip below rewrites this command
   251				// to run in a different directory, to work around a fossil bug.
   252				return str.StringList("fossil", "zip", "-R", ".fossil", "--name", "prefix", extra, "--", rev, target)
   253			},
   254		},
   255	}
   256	
   257	func (r *vcsRepo) loadTags() {
   258		out, err := Run(r.dir, r.cmd.tags(r.remote))
   259		if err != nil {
   260			return
   261		}
   262	
   263		// Run tag-listing command and extract tags.
   264		r.tags = make(map[string]bool)
   265		for _, tag := range r.cmd.tagRE.FindAllString(string(out), -1) {
   266			if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(tag) {
   267				continue
   268			}
   269			r.tags[tag] = true
   270		}
   271	}
   272	
   273	func (r *vcsRepo) loadBranches() {
   274		if r.cmd.branches == nil {
   275			return
   276		}
   277	
   278		out, err := Run(r.dir, r.cmd.branches(r.remote))
   279		if err != nil {
   280			return
   281		}
   282	
   283		r.branches = make(map[string]bool)
   284		for _, branch := range r.cmd.branchRE.FindAllString(string(out), -1) {
   285			if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(branch) {
   286				continue
   287			}
   288			r.branches[branch] = true
   289		}
   290	}
   291	
   292	func (r *vcsRepo) Tags(prefix string) ([]string, error) {
   293		unlock, err := r.mu.Lock()
   294		if err != nil {
   295			return nil, err
   296		}
   297		defer unlock()
   298	
   299		r.tagsOnce.Do(r.loadTags)
   300	
   301		tags := []string{}
   302		for tag := range r.tags {
   303			if strings.HasPrefix(tag, prefix) {
   304				tags = append(tags, tag)
   305			}
   306		}
   307		sort.Strings(tags)
   308		return tags, nil
   309	}
   310	
   311	func (r *vcsRepo) Stat(rev string) (*RevInfo, error) {
   312		unlock, err := r.mu.Lock()
   313		if err != nil {
   314			return nil, err
   315		}
   316		defer unlock()
   317	
   318		if rev == "latest" {
   319			rev = r.cmd.latest
   320		}
   321		r.branchesOnce.Do(r.loadBranches)
   322		revOK := (r.cmd.badLocalRevRE == nil || !r.cmd.badLocalRevRE.MatchString(rev)) && !r.branches[rev]
   323		if revOK {
   324			if info, err := r.statLocal(rev); err == nil {
   325				return info, nil
   326			}
   327		}
   328	
   329		r.fetchOnce.Do(r.fetch)
   330		if r.fetchErr != nil {
   331			return nil, r.fetchErr
   332		}
   333		info, err := r.statLocal(rev)
   334		if err != nil {
   335			return nil, err
   336		}
   337		if !revOK {
   338			info.Version = info.Name
   339		}
   340		return info, nil
   341	}
   342	
   343	func (r *vcsRepo) fetch() {
   344		if len(r.cmd.fetch) > 0 {
   345			_, r.fetchErr = Run(r.dir, r.cmd.fetch)
   346		}
   347	}
   348	
   349	func (r *vcsRepo) statLocal(rev string) (*RevInfo, error) {
   350		out, err := Run(r.dir, r.cmd.statLocal(rev, r.remote))
   351		if err != nil {
   352			return nil, &UnknownRevisionError{Rev: rev}
   353		}
   354		return r.cmd.parseStat(rev, string(out))
   355	}
   356	
   357	func (r *vcsRepo) Latest() (*RevInfo, error) {
   358		return r.Stat("latest")
   359	}
   360	
   361	func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
   362		if rev == "latest" {
   363			rev = r.cmd.latest
   364		}
   365		_, err := r.Stat(rev) // download rev into local repo
   366		if err != nil {
   367			return nil, err
   368		}
   369	
   370		// r.Stat acquires r.mu, so lock after that.
   371		unlock, err := r.mu.Lock()
   372		if err != nil {
   373			return nil, err
   374		}
   375		defer unlock()
   376	
   377		out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote))
   378		if err != nil {
   379			return nil, os.ErrNotExist
   380		}
   381		return out, nil
   382	}
   383	
   384	func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[string]*FileRev, error) {
   385		// We don't technically need to lock here since we're returning an error
   386		// uncondititonally, but doing so anyway will help to avoid baking in
   387		// lock-inversion bugs.
   388		unlock, err := r.mu.Lock()
   389		if err != nil {
   390			return nil, err
   391		}
   392		defer unlock()
   393	
   394		return nil, vcsErrorf("ReadFileRevs not implemented")
   395	}
   396	
   397	func (r *vcsRepo) RecentTag(rev, prefix, major string) (tag string, err error) {
   398		// We don't technically need to lock here since we're returning an error
   399		// uncondititonally, but doing so anyway will help to avoid baking in
   400		// lock-inversion bugs.
   401		unlock, err := r.mu.Lock()
   402		if err != nil {
   403			return "", err
   404		}
   405		defer unlock()
   406	
   407		return "", vcsErrorf("RecentTag not implemented")
   408	}
   409	
   410	func (r *vcsRepo) DescendsFrom(rev, tag string) (bool, error) {
   411		unlock, err := r.mu.Lock()
   412		if err != nil {
   413			return false, err
   414		}
   415		defer unlock()
   416	
   417		return false, vcsErrorf("DescendsFrom not implemented")
   418	}
   419	
   420	func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) {
   421		if r.cmd.readZip == nil {
   422			return nil, "", vcsErrorf("ReadZip not implemented for %s", r.cmd.vcs)
   423		}
   424	
   425		unlock, err := r.mu.Lock()
   426		if err != nil {
   427			return nil, "", err
   428		}
   429		defer unlock()
   430	
   431		if rev == "latest" {
   432			rev = r.cmd.latest
   433		}
   434		f, err := ioutil.TempFile("", "go-readzip-*.zip")
   435		if err != nil {
   436			return nil, "", err
   437		}
   438		if r.cmd.vcs == "fossil" {
   439			// If you run
   440			//	fossil zip -R .fossil --name prefix trunk /tmp/x.zip
   441			// fossil fails with "unable to create directory /tmp" [sic].
   442			// Change the command to run in /tmp instead,
   443			// replacing the -R argument with an absolute path.
   444			args := r.cmd.readZip(rev, subdir, r.remote, filepath.Base(f.Name()))
   445			for i := range args {
   446				if args[i] == ".fossil" {
   447					args[i] = filepath.Join(r.dir, ".fossil")
   448				}
   449			}
   450			_, err = Run(filepath.Dir(f.Name()), args)
   451		} else {
   452			_, err = Run(r.dir, r.cmd.readZip(rev, subdir, r.remote, f.Name()))
   453		}
   454		if err != nil {
   455			f.Close()
   456			os.Remove(f.Name())
   457			return nil, "", err
   458		}
   459		return &deleteCloser{f}, "", nil
   460	}
   461	
   462	// deleteCloser is a file that gets deleted on Close.
   463	type deleteCloser struct {
   464		*os.File
   465	}
   466	
   467	func (d *deleteCloser) Close() error {
   468		defer os.Remove(d.File.Name())
   469		return d.File.Close()
   470	}
   471	
   472	func hgParseStat(rev, out string) (*RevInfo, error) {
   473		f := strings.Fields(string(out))
   474		if len(f) < 3 {
   475			return nil, vcsErrorf("unexpected response from hg log: %q", out)
   476		}
   477		hash := f[0]
   478		version := rev
   479		if strings.HasPrefix(hash, version) {
   480			version = hash // extend to full hash
   481		}
   482		t, err := strconv.ParseInt(f[1], 10, 64)
   483		if err != nil {
   484			return nil, vcsErrorf("invalid time from hg log: %q", out)
   485		}
   486	
   487		var tags []string
   488		for _, tag := range f[3:] {
   489			if tag != "tip" {
   490				tags = append(tags, tag)
   491			}
   492		}
   493		sort.Strings(tags)
   494	
   495		info := &RevInfo{
   496			Name:    hash,
   497			Short:   ShortenSHA1(hash),
   498			Time:    time.Unix(t, 0).UTC(),
   499			Version: version,
   500			Tags:    tags,
   501		}
   502		return info, nil
   503	}
   504	
   505	func svnParseStat(rev, out string) (*RevInfo, error) {
   506		var log struct {
   507			Logentry struct {
   508				Revision int64  `xml:"revision,attr"`
   509				Date     string `xml:"date"`
   510			} `xml:"logentry"`
   511		}
   512		if err := xml.Unmarshal([]byte(out), &log); err != nil {
   513			return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out)
   514		}
   515	
   516		t, err := time.Parse(time.RFC3339, log.Logentry.Date)
   517		if err != nil {
   518			return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out)
   519		}
   520	
   521		info := &RevInfo{
   522			Name:    fmt.Sprintf("%d", log.Logentry.Revision),
   523			Short:   fmt.Sprintf("%012d", log.Logentry.Revision),
   524			Time:    t.UTC(),
   525			Version: rev,
   526		}
   527		return info, nil
   528	}
   529	
   530	func bzrParseStat(rev, out string) (*RevInfo, error) {
   531		var revno int64
   532		var tm time.Time
   533		for _, line := range strings.Split(out, "\n") {
   534			if line == "" || line[0] == ' ' || line[0] == '\t' {
   535				// End of header, start of commit message.
   536				break
   537			}
   538			if line[0] == '-' {
   539				continue
   540			}
   541			i := strings.Index(line, ":")
   542			if i < 0 {
   543				// End of header, start of commit message.
   544				break
   545			}
   546			key, val := line[:i], strings.TrimSpace(line[i+1:])
   547			switch key {
   548			case "revno":
   549				if j := strings.Index(val, " "); j >= 0 {
   550					val = val[:j]
   551				}
   552				i, err := strconv.ParseInt(val, 10, 64)
   553				if err != nil {
   554					return nil, vcsErrorf("unexpected revno from bzr log: %q", line)
   555				}
   556				revno = i
   557			case "timestamp":
   558				j := strings.Index(val, " ")
   559				if j < 0 {
   560					return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
   561				}
   562				t, err := time.Parse("2006-01-02 15:04:05 -0700", val[j+1:])
   563				if err != nil {
   564					return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
   565				}
   566				tm = t.UTC()
   567			}
   568		}
   569		if revno == 0 || tm.IsZero() {
   570			return nil, vcsErrorf("unexpected response from bzr log: %q", out)
   571		}
   572	
   573		info := &RevInfo{
   574			Name:    fmt.Sprintf("%d", revno),
   575			Short:   fmt.Sprintf("%012d", revno),
   576			Time:    tm,
   577			Version: rev,
   578		}
   579		return info, nil
   580	}
   581	
   582	func fossilParseStat(rev, out string) (*RevInfo, error) {
   583		for _, line := range strings.Split(out, "\n") {
   584			if strings.HasPrefix(line, "uuid:") {
   585				f := strings.Fields(line)
   586				if len(f) != 5 || len(f[1]) != 40 || f[4] != "UTC" {
   587					return nil, vcsErrorf("unexpected response from fossil info: %q", line)
   588				}
   589				t, err := time.Parse("2006-01-02 15:04:05", f[2]+" "+f[3])
   590				if err != nil {
   591					return nil, vcsErrorf("unexpected response from fossil info: %q", line)
   592				}
   593				hash := f[1]
   594				version := rev
   595				if strings.HasPrefix(hash, version) {
   596					version = hash // extend to full hash
   597				}
   598				info := &RevInfo{
   599					Name:    hash,
   600					Short:   ShortenSHA1(hash),
   601					Time:    t,
   602					Version: version,
   603				}
   604				return info, nil
   605			}
   606		}
   607		return nil, vcsErrorf("unexpected response from fossil info: %q", out)
   608	}
   609	

View as plain text