Source file src/cmd/go/internal/modfetch/sumdb.go

     1	// Copyright 2019 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// Go checksum database lookup
     6	
     7	// +build !cmd_go_bootstrap
     8	
     9	package modfetch
    10	
    11	import (
    12		"bytes"
    13		"errors"
    14		"fmt"
    15		"io/ioutil"
    16		"net/url"
    17		"os"
    18		"path/filepath"
    19		"strings"
    20		"sync"
    21		"time"
    22	
    23		"cmd/go/internal/base"
    24		"cmd/go/internal/cfg"
    25		"cmd/go/internal/get"
    26		"cmd/go/internal/lockedfile"
    27		"cmd/go/internal/module"
    28		"cmd/go/internal/note"
    29		"cmd/go/internal/str"
    30		"cmd/go/internal/sumweb"
    31		"cmd/go/internal/web"
    32	)
    33	
    34	// useSumDB reports whether to use the Go checksum database for the given module.
    35	func useSumDB(mod module.Version) bool {
    36		return cfg.GOSUMDB != "off" && !get.Insecure && !str.GlobsMatchPath(cfg.GONOSUMDB, mod.Path)
    37	}
    38	
    39	// lookupSumDB returns the Go checksum database's go.sum lines for the given module,
    40	// along with the name of the database.
    41	func lookupSumDB(mod module.Version) (dbname string, lines []string, err error) {
    42		dbOnce.Do(func() {
    43			dbName, db, dbErr = dbDial()
    44		})
    45		if dbErr != nil {
    46			return "", nil, dbErr
    47		}
    48		lines, err = db.Lookup(mod.Path, mod.Version)
    49		return dbName, lines, err
    50	}
    51	
    52	var (
    53		dbOnce sync.Once
    54		dbName string
    55		db     *sumweb.Conn
    56		dbErr  error
    57	)
    58	
    59	func dbDial() (dbName string, db *sumweb.Conn, err error) {
    60		// $GOSUMDB can be "key" or "key url",
    61		// and the key can be a full verifier key
    62		// or a host on our list of known keys.
    63	
    64		// Special case: sum.golang.google.cn
    65		// is an alias, reachable inside mainland China,
    66		// for sum.golang.org. If there are more
    67		// of these we should add a map like knownGOSUMDB.
    68		gosumdb := cfg.GOSUMDB
    69		if gosumdb == "sum.golang.google.cn" {
    70			gosumdb = "sum.golang.org https://sum.golang.google.cn"
    71		}
    72	
    73		key := strings.Fields(gosumdb)
    74		if len(key) >= 1 {
    75			if k := knownGOSUMDB[key[0]]; k != "" {
    76				key[0] = k
    77			}
    78		}
    79		if len(key) == 0 {
    80			return "", nil, fmt.Errorf("missing GOSUMDB")
    81		}
    82		if len(key) > 2 {
    83			return "", nil, fmt.Errorf("invalid GOSUMDB: too many fields")
    84		}
    85		vkey, err := note.NewVerifier(key[0])
    86		if err != nil {
    87			return "", nil, fmt.Errorf("invalid GOSUMDB: %v", err)
    88		}
    89		name := vkey.Name()
    90	
    91		// No funny business in the database name.
    92		direct, err := url.Parse("https://" + name)
    93		if err != nil || strings.HasSuffix(name, "/") || *direct != (url.URL{Scheme: "https", Host: direct.Host, Path: direct.Path, RawPath: direct.RawPath}) || direct.RawPath != "" || direct.Host == "" {
    94			return "", nil, fmt.Errorf("invalid sumdb name (must be host[/path]): %s %+v", name, *direct)
    95		}
    96	
    97		// Determine how to get to database.
    98		var base *url.URL
    99		if len(key) >= 2 {
   100			// Use explicit alternate URL listed in $GOSUMDB,
   101			// bypassing both the default URL derivation and any proxies.
   102			u, err := url.Parse(key[1])
   103			if err != nil {
   104				return "", nil, fmt.Errorf("invalid GOSUMDB URL: %v", err)
   105			}
   106			base = u
   107		}
   108	
   109		return name, sumweb.NewConn(&dbClient{key: key[0], name: name, direct: direct, base: base}), nil
   110	}
   111	
   112	type dbClient struct {
   113		key    string
   114		name   string
   115		direct *url.URL
   116	
   117		once    sync.Once
   118		base    *url.URL
   119		baseErr error
   120	}
   121	
   122	func (c *dbClient) ReadRemote(path string) ([]byte, error) {
   123		c.once.Do(c.initBase)
   124		if c.baseErr != nil {
   125			return nil, c.baseErr
   126		}
   127	
   128		var data []byte
   129		start := time.Now()
   130		targ := web.Join(c.base, path)
   131		data, err := web.GetBytes(targ)
   132		if false {
   133			fmt.Fprintf(os.Stderr, "%.3fs %s\n", time.Since(start).Seconds(), web.Redacted(targ))
   134		}
   135		return data, err
   136	}
   137	
   138	// initBase determines the base URL for connecting to the database.
   139	// Determining the URL requires sending network traffic to proxies,
   140	// so this work is delayed until we need to download something from
   141	// the database. If everything we need is in the local cache and
   142	// c.ReadRemote is never called, we will never do this work.
   143	func (c *dbClient) initBase() {
   144		if c.base != nil {
   145			return
   146		}
   147	
   148		// Try proxies in turn until we find out how to connect to this database.
   149		urls, err := proxyURLs()
   150		if err != nil {
   151			c.baseErr = err
   152			return
   153		}
   154		for _, proxyURL := range urls {
   155			if proxyURL == "noproxy" {
   156				continue
   157			}
   158			if proxyURL == "direct" || proxyURL == "off" {
   159				break
   160			}
   161			proxy, err := url.Parse(proxyURL)
   162			if err != nil {
   163				c.baseErr = err
   164				return
   165			}
   166			// Quoting https://golang.org/design/25530-sumdb#proxying-a-checksum-database:
   167			//
   168			// Before accessing any checksum database URL using a proxy,
   169			// the proxy client should first fetch <proxyURL>/sumdb/<sumdb-name>/supported.
   170			// If that request returns a successful (HTTP 200) response, then the proxy supports
   171			// proxying checksum database requests. In that case, the client should use
   172			// the proxied access method only, never falling back to a direct connection to the database.
   173			// If the /sumdb/<sumdb-name>/supported check fails with a “not found” (HTTP 404)
   174			// or “gone” (HTTP 410) response, the proxy is unwilling to proxy the checksum database,
   175			// and the client should connect directly to the database.
   176			// Any other response is treated as the database being unavailable.
   177			_, err = web.GetBytes(web.Join(proxy, "sumdb/"+c.name+"/supported"))
   178			if err == nil {
   179				// Success! This proxy will help us.
   180				c.base = web.Join(proxy, "sumdb/"+c.name)
   181				return
   182			}
   183			// If the proxy serves a non-404/410, give up.
   184			if !errors.Is(err, os.ErrNotExist) {
   185				c.baseErr = err
   186				return
   187			}
   188		}
   189	
   190		// No proxies, or all proxies said 404, or we reached an explicit "direct".
   191		c.base = c.direct
   192	}
   193	
   194	// ReadConfig reads the key from c.key
   195	// and otherwise reads the config (a latest tree head) from GOPATH/pkg/sumdb/<file>.
   196	func (c *dbClient) ReadConfig(file string) (data []byte, err error) {
   197		if file == "key" {
   198			return []byte(c.key), nil
   199		}
   200	
   201		// GOPATH/pkg is PkgMod/..
   202		targ := filepath.Join(PkgMod, "../sumdb/"+file)
   203		data, err = lockedfile.Read(targ)
   204		if errors.Is(err, os.ErrNotExist) {
   205			// Treat non-existent as empty, to bootstrap the "latest" file
   206			// the first time we connect to a given database.
   207			return []byte{}, nil
   208		}
   209		return data, err
   210	}
   211	
   212	// WriteConfig rewrites the latest tree head.
   213	func (*dbClient) WriteConfig(file string, old, new []byte) error {
   214		if file == "key" {
   215			// Should not happen.
   216			return fmt.Errorf("cannot write key")
   217		}
   218		targ := filepath.Join(PkgMod, "../sumdb/"+file)
   219		os.MkdirAll(filepath.Dir(targ), 0777)
   220		f, err := lockedfile.Edit(targ)
   221		if err != nil {
   222			return err
   223		}
   224		defer f.Close()
   225		data, err := ioutil.ReadAll(f)
   226		if err != nil {
   227			return err
   228		}
   229		if len(data) > 0 && !bytes.Equal(data, old) {
   230			return sumweb.ErrWriteConflict
   231		}
   232		if _, err := f.Seek(0, 0); err != nil {
   233			return err
   234		}
   235		if err := f.Truncate(0); err != nil {
   236			return err
   237		}
   238		if _, err := f.Write(new); err != nil {
   239			return err
   240		}
   241		return f.Close()
   242	}
   243	
   244	// ReadCache reads cached lookups or tiles from
   245	// GOPATH/pkg/mod/cache/download/sumdb,
   246	// which will be deleted by "go clean -modcache".
   247	func (*dbClient) ReadCache(file string) ([]byte, error) {
   248		targ := filepath.Join(PkgMod, "cache/download/sumdb", file)
   249		data, err := lockedfile.Read(targ)
   250		// lockedfile.Write does not atomically create the file with contents.
   251		// There is a moment between file creation and locking the file for writing,
   252		// during which the empty file can be locked for reading.
   253		// Treat observing an empty file as file not found.
   254		if err == nil && len(data) == 0 {
   255			err = &os.PathError{Op: "read", Path: targ, Err: os.ErrNotExist}
   256		}
   257		return data, err
   258	}
   259	
   260	// WriteCache updates cached lookups or tiles.
   261	func (*dbClient) WriteCache(file string, data []byte) {
   262		targ := filepath.Join(PkgMod, "cache/download/sumdb", file)
   263		os.MkdirAll(filepath.Dir(targ), 0777)
   264		lockedfile.Write(targ, bytes.NewReader(data), 0666)
   265	}
   266	
   267	func (*dbClient) Log(msg string) {
   268		// nothing for now
   269	}
   270	
   271	func (*dbClient) SecurityError(msg string) {
   272		base.Fatalf("%s", msg)
   273	}
   274	

View as plain text