Source file src/pkg/cmd/go/internal/modfetch/proxy.go
1
2
3
4
5 package modfetch
6
7 import (
8 "encoding/json"
9 "errors"
10 "fmt"
11 "io"
12 "io/ioutil"
13 "net/url"
14 "os"
15 "path"
16 pathpkg "path"
17 "path/filepath"
18 "strings"
19 "sync"
20 "time"
21
22 "cmd/go/internal/base"
23 "cmd/go/internal/cfg"
24 "cmd/go/internal/modfetch/codehost"
25 "cmd/go/internal/module"
26 "cmd/go/internal/semver"
27 "cmd/go/internal/web"
28 )
29
30 var HelpGoproxy = &base.Command{
31 UsageLine: "goproxy",
32 Short: "module proxy protocol",
33 Long: `
34 A Go module proxy is any web server that can respond to GET requests for
35 URLs of a specified form. The requests have no query parameters, so even
36 a site serving from a fixed file system (including a file:/// URL)
37 can be a module proxy.
38
39 The GET requests sent to a Go module proxy are:
40
41 GET $GOPROXY/<module>/@v/list returns a list of all known versions of the
42 given module, one per line.
43
44 GET $GOPROXY/<module>/@v/<version>.info returns JSON-formatted metadata
45 about that version of the given module.
46
47 GET $GOPROXY/<module>/@v/<version>.mod returns the go.mod file
48 for that version of the given module.
49
50 GET $GOPROXY/<module>/@v/<version>.zip returns the zip archive
51 for that version of the given module.
52
53 To avoid problems when serving from case-sensitive file systems,
54 the <module> and <version> elements are case-encoded, replacing every
55 uppercase letter with an exclamation mark followed by the corresponding
56 lower-case letter: github.com/Azure encodes as github.com/!azure.
57
58 The JSON-formatted metadata about a given module corresponds to
59 this Go data structure, which may be expanded in the future:
60
61 type Info struct {
62 Version string // version string
63 Time time.Time // commit time
64 }
65
66 The zip archive for a specific version of a given module is a
67 standard zip file that contains the file tree corresponding
68 to the module's source code and related files. The archive uses
69 slash-separated paths, and every file path in the archive must
70 begin with <module>@<version>/, where the module and version are
71 substituted directly, not case-encoded. The root of the module
72 file tree corresponds to the <module>@<version>/ prefix in the
73 archive.
74
75 Even when downloading directly from version control systems,
76 the go command synthesizes explicit info, mod, and zip files
77 and stores them in its local cache, $GOPATH/pkg/mod/cache/download,
78 the same as if it had downloaded them directly from a proxy.
79 The cache layout is the same as the proxy URL space, so
80 serving $GOPATH/pkg/mod/cache/download at (or copying it to)
81 https://example.com/proxy would let other users access those
82 cached module versions with GOPROXY=https://example.com/proxy.
83 `,
84 }
85
86 var proxyOnce struct {
87 sync.Once
88 list []string
89 err error
90 }
91
92 func proxyURLs() ([]string, error) {
93 proxyOnce.Do(func() {
94 if cfg.GONOPROXY != "" && cfg.GOPROXY != "direct" {
95 proxyOnce.list = append(proxyOnce.list, "noproxy")
96 }
97 for _, proxyURL := range strings.Split(cfg.GOPROXY, ",") {
98 proxyURL = strings.TrimSpace(proxyURL)
99 if proxyURL == "" {
100 continue
101 }
102 if proxyURL == "off" {
103
104 proxyOnce.list = append(proxyOnce.list, "off")
105 break
106 }
107 if proxyURL == "direct" {
108 proxyOnce.list = append(proxyOnce.list, "direct")
109
110
111
112 break
113 }
114
115
116
117
118 if strings.ContainsAny(proxyURL, ".:/") && !strings.Contains(proxyURL, ":/") && !filepath.IsAbs(proxyURL) && !path.IsAbs(proxyURL) {
119 proxyURL = "https://" + proxyURL
120 }
121
122
123
124 _, err := newProxyRepo(proxyURL, "golang.org/x/text")
125 if err != nil {
126 proxyOnce.err = err
127 return
128 }
129 proxyOnce.list = append(proxyOnce.list, proxyURL)
130 }
131 })
132
133 return proxyOnce.list, proxyOnce.err
134 }
135
136
137
138
139
140
141
142
143
144 func TryProxies(f func(proxy string) error) error {
145 proxies, err := proxyURLs()
146 if err != nil {
147 return err
148 }
149 if len(proxies) == 0 {
150 return f("off")
151 }
152
153 for _, proxy := range proxies {
154 err = f(proxy)
155 if !errors.Is(err, os.ErrNotExist) {
156 break
157 }
158 }
159 return err
160 }
161
162 type proxyRepo struct {
163 url *url.URL
164 path string
165 }
166
167 func newProxyRepo(baseURL, path string) (Repo, error) {
168 base, err := url.Parse(baseURL)
169 if err != nil {
170 return nil, err
171 }
172 switch base.Scheme {
173 case "http", "https":
174
175 case "file":
176 if *base != (url.URL{Scheme: base.Scheme, Path: base.Path, RawPath: base.RawPath}) {
177 return nil, fmt.Errorf("invalid file:// proxy URL with non-path elements: %s", web.Redacted(base))
178 }
179 case "":
180 return nil, fmt.Errorf("invalid proxy URL missing scheme: %s", web.Redacted(base))
181 default:
182 return nil, fmt.Errorf("invalid proxy URL scheme (must be https, http, file): %s", web.Redacted(base))
183 }
184
185 enc, err := module.EncodePath(path)
186 if err != nil {
187 return nil, err
188 }
189
190 base.Path = strings.TrimSuffix(base.Path, "/") + "/" + enc
191 base.RawPath = strings.TrimSuffix(base.RawPath, "/") + "/" + pathEscape(enc)
192 return &proxyRepo{base, path}, nil
193 }
194
195 func (p *proxyRepo) ModulePath() string {
196 return p.path
197 }
198
199
200 func (p *proxyRepo) versionError(version string, err error) error {
201 if version != "" && version != module.CanonicalVersion(version) {
202 return &module.ModuleError{
203 Path: p.path,
204 Err: &module.InvalidVersionError{
205 Version: version,
206 Pseudo: IsPseudoVersion(version),
207 Err: err,
208 },
209 }
210 }
211
212 return &module.ModuleError{
213 Path: p.path,
214 Version: version,
215 Err: err,
216 }
217 }
218
219 func (p *proxyRepo) getBytes(path string) ([]byte, error) {
220 body, err := p.getBody(path)
221 if err != nil {
222 return nil, err
223 }
224 defer body.Close()
225 return ioutil.ReadAll(body)
226 }
227
228 func (p *proxyRepo) getBody(path string) (io.ReadCloser, error) {
229 fullPath := pathpkg.Join(p.url.Path, path)
230
231 target := *p.url
232 target.Path = fullPath
233 target.RawPath = pathpkg.Join(target.RawPath, pathEscape(path))
234
235 resp, err := web.Get(web.DefaultSecurity, &target)
236 if err != nil {
237 return nil, err
238 }
239 if err := resp.Err(); err != nil {
240 resp.Body.Close()
241 return nil, err
242 }
243 return resp.Body, nil
244 }
245
246 func (p *proxyRepo) Versions(prefix string) ([]string, error) {
247 data, err := p.getBytes("@v/list")
248 if err != nil {
249 return nil, p.versionError("", err)
250 }
251 var list []string
252 for _, line := range strings.Split(string(data), "\n") {
253 f := strings.Fields(line)
254 if len(f) >= 1 && semver.IsValid(f[0]) && strings.HasPrefix(f[0], prefix) && !IsPseudoVersion(f[0]) {
255 list = append(list, f[0])
256 }
257 }
258 SortVersions(list)
259 return list, nil
260 }
261
262 func (p *proxyRepo) latest() (*RevInfo, error) {
263 data, err := p.getBytes("@v/list")
264 if err != nil {
265 return nil, p.versionError("", err)
266 }
267
268 var (
269 bestTime time.Time
270 bestTimeIsFromPseudo bool
271 bestVersion string
272 )
273
274 for _, line := range strings.Split(string(data), "\n") {
275 f := strings.Fields(line)
276 if len(f) >= 1 && semver.IsValid(f[0]) {
277
278
279 var (
280 ft time.Time
281 ftIsFromPseudo = false
282 )
283 if len(f) >= 2 {
284 ft, _ = time.Parse(time.RFC3339, f[1])
285 } else if IsPseudoVersion(f[0]) {
286 ft, _ = PseudoVersionTime(f[0])
287 ftIsFromPseudo = true
288 } else {
289
290
291
292 continue
293 }
294 if bestTime.Before(ft) {
295 bestTime = ft
296 bestTimeIsFromPseudo = ftIsFromPseudo
297 bestVersion = f[0]
298 }
299 }
300 }
301 if bestVersion == "" {
302 return nil, p.versionError("", codehost.ErrNoCommits)
303 }
304
305 if bestTimeIsFromPseudo {
306
307
308
309
310
311
312 return p.Stat(bestVersion)
313 }
314
315 return &RevInfo{
316 Version: bestVersion,
317 Name: bestVersion,
318 Short: bestVersion,
319 Time: bestTime,
320 }, nil
321 }
322
323 func (p *proxyRepo) Stat(rev string) (*RevInfo, error) {
324 encRev, err := module.EncodeVersion(rev)
325 if err != nil {
326 return nil, p.versionError(rev, err)
327 }
328 data, err := p.getBytes("@v/" + encRev + ".info")
329 if err != nil {
330 return nil, p.versionError(rev, err)
331 }
332 info := new(RevInfo)
333 if err := json.Unmarshal(data, info); err != nil {
334 return nil, p.versionError(rev, err)
335 }
336 if info.Version != rev && rev == module.CanonicalVersion(rev) && module.Check(p.path, rev) == nil {
337
338
339
340 return nil, p.versionError(rev, fmt.Errorf("proxy returned info for version %s instead of requested version", info.Version))
341 }
342 return info, nil
343 }
344
345 func (p *proxyRepo) Latest() (*RevInfo, error) {
346 data, err := p.getBytes("@latest")
347 if err != nil {
348 if !errors.Is(err, os.ErrNotExist) {
349 return nil, p.versionError("", err)
350 }
351 return p.latest()
352 }
353 info := new(RevInfo)
354 if err := json.Unmarshal(data, info); err != nil {
355 return nil, p.versionError("", err)
356 }
357 return info, nil
358 }
359
360 func (p *proxyRepo) GoMod(version string) ([]byte, error) {
361 if version != module.CanonicalVersion(version) {
362 return nil, p.versionError(version, fmt.Errorf("internal error: version passed to GoMod is not canonical"))
363 }
364
365 encVer, err := module.EncodeVersion(version)
366 if err != nil {
367 return nil, p.versionError(version, err)
368 }
369 data, err := p.getBytes("@v/" + encVer + ".mod")
370 if err != nil {
371 return nil, p.versionError(version, err)
372 }
373 return data, nil
374 }
375
376 func (p *proxyRepo) Zip(dst io.Writer, version string) error {
377 if version != module.CanonicalVersion(version) {
378 return p.versionError(version, fmt.Errorf("internal error: version passed to Zip is not canonical"))
379 }
380
381 encVer, err := module.EncodeVersion(version)
382 if err != nil {
383 return p.versionError(version, err)
384 }
385 body, err := p.getBody("@v/" + encVer + ".zip")
386 if err != nil {
387 return p.versionError(version, err)
388 }
389 defer body.Close()
390
391 lr := &io.LimitedReader{R: body, N: codehost.MaxZipFile + 1}
392 if _, err := io.Copy(dst, lr); err != nil {
393 return p.versionError(version, err)
394 }
395 if lr.N <= 0 {
396 return p.versionError(version, fmt.Errorf("downloaded zip file too large"))
397 }
398 return nil
399 }
400
401
402
403
404 func pathEscape(s string) string {
405 return strings.ReplaceAll(url.PathEscape(s), "%2F", "/")
406 }
407
View as plain text