Source file src/pkg/cmd/vendor/github.com/google/pprof/internal/driver/fetch.go

     1	// Copyright 2014 Google Inc. All Rights Reserved.
     2	//
     3	// Licensed under the Apache License, Version 2.0 (the "License");
     4	// you may not use this file except in compliance with the License.
     5	// You may obtain a copy of the License at
     6	//
     7	//     http://www.apache.org/licenses/LICENSE-2.0
     8	//
     9	// Unless required by applicable law or agreed to in writing, software
    10	// distributed under the License is distributed on an "AS IS" BASIS,
    11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12	// See the License for the specific language governing permissions and
    13	// limitations under the License.
    14	
    15	package driver
    16	
    17	import (
    18		"bytes"
    19		"fmt"
    20		"io"
    21		"io/ioutil"
    22		"net/http"
    23		"net/url"
    24		"os"
    25		"os/exec"
    26		"path/filepath"
    27		"runtime"
    28		"strconv"
    29		"strings"
    30		"sync"
    31		"time"
    32	
    33		"github.com/google/pprof/internal/measurement"
    34		"github.com/google/pprof/internal/plugin"
    35		"github.com/google/pprof/profile"
    36	)
    37	
    38	// fetchProfiles fetches and symbolizes the profiles specified by s.
    39	// It will merge all the profiles it is able to retrieve, even if
    40	// there are some failures. It will return an error if it is unable to
    41	// fetch any profiles.
    42	func fetchProfiles(s *source, o *plugin.Options) (*profile.Profile, error) {
    43		sources := make([]profileSource, 0, len(s.Sources))
    44		for _, src := range s.Sources {
    45			sources = append(sources, profileSource{
    46				addr:   src,
    47				source: s,
    48			})
    49		}
    50	
    51		bases := make([]profileSource, 0, len(s.Base))
    52		for _, src := range s.Base {
    53			bases = append(bases, profileSource{
    54				addr:   src,
    55				source: s,
    56			})
    57		}
    58	
    59		p, pbase, m, mbase, save, err := grabSourcesAndBases(sources, bases, o.Fetch, o.Obj, o.UI, o.HTTPTransport)
    60		if err != nil {
    61			return nil, err
    62		}
    63	
    64		if pbase != nil {
    65			if s.DiffBase {
    66				pbase.SetLabel("pprof::base", []string{"true"})
    67			}
    68			if s.Normalize {
    69				err := p.Normalize(pbase)
    70				if err != nil {
    71					return nil, err
    72				}
    73			}
    74			pbase.Scale(-1)
    75			p, m, err = combineProfiles([]*profile.Profile{p, pbase}, []plugin.MappingSources{m, mbase})
    76			if err != nil {
    77				return nil, err
    78			}
    79		}
    80	
    81		// Symbolize the merged profile.
    82		if err := o.Sym.Symbolize(s.Symbolize, m, p); err != nil {
    83			return nil, err
    84		}
    85		p.RemoveUninteresting()
    86		unsourceMappings(p)
    87	
    88		if s.Comment != "" {
    89			p.Comments = append(p.Comments, s.Comment)
    90		}
    91	
    92		// Save a copy of the merged profile if there is at least one remote source.
    93		if save {
    94			dir, err := setTmpDir(o.UI)
    95			if err != nil {
    96				return nil, err
    97			}
    98	
    99			prefix := "pprof."
   100			if len(p.Mapping) > 0 && p.Mapping[0].File != "" {
   101				prefix += filepath.Base(p.Mapping[0].File) + "."
   102			}
   103			for _, s := range p.SampleType {
   104				prefix += s.Type + "."
   105			}
   106	
   107			tempFile, err := newTempFile(dir, prefix, ".pb.gz")
   108			if err == nil {
   109				if err = p.Write(tempFile); err == nil {
   110					o.UI.PrintErr("Saved profile in ", tempFile.Name())
   111				}
   112			}
   113			if err != nil {
   114				o.UI.PrintErr("Could not save profile: ", err)
   115			}
   116		}
   117	
   118		if err := p.CheckValid(); err != nil {
   119			return nil, err
   120		}
   121	
   122		return p, nil
   123	}
   124	
   125	func grabSourcesAndBases(sources, bases []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, *profile.Profile, plugin.MappingSources, plugin.MappingSources, bool, error) {
   126		wg := sync.WaitGroup{}
   127		wg.Add(2)
   128		var psrc, pbase *profile.Profile
   129		var msrc, mbase plugin.MappingSources
   130		var savesrc, savebase bool
   131		var errsrc, errbase error
   132		var countsrc, countbase int
   133		go func() {
   134			defer wg.Done()
   135			psrc, msrc, savesrc, countsrc, errsrc = chunkedGrab(sources, fetch, obj, ui, tr)
   136		}()
   137		go func() {
   138			defer wg.Done()
   139			pbase, mbase, savebase, countbase, errbase = chunkedGrab(bases, fetch, obj, ui, tr)
   140		}()
   141		wg.Wait()
   142		save := savesrc || savebase
   143	
   144		if errsrc != nil {
   145			return nil, nil, nil, nil, false, fmt.Errorf("problem fetching source profiles: %v", errsrc)
   146		}
   147		if errbase != nil {
   148			return nil, nil, nil, nil, false, fmt.Errorf("problem fetching base profiles: %v,", errbase)
   149		}
   150		if countsrc == 0 {
   151			return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any source profiles")
   152		}
   153		if countbase == 0 && len(bases) > 0 {
   154			return nil, nil, nil, nil, false, fmt.Errorf("failed to fetch any base profiles")
   155		}
   156		if want, got := len(sources), countsrc; want != got {
   157			ui.PrintErr(fmt.Sprintf("Fetched %d source profiles out of %d", got, want))
   158		}
   159		if want, got := len(bases), countbase; want != got {
   160			ui.PrintErr(fmt.Sprintf("Fetched %d base profiles out of %d", got, want))
   161		}
   162	
   163		return psrc, pbase, msrc, mbase, save, nil
   164	}
   165	
   166	// chunkedGrab fetches the profiles described in source and merges them into
   167	// a single profile. It fetches a chunk of profiles concurrently, with a maximum
   168	// chunk size to limit its memory usage.
   169	func chunkedGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
   170		const chunkSize = 64
   171	
   172		var p *profile.Profile
   173		var msrc plugin.MappingSources
   174		var save bool
   175		var count int
   176	
   177		for start := 0; start < len(sources); start += chunkSize {
   178			end := start + chunkSize
   179			if end > len(sources) {
   180				end = len(sources)
   181			}
   182			chunkP, chunkMsrc, chunkSave, chunkCount, chunkErr := concurrentGrab(sources[start:end], fetch, obj, ui, tr)
   183			switch {
   184			case chunkErr != nil:
   185				return nil, nil, false, 0, chunkErr
   186			case chunkP == nil:
   187				continue
   188			case p == nil:
   189				p, msrc, save, count = chunkP, chunkMsrc, chunkSave, chunkCount
   190			default:
   191				p, msrc, chunkErr = combineProfiles([]*profile.Profile{p, chunkP}, []plugin.MappingSources{msrc, chunkMsrc})
   192				if chunkErr != nil {
   193					return nil, nil, false, 0, chunkErr
   194				}
   195				if chunkSave {
   196					save = true
   197				}
   198				count += chunkCount
   199			}
   200		}
   201	
   202		return p, msrc, save, count, nil
   203	}
   204	
   205	// concurrentGrab fetches multiple profiles concurrently
   206	func concurrentGrab(sources []profileSource, fetch plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (*profile.Profile, plugin.MappingSources, bool, int, error) {
   207		wg := sync.WaitGroup{}
   208		wg.Add(len(sources))
   209		for i := range sources {
   210			go func(s *profileSource) {
   211				defer wg.Done()
   212				s.p, s.msrc, s.remote, s.err = grabProfile(s.source, s.addr, fetch, obj, ui, tr)
   213			}(&sources[i])
   214		}
   215		wg.Wait()
   216	
   217		var save bool
   218		profiles := make([]*profile.Profile, 0, len(sources))
   219		msrcs := make([]plugin.MappingSources, 0, len(sources))
   220		for i := range sources {
   221			s := &sources[i]
   222			if err := s.err; err != nil {
   223				ui.PrintErr(s.addr + ": " + err.Error())
   224				continue
   225			}
   226			save = save || s.remote
   227			profiles = append(profiles, s.p)
   228			msrcs = append(msrcs, s.msrc)
   229			*s = profileSource{}
   230		}
   231	
   232		if len(profiles) == 0 {
   233			return nil, nil, false, 0, nil
   234		}
   235	
   236		p, msrc, err := combineProfiles(profiles, msrcs)
   237		if err != nil {
   238			return nil, nil, false, 0, err
   239		}
   240		return p, msrc, save, len(profiles), nil
   241	}
   242	
   243	func combineProfiles(profiles []*profile.Profile, msrcs []plugin.MappingSources) (*profile.Profile, plugin.MappingSources, error) {
   244		// Merge profiles.
   245		if err := measurement.ScaleProfiles(profiles); err != nil {
   246			return nil, nil, err
   247		}
   248	
   249		p, err := profile.Merge(profiles)
   250		if err != nil {
   251			return nil, nil, err
   252		}
   253	
   254		// Combine mapping sources.
   255		msrc := make(plugin.MappingSources)
   256		for _, ms := range msrcs {
   257			for m, s := range ms {
   258				msrc[m] = append(msrc[m], s...)
   259			}
   260		}
   261		return p, msrc, nil
   262	}
   263	
   264	type profileSource struct {
   265		addr   string
   266		source *source
   267	
   268		p      *profile.Profile
   269		msrc   plugin.MappingSources
   270		remote bool
   271		err    error
   272	}
   273	
   274	func homeEnv() string {
   275		switch runtime.GOOS {
   276		case "windows":
   277			return "USERPROFILE"
   278		case "plan9":
   279			return "home"
   280		default:
   281			return "HOME"
   282		}
   283	}
   284	
   285	// setTmpDir prepares the directory to use to save profiles retrieved
   286	// remotely. It is selected from PPROF_TMPDIR, defaults to $HOME/pprof, and, if
   287	// $HOME is not set, falls back to os.TempDir().
   288	func setTmpDir(ui plugin.UI) (string, error) {
   289		var dirs []string
   290		if profileDir := os.Getenv("PPROF_TMPDIR"); profileDir != "" {
   291			dirs = append(dirs, profileDir)
   292		}
   293		if homeDir := os.Getenv(homeEnv()); homeDir != "" {
   294			dirs = append(dirs, filepath.Join(homeDir, "pprof"))
   295		}
   296		dirs = append(dirs, os.TempDir())
   297		for _, tmpDir := range dirs {
   298			if err := os.MkdirAll(tmpDir, 0755); err != nil {
   299				ui.PrintErr("Could not use temp dir ", tmpDir, ": ", err.Error())
   300				continue
   301			}
   302			return tmpDir, nil
   303		}
   304		return "", fmt.Errorf("failed to identify temp dir")
   305	}
   306	
   307	const testSourceAddress = "pproftest.local"
   308	
   309	// grabProfile fetches a profile. Returns the profile, sources for the
   310	// profile mappings, a bool indicating if the profile was fetched
   311	// remotely, and an error.
   312	func grabProfile(s *source, source string, fetcher plugin.Fetcher, obj plugin.ObjTool, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, msrc plugin.MappingSources, remote bool, err error) {
   313		var src string
   314		duration, timeout := time.Duration(s.Seconds)*time.Second, time.Duration(s.Timeout)*time.Second
   315		if fetcher != nil {
   316			p, src, err = fetcher.Fetch(source, duration, timeout)
   317			if err != nil {
   318				return
   319			}
   320		}
   321		if err != nil || p == nil {
   322			// Fetch the profile over HTTP or from a file.
   323			p, src, err = fetch(source, duration, timeout, ui, tr)
   324			if err != nil {
   325				return
   326			}
   327		}
   328	
   329		if err = p.CheckValid(); err != nil {
   330			return
   331		}
   332	
   333		// Update the binary locations from command line and paths.
   334		locateBinaries(p, s, obj, ui)
   335	
   336		// Collect the source URL for all mappings.
   337		if src != "" {
   338			msrc = collectMappingSources(p, src)
   339			remote = true
   340			if strings.HasPrefix(src, "http://"+testSourceAddress) {
   341				// Treat test inputs as local to avoid saving
   342				// testcase profiles during driver testing.
   343				remote = false
   344			}
   345		}
   346		return
   347	}
   348	
   349	// collectMappingSources saves the mapping sources of a profile.
   350	func collectMappingSources(p *profile.Profile, source string) plugin.MappingSources {
   351		ms := plugin.MappingSources{}
   352		for _, m := range p.Mapping {
   353			src := struct {
   354				Source string
   355				Start  uint64
   356			}{
   357				source, m.Start,
   358			}
   359			key := m.BuildID
   360			if key == "" {
   361				key = m.File
   362			}
   363			if key == "" {
   364				// If there is no build id or source file, use the source as the
   365				// mapping file. This will enable remote symbolization for this
   366				// mapping, in particular for Go profiles on the legacy format.
   367				// The source is reset back to empty string by unsourceMapping
   368				// which is called after symbolization is finished.
   369				m.File = source
   370				key = source
   371			}
   372			ms[key] = append(ms[key], src)
   373		}
   374		return ms
   375	}
   376	
   377	// unsourceMappings iterates over the mappings in a profile and replaces file
   378	// set to the remote source URL by collectMappingSources back to empty string.
   379	func unsourceMappings(p *profile.Profile) {
   380		for _, m := range p.Mapping {
   381			if m.BuildID == "" {
   382				if u, err := url.Parse(m.File); err == nil && u.IsAbs() {
   383					m.File = ""
   384				}
   385			}
   386		}
   387	}
   388	
   389	// locateBinaries searches for binary files listed in the profile and, if found,
   390	// updates the profile accordingly.
   391	func locateBinaries(p *profile.Profile, s *source, obj plugin.ObjTool, ui plugin.UI) {
   392		// Construct search path to examine
   393		searchPath := os.Getenv("PPROF_BINARY_PATH")
   394		if searchPath == "" {
   395			// Use $HOME/pprof/binaries as default directory for local symbolization binaries
   396			searchPath = filepath.Join(os.Getenv(homeEnv()), "pprof", "binaries")
   397		}
   398	mapping:
   399		for _, m := range p.Mapping {
   400			var baseName string
   401			if m.File != "" {
   402				baseName = filepath.Base(m.File)
   403			}
   404	
   405			for _, path := range filepath.SplitList(searchPath) {
   406				var fileNames []string
   407				if m.BuildID != "" {
   408					fileNames = []string{filepath.Join(path, m.BuildID, baseName)}
   409					if matches, err := filepath.Glob(filepath.Join(path, m.BuildID, "*")); err == nil {
   410						fileNames = append(fileNames, matches...)
   411					}
   412					fileNames = append(fileNames, filepath.Join(path, m.File, m.BuildID)) // perf path format
   413				}
   414				if m.File != "" {
   415					// Try both the basename and the full path, to support the same directory
   416					// structure as the perf symfs option.
   417					if baseName != "" {
   418						fileNames = append(fileNames, filepath.Join(path, baseName))
   419					}
   420					fileNames = append(fileNames, filepath.Join(path, m.File))
   421				}
   422				for _, name := range fileNames {
   423					if f, err := obj.Open(name, m.Start, m.Limit, m.Offset); err == nil {
   424						defer f.Close()
   425						fileBuildID := f.BuildID()
   426						if m.BuildID != "" && m.BuildID != fileBuildID {
   427							ui.PrintErr("Ignoring local file " + name + ": build-id mismatch (" + m.BuildID + " != " + fileBuildID + ")")
   428						} else {
   429							m.File = name
   430							continue mapping
   431						}
   432					}
   433				}
   434			}
   435		}
   436		if len(p.Mapping) == 0 {
   437			// If there are no mappings, add a fake mapping to attempt symbolization.
   438			// This is useful for some profiles generated by the golang runtime, which
   439			// do not include any mappings. Symbolization with a fake mapping will only
   440			// be successful against a non-PIE binary.
   441			m := &profile.Mapping{ID: 1}
   442			p.Mapping = []*profile.Mapping{m}
   443			for _, l := range p.Location {
   444				l.Mapping = m
   445			}
   446		}
   447		// Replace executable filename/buildID with the overrides from source.
   448		// Assumes the executable is the first Mapping entry.
   449		if execName, buildID := s.ExecName, s.BuildID; execName != "" || buildID != "" {
   450			m := p.Mapping[0]
   451			if execName != "" {
   452				m.File = execName
   453			}
   454			if buildID != "" {
   455				m.BuildID = buildID
   456			}
   457		}
   458	}
   459	
   460	// fetch fetches a profile from source, within the timeout specified,
   461	// producing messages through the ui. It returns the profile and the
   462	// url of the actual source of the profile for remote profiles.
   463	func fetch(source string, duration, timeout time.Duration, ui plugin.UI, tr http.RoundTripper) (p *profile.Profile, src string, err error) {
   464		var f io.ReadCloser
   465	
   466		if sourceURL, timeout := adjustURL(source, duration, timeout); sourceURL != "" {
   467			ui.Print("Fetching profile over HTTP from " + sourceURL)
   468			if duration > 0 {
   469				ui.Print(fmt.Sprintf("Please wait... (%v)", duration))
   470			}
   471			f, err = fetchURL(sourceURL, timeout, tr)
   472			src = sourceURL
   473		} else if isPerfFile(source) {
   474			f, err = convertPerfData(source, ui)
   475		} else {
   476			f, err = os.Open(source)
   477		}
   478		if err == nil {
   479			defer f.Close()
   480			p, err = profile.Parse(f)
   481		}
   482		return
   483	}
   484	
   485	// fetchURL fetches a profile from a URL using HTTP.
   486	func fetchURL(source string, timeout time.Duration, tr http.RoundTripper) (io.ReadCloser, error) {
   487		client := &http.Client{
   488			Transport: tr,
   489			Timeout:   timeout + 5*time.Second,
   490		}
   491		resp, err := client.Get(source)
   492		if err != nil {
   493			return nil, fmt.Errorf("http fetch: %v", err)
   494		}
   495		if resp.StatusCode != http.StatusOK {
   496			defer resp.Body.Close()
   497			return nil, statusCodeError(resp)
   498		}
   499	
   500		return resp.Body, nil
   501	}
   502	
   503	func statusCodeError(resp *http.Response) error {
   504		if resp.Header.Get("X-Go-Pprof") != "" && strings.Contains(resp.Header.Get("Content-Type"), "text/plain") {
   505			// error is from pprof endpoint
   506			if body, err := ioutil.ReadAll(resp.Body); err == nil {
   507				return fmt.Errorf("server response: %s - %s", resp.Status, body)
   508			}
   509		}
   510		return fmt.Errorf("server response: %s", resp.Status)
   511	}
   512	
   513	// isPerfFile checks if a file is in perf.data format. It also returns false
   514	// if it encounters an error during the check.
   515	func isPerfFile(path string) bool {
   516		sourceFile, openErr := os.Open(path)
   517		if openErr != nil {
   518			return false
   519		}
   520		defer sourceFile.Close()
   521	
   522		// If the file is the output of a perf record command, it should begin
   523		// with the string PERFILE2.
   524		perfHeader := []byte("PERFILE2")
   525		actualHeader := make([]byte, len(perfHeader))
   526		if _, readErr := sourceFile.Read(actualHeader); readErr != nil {
   527			return false
   528		}
   529		return bytes.Equal(actualHeader, perfHeader)
   530	}
   531	
   532	// convertPerfData converts the file at path which should be in perf.data format
   533	// using the perf_to_profile tool and returns the file containing the
   534	// profile.proto formatted data.
   535	func convertPerfData(perfPath string, ui plugin.UI) (*os.File, error) {
   536		ui.Print(fmt.Sprintf(
   537			"Converting %s to a profile.proto... (May take a few minutes)",
   538			perfPath))
   539		profile, err := newTempFile(os.TempDir(), "pprof_", ".pb.gz")
   540		if err != nil {
   541			return nil, err
   542		}
   543		deferDeleteTempFile(profile.Name())
   544		cmd := exec.Command("perf_to_profile", "-i", perfPath, "-o", profile.Name(), "-f")
   545		cmd.Stdout, cmd.Stderr = os.Stdout, os.Stderr
   546		if err := cmd.Run(); err != nil {
   547			profile.Close()
   548			return nil, fmt.Errorf("failed to convert perf.data file. Try github.com/google/perf_data_converter: %v", err)
   549		}
   550		return profile, nil
   551	}
   552	
   553	// adjustURL validates if a profile source is a URL and returns an
   554	// cleaned up URL and the timeout to use for retrieval over HTTP.
   555	// If the source cannot be recognized as a URL it returns an empty string.
   556	func adjustURL(source string, duration, timeout time.Duration) (string, time.Duration) {
   557		u, err := url.Parse(source)
   558		if err != nil || (u.Host == "" && u.Scheme != "" && u.Scheme != "file") {
   559			// Try adding http:// to catch sources of the form hostname:port/path.
   560			// url.Parse treats "hostname" as the scheme.
   561			u, err = url.Parse("http://" + source)
   562		}
   563		if err != nil || u.Host == "" {
   564			return "", 0
   565		}
   566	
   567		// Apply duration/timeout overrides to URL.
   568		values := u.Query()
   569		if duration > 0 {
   570			values.Set("seconds", fmt.Sprint(int(duration.Seconds())))
   571		} else {
   572			if urlSeconds := values.Get("seconds"); urlSeconds != "" {
   573				if us, err := strconv.ParseInt(urlSeconds, 10, 32); err == nil {
   574					duration = time.Duration(us) * time.Second
   575				}
   576			}
   577		}
   578		if timeout <= 0 {
   579			if duration > 0 {
   580				timeout = duration + duration/2
   581			} else {
   582				timeout = 60 * time.Second
   583			}
   584		}
   585		u.RawQuery = values.Encode()
   586		return u.String(), timeout
   587	}
   588	

View as plain text