Source file src/runtime/pprof/internal/profile/legacy_profile.go

     1	// Copyright 2014 The Go Authors. All rights reserved.
     2	// Use of this source code is governed by a BSD-style
     3	// license that can be found in the LICENSE file.
     4	
     5	// This file implements parsers to convert legacy profiles into the
     6	// profile.proto format.
     7	
     8	package profile
     9	
    10	import (
    11		"bufio"
    12		"bytes"
    13		"fmt"
    14		"io"
    15		"math"
    16		"regexp"
    17		"strconv"
    18		"strings"
    19	)
    20	
    21	var (
    22		countStartRE = regexp.MustCompile(`\A(\w+) profile: total \d+\n\z`)
    23		countRE      = regexp.MustCompile(`\A(\d+) @(( 0x[0-9a-f]+)+)\n\z`)
    24	
    25		heapHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] *@ *(heap[_a-z0-9]*)/?(\d*)`)
    26		heapSampleRE = regexp.MustCompile(`(-?\d+): *(-?\d+) *\[ *(\d+): *(\d+) *] @([ x0-9a-f]*)`)
    27	
    28		contentionSampleRE = regexp.MustCompile(`(\d+) *(\d+) @([ x0-9a-f]*)`)
    29	
    30		hexNumberRE = regexp.MustCompile(`0x[0-9a-f]+`)
    31	
    32		growthHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ growthz`)
    33	
    34		fragmentationHeaderRE = regexp.MustCompile(`heap profile: *(\d+): *(\d+) *\[ *(\d+): *(\d+) *\] @ fragmentationz`)
    35	
    36		threadzStartRE = regexp.MustCompile(`--- threadz \d+ ---`)
    37		threadStartRE  = regexp.MustCompile(`--- Thread ([[:xdigit:]]+) \(name: (.*)/(\d+)\) stack: ---`)
    38	
    39		procMapsRE = regexp.MustCompile(`([[:xdigit:]]+)-([[:xdigit:]]+)\s+([-rwxp]+)\s+([[:xdigit:]]+)\s+([[:xdigit:]]+):([[:xdigit:]]+)\s+([[:digit:]]+)\s*(\S+)?`)
    40	
    41		briefMapsRE = regexp.MustCompile(`\s*([[:xdigit:]]+)-([[:xdigit:]]+):\s*(\S+)(\s.*@)?([[:xdigit:]]+)?`)
    42	
    43		// LegacyHeapAllocated instructs the heapz parsers to use the
    44		// allocated memory stats instead of the default in-use memory. Note
    45		// that tcmalloc doesn't provide all allocated memory, only in-use
    46		// stats.
    47		LegacyHeapAllocated bool
    48	)
    49	
    50	func isSpaceOrComment(line string) bool {
    51		trimmed := strings.TrimSpace(line)
    52		return len(trimmed) == 0 || trimmed[0] == '#'
    53	}
    54	
    55	// parseGoCount parses a Go count profile (e.g., threadcreate or
    56	// goroutine) and returns a new Profile.
    57	func parseGoCount(b []byte) (*Profile, error) {
    58		r := bytes.NewBuffer(b)
    59	
    60		var line string
    61		var err error
    62		for {
    63			// Skip past comments and empty lines seeking a real header.
    64			line, err = r.ReadString('\n')
    65			if err != nil {
    66				return nil, err
    67			}
    68			if !isSpaceOrComment(line) {
    69				break
    70			}
    71		}
    72	
    73		m := countStartRE.FindStringSubmatch(line)
    74		if m == nil {
    75			return nil, errUnrecognized
    76		}
    77		profileType := m[1]
    78		p := &Profile{
    79			PeriodType: &ValueType{Type: profileType, Unit: "count"},
    80			Period:     1,
    81			SampleType: []*ValueType{{Type: profileType, Unit: "count"}},
    82		}
    83		locations := make(map[uint64]*Location)
    84		for {
    85			line, err = r.ReadString('\n')
    86			if err != nil {
    87				if err == io.EOF {
    88					break
    89				}
    90				return nil, err
    91			}
    92			if isSpaceOrComment(line) {
    93				continue
    94			}
    95			if strings.HasPrefix(line, "---") {
    96				break
    97			}
    98			m := countRE.FindStringSubmatch(line)
    99			if m == nil {
   100				return nil, errMalformed
   101			}
   102			n, err := strconv.ParseInt(m[1], 0, 64)
   103			if err != nil {
   104				return nil, errMalformed
   105			}
   106			fields := strings.Fields(m[2])
   107			locs := make([]*Location, 0, len(fields))
   108			for _, stk := range fields {
   109				addr, err := strconv.ParseUint(stk, 0, 64)
   110				if err != nil {
   111					return nil, errMalformed
   112				}
   113				// Adjust all frames by -1 to land on the call instruction.
   114				addr--
   115				loc := locations[addr]
   116				if loc == nil {
   117					loc = &Location{
   118						Address: addr,
   119					}
   120					locations[addr] = loc
   121					p.Location = append(p.Location, loc)
   122				}
   123				locs = append(locs, loc)
   124			}
   125			p.Sample = append(p.Sample, &Sample{
   126				Location: locs,
   127				Value:    []int64{n},
   128			})
   129		}
   130	
   131		if err = parseAdditionalSections(strings.TrimSpace(line), r, p); err != nil {
   132			return nil, err
   133		}
   134		return p, nil
   135	}
   136	
   137	// remapLocationIDs ensures there is a location for each address
   138	// referenced by a sample, and remaps the samples to point to the new
   139	// location ids.
   140	func (p *Profile) remapLocationIDs() {
   141		seen := make(map[*Location]bool, len(p.Location))
   142		var locs []*Location
   143	
   144		for _, s := range p.Sample {
   145			for _, l := range s.Location {
   146				if seen[l] {
   147					continue
   148				}
   149				l.ID = uint64(len(locs) + 1)
   150				locs = append(locs, l)
   151				seen[l] = true
   152			}
   153		}
   154		p.Location = locs
   155	}
   156	
   157	func (p *Profile) remapFunctionIDs() {
   158		seen := make(map[*Function]bool, len(p.Function))
   159		var fns []*Function
   160	
   161		for _, l := range p.Location {
   162			for _, ln := range l.Line {
   163				fn := ln.Function
   164				if fn == nil || seen[fn] {
   165					continue
   166				}
   167				fn.ID = uint64(len(fns) + 1)
   168				fns = append(fns, fn)
   169				seen[fn] = true
   170			}
   171		}
   172		p.Function = fns
   173	}
   174	
   175	// remapMappingIDs matches location addresses with existing mappings
   176	// and updates them appropriately. This is O(N*M), if this ever shows
   177	// up as a bottleneck, evaluate sorting the mappings and doing a
   178	// binary search, which would make it O(N*log(M)).
   179	func (p *Profile) remapMappingIDs() {
   180		if len(p.Mapping) == 0 {
   181			return
   182		}
   183	
   184		// Some profile handlers will incorrectly set regions for the main
   185		// executable if its section is remapped. Fix them through heuristics.
   186	
   187		// Remove the initial mapping if named '/anon_hugepage' and has a
   188		// consecutive adjacent mapping.
   189		if m := p.Mapping[0]; strings.HasPrefix(m.File, "/anon_hugepage") {
   190			if len(p.Mapping) > 1 && m.Limit == p.Mapping[1].Start {
   191				p.Mapping = p.Mapping[1:]
   192			}
   193		}
   194	
   195		// Subtract the offset from the start of the main mapping if it
   196		// ends up at a recognizable start address.
   197		const expectedStart = 0x400000
   198		if m := p.Mapping[0]; m.Start-m.Offset == expectedStart {
   199			m.Start = expectedStart
   200			m.Offset = 0
   201		}
   202	
   203		for _, l := range p.Location {
   204			if a := l.Address; a != 0 {
   205				for _, m := range p.Mapping {
   206					if m.Start <= a && a < m.Limit {
   207						l.Mapping = m
   208						break
   209					}
   210				}
   211			}
   212		}
   213	
   214		// Reset all mapping IDs.
   215		for i, m := range p.Mapping {
   216			m.ID = uint64(i + 1)
   217		}
   218	}
   219	
   220	var cpuInts = []func([]byte) (uint64, []byte){
   221		get32l,
   222		get32b,
   223		get64l,
   224		get64b,
   225	}
   226	
   227	func get32l(b []byte) (uint64, []byte) {
   228		if len(b) < 4 {
   229			return 0, nil
   230		}
   231		return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24, b[4:]
   232	}
   233	
   234	func get32b(b []byte) (uint64, []byte) {
   235		if len(b) < 4 {
   236			return 0, nil
   237		}
   238		return uint64(b[3]) | uint64(b[2])<<8 | uint64(b[1])<<16 | uint64(b[0])<<24, b[4:]
   239	}
   240	
   241	func get64l(b []byte) (uint64, []byte) {
   242		if len(b) < 8 {
   243			return 0, nil
   244		}
   245		return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56, b[8:]
   246	}
   247	
   248	func get64b(b []byte) (uint64, []byte) {
   249		if len(b) < 8 {
   250			return 0, nil
   251		}
   252		return uint64(b[7]) | uint64(b[6])<<8 | uint64(b[5])<<16 | uint64(b[4])<<24 | uint64(b[3])<<32 | uint64(b[2])<<40 | uint64(b[1])<<48 | uint64(b[0])<<56, b[8:]
   253	}
   254	
   255	// ParseTracebacks parses a set of tracebacks and returns a newly
   256	// populated profile. It will accept any text file and generate a
   257	// Profile out of it with any hex addresses it can identify, including
   258	// a process map if it can recognize one. Each sample will include a
   259	// tag "source" with the addresses recognized in string format.
   260	func ParseTracebacks(b []byte) (*Profile, error) {
   261		r := bytes.NewBuffer(b)
   262	
   263		p := &Profile{
   264			PeriodType: &ValueType{Type: "trace", Unit: "count"},
   265			Period:     1,
   266			SampleType: []*ValueType{
   267				{Type: "trace", Unit: "count"},
   268			},
   269		}
   270	
   271		var sources []string
   272		var sloc []*Location
   273	
   274		locs := make(map[uint64]*Location)
   275		for {
   276			l, err := r.ReadString('\n')
   277			if err != nil {
   278				if err != io.EOF {
   279					return nil, err
   280				}
   281				if l == "" {
   282					break
   283				}
   284			}
   285			if sectionTrigger(l) == memoryMapSection {
   286				break
   287			}
   288			if s, addrs := extractHexAddresses(l); len(s) > 0 {
   289				for _, addr := range addrs {
   290					// Addresses from stack traces point to the next instruction after
   291					// each call. Adjust by -1 to land somewhere on the actual call.
   292					addr--
   293					loc := locs[addr]
   294					if locs[addr] == nil {
   295						loc = &Location{
   296							Address: addr,
   297						}
   298						p.Location = append(p.Location, loc)
   299						locs[addr] = loc
   300					}
   301					sloc = append(sloc, loc)
   302				}
   303	
   304				sources = append(sources, s...)
   305			} else {
   306				if len(sources) > 0 || len(sloc) > 0 {
   307					addTracebackSample(sloc, sources, p)
   308					sloc, sources = nil, nil
   309				}
   310			}
   311		}
   312	
   313		// Add final sample to save any leftover data.
   314		if len(sources) > 0 || len(sloc) > 0 {
   315			addTracebackSample(sloc, sources, p)
   316		}
   317	
   318		if err := p.ParseMemoryMap(r); err != nil {
   319			return nil, err
   320		}
   321		return p, nil
   322	}
   323	
   324	func addTracebackSample(l []*Location, s []string, p *Profile) {
   325		p.Sample = append(p.Sample,
   326			&Sample{
   327				Value:    []int64{1},
   328				Location: l,
   329				Label:    map[string][]string{"source": s},
   330			})
   331	}
   332	
   333	// parseCPU parses a profilez legacy profile and returns a newly
   334	// populated Profile.
   335	//
   336	// The general format for profilez samples is a sequence of words in
   337	// binary format. The first words are a header with the following data:
   338	//   1st word -- 0
   339	//   2nd word -- 3
   340	//   3rd word -- 0 if a c++ application, 1 if a java application.
   341	//   4th word -- Sampling period (in microseconds).
   342	//   5th word -- Padding.
   343	func parseCPU(b []byte) (*Profile, error) {
   344		var parse func([]byte) (uint64, []byte)
   345		var n1, n2, n3, n4, n5 uint64
   346		for _, parse = range cpuInts {
   347			var tmp []byte
   348			n1, tmp = parse(b)
   349			n2, tmp = parse(tmp)
   350			n3, tmp = parse(tmp)
   351			n4, tmp = parse(tmp)
   352			n5, tmp = parse(tmp)
   353	
   354			if tmp != nil && n1 == 0 && n2 == 3 && n3 == 0 && n4 > 0 && n5 == 0 {
   355				b = tmp
   356				return cpuProfile(b, int64(n4), parse)
   357			}
   358		}
   359		return nil, errUnrecognized
   360	}
   361	
   362	// cpuProfile returns a new Profile from C++ profilez data.
   363	// b is the profile bytes after the header, period is the profiling
   364	// period, and parse is a function to parse 8-byte chunks from the
   365	// profile in its native endianness.
   366	func cpuProfile(b []byte, period int64, parse func(b []byte) (uint64, []byte)) (*Profile, error) {
   367		p := &Profile{
   368			Period:     period * 1000,
   369			PeriodType: &ValueType{Type: "cpu", Unit: "nanoseconds"},
   370			SampleType: []*ValueType{
   371				{Type: "samples", Unit: "count"},
   372				{Type: "cpu", Unit: "nanoseconds"},
   373			},
   374		}
   375		var err error
   376		if b, _, err = parseCPUSamples(b, parse, true, p); err != nil {
   377			return nil, err
   378		}
   379	
   380		// If all samples have the same second-to-the-bottom frame, it
   381		// strongly suggests that it is an uninteresting artifact of
   382		// measurement -- a stack frame pushed by the signal handler. The
   383		// bottom frame is always correct as it is picked up from the signal
   384		// structure, not the stack. Check if this is the case and if so,
   385		// remove.
   386		if len(p.Sample) > 1 && len(p.Sample[0].Location) > 1 {
   387			allSame := true
   388			id1 := p.Sample[0].Location[1].Address
   389			for _, s := range p.Sample {
   390				if len(s.Location) < 2 || id1 != s.Location[1].Address {
   391					allSame = false
   392					break
   393				}
   394			}
   395			if allSame {
   396				for _, s := range p.Sample {
   397					s.Location = append(s.Location[:1], s.Location[2:]...)
   398				}
   399			}
   400		}
   401	
   402		if err := p.ParseMemoryMap(bytes.NewBuffer(b)); err != nil {
   403			return nil, err
   404		}
   405		return p, nil
   406	}
   407	
   408	// parseCPUSamples parses a collection of profilez samples from a
   409	// profile.
   410	//
   411	// profilez samples are a repeated sequence of stack frames of the
   412	// form:
   413	//    1st word -- The number of times this stack was encountered.
   414	//    2nd word -- The size of the stack (StackSize).
   415	//    3rd word -- The first address on the stack.
   416	//    ...
   417	//    StackSize + 2 -- The last address on the stack
   418	// The last stack trace is of the form:
   419	//   1st word -- 0
   420	//   2nd word -- 1
   421	//   3rd word -- 0
   422	//
   423	// Addresses from stack traces may point to the next instruction after
   424	// each call. Optionally adjust by -1 to land somewhere on the actual
   425	// call (except for the leaf, which is not a call).
   426	func parseCPUSamples(b []byte, parse func(b []byte) (uint64, []byte), adjust bool, p *Profile) ([]byte, map[uint64]*Location, error) {
   427		locs := make(map[uint64]*Location)
   428		for len(b) > 0 {
   429			var count, nstk uint64
   430			count, b = parse(b)
   431			nstk, b = parse(b)
   432			if b == nil || nstk > uint64(len(b)/4) {
   433				return nil, nil, errUnrecognized
   434			}
   435			var sloc []*Location
   436			addrs := make([]uint64, nstk)
   437			for i := 0; i < int(nstk); i++ {
   438				addrs[i], b = parse(b)
   439			}
   440	
   441			if count == 0 && nstk == 1 && addrs[0] == 0 {
   442				// End of data marker
   443				break
   444			}
   445			for i, addr := range addrs {
   446				if adjust && i > 0 {
   447					addr--
   448				}
   449				loc := locs[addr]
   450				if loc == nil {
   451					loc = &Location{
   452						Address: addr,
   453					}
   454					locs[addr] = loc
   455					p.Location = append(p.Location, loc)
   456				}
   457				sloc = append(sloc, loc)
   458			}
   459			p.Sample = append(p.Sample,
   460				&Sample{
   461					Value:    []int64{int64(count), int64(count) * p.Period},
   462					Location: sloc,
   463				})
   464		}
   465		// Reached the end without finding the EOD marker.
   466		return b, locs, nil
   467	}
   468	
   469	// parseHeap parses a heapz legacy or a growthz profile and
   470	// returns a newly populated Profile.
   471	func parseHeap(b []byte) (p *Profile, err error) {
   472		r := bytes.NewBuffer(b)
   473		l, err := r.ReadString('\n')
   474		if err != nil {
   475			return nil, errUnrecognized
   476		}
   477	
   478		sampling := ""
   479	
   480		if header := heapHeaderRE.FindStringSubmatch(l); header != nil {
   481			p = &Profile{
   482				SampleType: []*ValueType{
   483					{Type: "objects", Unit: "count"},
   484					{Type: "space", Unit: "bytes"},
   485				},
   486				PeriodType: &ValueType{Type: "objects", Unit: "bytes"},
   487			}
   488	
   489			var period int64
   490			if len(header[6]) > 0 {
   491				if period, err = strconv.ParseInt(header[6], 10, 64); err != nil {
   492					return nil, errUnrecognized
   493				}
   494			}
   495	
   496			switch header[5] {
   497			case "heapz_v2", "heap_v2":
   498				sampling, p.Period = "v2", period
   499			case "heapprofile":
   500				sampling, p.Period = "", 1
   501			case "heap":
   502				sampling, p.Period = "v2", period/2
   503			default:
   504				return nil, errUnrecognized
   505			}
   506		} else if header = growthHeaderRE.FindStringSubmatch(l); header != nil {
   507			p = &Profile{
   508				SampleType: []*ValueType{
   509					{Type: "objects", Unit: "count"},
   510					{Type: "space", Unit: "bytes"},
   511				},
   512				PeriodType: &ValueType{Type: "heapgrowth", Unit: "count"},
   513				Period:     1,
   514			}
   515		} else if header = fragmentationHeaderRE.FindStringSubmatch(l); header != nil {
   516			p = &Profile{
   517				SampleType: []*ValueType{
   518					{Type: "objects", Unit: "count"},
   519					{Type: "space", Unit: "bytes"},
   520				},
   521				PeriodType: &ValueType{Type: "allocations", Unit: "count"},
   522				Period:     1,
   523			}
   524		} else {
   525			return nil, errUnrecognized
   526		}
   527	
   528		if LegacyHeapAllocated {
   529			for _, st := range p.SampleType {
   530				st.Type = "alloc_" + st.Type
   531			}
   532		} else {
   533			for _, st := range p.SampleType {
   534				st.Type = "inuse_" + st.Type
   535			}
   536		}
   537	
   538		locs := make(map[uint64]*Location)
   539		for {
   540			l, err = r.ReadString('\n')
   541			if err != nil {
   542				if err != io.EOF {
   543					return nil, err
   544				}
   545	
   546				if l == "" {
   547					break
   548				}
   549			}
   550	
   551			if isSpaceOrComment(l) {
   552				continue
   553			}
   554			l = strings.TrimSpace(l)
   555	
   556			if sectionTrigger(l) != unrecognizedSection {
   557				break
   558			}
   559	
   560			value, blocksize, addrs, err := parseHeapSample(l, p.Period, sampling)
   561			if err != nil {
   562				return nil, err
   563			}
   564			var sloc []*Location
   565			for _, addr := range addrs {
   566				// Addresses from stack traces point to the next instruction after
   567				// each call. Adjust by -1 to land somewhere on the actual call.
   568				addr--
   569				loc := locs[addr]
   570				if locs[addr] == nil {
   571					loc = &Location{
   572						Address: addr,
   573					}
   574					p.Location = append(p.Location, loc)
   575					locs[addr] = loc
   576				}
   577				sloc = append(sloc, loc)
   578			}
   579	
   580			p.Sample = append(p.Sample, &Sample{
   581				Value:    value,
   582				Location: sloc,
   583				NumLabel: map[string][]int64{"bytes": {blocksize}},
   584			})
   585		}
   586	
   587		if err = parseAdditionalSections(l, r, p); err != nil {
   588			return nil, err
   589		}
   590		return p, nil
   591	}
   592	
   593	// parseHeapSample parses a single row from a heap profile into a new Sample.
   594	func parseHeapSample(line string, rate int64, sampling string) (value []int64, blocksize int64, addrs []uint64, err error) {
   595		sampleData := heapSampleRE.FindStringSubmatch(line)
   596		if len(sampleData) != 6 {
   597			return value, blocksize, addrs, fmt.Errorf("unexpected number of sample values: got %d, want 6", len(sampleData))
   598		}
   599	
   600		// Use first two values by default; tcmalloc sampling generates the
   601		// same value for both, only the older heap-profile collect separate
   602		// stats for in-use and allocated objects.
   603		valueIndex := 1
   604		if LegacyHeapAllocated {
   605			valueIndex = 3
   606		}
   607	
   608		var v1, v2 int64
   609		if v1, err = strconv.ParseInt(sampleData[valueIndex], 10, 64); err != nil {
   610			return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err)
   611		}
   612		if v2, err = strconv.ParseInt(sampleData[valueIndex+1], 10, 64); err != nil {
   613			return value, blocksize, addrs, fmt.Errorf("malformed sample: %s: %v", line, err)
   614		}
   615	
   616		if v1 == 0 {
   617			if v2 != 0 {
   618				return value, blocksize, addrs, fmt.Errorf("allocation count was 0 but allocation bytes was %d", v2)
   619			}
   620		} else {
   621			blocksize = v2 / v1
   622			if sampling == "v2" {
   623				v1, v2 = scaleHeapSample(v1, v2, rate)
   624			}
   625		}
   626	
   627		value = []int64{v1, v2}
   628		addrs = parseHexAddresses(sampleData[5])
   629	
   630		return value, blocksize, addrs, nil
   631	}
   632	
   633	// extractHexAddresses extracts hex numbers from a string and returns
   634	// them, together with their numeric value, in a slice.
   635	func extractHexAddresses(s string) ([]string, []uint64) {
   636		hexStrings := hexNumberRE.FindAllString(s, -1)
   637		var ids []uint64
   638		for _, s := range hexStrings {
   639			if id, err := strconv.ParseUint(s, 0, 64); err == nil {
   640				ids = append(ids, id)
   641			} else {
   642				// Do not expect any parsing failures due to the regexp matching.
   643				panic("failed to parse hex value:" + s)
   644			}
   645		}
   646		return hexStrings, ids
   647	}
   648	
   649	// parseHexAddresses parses hex numbers from a string and returns them
   650	// in a slice.
   651	func parseHexAddresses(s string) []uint64 {
   652		_, ids := extractHexAddresses(s)
   653		return ids
   654	}
   655	
   656	// scaleHeapSample adjusts the data from a heapz Sample to
   657	// account for its probability of appearing in the collected
   658	// data. heapz profiles are a sampling of the memory allocations
   659	// requests in a program. We estimate the unsampled value by dividing
   660	// each collected sample by its probability of appearing in the
   661	// profile. heapz v2 profiles rely on a poisson process to determine
   662	// which samples to collect, based on the desired average collection
   663	// rate R. The probability of a sample of size S to appear in that
   664	// profile is 1-exp(-S/R).
   665	func scaleHeapSample(count, size, rate int64) (int64, int64) {
   666		if count == 0 || size == 0 {
   667			return 0, 0
   668		}
   669	
   670		if rate <= 1 {
   671			// if rate==1 all samples were collected so no adjustment is needed.
   672			// if rate<1 treat as unknown and skip scaling.
   673			return count, size
   674		}
   675	
   676		avgSize := float64(size) / float64(count)
   677		scale := 1 / (1 - math.Exp(-avgSize/float64(rate)))
   678	
   679		return int64(float64(count) * scale), int64(float64(size) * scale)
   680	}
   681	
   682	// parseContention parses a mutex or contention profile. There are 2 cases:
   683	// "--- contentionz " for legacy C++ profiles (and backwards compatibility)
   684	// "--- mutex:" or "--- contention:" for profiles generated by the Go runtime.
   685	// This code converts the text output from runtime into a *Profile. (In the future
   686	// the runtime might write a serialized Profile directly making this unnecessary.)
   687	func parseContention(b []byte) (*Profile, error) {
   688		r := bytes.NewBuffer(b)
   689		var l string
   690		var err error
   691		for {
   692			// Skip past comments and empty lines seeking a real header.
   693			l, err = r.ReadString('\n')
   694			if err != nil {
   695				return nil, err
   696			}
   697			if !isSpaceOrComment(l) {
   698				break
   699			}
   700		}
   701	
   702		if strings.HasPrefix(l, "--- contentionz ") {
   703			return parseCppContention(r)
   704		} else if strings.HasPrefix(l, "--- mutex:") {
   705			return parseCppContention(r)
   706		} else if strings.HasPrefix(l, "--- contention:") {
   707			return parseCppContention(r)
   708		}
   709		return nil, errUnrecognized
   710	}
   711	
   712	// parseCppContention parses the output from synchronization_profiling.cc
   713	// for backward compatibility, and the compatible (non-debug) block profile
   714	// output from the Go runtime.
   715	func parseCppContention(r *bytes.Buffer) (*Profile, error) {
   716		p := &Profile{
   717			PeriodType: &ValueType{Type: "contentions", Unit: "count"},
   718			Period:     1,
   719			SampleType: []*ValueType{
   720				{Type: "contentions", Unit: "count"},
   721				{Type: "delay", Unit: "nanoseconds"},
   722			},
   723		}
   724	
   725		var cpuHz int64
   726		var l string
   727		var err error
   728		// Parse text of the form "attribute = value" before the samples.
   729		const delimiter = "="
   730		for {
   731			l, err = r.ReadString('\n')
   732			if err != nil {
   733				if err != io.EOF {
   734					return nil, err
   735				}
   736	
   737				if l == "" {
   738					break
   739				}
   740			}
   741			if isSpaceOrComment(l) {
   742				continue
   743			}
   744	
   745			if l = strings.TrimSpace(l); l == "" {
   746				continue
   747			}
   748	
   749			if strings.HasPrefix(l, "---") {
   750				break
   751			}
   752	
   753			attr := strings.SplitN(l, delimiter, 2)
   754			if len(attr) != 2 {
   755				break
   756			}
   757			key, val := strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1])
   758			var err error
   759			switch key {
   760			case "cycles/second":
   761				if cpuHz, err = strconv.ParseInt(val, 0, 64); err != nil {
   762					return nil, errUnrecognized
   763				}
   764			case "sampling period":
   765				if p.Period, err = strconv.ParseInt(val, 0, 64); err != nil {
   766					return nil, errUnrecognized
   767				}
   768			case "ms since reset":
   769				ms, err := strconv.ParseInt(val, 0, 64)
   770				if err != nil {
   771					return nil, errUnrecognized
   772				}
   773				p.DurationNanos = ms * 1000 * 1000
   774			case "format":
   775				// CPP contentionz profiles don't have format.
   776				return nil, errUnrecognized
   777			case "resolution":
   778				// CPP contentionz profiles don't have resolution.
   779				return nil, errUnrecognized
   780			case "discarded samples":
   781			default:
   782				return nil, errUnrecognized
   783			}
   784		}
   785	
   786		locs := make(map[uint64]*Location)
   787		for {
   788			if !isSpaceOrComment(l) {
   789				if l = strings.TrimSpace(l); strings.HasPrefix(l, "---") {
   790					break
   791				}
   792				value, addrs, err := parseContentionSample(l, p.Period, cpuHz)
   793				if err != nil {
   794					return nil, err
   795				}
   796				var sloc []*Location
   797				for _, addr := range addrs {
   798					// Addresses from stack traces point to the next instruction after
   799					// each call. Adjust by -1 to land somewhere on the actual call.
   800					addr--
   801					loc := locs[addr]
   802					if locs[addr] == nil {
   803						loc = &Location{
   804							Address: addr,
   805						}
   806						p.Location = append(p.Location, loc)
   807						locs[addr] = loc
   808					}
   809					sloc = append(sloc, loc)
   810				}
   811				p.Sample = append(p.Sample, &Sample{
   812					Value:    value,
   813					Location: sloc,
   814				})
   815			}
   816	
   817			if l, err = r.ReadString('\n'); err != nil {
   818				if err != io.EOF {
   819					return nil, err
   820				}
   821				if l == "" {
   822					break
   823				}
   824			}
   825		}
   826	
   827		if err = parseAdditionalSections(l, r, p); err != nil {
   828			return nil, err
   829		}
   830	
   831		return p, nil
   832	}
   833	
   834	// parseContentionSample parses a single row from a contention profile
   835	// into a new Sample.
   836	func parseContentionSample(line string, period, cpuHz int64) (value []int64, addrs []uint64, err error) {
   837		sampleData := contentionSampleRE.FindStringSubmatch(line)
   838		if sampleData == nil {
   839			return value, addrs, errUnrecognized
   840		}
   841	
   842		v1, err := strconv.ParseInt(sampleData[1], 10, 64)
   843		if err != nil {
   844			return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err)
   845		}
   846		v2, err := strconv.ParseInt(sampleData[2], 10, 64)
   847		if err != nil {
   848			return value, addrs, fmt.Errorf("malformed sample: %s: %v", line, err)
   849		}
   850	
   851		// Unsample values if period and cpuHz are available.
   852		// - Delays are scaled to cycles and then to nanoseconds.
   853		// - Contentions are scaled to cycles.
   854		if period > 0 {
   855			if cpuHz > 0 {
   856				cpuGHz := float64(cpuHz) / 1e9
   857				v1 = int64(float64(v1) * float64(period) / cpuGHz)
   858			}
   859			v2 = v2 * period
   860		}
   861	
   862		value = []int64{v2, v1}
   863		addrs = parseHexAddresses(sampleData[3])
   864	
   865		return value, addrs, nil
   866	}
   867	
   868	// parseThread parses a Threadz profile and returns a new Profile.
   869	func parseThread(b []byte) (*Profile, error) {
   870		r := bytes.NewBuffer(b)
   871	
   872		var line string
   873		var err error
   874		for {
   875			// Skip past comments and empty lines seeking a real header.
   876			line, err = r.ReadString('\n')
   877			if err != nil {
   878				return nil, err
   879			}
   880			if !isSpaceOrComment(line) {
   881				break
   882			}
   883		}
   884	
   885		if m := threadzStartRE.FindStringSubmatch(line); m != nil {
   886			// Advance over initial comments until first stack trace.
   887			for {
   888				line, err = r.ReadString('\n')
   889				if err != nil {
   890					if err != io.EOF {
   891						return nil, err
   892					}
   893	
   894					if line == "" {
   895						break
   896					}
   897				}
   898				if sectionTrigger(line) != unrecognizedSection || line[0] == '-' {
   899					break
   900				}
   901			}
   902		} else if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 {
   903			return nil, errUnrecognized
   904		}
   905	
   906		p := &Profile{
   907			SampleType: []*ValueType{{Type: "thread", Unit: "count"}},
   908			PeriodType: &ValueType{Type: "thread", Unit: "count"},
   909			Period:     1,
   910		}
   911	
   912		locs := make(map[uint64]*Location)
   913		// Recognize each thread and populate profile samples.
   914		for sectionTrigger(line) == unrecognizedSection {
   915			if strings.HasPrefix(line, "---- no stack trace for") {
   916				line = ""
   917				break
   918			}
   919			if t := threadStartRE.FindStringSubmatch(line); len(t) != 4 {
   920				return nil, errUnrecognized
   921			}
   922	
   923			var addrs []uint64
   924			line, addrs, err = parseThreadSample(r)
   925			if err != nil {
   926				return nil, errUnrecognized
   927			}
   928			if len(addrs) == 0 {
   929				// We got a --same as previous threads--. Bump counters.
   930				if len(p.Sample) > 0 {
   931					s := p.Sample[len(p.Sample)-1]
   932					s.Value[0]++
   933				}
   934				continue
   935			}
   936	
   937			var sloc []*Location
   938			for _, addr := range addrs {
   939				// Addresses from stack traces point to the next instruction after
   940				// each call. Adjust by -1 to land somewhere on the actual call.
   941				addr--
   942				loc := locs[addr]
   943				if locs[addr] == nil {
   944					loc = &Location{
   945						Address: addr,
   946					}
   947					p.Location = append(p.Location, loc)
   948					locs[addr] = loc
   949				}
   950				sloc = append(sloc, loc)
   951			}
   952	
   953			p.Sample = append(p.Sample, &Sample{
   954				Value:    []int64{1},
   955				Location: sloc,
   956			})
   957		}
   958	
   959		if err = parseAdditionalSections(line, r, p); err != nil {
   960			return nil, err
   961		}
   962	
   963		return p, nil
   964	}
   965	
   966	// parseThreadSample parses a symbolized or unsymbolized stack trace.
   967	// Returns the first line after the traceback, the sample (or nil if
   968	// it hits a 'same-as-previous' marker) and an error.
   969	func parseThreadSample(b *bytes.Buffer) (nextl string, addrs []uint64, err error) {
   970		var l string
   971		sameAsPrevious := false
   972		for {
   973			if l, err = b.ReadString('\n'); err != nil {
   974				if err != io.EOF {
   975					return "", nil, err
   976				}
   977				if l == "" {
   978					break
   979				}
   980			}
   981			if l = strings.TrimSpace(l); l == "" {
   982				continue
   983			}
   984	
   985			if strings.HasPrefix(l, "---") {
   986				break
   987			}
   988			if strings.Contains(l, "same as previous thread") {
   989				sameAsPrevious = true
   990				continue
   991			}
   992	
   993			addrs = append(addrs, parseHexAddresses(l)...)
   994		}
   995	
   996		if sameAsPrevious {
   997			return l, nil, nil
   998		}
   999		return l, addrs, nil
  1000	}
  1001	
  1002	// parseAdditionalSections parses any additional sections in the
  1003	// profile, ignoring any unrecognized sections.
  1004	func parseAdditionalSections(l string, b *bytes.Buffer, p *Profile) (err error) {
  1005		for {
  1006			if sectionTrigger(l) == memoryMapSection {
  1007				break
  1008			}
  1009			// Ignore any unrecognized sections.
  1010			if l, err := b.ReadString('\n'); err != nil {
  1011				if err != io.EOF {
  1012					return err
  1013				}
  1014				if l == "" {
  1015					break
  1016				}
  1017			}
  1018		}
  1019		return p.ParseMemoryMap(b)
  1020	}
  1021	
  1022	// ParseMemoryMap parses a memory map in the format of
  1023	// /proc/self/maps, and overrides the mappings in the current profile.
  1024	// It renumbers the samples and locations in the profile correspondingly.
  1025	func (p *Profile) ParseMemoryMap(rd io.Reader) error {
  1026		b := bufio.NewReader(rd)
  1027	
  1028		var attrs []string
  1029		var r *strings.Replacer
  1030		const delimiter = "="
  1031		for {
  1032			l, err := b.ReadString('\n')
  1033			if err != nil {
  1034				if err != io.EOF {
  1035					return err
  1036				}
  1037				if l == "" {
  1038					break
  1039				}
  1040			}
  1041			if l = strings.TrimSpace(l); l == "" {
  1042				continue
  1043			}
  1044	
  1045			if r != nil {
  1046				l = r.Replace(l)
  1047			}
  1048			m, err := parseMappingEntry(l)
  1049			if err != nil {
  1050				if err == errUnrecognized {
  1051					// Recognize assignments of the form: attr=value, and replace
  1052					// $attr with value on subsequent mappings.
  1053					if attr := strings.SplitN(l, delimiter, 2); len(attr) == 2 {
  1054						attrs = append(attrs, "$"+strings.TrimSpace(attr[0]), strings.TrimSpace(attr[1]))
  1055						r = strings.NewReplacer(attrs...)
  1056					}
  1057					// Ignore any unrecognized entries
  1058					continue
  1059				}
  1060				return err
  1061			}
  1062			if m == nil || (m.File == "" && len(p.Mapping) != 0) {
  1063				// In some cases the first entry may include the address range
  1064				// but not the name of the file. It should be followed by
  1065				// another entry with the name.
  1066				continue
  1067			}
  1068			if len(p.Mapping) == 1 && p.Mapping[0].File == "" {
  1069				// Update the name if this is the entry following that empty one.
  1070				p.Mapping[0].File = m.File
  1071				continue
  1072			}
  1073			p.Mapping = append(p.Mapping, m)
  1074		}
  1075		p.remapLocationIDs()
  1076		p.remapFunctionIDs()
  1077		p.remapMappingIDs()
  1078		return nil
  1079	}
  1080	
  1081	func parseMappingEntry(l string) (*Mapping, error) {
  1082		mapping := &Mapping{}
  1083		var err error
  1084		if me := procMapsRE.FindStringSubmatch(l); len(me) == 9 {
  1085			if !strings.Contains(me[3], "x") {
  1086				// Skip non-executable entries.
  1087				return nil, nil
  1088			}
  1089			if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil {
  1090				return nil, errUnrecognized
  1091			}
  1092			if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil {
  1093				return nil, errUnrecognized
  1094			}
  1095			if me[4] != "" {
  1096				if mapping.Offset, err = strconv.ParseUint(me[4], 16, 64); err != nil {
  1097					return nil, errUnrecognized
  1098				}
  1099			}
  1100			mapping.File = me[8]
  1101			return mapping, nil
  1102		}
  1103	
  1104		if me := briefMapsRE.FindStringSubmatch(l); len(me) == 6 {
  1105			if mapping.Start, err = strconv.ParseUint(me[1], 16, 64); err != nil {
  1106				return nil, errUnrecognized
  1107			}
  1108			if mapping.Limit, err = strconv.ParseUint(me[2], 16, 64); err != nil {
  1109				return nil, errUnrecognized
  1110			}
  1111			mapping.File = me[3]
  1112			if me[5] != "" {
  1113				if mapping.Offset, err = strconv.ParseUint(me[5], 16, 64); err != nil {
  1114					return nil, errUnrecognized
  1115				}
  1116			}
  1117			return mapping, nil
  1118		}
  1119	
  1120		return nil, errUnrecognized
  1121	}
  1122	
  1123	type sectionType int
  1124	
  1125	const (
  1126		unrecognizedSection sectionType = iota
  1127		memoryMapSection
  1128	)
  1129	
  1130	var memoryMapTriggers = []string{
  1131		"--- Memory map: ---",
  1132		"MAPPED_LIBRARIES:",
  1133	}
  1134	
  1135	func sectionTrigger(line string) sectionType {
  1136		for _, trigger := range memoryMapTriggers {
  1137			if strings.Contains(line, trigger) {
  1138				return memoryMapSection
  1139			}
  1140		}
  1141		return unrecognizedSection
  1142	}
  1143	
  1144	func (p *Profile) addLegacyFrameInfo() {
  1145		switch {
  1146		case isProfileType(p, heapzSampleTypes) ||
  1147			isProfileType(p, heapzInUseSampleTypes) ||
  1148			isProfileType(p, heapzAllocSampleTypes):
  1149			p.DropFrames, p.KeepFrames = allocRxStr, allocSkipRxStr
  1150		case isProfileType(p, contentionzSampleTypes):
  1151			p.DropFrames, p.KeepFrames = lockRxStr, ""
  1152		default:
  1153			p.DropFrames, p.KeepFrames = cpuProfilerRxStr, ""
  1154		}
  1155	}
  1156	
  1157	var heapzSampleTypes = []string{"allocations", "size"} // early Go pprof profiles
  1158	var heapzInUseSampleTypes = []string{"inuse_objects", "inuse_space"}
  1159	var heapzAllocSampleTypes = []string{"alloc_objects", "alloc_space"}
  1160	var contentionzSampleTypes = []string{"contentions", "delay"}
  1161	
  1162	func isProfileType(p *Profile, t []string) bool {
  1163		st := p.SampleType
  1164		if len(st) != len(t) {
  1165			return false
  1166		}
  1167	
  1168		for i := range st {
  1169			if st[i].Type != t[i] {
  1170				return false
  1171			}
  1172		}
  1173		return true
  1174	}
  1175	
  1176	var allocRxStr = strings.Join([]string{
  1177		// POSIX entry points.
  1178		`calloc`,
  1179		`cfree`,
  1180		`malloc`,
  1181		`free`,
  1182		`memalign`,
  1183		`do_memalign`,
  1184		`(__)?posix_memalign`,
  1185		`pvalloc`,
  1186		`valloc`,
  1187		`realloc`,
  1188	
  1189		// TC malloc.
  1190		`tcmalloc::.*`,
  1191		`tc_calloc`,
  1192		`tc_cfree`,
  1193		`tc_malloc`,
  1194		`tc_free`,
  1195		`tc_memalign`,
  1196		`tc_posix_memalign`,
  1197		`tc_pvalloc`,
  1198		`tc_valloc`,
  1199		`tc_realloc`,
  1200		`tc_new`,
  1201		`tc_delete`,
  1202		`tc_newarray`,
  1203		`tc_deletearray`,
  1204		`tc_new_nothrow`,
  1205		`tc_newarray_nothrow`,
  1206	
  1207		// Memory-allocation routines on OS X.
  1208		`malloc_zone_malloc`,
  1209		`malloc_zone_calloc`,
  1210		`malloc_zone_valloc`,
  1211		`malloc_zone_realloc`,
  1212		`malloc_zone_memalign`,
  1213		`malloc_zone_free`,
  1214	
  1215		// Go runtime
  1216		`runtime\..*`,
  1217	
  1218		// Other misc. memory allocation routines
  1219		`BaseArena::.*`,
  1220		`(::)?do_malloc_no_errno`,
  1221		`(::)?do_malloc_pages`,
  1222		`(::)?do_malloc`,
  1223		`DoSampledAllocation`,
  1224		`MallocedMemBlock::MallocedMemBlock`,
  1225		`_M_allocate`,
  1226		`__builtin_(vec_)?delete`,
  1227		`__builtin_(vec_)?new`,
  1228		`__gnu_cxx::new_allocator::allocate`,
  1229		`__libc_malloc`,
  1230		`__malloc_alloc_template::allocate`,
  1231		`allocate`,
  1232		`cpp_alloc`,
  1233		`operator new(\[\])?`,
  1234		`simple_alloc::allocate`,
  1235	}, `|`)
  1236	
  1237	var allocSkipRxStr = strings.Join([]string{
  1238		// Preserve Go runtime frames that appear in the middle/bottom of
  1239		// the stack.
  1240		`runtime\.panic`,
  1241		`runtime\.reflectcall`,
  1242		`runtime\.call[0-9]*`,
  1243	}, `|`)
  1244	
  1245	var cpuProfilerRxStr = strings.Join([]string{
  1246		`ProfileData::Add`,
  1247		`ProfileData::prof_handler`,
  1248		`CpuProfiler::prof_handler`,
  1249		`__pthread_sighandler`,
  1250		`__restore`,
  1251	}, `|`)
  1252	
  1253	var lockRxStr = strings.Join([]string{
  1254		`RecordLockProfileData`,
  1255		`(base::)?RecordLockProfileData.*`,
  1256		`(base::)?SubmitMutexProfileData.*`,
  1257		`(base::)?SubmitSpinLockProfileData.*`,
  1258		`(Mutex::)?AwaitCommon.*`,
  1259		`(Mutex::)?Unlock.*`,
  1260		`(Mutex::)?UnlockSlow.*`,
  1261		`(Mutex::)?ReaderUnlock.*`,
  1262		`(MutexLock::)?~MutexLock.*`,
  1263		`(SpinLock::)?Unlock.*`,
  1264		`(SpinLock::)?SlowUnlock.*`,
  1265		`(SpinLockHolder::)?~SpinLockHolder.*`,
  1266	}, `|`)
  1267	

View as plain text