...

Source file src/pkg/cmd/vendor/github.com/google/pprof/internal/report/source.go

     1	// Copyright 2014 Google Inc. All Rights Reserved.
     2	//
     3	// Licensed under the Apache License, Version 2.0 (the "License");
     4	// you may not use this file except in compliance with the License.
     5	// You may obtain a copy of the License at
     6	//
     7	//     http://www.apache.org/licenses/LICENSE-2.0
     8	//
     9	// Unless required by applicable law or agreed to in writing, software
    10	// distributed under the License is distributed on an "AS IS" BASIS,
    11	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    12	// See the License for the specific language governing permissions and
    13	// limitations under the License.
    14	
    15	package report
    16	
    17	// This file contains routines related to the generation of annotated
    18	// source listings.
    19	
    20	import (
    21		"bufio"
    22		"fmt"
    23		"html/template"
    24		"io"
    25		"os"
    26		"path/filepath"
    27		"strconv"
    28		"strings"
    29	
    30		"github.com/google/pprof/internal/graph"
    31		"github.com/google/pprof/internal/measurement"
    32		"github.com/google/pprof/internal/plugin"
    33	)
    34	
    35	// printSource prints an annotated source listing, include all
    36	// functions with samples that match the regexp rpt.options.symbol.
    37	// The sources are sorted by function name and then by filename to
    38	// eliminate potential nondeterminism.
    39	func printSource(w io.Writer, rpt *Report) error {
    40		o := rpt.options
    41		g := rpt.newGraph(nil)
    42	
    43		// Identify all the functions that match the regexp provided.
    44		// Group nodes for each matching function.
    45		var functions graph.Nodes
    46		functionNodes := make(map[string]graph.Nodes)
    47		for _, n := range g.Nodes {
    48			if !o.Symbol.MatchString(n.Info.Name) {
    49				continue
    50			}
    51			if functionNodes[n.Info.Name] == nil {
    52				functions = append(functions, n)
    53			}
    54			functionNodes[n.Info.Name] = append(functionNodes[n.Info.Name], n)
    55		}
    56		functions.Sort(graph.NameOrder)
    57	
    58		sourcePath := o.SourcePath
    59		if sourcePath == "" {
    60			wd, err := os.Getwd()
    61			if err != nil {
    62				return fmt.Errorf("could not stat current dir: %v", err)
    63			}
    64			sourcePath = wd
    65		}
    66		reader := newSourceReader(sourcePath, o.TrimPath)
    67	
    68		fmt.Fprintf(w, "Total: %s\n", rpt.formatValue(rpt.total))
    69		for _, fn := range functions {
    70			name := fn.Info.Name
    71	
    72			// Identify all the source files associated to this function.
    73			// Group nodes for each source file.
    74			var sourceFiles graph.Nodes
    75			fileNodes := make(map[string]graph.Nodes)
    76			for _, n := range functionNodes[name] {
    77				if n.Info.File == "" {
    78					continue
    79				}
    80				if fileNodes[n.Info.File] == nil {
    81					sourceFiles = append(sourceFiles, n)
    82				}
    83				fileNodes[n.Info.File] = append(fileNodes[n.Info.File], n)
    84			}
    85	
    86			if len(sourceFiles) == 0 {
    87				fmt.Fprintf(w, "No source information for %s\n", name)
    88				continue
    89			}
    90	
    91			sourceFiles.Sort(graph.FileOrder)
    92	
    93			// Print each file associated with this function.
    94			for _, fl := range sourceFiles {
    95				filename := fl.Info.File
    96				fns := fileNodes[filename]
    97				flatSum, cumSum := fns.Sum()
    98	
    99				fnodes, _, err := getSourceFromFile(filename, reader, fns, 0, 0)
   100				fmt.Fprintf(w, "ROUTINE ======================== %s in %s\n", name, filename)
   101				fmt.Fprintf(w, "%10s %10s (flat, cum) %s of Total\n",
   102					rpt.formatValue(flatSum), rpt.formatValue(cumSum),
   103					measurement.Percentage(cumSum, rpt.total))
   104	
   105				if err != nil {
   106					fmt.Fprintf(w, " Error: %v\n", err)
   107					continue
   108				}
   109	
   110				for _, fn := range fnodes {
   111					fmt.Fprintf(w, "%10s %10s %6d:%s\n", valueOrDot(fn.Flat, rpt), valueOrDot(fn.Cum, rpt), fn.Info.Lineno, fn.Info.Name)
   112				}
   113			}
   114		}
   115		return nil
   116	}
   117	
   118	// printWebSource prints an annotated source listing, include all
   119	// functions with samples that match the regexp rpt.options.symbol.
   120	func printWebSource(w io.Writer, rpt *Report, obj plugin.ObjTool) error {
   121		printHeader(w, rpt)
   122		if err := PrintWebList(w, rpt, obj, -1); err != nil {
   123			return err
   124		}
   125		printPageClosing(w)
   126		return nil
   127	}
   128	
   129	// PrintWebList prints annotated source listing of rpt to w.
   130	func PrintWebList(w io.Writer, rpt *Report, obj plugin.ObjTool, maxFiles int) error {
   131		o := rpt.options
   132		g := rpt.newGraph(nil)
   133	
   134		// If the regexp source can be parsed as an address, also match
   135		// functions that land on that address.
   136		var address *uint64
   137		if hex, err := strconv.ParseUint(o.Symbol.String(), 0, 64); err == nil {
   138			address = &hex
   139		}
   140	
   141		sourcePath := o.SourcePath
   142		if sourcePath == "" {
   143			wd, err := os.Getwd()
   144			if err != nil {
   145				return fmt.Errorf("could not stat current dir: %v", err)
   146			}
   147			sourcePath = wd
   148		}
   149		reader := newSourceReader(sourcePath, o.TrimPath)
   150	
   151		type fileFunction struct {
   152			fileName, functionName string
   153		}
   154	
   155		// Extract interesting symbols from binary files in the profile and
   156		// classify samples per symbol.
   157		symbols := symbolsFromBinaries(rpt.prof, g, o.Symbol, address, obj)
   158		symNodes := nodesPerSymbol(g.Nodes, symbols)
   159	
   160		// Identify sources associated to a symbol by examining
   161		// symbol samples. Classify samples per source file.
   162		fileNodes := make(map[fileFunction]graph.Nodes)
   163		if len(symNodes) == 0 {
   164			for _, n := range g.Nodes {
   165				if n.Info.File == "" || !o.Symbol.MatchString(n.Info.Name) {
   166					continue
   167				}
   168				ff := fileFunction{n.Info.File, n.Info.Name}
   169				fileNodes[ff] = append(fileNodes[ff], n)
   170			}
   171		} else {
   172			for _, nodes := range symNodes {
   173				for _, n := range nodes {
   174					if n.Info.File != "" {
   175						ff := fileFunction{n.Info.File, n.Info.Name}
   176						fileNodes[ff] = append(fileNodes[ff], n)
   177					}
   178				}
   179			}
   180		}
   181	
   182		if len(fileNodes) == 0 {
   183			return fmt.Errorf("no source information for %s", o.Symbol.String())
   184		}
   185	
   186		sourceFiles := make(graph.Nodes, 0, len(fileNodes))
   187		for _, nodes := range fileNodes {
   188			sNode := *nodes[0]
   189			sNode.Flat, sNode.Cum = nodes.Sum()
   190			sourceFiles = append(sourceFiles, &sNode)
   191		}
   192	
   193		// Limit number of files printed?
   194		if maxFiles < 0 {
   195			sourceFiles.Sort(graph.FileOrder)
   196		} else {
   197			sourceFiles.Sort(graph.FlatNameOrder)
   198			if maxFiles < len(sourceFiles) {
   199				sourceFiles = sourceFiles[:maxFiles]
   200			}
   201		}
   202	
   203		// Print each file associated with this function.
   204		for _, n := range sourceFiles {
   205			ff := fileFunction{n.Info.File, n.Info.Name}
   206			fns := fileNodes[ff]
   207	
   208			asm := assemblyPerSourceLine(symbols, fns, ff.fileName, obj)
   209			start, end := sourceCoordinates(asm)
   210	
   211			fnodes, path, err := getSourceFromFile(ff.fileName, reader, fns, start, end)
   212			if err != nil {
   213				fnodes, path = getMissingFunctionSource(ff.fileName, asm, start, end)
   214			}
   215	
   216			printFunctionHeader(w, ff.functionName, path, n.Flat, n.Cum, rpt)
   217			for _, fn := range fnodes {
   218				printFunctionSourceLine(w, fn, asm[fn.Info.Lineno], reader, rpt)
   219			}
   220			printFunctionClosing(w)
   221		}
   222		return nil
   223	}
   224	
   225	// sourceCoordinates returns the lowest and highest line numbers from
   226	// a set of assembly statements.
   227	func sourceCoordinates(asm map[int][]assemblyInstruction) (start, end int) {
   228		for l := range asm {
   229			if start == 0 || l < start {
   230				start = l
   231			}
   232			if end == 0 || l > end {
   233				end = l
   234			}
   235		}
   236		return start, end
   237	}
   238	
   239	// assemblyPerSourceLine disassembles the binary containing a symbol
   240	// and classifies the assembly instructions according to its
   241	// corresponding source line, annotating them with a set of samples.
   242	func assemblyPerSourceLine(objSyms []*objSymbol, rs graph.Nodes, src string, obj plugin.ObjTool) map[int][]assemblyInstruction {
   243		assembly := make(map[int][]assemblyInstruction)
   244		// Identify symbol to use for this collection of samples.
   245		o := findMatchingSymbol(objSyms, rs)
   246		if o == nil {
   247			return assembly
   248		}
   249	
   250		// Extract assembly for matched symbol
   251		insts, err := obj.Disasm(o.sym.File, o.sym.Start, o.sym.End)
   252		if err != nil {
   253			return assembly
   254		}
   255	
   256		srcBase := filepath.Base(src)
   257		anodes := annotateAssembly(insts, rs, o.base)
   258		var lineno = 0
   259		var prevline = 0
   260		for _, an := range anodes {
   261			// Do not rely solely on the line number produced by Disasm
   262			// since it is not what we want in the presence of inlining.
   263			//
   264			// E.g., suppose we are printing source code for F and this
   265			// instruction is from H where F called G called H and both
   266			// of those calls were inlined. We want to use the line
   267			// number from F, not from H (which is what Disasm gives us).
   268			//
   269			// So find the outer-most linenumber in the source file.
   270			found := false
   271			if frames, err := o.file.SourceLine(an.address + o.base); err == nil {
   272				for i := len(frames) - 1; i >= 0; i-- {
   273					if filepath.Base(frames[i].File) == srcBase {
   274						for j := i - 1; j >= 0; j-- {
   275							an.inlineCalls = append(an.inlineCalls, callID{frames[j].File, frames[j].Line})
   276						}
   277						lineno = frames[i].Line
   278						found = true
   279						break
   280					}
   281				}
   282			}
   283			if !found && filepath.Base(an.file) == srcBase {
   284				lineno = an.line
   285			}
   286	
   287			if lineno != 0 {
   288				if lineno != prevline {
   289					// This instruction starts a new block
   290					// of contiguous instructions on this line.
   291					an.startsBlock = true
   292				}
   293				prevline = lineno
   294				assembly[lineno] = append(assembly[lineno], an)
   295			}
   296		}
   297	
   298		return assembly
   299	}
   300	
   301	// findMatchingSymbol looks for the symbol that corresponds to a set
   302	// of samples, by comparing their addresses.
   303	func findMatchingSymbol(objSyms []*objSymbol, ns graph.Nodes) *objSymbol {
   304		for _, n := range ns {
   305			for _, o := range objSyms {
   306				if filepath.Base(o.sym.File) == filepath.Base(n.Info.Objfile) &&
   307					o.sym.Start <= n.Info.Address-o.base &&
   308					n.Info.Address-o.base <= o.sym.End {
   309					return o
   310				}
   311			}
   312		}
   313		return nil
   314	}
   315	
   316	// printHeader prints the page header for a weblist report.
   317	func printHeader(w io.Writer, rpt *Report) {
   318		fmt.Fprintln(w, `
   319	<!DOCTYPE html>
   320	<html>
   321	<head>
   322	<meta charset="UTF-8">
   323	<title>Pprof listing</title>`)
   324		fmt.Fprintln(w, weblistPageCSS)
   325		fmt.Fprintln(w, weblistPageScript)
   326		fmt.Fprint(w, "</head>\n<body>\n\n")
   327	
   328		var labels []string
   329		for _, l := range ProfileLabels(rpt) {
   330			labels = append(labels, template.HTMLEscapeString(l))
   331		}
   332	
   333		fmt.Fprintf(w, `<div class="legend">%s<br>Total: %s</div>`,
   334			strings.Join(labels, "<br>\n"),
   335			rpt.formatValue(rpt.total),
   336		)
   337	}
   338	
   339	// printFunctionHeader prints a function header for a weblist report.
   340	func printFunctionHeader(w io.Writer, name, path string, flatSum, cumSum int64, rpt *Report) {
   341		fmt.Fprintf(w, `<h2>%s</h2><p class="filename">%s</p>
   342	<pre onClick="pprof_toggle_asm(event)">
   343	  Total:  %10s %10s (flat, cum) %s
   344	`,
   345			template.HTMLEscapeString(name), template.HTMLEscapeString(path),
   346			rpt.formatValue(flatSum), rpt.formatValue(cumSum),
   347			measurement.Percentage(cumSum, rpt.total))
   348	}
   349	
   350	// printFunctionSourceLine prints a source line and the corresponding assembly.
   351	func printFunctionSourceLine(w io.Writer, fn *graph.Node, assembly []assemblyInstruction, reader *sourceReader, rpt *Report) {
   352		if len(assembly) == 0 {
   353			fmt.Fprintf(w,
   354				"<span class=line> %6d</span> <span class=nop>  %10s %10s %8s  %s </span>\n",
   355				fn.Info.Lineno,
   356				valueOrDot(fn.Flat, rpt), valueOrDot(fn.Cum, rpt),
   357				"", template.HTMLEscapeString(fn.Info.Name))
   358			return
   359		}
   360	
   361		fmt.Fprintf(w,
   362			"<span class=line> %6d</span> <span class=deadsrc>  %10s %10s %8s  %s </span>",
   363			fn.Info.Lineno,
   364			valueOrDot(fn.Flat, rpt), valueOrDot(fn.Cum, rpt),
   365			"", template.HTMLEscapeString(fn.Info.Name))
   366		srcIndent := indentation(fn.Info.Name)
   367		fmt.Fprint(w, "<span class=asm>")
   368		var curCalls []callID
   369		for i, an := range assembly {
   370			if an.startsBlock && i != 0 {
   371				// Insert a separator between discontiguous blocks.
   372				fmt.Fprintf(w, " %8s %28s\n", "", "⋮")
   373			}
   374	
   375			var fileline string
   376			if an.file != "" {
   377				fileline = fmt.Sprintf("%s:%d", template.HTMLEscapeString(an.file), an.line)
   378			}
   379			flat, cum := an.flat, an.cum
   380			if an.flatDiv != 0 {
   381				flat = flat / an.flatDiv
   382			}
   383			if an.cumDiv != 0 {
   384				cum = cum / an.cumDiv
   385			}
   386	
   387			// Print inlined call context.
   388			for j, c := range an.inlineCalls {
   389				if j < len(curCalls) && curCalls[j] == c {
   390					// Skip if same as previous instruction.
   391					continue
   392				}
   393				curCalls = nil
   394				fline, ok := reader.line(c.file, c.line)
   395				if !ok {
   396					fline = ""
   397				}
   398				text := strings.Repeat(" ", srcIndent+4+4*j) + strings.TrimSpace(fline)
   399				fmt.Fprintf(w, " %8s %10s %10s %8s  <span class=inlinesrc>%s</span> <span class=unimportant>%s:%d</span>\n",
   400					"", "", "", "",
   401					template.HTMLEscapeString(fmt.Sprintf("%-80s", text)),
   402					template.HTMLEscapeString(filepath.Base(c.file)), c.line)
   403			}
   404			curCalls = an.inlineCalls
   405			text := strings.Repeat(" ", srcIndent+4+4*len(curCalls)) + an.instruction
   406			fmt.Fprintf(w, " %8s %10s %10s %8x: %s <span class=unimportant>%s</span>\n",
   407				"", valueOrDot(flat, rpt), valueOrDot(cum, rpt), an.address,
   408				template.HTMLEscapeString(fmt.Sprintf("%-80s", text)),
   409				template.HTMLEscapeString(fileline))
   410		}
   411		fmt.Fprintln(w, "</span>")
   412	}
   413	
   414	// printFunctionClosing prints the end of a function in a weblist report.
   415	func printFunctionClosing(w io.Writer) {
   416		fmt.Fprintln(w, "</pre>")
   417	}
   418	
   419	// printPageClosing prints the end of the page in a weblist report.
   420	func printPageClosing(w io.Writer) {
   421		fmt.Fprintln(w, weblistPageClosing)
   422	}
   423	
   424	// getSourceFromFile collects the sources of a function from a source
   425	// file and annotates it with the samples in fns. Returns the sources
   426	// as nodes, using the info.name field to hold the source code.
   427	func getSourceFromFile(file string, reader *sourceReader, fns graph.Nodes, start, end int) (graph.Nodes, string, error) {
   428		lineNodes := make(map[int]graph.Nodes)
   429	
   430		// Collect source coordinates from profile.
   431		const margin = 5 // Lines before first/after last sample.
   432		if start == 0 {
   433			if fns[0].Info.StartLine != 0 {
   434				start = fns[0].Info.StartLine
   435			} else {
   436				start = fns[0].Info.Lineno - margin
   437			}
   438		} else {
   439			start -= margin
   440		}
   441		if end == 0 {
   442			end = fns[0].Info.Lineno
   443		}
   444		end += margin
   445		for _, n := range fns {
   446			lineno := n.Info.Lineno
   447			nodeStart := n.Info.StartLine
   448			if nodeStart == 0 {
   449				nodeStart = lineno - margin
   450			}
   451			nodeEnd := lineno + margin
   452			if nodeStart < start {
   453				start = nodeStart
   454			} else if nodeEnd > end {
   455				end = nodeEnd
   456			}
   457			lineNodes[lineno] = append(lineNodes[lineno], n)
   458		}
   459		if start < 1 {
   460			start = 1
   461		}
   462	
   463		var src graph.Nodes
   464		for lineno := start; lineno <= end; lineno++ {
   465			line, ok := reader.line(file, lineno)
   466			if !ok {
   467				break
   468			}
   469			flat, cum := lineNodes[lineno].Sum()
   470			src = append(src, &graph.Node{
   471				Info: graph.NodeInfo{
   472					Name:   strings.TrimRight(line, "\n"),
   473					Lineno: lineno,
   474				},
   475				Flat: flat,
   476				Cum:  cum,
   477			})
   478		}
   479		if err := reader.fileError(file); err != nil {
   480			return nil, file, err
   481		}
   482		return src, file, nil
   483	}
   484	
   485	// getMissingFunctionSource creates a dummy function body to point to
   486	// the source file and annotates it with the samples in asm.
   487	func getMissingFunctionSource(filename string, asm map[int][]assemblyInstruction, start, end int) (graph.Nodes, string) {
   488		var fnodes graph.Nodes
   489		for i := start; i <= end; i++ {
   490			insts := asm[i]
   491			if len(insts) == 0 {
   492				continue
   493			}
   494			var group assemblyInstruction
   495			for _, insn := range insts {
   496				group.flat += insn.flat
   497				group.cum += insn.cum
   498				group.flatDiv += insn.flatDiv
   499				group.cumDiv += insn.cumDiv
   500			}
   501			flat := group.flatValue()
   502			cum := group.cumValue()
   503			fnodes = append(fnodes, &graph.Node{
   504				Info: graph.NodeInfo{
   505					Name:   "???",
   506					Lineno: i,
   507				},
   508				Flat: flat,
   509				Cum:  cum,
   510			})
   511		}
   512		return fnodes, filename
   513	}
   514	
   515	// sourceReader provides access to source code with caching of file contents.
   516	type sourceReader struct {
   517		// searchPath is a filepath.ListSeparator-separated list of directories where
   518		// source files should be searched.
   519		searchPath string
   520	
   521		// trimPath is a filepath.ListSeparator-separated list of paths to trim.
   522		trimPath string
   523	
   524		// files maps from path name to a list of lines.
   525		// files[*][0] is unused since line numbering starts at 1.
   526		files map[string][]string
   527	
   528		// errors collects errors encountered per file. These errors are
   529		// consulted before returning out of these module.
   530		errors map[string]error
   531	}
   532	
   533	func newSourceReader(searchPath, trimPath string) *sourceReader {
   534		return &sourceReader{
   535			searchPath,
   536			trimPath,
   537			make(map[string][]string),
   538			make(map[string]error),
   539		}
   540	}
   541	
   542	func (reader *sourceReader) fileError(path string) error {
   543		return reader.errors[path]
   544	}
   545	
   546	func (reader *sourceReader) line(path string, lineno int) (string, bool) {
   547		lines, ok := reader.files[path]
   548		if !ok {
   549			// Read and cache file contents.
   550			lines = []string{""} // Skip 0th line
   551			f, err := openSourceFile(path, reader.searchPath, reader.trimPath)
   552			if err != nil {
   553				reader.errors[path] = err
   554			} else {
   555				s := bufio.NewScanner(f)
   556				for s.Scan() {
   557					lines = append(lines, s.Text())
   558				}
   559				f.Close()
   560				if s.Err() != nil {
   561					reader.errors[path] = err
   562				}
   563			}
   564			reader.files[path] = lines
   565		}
   566		if lineno <= 0 || lineno >= len(lines) {
   567			return "", false
   568		}
   569		return lines[lineno], true
   570	}
   571	
   572	// openSourceFile opens a source file from a name encoded in a profile. File
   573	// names in a profile after can be relative paths, so search them in each of
   574	// the paths in searchPath and their parents. In case the profile contains
   575	// absolute paths, additional paths may be configured to trim from the source
   576	// paths in the profile. This effectively turns the path into a relative path
   577	// searching it using searchPath as usual).
   578	func openSourceFile(path, searchPath, trim string) (*os.File, error) {
   579		path = trimPath(path, trim, searchPath)
   580		// If file is still absolute, require file to exist.
   581		if filepath.IsAbs(path) {
   582			f, err := os.Open(path)
   583			return f, err
   584		}
   585		// Scan each component of the path.
   586		for _, dir := range filepath.SplitList(searchPath) {
   587			// Search up for every parent of each possible path.
   588			for {
   589				filename := filepath.Join(dir, path)
   590				if f, err := os.Open(filename); err == nil {
   591					return f, nil
   592				}
   593				parent := filepath.Dir(dir)
   594				if parent == dir {
   595					break
   596				}
   597				dir = parent
   598			}
   599		}
   600	
   601		return nil, fmt.Errorf("could not find file %s on path %s", path, searchPath)
   602	}
   603	
   604	// trimPath cleans up a path by removing prefixes that are commonly
   605	// found on profiles plus configured prefixes.
   606	// TODO(aalexand): Consider optimizing out the redundant work done in this
   607	// function if it proves to matter.
   608	func trimPath(path, trimPath, searchPath string) string {
   609		// Keep path variable intact as it's used below to form the return value.
   610		sPath, searchPath := filepath.ToSlash(path), filepath.ToSlash(searchPath)
   611		if trimPath == "" {
   612			// If the trim path is not configured, try to guess it heuristically:
   613			// search for basename of each search path in the original path and, if
   614			// found, strip everything up to and including the basename. So, for
   615			// example, given original path "/some/remote/path/my-project/foo/bar.c"
   616			// and search path "/my/local/path/my-project" the heuristic will return
   617			// "/my/local/path/my-project/foo/bar.c".
   618			for _, dir := range filepath.SplitList(searchPath) {
   619				want := "/" + filepath.Base(dir) + "/"
   620				if found := strings.Index(sPath, want); found != -1 {
   621					return path[found+len(want):]
   622				}
   623			}
   624		}
   625		// Trim configured trim prefixes.
   626		trimPaths := append(filepath.SplitList(filepath.ToSlash(trimPath)), "/proc/self/cwd/./", "/proc/self/cwd/")
   627		for _, trimPath := range trimPaths {
   628			if !strings.HasSuffix(trimPath, "/") {
   629				trimPath += "/"
   630			}
   631			if strings.HasPrefix(sPath, trimPath) {
   632				return path[len(trimPath):]
   633			}
   634		}
   635		return path
   636	}
   637	
   638	func indentation(line string) int {
   639		column := 0
   640		for _, c := range line {
   641			if c == ' ' {
   642				column++
   643			} else if c == '\t' {
   644				column++
   645				for column%8 != 0 {
   646					column++
   647				}
   648			} else {
   649				break
   650			}
   651		}
   652		return column
   653	}
   654	

View as plain text