From 4ddcd54db3aaebefc3ad7701b7d5ed798ae63dc2 Mon Sep 17 00:00:00 2001 From: Dmitry Filimonov Date: Wed, 6 Dec 2023 00:26:24 -0800 Subject: [PATCH] chore: adds ability to render profile in dot format (#2808) * Add API for fetching a profile in the DOT format * Remove unneeded comments * Remove unused code, fix warnings * create dot profile in /render --------- Co-authored-by: Aleksandar Petrov <8142643+aleks-p@users.noreply.github.com> Co-authored-by: Anton Kolesnikov --- pkg/frontend/dot/graph/dotgraph.go | 494 ++++++++ pkg/frontend/dot/graph/graph.go | 1170 +++++++++++++++++++ pkg/frontend/dot/measurement/measurement.go | 293 +++++ pkg/frontend/dot/report/report.go | 451 +++++++ pkg/querier/http.go | 51 + 5 files changed, 2459 insertions(+) create mode 100644 pkg/frontend/dot/graph/dotgraph.go create mode 100644 pkg/frontend/dot/graph/graph.go create mode 100644 pkg/frontend/dot/measurement/measurement.go create mode 100644 pkg/frontend/dot/report/report.go diff --git a/pkg/frontend/dot/graph/dotgraph.go b/pkg/frontend/dot/graph/dotgraph.go new file mode 100644 index 0000000000..0de5532590 --- /dev/null +++ b/pkg/frontend/dot/graph/dotgraph.go @@ -0,0 +1,494 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package graph + +import ( + "fmt" + "io" + "math" + "path/filepath" + "strings" + + "github.com/grafana/pyroscope/pkg/frontend/dot/measurement" +) + +// DotAttributes contains details about the graph itself, giving +// insight into how its elements should be rendered. +type DotAttributes struct { + Nodes map[*Node]*DotNodeAttributes // A map allowing each Node to have its own visualization option +} + +// DotNodeAttributes contains Node specific visualization options. +type DotNodeAttributes struct { + Shape string // The optional shape of the node when rendered visually + Bold bool // If the node should be bold or not + Peripheries int // An optional number of borders to place around a node + URL string // An optional url link to add to a node + Formatter func(*NodeInfo) string // An optional formatter for the node's label +} + +// DotConfig contains attributes about how a graph should be +// constructed and how it should look. +type DotConfig struct { + Title string // The title of the DOT graph + LegendURL string // The URL to link to from the legend. + Labels []string // The labels for the DOT's legend + + FormatValue func(int64) string // A formatting function for values + Total int64 // The total weight of the graph, used to compute percentages +} + +const maxNodelets = 4 // Number of nodelets for labels (both numeric and non) + +// ComposeDot creates and writes a in the DOT format to the writer, using +// the configurations given. +func ComposeDot(w io.Writer, g *Graph, a *DotAttributes, c *DotConfig) { + builder := &builder{w, a, c} + + // Begin constructing DOT by adding a title and legend. + builder.start() + defer builder.finish() + builder.addLegend() + + if len(g.Nodes) == 0 { + return + } + + // Preprocess graph to get id map and find max flat. + nodeIDMap := make(map[*Node]int) + hasNodelets := make(map[*Node]bool) + + maxFlat := float64(abs64(g.Nodes[0].FlatValue())) + for i, n := range g.Nodes { + nodeIDMap[n] = i + 1 + if float64(abs64(n.FlatValue())) > maxFlat { + maxFlat = float64(abs64(n.FlatValue())) + } + } + + edges := EdgeMap{} + + // Add nodes and nodelets to DOT builder. + for _, n := range g.Nodes { + builder.addNode(n, nodeIDMap[n], maxFlat) + hasNodelets[n] = builder.addNodelets(n, nodeIDMap[n]) + + // Collect all edges. Use a fake node to support multiple incoming edges. + for _, e := range n.Out { + edges[&Node{}] = e + } + } + + // Add edges to DOT builder. Sort edges by frequency as a hint to the graph layout engine. + for _, e := range edges.Sort() { + builder.addEdge(e, nodeIDMap[e.Src], nodeIDMap[e.Dest], hasNodelets[e.Src]) + } +} + +// builder wraps an io.Writer and understands how to compose DOT formatted elements. +type builder struct { + io.Writer + attributes *DotAttributes + config *DotConfig +} + +// start generates a title and initial node in DOT format. +func (b *builder) start() { + graphname := "unnamed" + if b.config.Title != "" { + graphname = b.config.Title + } + fmt.Fprintln(b, `digraph "`+graphname+`" {`) + fmt.Fprintln(b, `node [style=filled fillcolor="#f8f8f8"]`) +} + +// finish closes the opening curly bracket in the constructed DOT buffer. +func (b *builder) finish() { + fmt.Fprintln(b, "}") +} + +// addLegend generates a legend in DOT format. +func (b *builder) addLegend() { + labels := b.config.Labels + if len(labels) == 0 { + return + } + title := labels[0] + fmt.Fprintf(b, `subgraph cluster_L { "%s" [shape=box fontsize=16`, escapeForDot(title)) + fmt.Fprintf(b, ` label="%s\l"`, strings.Join(escapeAllForDot(labels), `\l`)) + if b.config.LegendURL != "" { + fmt.Fprintf(b, ` URL="%s" target="_blank"`, b.config.LegendURL) + } + if b.config.Title != "" { + fmt.Fprintf(b, ` tooltip="%s"`, b.config.Title) + } + fmt.Fprintf(b, "] }\n") +} + +// addNode generates a graph node in DOT format. +func (b *builder) addNode(node *Node, nodeID int, maxFlat float64) { + flat, cum := node.FlatValue(), node.CumValue() + attrs := b.attributes.Nodes[node] + + // Populate label for node. + var label string + if attrs != nil && attrs.Formatter != nil { + label = attrs.Formatter(&node.Info) + } else { + label = multilinePrintableName(&node.Info) + } + + flatValue := b.config.FormatValue(flat) + if flat != 0 { + label = label + fmt.Sprintf(`%s (%s)`, + flatValue, + strings.TrimSpace(measurement.Percentage(flat, b.config.Total))) + } else { + label = label + "0" + } + cumValue := flatValue + if cum != flat { + if flat != 0 { + label = label + `\n` + } else { + label = label + " " + } + cumValue = b.config.FormatValue(cum) + label = label + fmt.Sprintf(`of %s (%s)`, + cumValue, + strings.TrimSpace(measurement.Percentage(cum, b.config.Total))) + } + + // Scale font sizes from 8 to 24 based on percentage of flat frequency. + // Use non linear growth to emphasize the size difference. + baseFontSize, maxFontGrowth := 8, 16.0 + fontSize := baseFontSize + if maxFlat != 0 && flat != 0 && float64(abs64(flat)) <= maxFlat { + fontSize += int(math.Ceil(maxFontGrowth * math.Sqrt(float64(abs64(flat))/maxFlat))) + } + + // Determine node shape. + shape := "box" + if attrs != nil && attrs.Shape != "" { + shape = attrs.Shape + } + + // Create DOT attribute for node. + attr := fmt.Sprintf(`label="%s" id="node%d" fontsize=%d shape=%s tooltip="%s (%s)" color="%s" fillcolor="%s"`, + label, nodeID, fontSize, shape, escapeForDot(node.Info.PrintableName()), cumValue, + dotColor(float64(node.CumValue())/float64(abs64(b.config.Total)), false), + dotColor(float64(node.CumValue())/float64(abs64(b.config.Total)), true)) + + // Add on extra attributes if provided. + if attrs != nil { + // Make bold if specified. + if attrs.Bold { + attr += ` style="bold,filled"` + } + + // Add peripheries if specified. + if attrs.Peripheries != 0 { + attr += fmt.Sprintf(` peripheries=%d`, attrs.Peripheries) + } + + // Add URL if specified. target="_blank" forces the link to open in a new tab. + if attrs.URL != "" { + attr += fmt.Sprintf(` URL="%s" target="_blank"`, attrs.URL) + } + } + + fmt.Fprintf(b, "N%d [%s]\n", nodeID, attr) +} + +// addNodelets generates the DOT boxes for the node tags if they exist. +func (b *builder) addNodelets(node *Node, nodeID int) bool { + var nodelets string + + // Populate two Tag slices, one for LabelTags and one for NumericTags. + var ts []*Tag + lnts := make(map[string][]*Tag) + for _, t := range node.LabelTags { + ts = append(ts, t) + } + for l, tm := range node.NumericTags { + for _, t := range tm { + lnts[l] = append(lnts[l], t) + } + } + + // For leaf nodes, print cumulative tags (includes weight from + // children that have been deleted). + // For internal nodes, print only flat tags. + flatTags := len(node.Out) > 0 + + // Select the top maxNodelets alphanumeric labels by weight. + SortTags(ts, flatTags) + if len(ts) > maxNodelets { + ts = ts[:maxNodelets] + } + for i, t := range ts { + w := t.CumValue() + if flatTags { + w = t.FlatValue() + } + if w == 0 { + continue + } + weight := b.config.FormatValue(w) + nodelets += fmt.Sprintf(`N%d_%d [label = "%s" id="N%d_%d" fontsize=8 shape=box3d tooltip="%s"]`+"\n", nodeID, i, t.Name, nodeID, i, weight) + nodelets += fmt.Sprintf(`N%d -> N%d_%d [label=" %s" weight=100 tooltip="%s" labeltooltip="%s"]`+"\n", nodeID, nodeID, i, weight, weight, weight) + if nts := lnts[t.Name]; nts != nil { + nodelets += b.numericNodelets(nts, maxNodelets, flatTags, fmt.Sprintf(`N%d_%d`, nodeID, i)) + } + } + + if nts := lnts[""]; nts != nil { + nodelets += b.numericNodelets(nts, maxNodelets, flatTags, fmt.Sprintf(`N%d`, nodeID)) + } + + fmt.Fprint(b, nodelets) + return nodelets != "" +} + +func (b *builder) numericNodelets(nts []*Tag, maxNumNodelets int, flatTags bool, source string) string { + nodelets := "" + + // Collapse numeric labels into maxNumNodelets buckets, of the form: + // 1MB..2MB, 3MB..5MB, ... + for j, t := range b.collapsedTags(nts, maxNumNodelets, flatTags) { + w, attr := t.CumValue(), ` style="dotted"` + if flatTags || t.FlatValue() == t.CumValue() { + w, attr = t.FlatValue(), "" + } + if w != 0 { + weight := b.config.FormatValue(w) + nodelets += fmt.Sprintf(`N%s_%d [label = "%s" id="N%s_%d" fontsize=8 shape=box3d tooltip="%s"]`+"\n", source, j, t.Name, source, j, weight) + nodelets += fmt.Sprintf(`%s -> N%s_%d [label=" %s" weight=100 tooltip="%s" labeltooltip="%s"%s]`+"\n", source, source, j, weight, weight, weight, attr) + } + } + return nodelets +} + +// addEdge generates a graph edge in DOT format. +func (b *builder) addEdge(edge *Edge, from, to int, hasNodelets bool) { + var inline string + if edge.Inline { + inline = `\n (inline)` + } + w := b.config.FormatValue(edge.WeightValue()) + attr := fmt.Sprintf(`label=" %s%s"`, w, inline) + if b.config.Total != 0 { + // Note: edge.weight > b.config.Total is possible for profile diffs. + if weight := 1 + int(min64(abs64(edge.WeightValue()*100/b.config.Total), 100)); weight > 1 { + attr = fmt.Sprintf(`%s weight=%d`, attr, weight) + } + if width := 1 + int(min64(abs64(edge.WeightValue()*5/b.config.Total), 5)); width > 1 { + attr = fmt.Sprintf(`%s penwidth=%d`, attr, width) + } + attr = fmt.Sprintf(`%s color="%s"`, attr, + dotColor(float64(edge.WeightValue())/float64(abs64(b.config.Total)), false)) + } + arrow := "->" + if edge.Residual { + arrow = "..." + } + tooltip := fmt.Sprintf(`"%s %s %s (%s)"`, + escapeForDot(edge.Src.Info.PrintableName()), arrow, + escapeForDot(edge.Dest.Info.PrintableName()), w) + attr = fmt.Sprintf(`%s tooltip=%s labeltooltip=%s`, attr, tooltip, tooltip) + + if edge.Residual { + attr = attr + ` style="dotted"` + } + + if hasNodelets { + // Separate children further if source has tags. + attr = attr + " minlen=2" + } + + fmt.Fprintf(b, "N%d -> N%d [%s]\n", from, to, attr) +} + +// dotColor returns a color for the given score (between -1.0 and +// 1.0), with -1.0 colored green, 0.0 colored grey, and 1.0 colored +// red. If isBackground is true, then a light (low-saturation) +// color is returned (suitable for use as a background color); +// otherwise, a darker color is returned (suitable for use as a +// foreground color). +func dotColor(score float64, isBackground bool) string { + // A float between 0.0 and 1.0, indicating the extent to which + // colors should be shifted away from grey (to make positive and + // negative values easier to distinguish, and to make more use of + // the color range.) + const shift = 0.7 + + // Saturation and value (in hsv colorspace) for background colors. + const bgSaturation = 0.1 + const bgValue = 0.93 + + // Saturation and value (in hsv colorspace) for foreground colors. + const fgSaturation = 1.0 + const fgValue = 0.7 + + // Choose saturation and value based on isBackground. + var saturation float64 + var value float64 + if isBackground { + saturation = bgSaturation + value = bgValue + } else { + saturation = fgSaturation + value = fgValue + } + + // Limit the score values to the range [-1.0, 1.0]. + score = math.Max(-1.0, math.Min(1.0, score)) + + // Reduce saturation near score=0 (so it is colored grey, rather than yellow). + if math.Abs(score) < 0.2 { + saturation *= math.Abs(score) / 0.2 + } + + // Apply 'shift' to move scores away from 0.0 (grey). + if score > 0.0 { + score = math.Pow(score, (1.0 - shift)) + } + if score < 0.0 { + score = -math.Pow(-score, (1.0 - shift)) + } + + var r, g, b float64 // red, green, blue + if score < 0.0 { + g = value + r = value * (1 + saturation*score) + } else { + r = value + g = value * (1 - saturation*score) + } + b = value * (1 - saturation) + return fmt.Sprintf("#%02x%02x%02x", uint8(r*255.0), uint8(g*255.0), uint8(b*255.0)) +} + +func multilinePrintableName(info *NodeInfo) string { + infoCopy := *info + infoCopy.Name = escapeForDot(ShortenFunctionName(infoCopy.Name)) + infoCopy.Name = strings.Replace(infoCopy.Name, "::", `\n`, -1) + // Go type parameters are reported as "[...]" by Go pprof profiles. + // Keep this ellipsis rather than replacing with newlines below. + infoCopy.Name = strings.Replace(infoCopy.Name, "[...]", "[…]", -1) + infoCopy.Name = strings.Replace(infoCopy.Name, ".", `\n`, -1) + if infoCopy.File != "" { + infoCopy.File = filepath.Base(infoCopy.File) + } + return strings.Join(infoCopy.NameComponents(), `\n`) + `\n` +} + +// collapsedTags trims and sorts a slice of tags. +func (b *builder) collapsedTags(ts []*Tag, count int, flatTags bool) []*Tag { + ts = SortTags(ts, flatTags) + if len(ts) <= count { + return ts + } + + tagGroups := make([][]*Tag, count) + for i, t := range (ts)[:count] { + tagGroups[i] = []*Tag{t} + } + for _, t := range (ts)[count:] { + g, d := 0, tagDistance(t, tagGroups[0][0]) + for i := 1; i < count; i++ { + if nd := tagDistance(t, tagGroups[i][0]); nd < d { + g, d = i, nd + } + } + tagGroups[g] = append(tagGroups[g], t) + } + + var nts []*Tag + for _, g := range tagGroups { + l, w, c := b.tagGroupLabel(g) + nts = append(nts, &Tag{ + Name: l, + Flat: w, + Cum: c, + }) + } + return SortTags(nts, flatTags) +} + +func tagDistance(t, u *Tag) float64 { + v, _ := measurement.Scale(u.Value, u.Unit, t.Unit) + if v < float64(t.Value) { + return float64(t.Value) - v + } + return v - float64(t.Value) +} + +func (b *builder) tagGroupLabel(g []*Tag) (label string, flat, cum int64) { + if len(g) == 1 { + t := g[0] + return measurement.Label(t.Value, t.Unit), t.FlatValue(), t.CumValue() + } + min := g[0] + max := g[0] + df, f := min.FlatDiv, min.Flat + dc, c := min.CumDiv, min.Cum + for _, t := range g[1:] { + if v, _ := measurement.Scale(t.Value, t.Unit, min.Unit); int64(v) < min.Value { + min = t + } + if v, _ := measurement.Scale(t.Value, t.Unit, max.Unit); int64(v) > max.Value { + max = t + } + f += t.Flat + df += t.FlatDiv + c += t.Cum + dc += t.CumDiv + } + if df != 0 { + f = f / df + } + if dc != 0 { + c = c / dc + } + + // Tags are not scaled with the selected output unit because tags are often + // much smaller than other values which appear, so the range of tag sizes + // sometimes would appear to be "0..0" when scaled to the selected output unit. + return measurement.Label(min.Value, min.Unit) + ".." + measurement.Label(max.Value, max.Unit), f, c +} + +func min64(a, b int64) int64 { + if a < b { + return a + } + return b +} + +// escapeAllForDot applies escapeForDot to all strings in the given slice. +func escapeAllForDot(in []string) []string { + var out = make([]string, len(in)) + for i := range in { + out[i] = escapeForDot(in[i]) + } + return out +} + +// escapeForDot escapes double quotes and backslashes, and replaces Graphviz's +// "center" character (\n) with a left-justified character. +// See https://graphviz.org/docs/attr-types/escString/ for more info. +func escapeForDot(str string) string { + return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(str, `\`, `\\`), `"`, `\"`), "\n", `\l`) +} diff --git a/pkg/frontend/dot/graph/graph.go b/pkg/frontend/dot/graph/graph.go new file mode 100644 index 0000000000..7ddea47c47 --- /dev/null +++ b/pkg/frontend/dot/graph/graph.go @@ -0,0 +1,1170 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package graph collects a set of samples into a directed graph. +package graph + +import ( + "fmt" + "math" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/google/pprof/profile" +) + +var ( + // Removes package name and method arguments for Java method names. + // See tests for examples. + javaRegExp = regexp.MustCompile(`^(?:[a-z]\w*\.)*([A-Z][\w\$]*\.(?:|[a-z][\w\$]*(?:\$\d+)?))(?:(?:\()|$)`) + // Removes package name and method arguments for Go function names. + // See tests for examples. + goRegExp = regexp.MustCompile(`^(?:[\w\-\.]+\/)+([^.]+\..+)`) + // Removes potential module versions in a package path. + goVerRegExp = regexp.MustCompile(`^(.*?)/v(?:[2-9]|[1-9][0-9]+)([./].*)$`) + // Strips C++ namespace prefix from a C++ function / method name. + // NOTE: Make sure to keep the template parameters in the name. Normally, + // template parameters are stripped from the C++ names but when + // -symbolize=demangle=templates flag is used, they will not be. + // See tests for examples. + cppRegExp = regexp.MustCompile(`^(?:[_a-zA-Z]\w*::)+(_*[A-Z]\w*::~?[_a-zA-Z]\w*(?:<.*>)?)`) + cppAnonymousPrefixRegExp = regexp.MustCompile(`^\(anonymous namespace\)::`) +) + +// Graph summarizes a performance profile into a format that is +// suitable for visualization. +type Graph struct { + Nodes Nodes +} + +// Options encodes the options for constructing a graph +type Options struct { + SampleValue func(s []int64) int64 // Function to compute the value of a sample + SampleMeanDivisor func(s []int64) int64 // Function to compute the divisor for mean graphs, or nil + FormatTag func(int64, string) string // Function to format a sample tag value into a string + ObjNames bool // Always preserve obj filename + OrigFnNames bool // Preserve original (eg mangled) function names + + CallTree bool // Build a tree instead of a graph + DropNegative bool // Drop nodes with overall negative values + + KeptNodes NodeSet // If non-nil, only use nodes in this set +} + +// Nodes is an ordered collection of graph nodes. +type Nodes []*Node + +// Node is an entry on a profiling report. It represents a unique +// program location. +type Node struct { + // Info describes the source location associated to this node. + Info NodeInfo + + // Function represents the function that this node belongs to. On + // graphs with sub-function resolution (eg line number or + // addresses), two nodes in a NodeMap that are part of the same + // function have the same value of Node.Function. If the Node + // represents the whole function, it points back to itself. + Function *Node + + // Values associated to this node. Flat is exclusive to this node, + // Cum includes all descendents. + Flat, FlatDiv, Cum, CumDiv int64 + + // In and out Contains the nodes immediately reaching or reached by + // this node. + In, Out EdgeMap + + // LabelTags provide additional information about subsets of a sample. + LabelTags TagMap + + // NumericTags provide additional values for subsets of a sample. + // Numeric tags are optionally associated to a label tag. The key + // for NumericTags is the name of the LabelTag they are associated + // to, or "" for numeric tags not associated to a label tag. + NumericTags map[string]TagMap +} + +// FlatValue returns the exclusive value for this node, computing the +// mean if a divisor is available. +func (n *Node) FlatValue() int64 { + if n.FlatDiv == 0 { + return n.Flat + } + return n.Flat / n.FlatDiv +} + +// CumValue returns the inclusive value for this node, computing the +// mean if a divisor is available. +func (n *Node) CumValue() int64 { + if n.CumDiv == 0 { + return n.Cum + } + return n.Cum / n.CumDiv +} + +// AddToEdge increases the weight of an edge between two nodes. If +// there isn't such an edge one is created. +func (n *Node) AddToEdge(to *Node, v int64, residual, inline bool) { + n.AddToEdgeDiv(to, 0, v, residual, inline) +} + +// AddToEdgeDiv increases the weight of an edge between two nodes. If +// there isn't such an edge one is created. +func (n *Node) AddToEdgeDiv(to *Node, dv, v int64, residual, inline bool) { + if n.Out[to] != to.In[n] { + panic(fmt.Errorf("asymmetric edges %v %v", *n, *to)) + } + + if e := n.Out[to]; e != nil { + e.WeightDiv += dv + e.Weight += v + if residual { + e.Residual = true + } + if !inline { + e.Inline = false + } + return + } + + info := &Edge{Src: n, Dest: to, WeightDiv: dv, Weight: v, Residual: residual, Inline: inline} + n.Out[to] = info + to.In[n] = info +} + +// NodeInfo contains the attributes for a node. +type NodeInfo struct { + Name string + OrigName string + Address uint64 + File string + StartLine, Lineno int + Objfile string +} + +// PrintableName calls the Node's Formatter function with a single space separator. +func (i *NodeInfo) PrintableName() string { + return strings.Join(i.NameComponents(), " ") +} + +// NameComponents returns the components of the printable name to be used for a node. +func (i *NodeInfo) NameComponents() []string { + var name []string + if i.Address != 0 { + name = append(name, fmt.Sprintf("%016x", i.Address)) + } + if fun := i.Name; fun != "" { + name = append(name, fun) + } + + switch { + case i.Lineno != 0: + // User requested line numbers, provide what we have. + name = append(name, fmt.Sprintf("%s:%d", i.File, i.Lineno)) + case i.File != "": + // User requested file name, provide it. + name = append(name, i.File) + case i.Name != "": + // User requested function name. It was already included. + case i.Objfile != "": + // Only binary name is available + name = append(name, "["+filepath.Base(i.Objfile)+"]") + default: + // Do not leave it empty if there is no information at all. + name = append(name, "") + } + return name +} + +// NodeMap maps from a node info struct to a node. It is used to merge +// report entries with the same info. +type NodeMap map[NodeInfo]*Node + +// NodeSet is a collection of node info structs. +type NodeSet map[NodeInfo]bool + +// NodePtrSet is a collection of nodes. Trimming a graph or tree requires a set +// of objects which uniquely identify the nodes to keep. In a graph, NodeInfo +// works as a unique identifier; however, in a tree multiple nodes may share +// identical NodeInfos. A *Node does uniquely identify a node so we can use that +// instead. Though a *Node also uniquely identifies a node in a graph, +// currently, during trimming, graphs are rebuilt from scratch using only the +// NodeSet, so there would not be the required context of the initial graph to +// allow for the use of *Node. +type NodePtrSet map[*Node]bool + +// FindOrInsertNode takes the info for a node and either returns a matching node +// from the node map if one exists, or adds one to the map if one does not. +// If kept is non-nil, nodes are only added if they can be located on it. +func (nm NodeMap) FindOrInsertNode(info NodeInfo, kept NodeSet) *Node { + if kept != nil { + if _, ok := kept[info]; !ok { + return nil + } + } + + if n, ok := nm[info]; ok { + return n + } + + n := &Node{ + Info: info, + In: make(EdgeMap), + Out: make(EdgeMap), + LabelTags: make(TagMap), + NumericTags: make(map[string]TagMap), + } + nm[info] = n + if info.Address == 0 && info.Lineno == 0 { + // This node represents the whole function, so point Function + // back to itself. + n.Function = n + return n + } + // Find a node that represents the whole function. + info.Address = 0 + info.Lineno = 0 + n.Function = nm.FindOrInsertNode(info, nil) + return n +} + +// EdgeMap is used to represent the incoming/outgoing edges from a node. +type EdgeMap map[*Node]*Edge + +// Edge contains any attributes to be represented about edges in a graph. +type Edge struct { + Src, Dest *Node + // The summary weight of the edge + Weight, WeightDiv int64 + + // residual edges connect nodes that were connected through a + // separate node, which has been removed from the report. + Residual bool + // An inline edge represents a call that was inlined into the caller. + Inline bool +} + +// WeightValue returns the weight value for this edge, normalizing if a +// divisor is available. +func (e *Edge) WeightValue() int64 { + if e.WeightDiv == 0 { + return e.Weight + } + return e.Weight / e.WeightDiv +} + +// Tag represent sample annotations +type Tag struct { + Name string + Unit string // Describe the value, "" for non-numeric tags + Value int64 + Flat, FlatDiv int64 + Cum, CumDiv int64 +} + +// FlatValue returns the exclusive value for this tag, computing the +// mean if a divisor is available. +func (t *Tag) FlatValue() int64 { + if t.FlatDiv == 0 { + return t.Flat + } + return t.Flat / t.FlatDiv +} + +// CumValue returns the inclusive value for this tag, computing the +// mean if a divisor is available. +func (t *Tag) CumValue() int64 { + if t.CumDiv == 0 { + return t.Cum + } + return t.Cum / t.CumDiv +} + +// TagMap is a collection of tags, classified by their name. +type TagMap map[string]*Tag + +// SortTags sorts a slice of tags based on their weight. +func SortTags(t []*Tag, flat bool) []*Tag { + ts := tags{t, flat} + sort.Sort(ts) + return ts.t +} + +// New summarizes performance data from a profile into a graph. +func New(prof *profile.Profile, o *Options) *Graph { + if o.CallTree { + return newTree(prof, o) + } + g, _ := newGraph(prof, o) + return g +} + +// newGraph computes a graph from a profile. It returns the graph, and +// a map from the profile location indices to the corresponding graph +// nodes. +func newGraph(prof *profile.Profile, o *Options) (*Graph, map[uint64]Nodes) { + nodes, locationMap := CreateNodes(prof, o) + seenNode := make(map[*Node]bool) + seenEdge := make(map[nodePair]bool) + for _, sample := range prof.Sample { + var w, dw int64 + w = o.SampleValue(sample.Value) + if o.SampleMeanDivisor != nil { + dw = o.SampleMeanDivisor(sample.Value) + } + if dw == 0 && w == 0 { + continue + } + for k := range seenNode { + delete(seenNode, k) + } + for k := range seenEdge { + delete(seenEdge, k) + } + var parent *Node + // A residual edge goes over one or more nodes that were not kept. + residual := false + + labels := joinLabels(sample) + // Group the sample frames, based on a global map. + for i := len(sample.Location) - 1; i >= 0; i-- { + l := sample.Location[i] + locNodes := locationMap[l.ID] + for ni := len(locNodes) - 1; ni >= 0; ni-- { + n := locNodes[ni] + if n == nil { + residual = true + continue + } + // Add cum weight to all nodes in stack, avoiding double counting. + if _, ok := seenNode[n]; !ok { + seenNode[n] = true + n.addSample(dw, w, labels, sample.NumLabel, sample.NumUnit, o.FormatTag, false) + } + // Update edge weights for all edges in stack, avoiding double counting. + if _, ok := seenEdge[nodePair{n, parent}]; !ok && parent != nil && n != parent { + seenEdge[nodePair{n, parent}] = true + parent.AddToEdgeDiv(n, dw, w, residual, ni != len(locNodes)-1) + } + parent = n + residual = false + } + } + if parent != nil && !residual { + // Add flat weight to leaf node. + parent.addSample(dw, w, labels, sample.NumLabel, sample.NumUnit, o.FormatTag, true) + } + } + + return selectNodesForGraph(nodes, o.DropNegative), locationMap +} + +func selectNodesForGraph(nodes Nodes, dropNegative bool) *Graph { + // Collect nodes into a graph. + gNodes := make(Nodes, 0, len(nodes)) + for _, n := range nodes { + if n == nil { + continue + } + if n.Cum == 0 && n.Flat == 0 { + continue + } + if dropNegative && isNegative(n) { + continue + } + gNodes = append(gNodes, n) + } + return &Graph{gNodes} +} + +type nodePair struct { + src, dest *Node +} + +func newTree(prof *profile.Profile, o *Options) (g *Graph) { + parentNodeMap := make(map[*Node]NodeMap, len(prof.Sample)) + for _, sample := range prof.Sample { + var w, dw int64 + w = o.SampleValue(sample.Value) + if o.SampleMeanDivisor != nil { + dw = o.SampleMeanDivisor(sample.Value) + } + if dw == 0 && w == 0 { + continue + } + var parent *Node + labels := joinLabels(sample) + // Group the sample frames, based on a per-node map. + for i := len(sample.Location) - 1; i >= 0; i-- { + l := sample.Location[i] + lines := l.Line + if len(lines) == 0 { + lines = []profile.Line{{}} // Create empty line to include location info. + } + for lidx := len(lines) - 1; lidx >= 0; lidx-- { + nodeMap := parentNodeMap[parent] + if nodeMap == nil { + nodeMap = make(NodeMap) + parentNodeMap[parent] = nodeMap + } + n := nodeMap.findOrInsertLine(l, lines[lidx], o) + if n == nil { + continue + } + n.addSample(dw, w, labels, sample.NumLabel, sample.NumUnit, o.FormatTag, false) + if parent != nil { + parent.AddToEdgeDiv(n, dw, w, false, lidx != len(lines)-1) + } + parent = n + } + } + if parent != nil { + parent.addSample(dw, w, labels, sample.NumLabel, sample.NumUnit, o.FormatTag, true) + } + } + + nodes := make(Nodes, len(prof.Location)) + for _, nm := range parentNodeMap { + nodes = append(nodes, nm.nodes()...) + } + return selectNodesForGraph(nodes, o.DropNegative) +} + +// ShortenFunctionName returns a shortened version of a function's name. +func ShortenFunctionName(f string) string { + f = cppAnonymousPrefixRegExp.ReplaceAllString(f, "") + f = goVerRegExp.ReplaceAllString(f, `${1}${2}`) + for _, re := range []*regexp.Regexp{goRegExp, javaRegExp, cppRegExp} { + if matches := re.FindStringSubmatch(f); len(matches) >= 2 { + return strings.Join(matches[1:], "") + } + } + return f +} + +// TrimTree trims a Graph in forest form, keeping only the nodes in kept. This +// will not work correctly if even a single node has multiple parents. +func (g *Graph) TrimTree(kept NodePtrSet) { + // Creates a new list of nodes + oldNodes := g.Nodes + g.Nodes = make(Nodes, 0, len(kept)) + + for _, cur := range oldNodes { + // A node may not have multiple parents + if len(cur.In) > 1 { + panic("TrimTree only works on trees") + } + + // If a node should be kept, add it to the new list of nodes + if _, ok := kept[cur]; ok { + g.Nodes = append(g.Nodes, cur) + continue + } + + // If a node has no parents, then delete all of the in edges of its + // children to make them each roots of their own trees. + if len(cur.In) == 0 { + for _, outEdge := range cur.Out { + delete(outEdge.Dest.In, cur) + } + continue + } + + // Get the parent. This works since at this point cur.In must contain only + // one element. + if len(cur.In) != 1 { + panic("Get parent assertion failed. cur.In expected to be of length 1.") + } + var parent *Node + for _, edge := range cur.In { + parent = edge.Src + } + + parentEdgeInline := parent.Out[cur].Inline + + // Remove the edge from the parent to this node + delete(parent.Out, cur) + + // Reconfigure every edge from the current node to now begin at the parent. + for _, outEdge := range cur.Out { + child := outEdge.Dest + + delete(child.In, cur) + child.In[parent] = outEdge + parent.Out[child] = outEdge + + outEdge.Src = parent + outEdge.Residual = true + // If the edge from the parent to the current node and the edge from the + // current node to the child are both inline, then this resulting residual + // edge should also be inline + outEdge.Inline = parentEdgeInline && outEdge.Inline + } + } + g.RemoveRedundantEdges() +} + +func joinLabels(s *profile.Sample) string { + if len(s.Label) == 0 { + return "" + } + + var labels []string + for key, vals := range s.Label { + for _, v := range vals { + labels = append(labels, key+":"+v) + } + } + sort.Strings(labels) + return strings.Join(labels, `\n`) +} + +// isNegative returns true if the node is considered as "negative" for the +// purposes of drop_negative. +func isNegative(n *Node) bool { + switch { + case n.Flat < 0: + return true + case n.Flat == 0 && n.Cum < 0: + return true + default: + return false + } +} + +// CreateNodes creates graph nodes for all locations in a profile. It +// returns set of all nodes, plus a mapping of each location to the +// set of corresponding nodes (one per location.Line). +func CreateNodes(prof *profile.Profile, o *Options) (Nodes, map[uint64]Nodes) { + locations := make(map[uint64]Nodes, len(prof.Location)) + nm := make(NodeMap, len(prof.Location)) + for _, l := range prof.Location { + lines := l.Line + if len(lines) == 0 { + lines = []profile.Line{{}} // Create empty line to include location info. + } + nodes := make(Nodes, len(lines)) + for ln := range lines { + nodes[ln] = nm.findOrInsertLine(l, lines[ln], o) + } + locations[l.ID] = nodes + } + return nm.nodes(), locations +} + +func (nm NodeMap) nodes() Nodes { + nodes := make(Nodes, 0, len(nm)) + for _, n := range nm { + nodes = append(nodes, n) + } + return nodes +} + +func (nm NodeMap) findOrInsertLine(l *profile.Location, li profile.Line, o *Options) *Node { + var objfile string + if m := l.Mapping; m != nil && m.File != "" { + objfile = m.File + } + + if ni := nodeInfo(l, li, objfile, o); ni != nil { + return nm.FindOrInsertNode(*ni, o.KeptNodes) + } + return nil +} + +func nodeInfo(l *profile.Location, line profile.Line, objfile string, o *Options) *NodeInfo { + if line.Function == nil { + return &NodeInfo{Address: l.Address, Objfile: objfile} + } + ni := &NodeInfo{ + Address: l.Address, + Lineno: int(line.Line), + Name: line.Function.Name, + } + if fname := line.Function.Filename; fname != "" { + ni.File = filepath.Clean(fname) + } + if o.OrigFnNames { + ni.OrigName = line.Function.SystemName + } + if o.ObjNames || (ni.Name == "" && ni.OrigName == "") { + ni.Objfile = objfile + ni.StartLine = int(line.Function.StartLine) + } + return ni +} + +type tags struct { + t []*Tag + flat bool +} + +func (t tags) Len() int { return len(t.t) } +func (t tags) Swap(i, j int) { t.t[i], t.t[j] = t.t[j], t.t[i] } +func (t tags) Less(i, j int) bool { + if !t.flat { + if t.t[i].Cum != t.t[j].Cum { + return abs64(t.t[i].Cum) > abs64(t.t[j].Cum) + } + } + if t.t[i].Flat != t.t[j].Flat { + return abs64(t.t[i].Flat) > abs64(t.t[j].Flat) + } + return t.t[i].Name < t.t[j].Name +} + +// Sum adds the flat and cum values of a set of nodes. +func (ns Nodes) Sum() (flat int64, cum int64) { + for _, n := range ns { + flat += n.Flat + cum += n.Cum + } + return +} + +func (n *Node) addSample(dw, w int64, labels string, numLabel map[string][]int64, numUnit map[string][]string, format func(int64, string) string, flat bool) { + // Update sample value + if flat { + n.FlatDiv += dw + n.Flat += w + } else { + n.CumDiv += dw + n.Cum += w + } + + // Add string tags + if labels != "" { + t := n.LabelTags.findOrAddTag(labels, "", 0) + if flat { + t.FlatDiv += dw + t.Flat += w + } else { + t.CumDiv += dw + t.Cum += w + } + } + + numericTags := n.NumericTags[labels] + if numericTags == nil { + numericTags = TagMap{} + n.NumericTags[labels] = numericTags + } + // Add numeric tags + if format == nil { + format = defaultLabelFormat + } + for k, nvals := range numLabel { + units := numUnit[k] + for i, v := range nvals { + var t *Tag + if len(units) > 0 { + t = numericTags.findOrAddTag(format(v, units[i]), units[i], v) + } else { + t = numericTags.findOrAddTag(format(v, k), k, v) + } + if flat { + t.FlatDiv += dw + t.Flat += w + } else { + t.CumDiv += dw + t.Cum += w + } + } + } +} + +func defaultLabelFormat(v int64, key string) string { + return strconv.FormatInt(v, 10) +} + +func (m TagMap) findOrAddTag(label, unit string, value int64) *Tag { + l := m[label] + if l == nil { + l = &Tag{ + Name: label, + Unit: unit, + Value: value, + } + m[label] = l + } + return l +} + +// String returns a text representation of a graph, for debugging purposes. +func (g *Graph) String() string { + var s []string + + nodeIndex := make(map[*Node]int, len(g.Nodes)) + + for i, n := range g.Nodes { + nodeIndex[n] = i + 1 + } + + for i, n := range g.Nodes { + name := n.Info.PrintableName() + var in, out []int + + for _, from := range n.In { + in = append(in, nodeIndex[from.Src]) + } + for _, to := range n.Out { + out = append(out, nodeIndex[to.Dest]) + } + s = append(s, fmt.Sprintf("%d: %s[flat=%d cum=%d] %x -> %v ", i+1, name, n.Flat, n.Cum, in, out)) + } + return strings.Join(s, "\n") +} + +// DiscardLowFrequencyNodes returns a set of the nodes at or over a +// specific cum value cutoff. +func (g *Graph) DiscardLowFrequencyNodes(nodeCutoff int64) NodeSet { + return makeNodeSet(g.Nodes, nodeCutoff) +} + +// DiscardLowFrequencyNodePtrs returns a NodePtrSet of nodes at or over a +// specific cum value cutoff. +func (g *Graph) DiscardLowFrequencyNodePtrs(nodeCutoff int64) NodePtrSet { + cutNodes := getNodesAboveCumCutoff(g.Nodes, nodeCutoff) + kept := make(NodePtrSet, len(cutNodes)) + for _, n := range cutNodes { + kept[n] = true + } + return kept +} + +func makeNodeSet(nodes Nodes, nodeCutoff int64) NodeSet { + cutNodes := getNodesAboveCumCutoff(nodes, nodeCutoff) + kept := make(NodeSet, len(cutNodes)) + for _, n := range cutNodes { + kept[n.Info] = true + } + return kept +} + +// getNodesAboveCumCutoff returns all the nodes which have a Cum value greater +// than or equal to cutoff. +func getNodesAboveCumCutoff(nodes Nodes, nodeCutoff int64) Nodes { + cutoffNodes := make(Nodes, 0, len(nodes)) + for _, n := range nodes { + if abs64(n.Cum) < nodeCutoff { + continue + } + cutoffNodes = append(cutoffNodes, n) + } + return cutoffNodes +} + +// TrimLowFrequencyTags removes tags that have less than +// the specified weight. +func (g *Graph) TrimLowFrequencyTags(tagCutoff int64) { + // Remove nodes with value <= total*nodeFraction + for _, n := range g.Nodes { + n.LabelTags = trimLowFreqTags(n.LabelTags, tagCutoff) + for s, nt := range n.NumericTags { + n.NumericTags[s] = trimLowFreqTags(nt, tagCutoff) + } + } +} + +func trimLowFreqTags(tags TagMap, minValue int64) TagMap { + kept := TagMap{} + for s, t := range tags { + if abs64(t.Flat) >= minValue || abs64(t.Cum) >= minValue { + kept[s] = t + } + } + return kept +} + +// TrimLowFrequencyEdges removes edges that have less than +// the specified weight. Returns the number of edges removed +func (g *Graph) TrimLowFrequencyEdges(edgeCutoff int64) int { + var droppedEdges int + for _, n := range g.Nodes { + for src, e := range n.In { + if abs64(e.Weight) < edgeCutoff { + delete(n.In, src) + delete(src.Out, n) + droppedEdges++ + } + } + } + return droppedEdges +} + +// SortNodes sorts the nodes in a graph based on a specific heuristic. +func (g *Graph) SortNodes(cum bool, visualMode bool) { + // Sort nodes based on requested mode + switch { + case visualMode: + // Specialized sort to produce a more visually-interesting graph + _ = g.Nodes.Sort(EntropyOrder) + case cum: + _ = g.Nodes.Sort(CumNameOrder) + default: + _ = g.Nodes.Sort(FlatNameOrder) + } +} + +// SelectTopNodePtrs returns a set of the top maxNodes *Node in a graph. +func (g *Graph) SelectTopNodePtrs(maxNodes int, visualMode bool) NodePtrSet { + set := make(NodePtrSet) + for _, node := range g.selectTopNodes(maxNodes, visualMode) { + set[node] = true + } + return set +} + +// SelectTopNodes returns a set of the top maxNodes nodes in a graph. +func (g *Graph) SelectTopNodes(maxNodes int, visualMode bool) NodeSet { + return makeNodeSet(g.selectTopNodes(maxNodes, visualMode), 0) +} + +// selectTopNodes returns a slice of the top maxNodes nodes in a graph. +func (g *Graph) selectTopNodes(maxNodes int, visualMode bool) Nodes { + if maxNodes > 0 { + if visualMode { + var count int + // If generating a visual graph, count tags as nodes. Update + // maxNodes to account for them. + for i, n := range g.Nodes { + tags := countTags(n) + if tags > maxNodelets { + tags = maxNodelets + } + if count += tags + 1; count >= maxNodes { + maxNodes = i + 1 + break + } + } + } + } + if maxNodes > len(g.Nodes) { + maxNodes = len(g.Nodes) + } + return g.Nodes[:maxNodes] +} + +// countTags counts the tags with flat count. This underestimates the +// number of tags being displayed, but in practice is close enough. +func countTags(n *Node) int { + count := 0 + for _, e := range n.LabelTags { + if e.Flat != 0 { + count++ + } + } + for _, t := range n.NumericTags { + for _, e := range t { + if e.Flat != 0 { + count++ + } + } + } + return count +} + +// RemoveRedundantEdges removes residual edges if the destination can +// be reached through another path. This is done to simplify the graph +// while preserving connectivity. +func (g *Graph) RemoveRedundantEdges() { + // Walk the nodes and outgoing edges in reverse order to prefer + // removing edges with the lowest weight. + for i := len(g.Nodes); i > 0; i-- { + n := g.Nodes[i-1] + in := n.In.Sort() + for j := len(in); j > 0; j-- { + e := in[j-1] + if !e.Residual { + // Do not remove edges heavier than a non-residual edge, to + // avoid potential confusion. + break + } + if isRedundantEdge(e) { + delete(e.Src.Out, e.Dest) + delete(e.Dest.In, e.Src) + } + } + } +} + +// isRedundantEdge determines if there is a path that allows e.Src +// to reach e.Dest after removing e. +func isRedundantEdge(e *Edge) bool { + src, n := e.Src, e.Dest + seen := map[*Node]bool{n: true} + queue := Nodes{n} + for len(queue) > 0 { + n := queue[0] + queue = queue[1:] + for _, ie := range n.In { + if e == ie || seen[ie.Src] { + continue + } + if ie.Src == src { + return true + } + seen[ie.Src] = true + queue = append(queue, ie.Src) + } + } + return false +} + +// nodeSorter is a mechanism used to allow a report to be sorted +// in different ways. +type nodeSorter struct { + rs Nodes + less func(l, r *Node) bool +} + +func (s nodeSorter) Len() int { return len(s.rs) } +func (s nodeSorter) Swap(i, j int) { s.rs[i], s.rs[j] = s.rs[j], s.rs[i] } +func (s nodeSorter) Less(i, j int) bool { return s.less(s.rs[i], s.rs[j]) } + +// Sort reorders a slice of nodes based on the specified ordering +// criteria. The result is sorted in decreasing order for (absolute) +// numeric quantities, alphabetically for text, and increasing for +// addresses. +func (ns Nodes) Sort(o NodeOrder) error { + var s nodeSorter + + switch o { + case FlatNameOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := abs64(l.Flat), abs64(r.Flat); iv != jv { + return iv > jv + } + if iv, jv := l.Info.PrintableName(), r.Info.PrintableName(); iv != jv { + return iv < jv + } + if iv, jv := abs64(l.Cum), abs64(r.Cum); iv != jv { + return iv > jv + } + return compareNodes(l, r) + }, + } + case FlatCumNameOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := abs64(l.Flat), abs64(r.Flat); iv != jv { + return iv > jv + } + if iv, jv := abs64(l.Cum), abs64(r.Cum); iv != jv { + return iv > jv + } + if iv, jv := l.Info.PrintableName(), r.Info.PrintableName(); iv != jv { + return iv < jv + } + return compareNodes(l, r) + }, + } + case NameOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := l.Info.Name, r.Info.Name; iv != jv { + return iv < jv + } + return compareNodes(l, r) + }, + } + case FileOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := l.Info.File, r.Info.File; iv != jv { + return iv < jv + } + if iv, jv := l.Info.StartLine, r.Info.StartLine; iv != jv { + return iv < jv + } + return compareNodes(l, r) + }, + } + case AddressOrder: + s = nodeSorter{ns, + func(l, r *Node) bool { + if iv, jv := l.Info.Address, r.Info.Address; iv != jv { + return iv < jv + } + return compareNodes(l, r) + }, + } + case CumNameOrder, EntropyOrder: + // Hold scoring for score-based ordering + var score map[*Node]int64 + scoreOrder := func(l, r *Node) bool { + if iv, jv := abs64(score[l]), abs64(score[r]); iv != jv { + return iv > jv + } + if iv, jv := l.Info.PrintableName(), r.Info.PrintableName(); iv != jv { + return iv < jv + } + if iv, jv := abs64(l.Flat), abs64(r.Flat); iv != jv { + return iv > jv + } + return compareNodes(l, r) + } + + switch o { + case CumNameOrder: + score = make(map[*Node]int64, len(ns)) + for _, n := range ns { + score[n] = n.Cum + } + s = nodeSorter{ns, scoreOrder} + case EntropyOrder: + score = make(map[*Node]int64, len(ns)) + for _, n := range ns { + score[n] = entropyScore(n) + } + s = nodeSorter{ns, scoreOrder} + } + default: + return fmt.Errorf("report: unrecognized sort ordering: %d", o) + } + sort.Sort(s) + return nil +} + +// compareNodes compares two nodes to provide a deterministic ordering +// between them. Two nodes cannot have the same Node.Info value. +func compareNodes(l, r *Node) bool { + return fmt.Sprint(l.Info) < fmt.Sprint(r.Info) +} + +// entropyScore computes a score for a node representing how important +// it is to include this node on a graph visualization. It is used to +// sort the nodes and select which ones to display if we have more +// nodes than desired in the graph. This number is computed by looking +// at the flat and cum weights of the node and the incoming/outgoing +// edges. The fundamental idea is to penalize nodes that have a simple +// fallthrough from their incoming to the outgoing edge. +func entropyScore(n *Node) int64 { + score := float64(0) + + if len(n.In) == 0 { + score++ // Favor entry nodes + } else { + score += edgeEntropyScore(n.In, 0) + } + + if len(n.Out) == 0 { + score++ // Favor leaf nodes + } else { + score += edgeEntropyScore(n.Out, n.Flat) + } + + return int64(score*float64(n.Cum)) + n.Flat +} + +// edgeEntropyScore computes the entropy value for a set of edges +// coming in or out of a node. Entropy (as defined in information +// theory) refers to the amount of information encoded by the set of +// edges. A set of edges that have a more interesting distribution of +// samples gets a higher score. +func edgeEntropyScore(edges EdgeMap, self int64) float64 { + score := float64(0) + total := self + for _, e := range edges { + if e.Weight > 0 { + total += abs64(e.Weight) + } + } + if total != 0 { + for _, e := range edges { + frac := float64(abs64(e.Weight)) / float64(total) + score += -frac * math.Log2(frac) + } + if self > 0 { + frac := float64(abs64(self)) / float64(total) + score += -frac * math.Log2(frac) + } + } + return score +} + +// NodeOrder sets the ordering for a Sort operation +type NodeOrder int + +// Sorting options for node sort. +const ( + FlatNameOrder NodeOrder = iota + FlatCumNameOrder + CumNameOrder + NameOrder + FileOrder + AddressOrder + EntropyOrder +) + +// Sort returns a slice of the edges in the map, in a consistent +// order. The sort order is first based on the edge weight +// (higher-to-lower) and then by the node names to avoid flakiness. +func (e EdgeMap) Sort() []*Edge { + el := make(edgeList, 0, len(e)) + for _, w := range e { + el = append(el, w) + } + + sort.Sort(el) + return el +} + +// Sum returns the total weight for a set of nodes. +func (e EdgeMap) Sum() int64 { + var ret int64 + for _, edge := range e { + ret += edge.Weight + } + return ret +} + +type edgeList []*Edge + +func (el edgeList) Len() int { + return len(el) +} + +func (el edgeList) Less(i, j int) bool { + if el[i].Weight != el[j].Weight { + return abs64(el[i].Weight) > abs64(el[j].Weight) + } + + from1 := el[i].Src.Info.PrintableName() + from2 := el[j].Src.Info.PrintableName() + if from1 != from2 { + return from1 < from2 + } + + to1 := el[i].Dest.Info.PrintableName() + to2 := el[j].Dest.Info.PrintableName() + + return to1 < to2 +} + +func (el edgeList) Swap(i, j int) { + el[i], el[j] = el[j], el[i] +} + +func abs64(i int64) int64 { + if i < 0 { + return -i + } + return i +} diff --git a/pkg/frontend/dot/measurement/measurement.go b/pkg/frontend/dot/measurement/measurement.go new file mode 100644 index 0000000000..d9644f9326 --- /dev/null +++ b/pkg/frontend/dot/measurement/measurement.go @@ -0,0 +1,293 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package measurement export utility functions to manipulate/format performance profile sample values. +package measurement + +import ( + "fmt" + "math" + "strings" + "time" + + "github.com/google/pprof/profile" +) + +// ScaleProfiles updates the units in a set of profiles to make them +// compatible. It scales the profiles to the smallest unit to preserve +// data. +func ScaleProfiles(profiles []*profile.Profile) error { + if len(profiles) == 0 { + return nil + } + periodTypes := make([]*profile.ValueType, 0, len(profiles)) + for _, p := range profiles { + if p.PeriodType != nil { + periodTypes = append(periodTypes, p.PeriodType) + } + } + periodType, err := CommonValueType(periodTypes) + if err != nil { + return fmt.Errorf("period type: %v", err) + } + + // Identify common sample types + numSampleTypes := len(profiles[0].SampleType) + for _, p := range profiles[1:] { + if numSampleTypes != len(p.SampleType) { + return fmt.Errorf("inconsistent samples type count: %d != %d", numSampleTypes, len(p.SampleType)) + } + } + sampleType := make([]*profile.ValueType, numSampleTypes) + for i := 0; i < numSampleTypes; i++ { + sampleTypes := make([]*profile.ValueType, len(profiles)) + for j, p := range profiles { + sampleTypes[j] = p.SampleType[i] + } + sampleType[i], err = CommonValueType(sampleTypes) + if err != nil { + return fmt.Errorf("sample types: %v", err) + } + } + + for _, p := range profiles { + if p.PeriodType != nil && periodType != nil { + period, _ := Scale(p.Period, p.PeriodType.Unit, periodType.Unit) + p.Period, p.PeriodType.Unit = int64(period), periodType.Unit + } + ratios := make([]float64, len(p.SampleType)) + for i, st := range p.SampleType { + if sampleType[i] == nil { + ratios[i] = 1 + continue + } + ratios[i], _ = Scale(1, st.Unit, sampleType[i].Unit) + p.SampleType[i].Unit = sampleType[i].Unit + } + if err := p.ScaleN(ratios); err != nil { + return fmt.Errorf("scale: %v", err) + } + } + return nil +} + +// CommonValueType returns the finest type from a set of compatible +// types. +func CommonValueType(ts []*profile.ValueType) (*profile.ValueType, error) { + if len(ts) <= 1 { + return nil, nil + } + minType := ts[0] + for _, t := range ts[1:] { + if !compatibleValueTypes(minType, t) { + return nil, fmt.Errorf("incompatible types: %v %v", *minType, *t) + } + if ratio, _ := Scale(1, t.Unit, minType.Unit); ratio < 1 { + minType = t + } + } + rcopy := *minType + return &rcopy, nil +} + +func compatibleValueTypes(v1, v2 *profile.ValueType) bool { + if v1 == nil || v2 == nil { + return true // No grounds to disqualify. + } + // Remove trailing 's' to permit minor mismatches. + if t1, t2 := strings.TrimSuffix(v1.Type, "s"), strings.TrimSuffix(v2.Type, "s"); t1 != t2 { + return false + } + + if v1.Unit == v2.Unit { + return true + } + for _, ut := range unitTypes { + if ut.sniffUnit(v1.Unit) != nil && ut.sniffUnit(v2.Unit) != nil { + return true + } + } + return false +} + +// Scale a measurement from a unit to a different unit and returns +// the scaled value and the target unit. The returned target unit +// will be empty if uninteresting (could be skipped). +func Scale(value int64, fromUnit, toUnit string) (float64, string) { + // Avoid infinite recursion on overflow. + if value < 0 && -value > 0 { + v, u := Scale(-value, fromUnit, toUnit) + return -v, u + } + for _, ut := range unitTypes { + if v, u, ok := ut.convertUnit(value, fromUnit, toUnit); ok { + return v, u + } + } + // Skip non-interesting units. + switch toUnit { + case "count", "sample", "unit", "minimum", "auto": + return float64(value), "" + default: + return float64(value), toUnit + } +} + +// Label returns the label used to describe a certain measurement. +func Label(value int64, unit string) string { + return ScaledLabel(value, unit, "auto") +} + +// ScaledLabel scales the passed-in measurement (if necessary) and +// returns the label used to describe a float measurement. +func ScaledLabel(value int64, fromUnit, toUnit string) string { + v, u := Scale(value, fromUnit, toUnit) + sv := strings.TrimSuffix(fmt.Sprintf("%.2f", v), ".00") + if sv == "0" || sv == "-0" { + return "0" + } + return sv + u +} + +// Percentage computes the percentage of total of a value, and encodes +// it as a string. At least two digits of precision are printed. +func Percentage(value, total int64) string { + var ratio float64 + if total != 0 { + ratio = math.Abs(float64(value)/float64(total)) * 100 + } + switch { + case math.Abs(ratio) >= 99.95 && math.Abs(ratio) <= 100.05: + return " 100%" + case math.Abs(ratio) >= 1.0: + return fmt.Sprintf("%5.2f%%", ratio) + default: + return fmt.Sprintf("%5.2g%%", ratio) + } +} + +// unit includes a list of aliases representing a specific unit and a factor +// which one can multiple a value in the specified unit by to get the value +// in terms of the base unit. +type unit struct { + canonicalName string + aliases []string + factor float64 +} + +// unitType includes a list of units that are within the same category (i.e. +// memory or time units) and a default unit to use for this type of unit. +type unitType struct { + defaultUnit unit + units []unit +} + +// findByAlias returns the unit associated with the specified alias. It returns +// nil if the unit with such alias is not found. +func (ut unitType) findByAlias(alias string) *unit { + for _, u := range ut.units { + for _, a := range u.aliases { + if alias == a { + return &u + } + } + } + return nil +} + +// sniffUnit simpifies the input alias and returns the unit associated with the +// specified alias. It returns nil if the unit with such alias is not found. +func (ut unitType) sniffUnit(unit string) *unit { + unit = strings.ToLower(unit) + if len(unit) > 2 { + unit = strings.TrimSuffix(unit, "s") + } + return ut.findByAlias(unit) +} + +// autoScale takes in the value with units of the base unit and returns +// that value scaled to a reasonable unit if a reasonable unit is +// found. +func (ut unitType) autoScale(value float64) (float64, string, bool) { + var f float64 + var unit string + for _, u := range ut.units { + if u.factor >= f && (value/u.factor) >= 1.0 { + f = u.factor + unit = u.canonicalName + } + } + if f == 0 { + return 0, "", false + } + return value / f, unit, true +} + +// convertUnit converts a value from the fromUnit to the toUnit, autoscaling +// the value if the toUnit is "minimum" or "auto". If the fromUnit is not +// included in the unitType, then a false boolean will be returned. If the +// toUnit is not in the unitType, the value will be returned in terms of the +// default unitType. +func (ut unitType) convertUnit(value int64, fromUnitStr, toUnitStr string) (float64, string, bool) { + fromUnit := ut.sniffUnit(fromUnitStr) + if fromUnit == nil { + return 0, "", false + } + v := float64(value) * fromUnit.factor + if toUnitStr == "minimum" || toUnitStr == "auto" { + if v, u, ok := ut.autoScale(v); ok { + return v, u, true + } + return v / ut.defaultUnit.factor, ut.defaultUnit.canonicalName, true + } + toUnit := ut.sniffUnit(toUnitStr) + if toUnit == nil { + return v / ut.defaultUnit.factor, ut.defaultUnit.canonicalName, true + } + return v / toUnit.factor, toUnit.canonicalName, true +} + +var unitTypes = []unitType{{ + units: []unit{ + {"B", []string{"b", "byte"}, 1}, + {"kB", []string{"kb", "kbyte", "kilobyte"}, float64(1 << 10)}, + {"MB", []string{"mb", "mbyte", "megabyte"}, float64(1 << 20)}, + {"GB", []string{"gb", "gbyte", "gigabyte"}, float64(1 << 30)}, + {"TB", []string{"tb", "tbyte", "terabyte"}, float64(1 << 40)}, + {"PB", []string{"pb", "pbyte", "petabyte"}, float64(1 << 50)}, + }, + defaultUnit: unit{"B", []string{"b", "byte"}, 1}, +}, { + units: []unit{ + {"ns", []string{"ns", "nanosecond"}, float64(time.Nanosecond)}, + {"us", []string{"μs", "us", "microsecond"}, float64(time.Microsecond)}, + {"ms", []string{"ms", "millisecond"}, float64(time.Millisecond)}, + {"s", []string{"s", "sec", "second"}, float64(time.Second)}, + {"hrs", []string{"hour", "hr"}, float64(time.Hour)}, + }, + defaultUnit: unit{"s", []string{}, float64(time.Second)}, +}, { + units: []unit{ + {"n*GCU", []string{"nanogcu"}, 1e-9}, + {"u*GCU", []string{"microgcu"}, 1e-6}, + {"m*GCU", []string{"milligcu"}, 1e-3}, + {"GCU", []string{"gcu"}, 1}, + {"k*GCU", []string{"kilogcu"}, 1e3}, + {"M*GCU", []string{"megagcu"}, 1e6}, + {"G*GCU", []string{"gigagcu"}, 1e9}, + {"T*GCU", []string{"teragcu"}, 1e12}, + {"P*GCU", []string{"petagcu"}, 1e15}, + }, + defaultUnit: unit{"GCU", []string{}, 1.0}, +}} diff --git a/pkg/frontend/dot/report/report.go b/pkg/frontend/dot/report/report.go new file mode 100644 index 0000000000..2c7ad740cb --- /dev/null +++ b/pkg/frontend/dot/report/report.go @@ -0,0 +1,451 @@ +// Copyright 2014 Google Inc. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package report summarizes a performance profile into a +// human-readable report. +package report + +import ( + "fmt" + "path/filepath" + "regexp" + "strings" + "time" + + "github.com/google/pprof/profile" + + "github.com/grafana/pyroscope/pkg/frontend/dot/graph" + "github.com/grafana/pyroscope/pkg/frontend/dot/measurement" +) + +// Options are the formatting and filtering options used to generate a +// profile. +type Options struct { + OutputFormat int + + CumSort bool + CallTree bool + DropNegative bool + CompactLabels bool + Ratio float64 + Title string + ProfileLabels []string + ActiveFilters []string + NumLabelUnits map[string]string + + NodeCount int + NodeFraction float64 + EdgeFraction float64 + + SampleValue func(s []int64) int64 + SampleMeanDivisor func(s []int64) int64 + SampleType string + SampleUnit string // Unit for the sample data from the profile. + + OutputUnit string // Units for data formatting in report. + + Symbol *regexp.Regexp // Symbols to include on disassembly report. + SourcePath string // Search path for source files. + TrimPath string // Paths to trim from source file paths. + + IntelSyntax bool // Whether to print assembly in Intel syntax. +} + +// newTrimmedGraph creates a graph for this report, trimmed according +// to the report options. +func (rpt *Report) newTrimmedGraph() (g *graph.Graph, origCount, droppedNodes, droppedEdges int) { + o := rpt.options + + // Build a graph and refine it. On each refinement step we must rebuild the graph from the samples, + // as the graph itself doesn't contain enough information to preserve full precision. + cumSort := o.CumSort + + // First step: Build complete graph to identify low frequency nodes, based on their cum weight. + g = rpt.newGraph(nil) + totalValue, _ := g.Nodes.Sum() + nodeCutoff := abs64(int64(float64(totalValue) * o.NodeFraction)) + edgeCutoff := abs64(int64(float64(totalValue) * o.EdgeFraction)) + + // Filter out nodes with cum value below nodeCutoff. + if nodeCutoff > 0 { + if nodesKept := g.DiscardLowFrequencyNodes(nodeCutoff); len(g.Nodes) != len(nodesKept) { + droppedNodes = len(g.Nodes) - len(nodesKept) + g = rpt.newGraph(nodesKept) + } + } + origCount = len(g.Nodes) + + // Second step: Limit the total number of nodes. Apply specialized heuristics to improve + // visualization when generating dot output. + g.SortNodes(cumSort, true) + if nodeCount := o.NodeCount; nodeCount > 0 { + // Remove low frequency tags and edges as they affect selection. + g.TrimLowFrequencyTags(nodeCutoff) + g.TrimLowFrequencyEdges(edgeCutoff) + if nodesKept := g.SelectTopNodes(nodeCount, true); len(g.Nodes) != len(nodesKept) { + g = rpt.newGraph(nodesKept) + g.SortNodes(cumSort, true) + } + } + + // Final step: Filter out low frequency tags and edges, and remove redundant edges that clutter + // the graph. + g.TrimLowFrequencyTags(nodeCutoff) + droppedEdges = g.TrimLowFrequencyEdges(edgeCutoff) + g.RemoveRedundantEdges() + return +} + +func (rpt *Report) selectOutputUnit(g *graph.Graph) { + o := rpt.options + + // Select best unit for profile output. + // Find the appropriate units for the smallest non-zero sample + if o.OutputUnit != "minimum" || len(g.Nodes) == 0 { + return + } + var minValue int64 + + for _, n := range g.Nodes { + nodeMin := abs64(n.FlatValue()) + if nodeMin == 0 { + nodeMin = abs64(n.CumValue()) + } + if nodeMin > 0 && (minValue == 0 || nodeMin < minValue) { + minValue = nodeMin + } + } + maxValue := rpt.total + if minValue == 0 { + minValue = maxValue + } + + if r := o.Ratio; r > 0 && r != 1 { + minValue = int64(float64(minValue) * r) + maxValue = int64(float64(maxValue) * r) + } + + _, minUnit := measurement.Scale(minValue, o.SampleUnit, "minimum") + _, maxUnit := measurement.Scale(maxValue, o.SampleUnit, "minimum") + + unit := minUnit + if minUnit != maxUnit && minValue*100 < maxValue { + // Minimum and maximum values have different units. Scale + // minimum by 100 to use larger units, allowing minimum value to + // be scaled down to 0.01, except for callgrind reports since + // they can only represent integer values. + _, unit = measurement.Scale(100*minValue, o.SampleUnit, "minimum") + } + + if unit != "" { + o.OutputUnit = unit + } else { + o.OutputUnit = o.SampleUnit + } +} + +// newGraph creates a new graph for this report. If nodes is non-nil, +// only nodes whose info matches are included. Otherwise, all nodes +// are included, without trimming. +func (rpt *Report) newGraph(nodes graph.NodeSet) *graph.Graph { + o := rpt.options + + // Clean up file paths using heuristics. + prof := rpt.prof + for _, f := range prof.Function { + f.Filename = trimPath(f.Filename, o.TrimPath, o.SourcePath) + } + // Removes all numeric tags except for the bytes tag prior + // to making graph. + // TODO: modify to select first numeric tag if no bytes tag + for _, s := range prof.Sample { + numLabels := make(map[string][]int64, len(s.NumLabel)) + numUnits := make(map[string][]string, len(s.NumLabel)) + for k, vs := range s.NumLabel { + if k == "bytes" { + unit := o.NumLabelUnits[k] + numValues := make([]int64, len(vs)) + numUnit := make([]string, len(vs)) + for i, v := range vs { + numValues[i] = v + numUnit[i] = unit + } + numLabels[k] = append(numLabels[k], numValues...) + numUnits[k] = append(numUnits[k], numUnit...) + } + } + s.NumLabel = numLabels + s.NumUnit = numUnits + } + + // Remove label marking samples from the base profiles, so it does not appear + // as a nodelet in the graph view. + prof.RemoveLabel("pprof::base") + + formatTag := func(v int64, key string) string { + return measurement.ScaledLabel(v, key, o.OutputUnit) + } + + gopt := &graph.Options{ + SampleValue: o.SampleValue, + SampleMeanDivisor: o.SampleMeanDivisor, + FormatTag: formatTag, + CallTree: false, + DropNegative: o.DropNegative, + KeptNodes: nodes, + } + + return graph.New(rpt.prof, gopt) +} + +// TextItem holds a single text report entry. +type TextItem struct { + Name string + InlineLabel string // Not empty if inlined + Flat, Cum int64 // Raw values + FlatFormat, CumFormat string // Formatted values +} + +// GetDOT returns a graph suitable for dot processing along with some +// configuration information. +func GetDOT(rpt *Report) (*graph.Graph, *graph.DotConfig) { + g, origCount, droppedNodes, droppedEdges := rpt.newTrimmedGraph() + rpt.selectOutputUnit(g) + labels := reportLabels(rpt, g, origCount, droppedNodes, droppedEdges, true) + + c := &graph.DotConfig{ + Title: rpt.options.Title, + Labels: labels, + FormatValue: rpt.formatValue, + Total: rpt.total, + } + return g, c +} + +// ProfileLabels returns printable labels for a profile. +func ProfileLabels(rpt *Report) []string { + var label []string + prof := rpt.prof + o := rpt.options + if len(prof.Mapping) > 0 { + if prof.Mapping[0].File != "" { + label = append(label, "File: "+filepath.Base(prof.Mapping[0].File)) + } + if prof.Mapping[0].BuildID != "" { + label = append(label, "Build ID: "+prof.Mapping[0].BuildID) + } + } + // Only include comments that do not start with '#'. + for _, c := range prof.Comments { + if !strings.HasPrefix(c, "#") { + label = append(label, c) + } + } + if o.SampleType != "" { + label = append(label, "Type: "+o.SampleType) + } + if prof.TimeNanos != 0 { + const layout = "Jan 2, 2006 at 3:04pm (MST)" + label = append(label, "Time: "+time.Unix(0, prof.TimeNanos).Format(layout)) + } + if prof.DurationNanos != 0 { + duration := measurement.Label(prof.DurationNanos, "nanoseconds") + totalNanos, totalUnit := measurement.Scale(rpt.total, o.SampleUnit, "nanoseconds") + var ratio string + if totalUnit == "ns" && totalNanos != 0 { + ratio = "(" + measurement.Percentage(int64(totalNanos), prof.DurationNanos) + ")" + } + label = append(label, fmt.Sprintf("Duration: %s, Total samples = %s %s", duration, rpt.formatValue(rpt.total), ratio)) + } + return label +} + +// reportLabels returns printable labels for a report. Includes +// profileLabels. +func reportLabels(rpt *Report, g *graph.Graph, origCount, droppedNodes, droppedEdges int, fullHeaders bool) []string { + nodeFraction := rpt.options.NodeFraction + edgeFraction := rpt.options.EdgeFraction + nodeCount := len(g.Nodes) + + var label []string + if len(rpt.options.ProfileLabels) > 0 { + label = append(label, rpt.options.ProfileLabels...) + } else if fullHeaders || !rpt.options.CompactLabels { + label = ProfileLabels(rpt) + } + + var flatSum int64 + for _, n := range g.Nodes { + flatSum = flatSum + n.FlatValue() + } + + if len(rpt.options.ActiveFilters) > 0 { + activeFilters := legendActiveFilters(rpt.options.ActiveFilters) + label = append(label, activeFilters...) + } + + label = append(label, fmt.Sprintf("Showing nodes accounting for %s, %s of %s total", rpt.formatValue(flatSum), strings.TrimSpace(measurement.Percentage(flatSum, rpt.total)), rpt.formatValue(rpt.total))) + + if rpt.total != 0 { + if droppedNodes > 0 { + label = append(label, genLabel(droppedNodes, "node", "cum", + rpt.formatValue(abs64(int64(float64(rpt.total)*nodeFraction))))) + } + if droppedEdges > 0 { + label = append(label, genLabel(droppedEdges, "edge", "freq", + rpt.formatValue(abs64(int64(float64(rpt.total)*edgeFraction))))) + } + if nodeCount > 0 && nodeCount < origCount { + label = append(label, fmt.Sprintf("Showing top %d nodes out of %d", + nodeCount, origCount)) + } + } + + // Help new users understand the graph. + // A new line is intentionally added here to better show this message. + if fullHeaders { + label = append(label, "\nSee https://git.io/JfYMW for how to read the graph") + } + + return label +} + +func legendActiveFilters(activeFilters []string) []string { + legendActiveFilters := make([]string, len(activeFilters)+1) + legendActiveFilters[0] = "Active filters:" + for i, s := range activeFilters { + if len(s) > 80 { + s = s[:80] + "…" + } + legendActiveFilters[i+1] = " " + s + } + return legendActiveFilters +} + +func genLabel(d int, n, l, f string) string { + if d > 1 { + n = n + "s" + } + return fmt.Sprintf("Dropped %d %s (%s <= %s)", d, n, l, f) +} + +// New builds a new report indexing the sample values interpreting the +// samples with the provided function. +func New(prof *profile.Profile, o *Options) *Report { + format := func(v int64) string { + if r := o.Ratio; r > 0 && r != 1 { + fv := float64(v) * r + v = int64(fv) + } + return measurement.ScaledLabel(v, o.SampleUnit, o.OutputUnit) + } + return &Report{prof, computeTotal(prof, o.SampleValue, o.SampleMeanDivisor), + o, format} +} + +// NewDefault builds a new report indexing the last sample value +// available. +func NewDefault(prof *profile.Profile, options Options) *Report { + index := len(prof.SampleType) - 1 + o := &options + if o.Title == "" && len(prof.Mapping) > 0 && prof.Mapping[0].File != "" { + o.Title = filepath.Base(prof.Mapping[0].File) + } + o.SampleType = prof.SampleType[index].Type + o.SampleUnit = strings.ToLower(prof.SampleType[index].Unit) + o.SampleValue = func(v []int64) int64 { + return v[index] + } + return New(prof, o) +} + +// computeTotal computes the sum of the absolute value of all sample values. +// If any samples have label indicating they belong to the diff base, then the +// total will only include samples with that label. +func computeTotal(prof *profile.Profile, value, meanDiv func(v []int64) int64) int64 { + var div, total, diffDiv, diffTotal int64 + for _, sample := range prof.Sample { + var d, v int64 + v = value(sample.Value) + if meanDiv != nil { + d = meanDiv(sample.Value) + } + if v < 0 { + v = -v + } + total += v + div += d + if sample.DiffBaseSample() { + diffTotal += v + diffDiv += d + } + } + if diffTotal > 0 { + total = diffTotal + div = diffDiv + } + if div != 0 { + return total / div + } + return total +} + +// Report contains the data and associated routines to extract a +// report from a profile. +type Report struct { + prof *profile.Profile + total int64 + options *Options + formatValue func(int64) string +} + +// Total returns the total number of samples in a report. +func (rpt *Report) Total() int64 { return rpt.total } + +func abs64(i int64) int64 { + if i < 0 { + return -i + } + return i +} + +func trimPath(path, trimPath, searchPath string) string { + // Keep path variable intact as it's used below to form the return value. + sPath, searchPath := filepath.ToSlash(path), filepath.ToSlash(searchPath) + if trimPath == "" { + // If the trim path is not configured, try to guess it heuristically: + // search for basename of each search path in the original path and, if + // found, strip everything up to and including the basename. So, for + // example, given original path "/some/remote/path/my-project/foo/bar.c" + // and search path "/my/local/path/my-project" the heuristic will return + // "/my/local/path/my-project/foo/bar.c". + for _, dir := range filepath.SplitList(searchPath) { + want := "/" + filepath.Base(dir) + "/" + if found := strings.Index(sPath, want); found != -1 { + return path[found+len(want):] + } + } + } + // Trim configured trim prefixes. + trimPaths := append(filepath.SplitList(filepath.ToSlash(trimPath)), "/proc/self/cwd/./", "/proc/self/cwd/") + for _, trimPath := range trimPaths { + if !strings.HasSuffix(trimPath, "/") { + trimPath += "/" + } + if strings.HasPrefix(sPath, trimPath) { + return path[len(trimPath):] + } + } + return path +} diff --git a/pkg/querier/http.go b/pkg/querier/http.go index a231ee80be..166c42be29 100644 --- a/pkg/querier/http.go +++ b/pkg/querier/http.go @@ -4,21 +4,26 @@ import ( "encoding/json" "errors" "fmt" + "io" "net/http" "strconv" "strings" "github.com/bufbuild/connect-go" "github.com/gogo/status" + "github.com/google/pprof/profile" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/promql/parser" "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" + profilev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1" querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1/querierv1connect" typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + "github.com/grafana/pyroscope/pkg/frontend/dot/graph" + "github.com/grafana/pyroscope/pkg/frontend/dot/report" phlaremodel "github.com/grafana/pyroscope/pkg/model" "github.com/grafana/pyroscope/pkg/og/structs/flamebearer" "github.com/grafana/pyroscope/pkg/og/util/attime" @@ -141,6 +146,37 @@ func (q *QueryHandlers) Render(w http.ResponseWriter, req *http.Request) { } } + format := req.URL.Query().Get("format") + if format == "dot" { + // We probably should distinguish max nodes of the source pprof + // profile and max nodes value for the output profile in dot format. + sourceProfileMaxNodes := int64(512) + dotProfileMaxNodes := int64(100) + if selectParams.MaxNodes != nil { + if v := *selectParams.MaxNodes; v > 0 { + dotProfileMaxNodes = v + } + if dotProfileMaxNodes > sourceProfileMaxNodes { + sourceProfileMaxNodes = dotProfileMaxNodes + } + } + resp, err := q.client.SelectMergeProfile(req.Context(), connect.NewRequest(&querierv1.SelectMergeProfileRequest{ + Start: selectParams.Start, + End: selectParams.End, + ProfileTypeID: selectParams.ProfileTypeID, + LabelSelector: selectParams.LabelSelector, + MaxNodes: &sourceProfileMaxNodes, + })) + if err != nil { + httputil.Error(w, connect.NewError(connect.CodeInternal, err)) + return + } + if err = pprofToDotProfile(w, resp.Msg, int(dotProfileMaxNodes)); err != nil { + httputil.Error(w, connect.NewError(connect.CodeInternal, err)) + } + return + } + var resFlame *connect.Response[querierv1.SelectMergeStacktracesResponse] g, ctx := errgroup.WithContext(req.Context()) selectParamsClone := selectParams.CloneVT() @@ -204,6 +240,21 @@ func (q *QueryHandlers) Render(w http.ResponseWriter, req *http.Request) { } } +func pprofToDotProfile(w io.Writer, p *profilev1.Profile, maxNodes int) error { + data, err := p.MarshalVT() + if err != nil { + return connect.NewError(connect.CodeInternal, err) + } + pr, err := profile.ParseData(data) + if err != nil { + return connect.NewError(connect.CodeInternal, err) + } + rpt := report.NewDefault(pr, report.Options{NodeCount: maxNodes}) + gr, cfg := report.GetDOT(rpt) + graph.ComposeDot(w, gr, &graph.DotAttributes{}, cfg) + return nil +} + type renderRequestFieldNames struct { query string from string