diff --git a/ebpf/go.mod b/ebpf/go.mod index f62a8ea10d..d4d7d9b008 100644 --- a/ebpf/go.mod +++ b/ebpf/go.mod @@ -8,7 +8,7 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 github.com/cilium/ebpf v0.11.0 github.com/go-kit/log v0.2.1 - github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 + github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 github.com/grafana/pyroscope/api v0.4.0 github.com/hashicorp/golang-lru/v2 v2.0.5 github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab diff --git a/ebpf/go.sum b/ebpf/go.sum index 9de04018f0..8f56806fb9 100644 --- a/ebpf/go.sum +++ b/ebpf/go.sum @@ -22,8 +22,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 h1:WzfWbQz/Ze8v6l++GGbGNFZnUShVpP/0xffCPLL+ax8= +github.com/google/pprof v0.0.0-20240117000934-35fc243c5815/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/grafana/pyroscope/api v0.4.0 h1:J86DxoNeLOvtJhB1Cn65JMZkXe682D+RqeoIUiYc/eo= diff --git a/go.mod b/go.mod index 47979ed076..180eceae0b 100644 --- a/go.mod +++ b/go.mod @@ -22,14 +22,14 @@ require ( github.com/gogo/status v1.1.1 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da github.com/google/go-cmp v0.6.0 - github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 + github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 github.com/google/uuid v1.4.0 github.com/gorilla/mux v1.8.0 github.com/grafana/dskit v0.0.0-20231221015914-de83901bf4d6 - github.com/grafana/jfr-parser/pprof v0.0.0-20240108135448-c7f61c1f689e + github.com/grafana/jfr-parser/pprof v0.0.0-20240126072739-986e71dc0361 github.com/grafana/pyroscope-go v1.0.3 github.com/grafana/pyroscope-go/godeltaprof v0.1.7 - github.com/grafana/pyroscope/api v0.3.0 + github.com/grafana/pyroscope/api v0.4.0 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db github.com/grafana/river v0.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 diff --git a/go.sum b/go.sum index aa04bffdfd..4ce6a4c9dc 100644 --- a/go.sum +++ b/go.sum @@ -365,8 +365,8 @@ github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20211214055906-6f57359322fd/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98 h1:pUa4ghanp6q4IJHwE9RwLgmVFfReJN+KbQ8ExNEUUoQ= -github.com/google/pprof v0.0.0-20230926050212-f7f687d19a98/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 h1:WzfWbQz/Ze8v6l++GGbGNFZnUShVpP/0xffCPLL+ax8= +github.com/google/pprof v0.0.0-20240117000934-35fc243c5815/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= @@ -388,8 +388,8 @@ github.com/grafana/dskit v0.0.0-20231221015914-de83901bf4d6 h1:Z78JZ7pa6InQ5BcMB github.com/grafana/dskit v0.0.0-20231221015914-de83901bf4d6/go.mod h1:kkWM4WUV230bNG3urVRWPBnSJHs64y/0RmWjftnnn0c= github.com/grafana/jfr-parser v0.8.0 h1:/uo2wZNXrxw7tKLFwP2omJ3EQGMkD9wzhPsRogVofc0= github.com/grafana/jfr-parser v0.8.0/go.mod h1:M5u1ux34Qo47ZBWksbMYVk40s7dvU3WMVYpxweEu4R0= -github.com/grafana/jfr-parser/pprof v0.0.0-20240108135448-c7f61c1f689e h1:Xpf1EsOY+ZMQYzusCLEixnWifv36J/jLz0GihnEyuNA= -github.com/grafana/jfr-parser/pprof v0.0.0-20240108135448-c7f61c1f689e/go.mod h1:SDOa+U4/vAkuGjYBWXJpHpiaesbeXuhaQW4xzHzTSL4= +github.com/grafana/jfr-parser/pprof v0.0.0-20240126072739-986e71dc0361 h1:TtNajaiSRfM2Mz8N7ouFQDFlviXbIEk9Hts0yoZnhGM= +github.com/grafana/jfr-parser/pprof v0.0.0-20240126072739-986e71dc0361/go.mod h1:P5406BrWxjahTzVF6aCSumNI1KPlZJc0zO0v+zKZ4gc= github.com/grafana/memberlist v0.3.1-0.20220708130638-bd88e10a3d91 h1:/NipyHnOmvRsVzj81j2qE0VxsvsqhOB0f4vJIhk2qCQ= github.com/grafana/memberlist v0.3.1-0.20220708130638-bd88e10a3d91/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= github.com/grafana/objstore v0.0.0-20231121154247-84f91ea90e72 h1:o22hsDMQ3kv/0N9PkzHQSd5xMmmmdA5UJR7Jb4xISZQ= diff --git a/go.work.sum b/go.work.sum index 64c5cd1384..f789e7aa9d 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1049,9 +1049,13 @@ github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmt github.com/chromedp/sysutil v1.0.0 h1:+ZxhTpfpZlmchB58ih/LBHX52ky7w2VhQVKQMucy3Ic= github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= github.com/chzyer/logex v1.1.10 h1:Swpa1K6QvQznwJRcfTfQJmTE72DqScAa40E+fbHEXEE= +github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1 h1:q763qf9huN11kDQavWsoZXJNW3xEE4JJyHa5Q25/sd8= +github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible h1:C29Ae4G5GtYyYMm1aztcyj/J5ckgJm2zwdDajFbx1NY= github.com/circonus-labs/circonusllhist v0.1.3 h1:TJH+oke8D16535+jHExHj4nQvzlZrj7ug5D7I/orNUA= github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI= @@ -1399,6 +1403,7 @@ github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9/go.mod h1:6eQoG github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20240117000934-35fc243c5815 h1:WzfWbQz/Ze8v6l++GGbGNFZnUShVpP/0xffCPLL+ax8= github.com/google/renameio v0.1.0 h1:GOZbcHa3HfsPKPlmyPyN2KEohoMXOhdMbHrvbpl2QaA= github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= diff --git a/pkg/ingester/pyroscope/ingest_handler_test.go b/pkg/ingester/pyroscope/ingest_handler_test.go index aa37ad6306..6ec43e67ef 100644 --- a/pkg/ingester/pyroscope/ingest_handler_test.go +++ b/pkg/ingester/pyroscope/ingest_handler_test.go @@ -8,7 +8,6 @@ import ( "net/http/httptest" "os" "sort" - "strings" "testing" "connectrpc.com/connect" @@ -142,54 +141,6 @@ const ( testdataDirJFR = repoRoot + "pkg/og/convert/jfr/testdata" ) -func TestIngestJFR(b *testing.T) { - testdata := []struct { - jfr string - labels string - }{ - {"cortex-dev-01__kafka-0__cpu__0.jfr.gz", ""}, - {"cortex-dev-01__kafka-0__cpu__1.jfr.gz", ""}, - {"cortex-dev-01__kafka-0__cpu__2.jfr.gz", ""}, - {"cortex-dev-01__kafka-0__cpu__3.jfr.gz", ""}, - {"cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz", ""}, - {"cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz", ""}, - {"cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz", ""}, - {"cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz", ""}, - {"cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz", ""}, - {"dump1.jfr.gz", "dump1.labels.pb.gz"}, - {"dump2.jfr.gz", "dump2.labels.pb.gz"}, - } - l := log.NewSyncLogger(log.NewLogfmtLogger(os.Stderr)) - - for _, jfr := range testdata { - td := jfr - b.Run(td.jfr, func(t *testing.T) { - src := testdataDirJFR + "/" + td.jfr - jfr, err := bench.ReadGzipFile(src) - require.NoError(t, err) - var labels []byte - if td.labels != "" { - labels, err = bench.ReadGzipFile(testdataDirJFR + "/" + td.labels) - } - require.NoError(t, err) - svc := &MockPushService{Keep: true, T: t} - h := NewPyroscopeIngestHandler(svc, l) - - res := httptest.NewRecorder() - body, ct := createJFRRequestBody(t, jfr, labels) - - req := httptest.NewRequest("POST", "/ingest?name=javaapp&format=jfr", bytes.NewReader(body)) - req.Header.Set("Content-Type", ct) - h.ServeHTTP(res, req) - assert.Equal(t, 200, res.Code) - - dst := strings.ReplaceAll(src, ".jfr.gz", ".pprof.json.gz") - // svc.DumpTo(dst) - svc.CompareDump(dst) - }) - } -} - func TestCorruptedJFR422(t *testing.T) { l := log.NewSyncLogger(log.NewLogfmtLogger(os.Stderr)) diff --git a/pkg/og/convert/jfr/parser_suite_test.go b/pkg/og/convert/jfr/parser_suite_test.go deleted file mode 100644 index 1a9dcf1d21..0000000000 --- a/pkg/og/convert/jfr/parser_suite_test.go +++ /dev/null @@ -1,300 +0,0 @@ -package jfr - -import ( - "encoding/json" - "fmt" - "os" - "sort" - "strings" - "testing" - "time" - - jfrPprof "github.com/grafana/jfr-parser/pprof" - jfrPprofPyroscope "github.com/grafana/jfr-parser/pprof/pyroscope" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "golang.org/x/exp/slices" - "google.golang.org/protobuf/proto" - - profilev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1" - v1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" - phlaremodel "github.com/grafana/pyroscope/pkg/model" - "github.com/grafana/pyroscope/pkg/og/convert/pprof/bench" - "github.com/grafana/pyroscope/pkg/og/storage/segment" - "github.com/grafana/pyroscope/pkg/pprof" -) - -func TestParseCompareExpectedData(t *testing.T) { - testdata := []struct { - jfr string - labels string - }{ - {"testdata/cortex-dev-01__kafka-0__cpu__0.jfr.gz", ""}, - {"testdata/cortex-dev-01__kafka-0__cpu__1.jfr.gz", ""}, - {"testdata/cortex-dev-01__kafka-0__cpu__2.jfr.gz", ""}, - {"testdata/cortex-dev-01__kafka-0__cpu__3.jfr.gz", ""}, - {"testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz", ""}, - {"testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz", ""}, - {"testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz", ""}, - {"testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz", ""}, - {"testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz", ""}, - {"testdata/dump1.jfr.gz", "testdata/dump1.labels.pb.gz"}, - {"testdata/dump2.jfr.gz", "testdata/dump2.labels.pb.gz"}, - } - for _, td := range testdata { - t.Run(td.jfr, func(t *testing.T) { - jfr, err := bench.ReadGzipFile(td.jfr) - require.NoError(t, err) - k, err := segment.ParseKey("kafka.app") - require.NoError(t, err) - - pi := &jfrPprof.ParseInput{ - StartTime: time.UnixMilli(1000), - EndTime: time.UnixMilli(2000), - SampleRate: 100, - } - var labels = new(jfrPprof.LabelsSnapshot) - if td.labels != "" { - labelsBytes, err := bench.ReadGzipFile(td.labels) - require.NoError(t, err) - err = proto.Unmarshal(labelsBytes, labels) - require.NoError(t, err) - } - req, err := jfrPprof.ParseJFR(jfr, pi, labels) - require.NoError(t, err) - if len(req.Profiles) == 0 { - t.Fatal(err) - } - jsonFile := strings.TrimSuffix(td.jfr, ".jfr.gz") + ".json.gz" - err = compareWithJson(t, req, k, jsonFile) - require.NoError(t, err) - }) - } -} - -func compareWithJson(t *testing.T, parsed *jfrPprof.Profiles, key *segment.Key, file string) error { - defer func() { - for _, profile := range parsed.Profiles { - profile.Profile.ReturnToVTPool() - } - }() - type flatProfileSeries struct { - Labels []*v1.LabelPair - Profile *profilev1.Profile - } - - var profiles []*flatProfileSeries - for _, s := range parsed.Profiles { - profile := pprof.RawFromProto(s.Profile) - profile.Normalize() - seriesLabels := jfrPprofPyroscope.Labels(key.Labels(), parsed.JFREvent, s.Metric, key.AppName(), "javaspy") - iterateProfileSeries(profile.CloneVT(), seriesLabels, func(p *profilev1.Profile, l phlaremodel.Labels) { - profiles = append(profiles, &flatProfileSeries{ - Labels: l, - Profile: p, - }) - }) - } - - goldBS, err := bench.ReadGzipFile(file) - if err != nil { - return err - } - trees := make(map[string]string) - err = json.Unmarshal(goldBS, &trees) - if err != nil { - return err - } - - checkedSeries := make(map[string]struct{}) - for _, profile := range profiles { - - var keys []string - scale := 1.0 - var valueIndices []int - ls := phlaremodel.Labels(profile.Labels) - metric := ls.Get(model.MetricNameLabel) - service_name := ls.Get("service_name") - typ := profile.Profile.StringTable[profile.Profile.SampleType[0].Type] - event := ls.Get("jfr_event") - keys = nil - switch metric { - case "process_cpu": - keys = nil - switch event { - case "cpu", "wall": - keys = []string{service_name + "." + "cpu"} - case "itimer": - keys = []string{service_name + "." + "itimer"} - default: - panic("unknown event: " + event) //todo wall - } - scale = 1.0 / (1e9 / 100.0) - valueIndices = []int{0} - case "memory": - if typ == "live" { - keys = []string{service_name + "." + "live"} - valueIndices = []int{0} - } else { - if strings.Contains(typ, "alloc_in_new_tlab_objects") { - keys = []string{ - service_name + "." + "alloc_in_new_tlab_objects", - service_name + "." + "alloc_in_new_tlab_bytes", - } - } else { - keys = []string{ - service_name + "." + "alloc_outside_tlab_objects", - service_name + "." + "alloc_outside_tlab_bytes", - } - } - valueIndices = []int{0, 1} - } - case "mutex": - keys = []string{ - service_name + "." + "lock_count", - service_name + "." + "lock_duration", - } - valueIndices = []int{0, 1} - case "block": - keys = []string{ - service_name + "." + "thread_park_count", - service_name + "." + "thread_park_duration", - } - valueIndices = []int{0, 1} - case "wall": - keys = []string{service_name + "." + "wall"} - valueIndices = []int{0} - scale = 1.0 / (1e9 / 100.0) - default: - panic("unknown metric: " + metric + " " + service_name) - } - if len(keys) == 0 { - return fmt.Errorf("no keys found for %s %s %s", metric, typ, service_name) - } - for i := range keys { - key := keys[i] - parseKey, err := segment.ParseKey(key) - if err != nil { - return err - } - for _, label := range profile.Labels { - if strings.HasPrefix(label.Name, "__") || label.Name == phlaremodel.LabelNameServiceName || label.Name == "jfr_event" || label.Name == phlaremodel.LabelNamePyroscopeSpy { - continue - } - parseKey.Add(label.Name, label.Value) - } - - // We used to duplicate samples with profile ID. - // Now we don't do it anymore, but have it in fixtures. - if _, ok := parseKey.ProfileID(); ok { - k := parseKey.Clone() - k.Add("profile_id", "") - delete(trees, k.Normalized()) - } - - key = parseKey.Normalized() - expectedTree := trees[key] - if expectedTree == "" { - return fmt.Errorf("no tree found for %s", key) - } - checkedSeries[key] = struct{}{} - expectedLines := strings.Split(expectedTree, "\n") - slices.Sort(expectedLines) - expectedTree = strings.Join(expectedLines, "\n") - expectedTree = strings.Trim(expectedTree, "\n") - - pp := pprof.Profile{Profile: profile.Profile} - pp.Normalize() - - collapseLines := bench.StackCollapseProto(pp.Profile, valueIndices[i], scale) - slices.Sort(collapseLines) - collapsedStr := strings.Join(collapseLines, "\n") - collapsedStr = strings.Trim(collapsedStr, "\n") - - if expectedTree != collapsedStr { - os.WriteFile(file+"_"+metric+"_"+typ+"_expected.txt", []byte(expectedTree), 0644) - os.WriteFile(file+"_"+metric+"_"+typ+"_actual.txt", []byte(collapsedStr), 0644) - return fmt.Errorf("expected tree:\n%s\ngot:\n%s", expectedTree, collapsedStr) - } - fmt.Printf("ok %s %d\n", key, len(collapsedStr)) - } - } - for k, v := range trees { - _ = v - if _, ok := checkedSeries[k]; !ok { - assert.Failf(t, "no profile found for ", "key=%s", k) - } - } - for k, v := range checkedSeries { - _ = v - if _, ok := trees[k]; !ok { - assert.Failf(t, "no tree found for ", "key=%s", k) - } - } - return nil -} - -func BenchmarkParser(b *testing.B) { - tests := []string{ - "testdata/cortex-dev-01__kafka-0__cpu__0.jfr.gz", - "testdata/cortex-dev-01__kafka-0__cpu__1.jfr.gz", - "testdata/cortex-dev-01__kafka-0__cpu__2.jfr.gz", - "testdata/cortex-dev-01__kafka-0__cpu__3.jfr.gz", - "testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz", - "testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz", - "testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz", - "testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz", - "testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz", - } - - for _, testdata := range tests { - f := testdata - b.Run(testdata, func(b *testing.B) { - jfr, err := bench.ReadGzipFile(f) - require.NoError(b, err) - pi := &jfrPprof.ParseInput{ - StartTime: time.UnixMilli(1000), - EndTime: time.UnixMilli(2000), - SampleRate: 100, - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - profiles, err := jfrPprof.ParseJFR(jfr, pi, nil) - if err != nil { - b.Fatal(err) - } - if len(profiles.Profiles) == 0 { - b.Fatal() - } - } - }) - } -} - -func iterateProfileSeries(p *profilev1.Profile, seriesLabels phlaremodel.Labels, fn func(*profilev1.Profile, phlaremodel.Labels)) { - for _, x := range p.Sample { - sort.Sort(pprof.LabelsByKeyValue(x.Label)) - } - sort.Sort(pprof.SamplesByLabels(p.Sample)) - groups := pprof.GroupSamplesByLabels(p) - e := pprof.NewSampleExporter(p) - for _, g := range groups { - ls := mergeSeriesAndSampleLabels(p, seriesLabels, g.Labels) - ps := e.ExportSamples(new(profilev1.Profile), g.Samples) - fn(ps, ls) - } -} - -func mergeSeriesAndSampleLabels(p *profilev1.Profile, sl []*v1.LabelPair, pl []*profilev1.Label) []*v1.LabelPair { - m := phlaremodel.Labels(sl).Clone() - for _, l := range pl { - m = append(m, &v1.LabelPair{ - Name: p.StringTable[l.Key], - Value: p.StringTable[l.Str], - }) - } - sort.Stable(m) - return m.Unique() -} diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__0.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__0.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..b3650f917b Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__0.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__0.pprof.json.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__0.pprof.json.gz deleted file mode 100644 index d5d955b662..0000000000 Binary files a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__0.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..2054b95890 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__1.pprof.json.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__1.pprof.json.gz deleted file mode 100644 index 5dd1453a57..0000000000 Binary files a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__1.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..31f786593f Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__2.pprof.json.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__2.pprof.json.gz deleted file mode 100644 index e566412331..0000000000 Binary files a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__2.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__3.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__3.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..1230675bcf Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__3.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__3.pprof.json.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__3.pprof.json.gz deleted file mode 100644 index b37bfb257a..0000000000 Binary files a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu__3.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.block:contentions:count:block:count.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.block:contentions:count:block:count.17241709254077376921.pb.gz new file mode 100644 index 0000000000..6f3b1b829b Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.block:contentions:count:block:count.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.block:delay:nanoseconds:block:count.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.block:delay:nanoseconds:block:count.17241709254077376921.pb.gz new file mode 100644 index 0000000000..6c31b6fdc3 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.block:delay:nanoseconds:block:count.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..4f83535d9e Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..62bc82462a Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_outside_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_outside_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..abfd0bddf8 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_outside_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_outside_tlab_objects:count:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_outside_tlab_objects:count:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..a8db40ef1a Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.memory:alloc_outside_tlab_objects:count:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.mutex:contentions:count:mutex:count.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.mutex:contentions:count:mutex:count.17241709254077376921.pb.gz new file mode 100644 index 0000000000..181ad8c6db Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.mutex:contentions:count:mutex:count.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.mutex:delay:nanoseconds:mutex:count.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.mutex:delay:nanoseconds:mutex:count.17241709254077376921.pb.gz new file mode 100644 index 0000000000..859e16423d Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.mutex:delay:nanoseconds:mutex:count.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..81a5e850bd Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.pprof.json.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.pprof.json.gz deleted file mode 100644 index fdf9db3540..0000000000 Binary files a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..157a91a0ff Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..fcc2100d17 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..0fcdca6933 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.pprof.json.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.pprof.json.gz deleted file mode 100644 index c9efe000a8..0000000000 Binary files a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__0.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..e14ed47c4a Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..ce503bd19b Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..bdd78386dc Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.pprof.json.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.pprof.json.gz deleted file mode 100644 index 73168c9eec..0000000000 Binary files a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__1.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..ccb51c33d9 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..5fed8b10ba Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..363d6ab1fb Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.pprof.json.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.pprof.json.gz deleted file mode 100644 index 7ac8e14873..0000000000 Binary files a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__2.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..dd77f2a1be Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..da1eb526be Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..f8446513f7 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.pprof.json.gz b/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.pprof.json.gz deleted file mode 100644 index faeb3b822c..0000000000 Binary files a/pkg/og/convert/jfr/testdata/cortex-dev-01__kafka-0__cpu_lock_alloc__3.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.15666661103234080825.pb.gz b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.15666661103234080825.pb.gz new file mode 100644 index 0000000000..2365909d36 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.15666661103234080825.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.16370823494798218517.pb.gz b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.16370823494798218517.pb.gz new file mode 100644 index 0000000000..b2f59be215 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.16370823494798218517.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..27b64fd779 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.2539015855607244867.pb.gz b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.2539015855607244867.pb.gz new file mode 100644 index 0000000000..965270a422 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.2539015855607244867.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.5571933481434853072.pb.gz b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.5571933481434853072.pb.gz new file mode 100644 index 0000000000..dc6d5440da Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump1.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.5571933481434853072.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump1.pprof.json.gz b/pkg/og/convert/jfr/testdata/dump1.pprof.json.gz deleted file mode 100644 index 18156df2aa..0000000000 Binary files a/pkg/og/convert/jfr/testdata/dump1.pprof.json.gz and /dev/null differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..8a25ff7c90 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.memory:alloc_in_new_tlab_bytes:bytes:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz new file mode 100644 index 0000000000..52ecb87999 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.memory:alloc_in_new_tlab_objects:count:space:bytes.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.memory:live:count:objects:count.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.memory:live:count:objects:count.17241709254077376921.pb.gz new file mode 100644 index 0000000000..6f64669515 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.memory:live:count:objects:count.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.10784330302778997279.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.10784330302778997279.pb.gz new file mode 100644 index 0000000000..ed482f104c Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.10784330302778997279.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.14685508092873724845.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.14685508092873724845.pb.gz new file mode 100644 index 0000000000..8ad25c951e Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.14685508092873724845.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.15666661103234080825.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.15666661103234080825.pb.gz new file mode 100644 index 0000000000..75da821614 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.15666661103234080825.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.16370823494798218517.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.16370823494798218517.pb.gz new file mode 100644 index 0000000000..8ac8300612 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.16370823494798218517.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..e69ff47d79 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.2539015855607244867.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.2539015855607244867.pb.gz new file mode 100644 index 0000000000..bfcda3f331 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.2539015855607244867.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.302387572117411911.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.302387572117411911.pb.gz new file mode 100644 index 0000000000..2101562268 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.302387572117411911.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.5571933481434853072.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.5571933481434853072.pb.gz new file mode 100644 index 0000000000..4fff5dd3af Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.5571933481434853072.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.6511396920643364271.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.6511396920643364271.pb.gz new file mode 100644 index 0000000000..861022c1aa Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.process_cpu:cpu:nanoseconds:cpu:nanoseconds.6511396920643364271.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.10784330302778997279.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.10784330302778997279.pb.gz new file mode 100644 index 0000000000..7ae5011a14 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.10784330302778997279.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.14685508092873724845.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.14685508092873724845.pb.gz new file mode 100644 index 0000000000..b9e92a8f28 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.14685508092873724845.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.15666661103234080825.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.15666661103234080825.pb.gz new file mode 100644 index 0000000000..3fc84f38e8 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.15666661103234080825.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.16370823494798218517.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.16370823494798218517.pb.gz new file mode 100644 index 0000000000..13f602a65e Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.16370823494798218517.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.17241709254077376921.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.17241709254077376921.pb.gz new file mode 100644 index 0000000000..3c2647a72b Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.17241709254077376921.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.2539015855607244867.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.2539015855607244867.pb.gz new file mode 100644 index 0000000000..ddc1f6898c Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.2539015855607244867.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.302387572117411911.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.302387572117411911.pb.gz new file mode 100644 index 0000000000..83aae7d356 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.302387572117411911.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.5571933481434853072.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.5571933481434853072.pb.gz new file mode 100644 index 0000000000..51bbb1d072 Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.5571933481434853072.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.6511396920643364271.pb.gz b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.6511396920643364271.pb.gz new file mode 100644 index 0000000000..bd7ce06eaa Binary files /dev/null and b/pkg/og/convert/jfr/testdata/dump2.jfr.gz.wall:wall:nanoseconds:wall:nanoseconds.6511396920643364271.pb.gz differ diff --git a/pkg/og/convert/jfr/testdata/dump2.pprof.json.gz b/pkg/og/convert/jfr/testdata/dump2.pprof.json.gz deleted file mode 100644 index 4885f65516..0000000000 Binary files a/pkg/og/convert/jfr/testdata/dump2.pprof.json.gz and /dev/null differ diff --git a/pkg/test/integration/helper.go b/pkg/test/integration/helper.go index c153bbcdf6..6de5a83071 100644 --- a/pkg/test/integration/helper.go +++ b/pkg/test/integration/helper.go @@ -2,23 +2,38 @@ package integration import ( "bytes" + "context" + "encoding/json" + "errors" "flag" "fmt" "io" + "math/rand" + "mime/multipart" "net" "net/http" + "os" "strings" "sync" "testing" "time" + "connectrpc.com/connect" "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + profilev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1" + pushv1 "github.com/grafana/pyroscope/api/gen/proto/go/push/v1" "github.com/grafana/pyroscope/api/gen/proto/go/push/v1/pushv1connect" + querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1/querierv1connect" + typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" "github.com/grafana/pyroscope/pkg/cfg" + "github.com/grafana/pyroscope/pkg/og/structs/flamebearer" "github.com/grafana/pyroscope/pkg/phlare" + "github.com/grafana/pyroscope/pkg/pprof" + "github.com/grafana/pyroscope/pkg/util/connectgrpc" ) // getFreePorts returns a number of free local port for the tests to listen on. Note this will make sure the returned ports do not overlap, by stopping to listen once all ports are allocated @@ -126,20 +141,6 @@ func (p *PyroscopeTest) URL() string { return fmt.Sprintf("http://%s:%d", address, p.httpPort) } -func (p *PyroscopeTest) queryClient() querierv1connect.QuerierServiceClient { - return querierv1connect.NewQuerierServiceClient( - http.DefaultClient, - p.URL(), - ) -} - -func (p *PyroscopeTest) pushClient() pushv1connect.PusherServiceClient { - return pushv1connect.NewPusherServiceClient( - http.DefaultClient, - p.URL(), - ) -} - func httpBodyContains(url string, needle string) bool { fmt.Println("httpBodyContains", url, needle) res, err := http.Get(url) @@ -157,3 +158,252 @@ func httpBodyContains(url string, needle string) bool { return strings.Contains(body.String(), needle) } + +func (p *PyroscopeTest) NewRequestBuilder(t *testing.T) *RequestBuilder { + return &RequestBuilder{ + t: t, + url: p.URL(), + AppName: p.TempAppName(), + spy: "foo239", + } +} + +func (p *PyroscopeTest) TempAppName() string { + return fmt.Sprintf("pprof.integration.%d", + rand.Uint64()) +} + +func createRenderQuery(metric, app string) string { + return metric + "{service_name=\"" + app + "\"}" +} + +type RequestBuilder struct { + AppName string + url string + spy string + t *testing.T +} + +func (b *RequestBuilder) Spy(spy string) *RequestBuilder { + b.spy = spy + return b +} + +func (b *RequestBuilder) IngestPPROFRequest(profilePath, prevProfilePath, sampleTypeConfigPath string) *http.Request { + var ( + profile, prevProfile, sampleTypeConfig []byte + err error + ) + profile, err = os.ReadFile(profilePath) + assert.NoError(b.t, err) + if prevProfilePath != "" { + prevProfile, err = os.ReadFile(prevProfilePath) + assert.NoError(b.t, err) + } + if sampleTypeConfigPath != "" { + sampleTypeConfig, err = os.ReadFile(sampleTypeConfigPath) + assert.NoError(b.t, err) + } + + const ( + formFieldProfile = "profile" + formFieldPreviousProfile = "prev_profile" + formFieldSampleTypeConfig = "sample_type_config" + ) + + var bb bytes.Buffer + w := multipart.NewWriter(&bb) + + profileW, err := w.CreateFormFile(formFieldProfile, "not used") + require.NoError(b.t, err) + _, err = profileW.Write(profile) + require.NoError(b.t, err) + + if sampleTypeConfig != nil { + + sampleTypeConfigW, err := w.CreateFormFile(formFieldSampleTypeConfig, "not used") + require.NoError(b.t, err) + _, err = sampleTypeConfigW.Write(sampleTypeConfig) + require.NoError(b.t, err) + } + + if prevProfile != nil { + prevProfileW, err := w.CreateFormFile(formFieldPreviousProfile, "not used") + require.NoError(b.t, err) + _, err = prevProfileW.Write(prevProfile) + require.NoError(b.t, err) + } + err = w.Close() + require.NoError(b.t, err) + + bs := bb.Bytes() + ct := w.FormDataContentType() + + url := b.url + "/ingest?name=" + b.AppName + "&spyName=" + b.spy + req, err := http.NewRequest("POST", url, bytes.NewReader(bs)) + require.NoError(b.t, err) + req.Header.Set("Content-Type", ct) + return req +} + +func (b *RequestBuilder) IngestJFRRequestFiles(jfrPath, labelsPath string) *http.Request { + var ( + jfr, labels []byte + err error + ) + jfr, err = os.ReadFile(jfrPath) + assert.NoError(b.t, err) + if labelsPath != "" { + labels, err = os.ReadFile(labelsPath) + assert.NoError(b.t, err) + } + + return b.IngestJFRRequestBody(jfr, labels) +} + +func (b *RequestBuilder) IngestJFRRequestBody(jfr []byte, labels []byte) *http.Request { + var bb bytes.Buffer + w := multipart.NewWriter(&bb) + jfrw, err := w.CreateFormFile("jfr", "jfr") + require.NoError(b.t, err) + _, err = jfrw.Write(jfr) + require.NoError(b.t, err) + if labels != nil { + labelsw, err := w.CreateFormFile("labels", "labels") + require.NoError(b.t, err) + _, err = labelsw.Write(labels) + require.NoError(b.t, err) + } + err = w.Close() + require.NoError(b.t, err) + ct := w.FormDataContentType() + bs := bb.Bytes() + + url := b.url + "/ingest?name=" + b.AppName + "&spyName=" + b.spy + "&format=jfr" + req, err := http.NewRequest("POST", url, bytes.NewReader(bs)) + require.NoError(b.t, err) + req.Header.Set("Content-Type", ct) + + return req +} + +func (b *RequestBuilder) Render(metric string) *flamebearer.FlamebearerProfile { + queryURL := b.url + "/pyroscope/render?query=" + createRenderQuery(metric, b.AppName) + "&from=946656000&until=now&format=collapsed" + fmt.Println(queryURL) + queryRes, err := http.Get(queryURL) + require.NoError(b.t, err) + body := bytes.NewBuffer(nil) + _, err = io.Copy(body, queryRes.Body) + assert.NoError(b.t, err) + fb := new(flamebearer.FlamebearerProfile) + err = json.Unmarshal(body.Bytes(), fb) + assert.NoError(b.t, err, body.String(), queryURL) + assert.Greater(b.t, len(fb.Flamebearer.Names), 1, body.String(), queryRes) + assert.Greater(b.t, fb.Flamebearer.NumTicks, 1, body.String(), queryRes) + // todo check actual stacktrace contents + return fb +} + +func (b *RequestBuilder) PushPPROFRequest(file string, metric string) *connect.Request[pushv1.PushRequest] { + updateTimestamp := func(rawProfile []byte) []byte { + expectedProfile, err := pprof.RawFromBytes(rawProfile) + require.NoError(b.t, err) + expectedProfile.Profile.TimeNanos = time.Now().Add(-time.Minute).UnixNano() + buf := bytes.NewBuffer(nil) + _, err = expectedProfile.WriteTo(buf) + require.NoError(b.t, err) + rawProfile = buf.Bytes() + return rawProfile + } + + rawProfile, err := os.ReadFile(file) + require.NoError(b.t, err) + + rawProfile = updateTimestamp(rawProfile) + + metricName := strings.Split(metric, ":")[0] + + req := connect.NewRequest(&pushv1.PushRequest{ + Series: []*pushv1.RawProfileSeries{{ + Labels: []*typesv1.LabelPair{ + {Name: "__name__", Value: metricName}, + {Name: "__delta__", Value: "false"}, + {Name: "service_name", Value: b.AppName}, + }, + Samples: []*pushv1.RawSample{{RawProfile: rawProfile}}, + }}, + }) + return req +} + +func (b *RequestBuilder) QueryClient() querierv1connect.QuerierServiceClient { + return querierv1connect.NewQuerierServiceClient( + http.DefaultClient, + b.url, + ) +} + +func (b *RequestBuilder) PushClient() pushv1connect.PusherServiceClient { + return pushv1connect.NewPusherServiceClient( + http.DefaultClient, + b.url, + ) +} + +func (p *PyroscopeTest) Ingest(t *testing.T, req *http.Request, expectStatus int) { + res, err := http.DefaultClient.Do(req) + require.NoError(t, err) + require.Equal(t, expectStatus, res.StatusCode) +} + +func (b *RequestBuilder) Push(request *connect.Request[pushv1.PushRequest], expectStatus int, expectedError string) { + cl := b.PushClient() + _, err := cl.Push(context.TODO(), request) + if expectStatus == 200 { + assert.NoError(b.t, err) + } else { + assert.Error(b.t, err) + var connectErr *connect.Error + if ok := errors.As(err, &connectErr); ok { + toHTTP := connectgrpc.CodeToHTTP(connectErr.Code()) + assert.Equal(b.t, expectStatus, int(toHTTP)) + if expectedError != "" { + assert.Contains(b.t, connectErr.Error(), expectedError) + } + } else { + assert.Fail(b.t, "unexpected error type", err) + } + } +} + +func (b *RequestBuilder) SelectMergeProfile(metric string, query map[string]string) *connect.Response[profilev1.Profile] { + + cnt := 0 + selector := strings.Builder{} + add := func(k, v string) { + if cnt > 0 { + selector.WriteString(", ") + } + selector.WriteString(k) + selector.WriteString("=") + selector.WriteString("\"") + selector.WriteString(v) + selector.WriteString("\"") + cnt++ + } + selector.WriteString("{") + add("service_name", b.AppName) + for k, v := range query { + add(k, v) + } + selector.WriteString("}") + qc := b.QueryClient() + resp, err := qc.SelectMergeProfile(context.Background(), connect.NewRequest(&querierv1.SelectMergeProfileRequest{ + ProfileTypeID: metric, + Start: time.Unix(0, 0).UnixMilli(), + End: time.Now().UnixMilli(), + LabelSelector: selector.String(), + })) + require.NoError(b.t, err) + return resp +} diff --git a/pkg/test/integration/ingest_jfr_test.go b/pkg/test/integration/ingest_jfr_test.go new file mode 100644 index 0000000000..1756844fc1 --- /dev/null +++ b/pkg/test/integration/ingest_jfr_test.go @@ -0,0 +1,223 @@ +package integration + +import ( + "fmt" + "os" + "testing" + + "connectrpc.com/connect" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + profilev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1" + phlaremodel "github.com/grafana/pyroscope/pkg/model" + "github.com/grafana/pyroscope/pkg/og/convert/pprof/bench" + "github.com/grafana/pyroscope/pkg/pprof" +) + +type jfrTestData struct { + jfr string + labels string + + expectedMetrics []expectedMetric + expectStatus int +} + +const ( + testdataDirJFR = repoRoot + "pkg/og/convert/jfr/testdata" +) + +var jfrTestDatas = []jfrTestData{ + { + jfr: testdataDirJFR + "/cortex-dev-01__kafka-0__cpu__0.jfr.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{{"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}}, + }, + { + jfr: testdataDirJFR + "/cortex-dev-01__kafka-0__cpu__1.jfr.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{{"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}}, + }, + { + jfr: testdataDirJFR + "/cortex-dev-01__kafka-0__cpu__2.jfr.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{{"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}}, + }, + { + jfr: testdataDirJFR + "/cortex-dev-01__kafka-0__cpu__3.jfr.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{{"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}}, + }, + { + jfr: testdataDirJFR + "/cortex-dev-01__kafka-0__cpu_lock_alloc__0.jfr.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{ + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}, + {"memory:alloc_in_new_tlab_objects:count:space:bytes", nil, 0}, + {"memory:alloc_in_new_tlab_bytes:bytes:space:bytes", nil, 0}, + }, + }, + { + jfr: testdataDirJFR + "/cortex-dev-01__kafka-0__cpu_lock_alloc__1.jfr.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{ + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}, + {"memory:alloc_in_new_tlab_objects:count:space:bytes", nil, 0}, + {"memory:alloc_in_new_tlab_bytes:bytes:space:bytes", nil, 0}, + }, + }, + { + jfr: testdataDirJFR + "/cortex-dev-01__kafka-0__cpu_lock_alloc__2.jfr.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{ + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}, + {"memory:alloc_in_new_tlab_objects:count:space:bytes", nil, 0}, + {"memory:alloc_in_new_tlab_bytes:bytes:space:bytes", nil, 0}, + }, + }, + { + jfr: testdataDirJFR + "/cortex-dev-01__kafka-0__cpu_lock_alloc__3.jfr.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{ + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}, + {"memory:alloc_in_new_tlab_objects:count:space:bytes", nil, 0}, + {"memory:alloc_in_new_tlab_bytes:bytes:space:bytes", nil, 0}, + }, + }, + { + jfr: testdataDirJFR + "/cortex-dev-01__kafka-0__cpu_lock0_alloc0__0.jfr.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{ + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}, + {"memory:alloc_in_new_tlab_objects:count:space:bytes", nil, 0}, + {"memory:alloc_in_new_tlab_bytes:bytes:space:bytes", nil, 0}, + {"memory:alloc_outside_tlab_objects:count:space:bytes", nil, 0}, + {"memory:alloc_outside_tlab_bytes:bytes:space:bytes", nil, 0}, + {"mutex:contentions:count:mutex:count", nil, 0}, + {"mutex:delay:nanoseconds:mutex:count", nil, 0}, + {"block:contentions:count:block:count", nil, 0}, + {"block:delay:nanoseconds:block:count", nil, 0}, + }, + }, + { + jfr: testdataDirJFR + "/dump1.jfr.gz", labels: testdataDirJFR + "/dump1.labels.pb.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{ + + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-7"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-3"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-5"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-1"}, 0}, + }, + }, + { + jfr: testdataDirJFR + "/dump2.jfr.gz", labels: testdataDirJFR + "/dump2.labels.pb.gz", + expectStatus: 200, + expectedMetrics: []expectedMetric{ + {"wall:wall:nanoseconds:wall:nanoseconds", map[string]string{}, 0}, + {"wall:wall:nanoseconds:wall:nanoseconds", map[string]string{"thread_name": "pool-2-thread-3"}, 0}, + {"wall:wall:nanoseconds:wall:nanoseconds", map[string]string{"thread_name": "pool-2-thread-6"}, 0}, + {"wall:wall:nanoseconds:wall:nanoseconds", map[string]string{"thread_name": "pool-2-thread-4"}, 0}, + {"wall:wall:nanoseconds:wall:nanoseconds", map[string]string{"thread_name": "pool-2-thread-5"}, 0}, + {"wall:wall:nanoseconds:wall:nanoseconds", map[string]string{"thread_name": "pool-2-thread-1"}, 0}, + {"wall:wall:nanoseconds:wall:nanoseconds", map[string]string{"thread_name": "pool-2-thread-8"}, 0}, + {"wall:wall:nanoseconds:wall:nanoseconds", map[string]string{"thread_name": "pool-2-thread-2"}, 0}, + {"wall:wall:nanoseconds:wall:nanoseconds", map[string]string{"thread_name": "pool-2-thread-7"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-3"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-6"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-4"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-5"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-1"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-8"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-2"}, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", map[string]string{"thread_name": "pool-2-thread-7"}, 0}, + {"memory:alloc_in_new_tlab_objects:count:space:bytes", map[string]string{}, 0}, + {"memory:alloc_in_new_tlab_bytes:bytes:space:bytes", map[string]string{}, 0}, + {"memory:live:count:objects:count", map[string]string{}, 0}, + }, + }, +} + +const dump = false + +func TestNOJFRDump(t *testing.T) { + assert.False(t, dump) +} + +func TestIngestJFR(t *testing.T) { + p := new(PyroscopeTest) + p.Start(t) + defer p.Stop(t) + + for _, testdatum := range jfrTestDatas { + t.Run(testdatum.jfr, func(t *testing.T) { + + rb := p.NewRequestBuilder(t) + req := rb.IngestJFRRequestFiles(testdatum.jfr, testdatum.labels) + p.Ingest(t, req, testdatum.expectStatus) + + if testdatum.expectStatus == 200 { + assert.NotEqual(t, len(testdatum.expectedMetrics), 0) + for _, metric := range testdatum.expectedMetrics { + goldFile := expectedPPROFFile(testdatum, metric) + t.Logf("%v gold file %s", metric, goldFile) + rb.Render(metric.name) + profile := rb.SelectMergeProfile(metric.name, metric.query) + verifyPPROF(t, profile, goldFile, metric) + } + } + }) + } +} + +func expectedPPROFFile(testdatum jfrTestData, metric expectedMetric) string { + lbls := phlaremodel.NewLabelsBuilder(nil) + for k, v := range metric.query { + lbls.Set(k, v) + } + ls := lbls.Labels() + h := ls.Hash() + + return fmt.Sprintf("%s.%s.%d.pb.gz", testdatum.jfr, metric.name, h) +} + +func TestCorruptedJFR422(t *testing.T) { + p := new(PyroscopeTest) + p.Start(t) + defer p.Stop(t) + + td := jfrTestDatas[0] + jfr, err := bench.ReadGzipFile(td.jfr) + require.NoError(t, err) + jfr[0] = 0 // corrupt jfr + + rb := p.NewRequestBuilder(t) + req := rb.IngestJFRRequestBody(jfr, nil) + p.Ingest(t, req, 422) +} + +func verifyPPROF(t *testing.T, resp *connect.Response[profilev1.Profile], expectedPPROF string, metric expectedMetric) { + var err error + + require.NoError(t, err) + assert.Equal(t, 1, len(resp.Msg.SampleType)) + + if dump { + bs, err := resp.Msg.MarshalVT() + require.NoError(t, err) + err = bench.WriteGzipFile(expectedPPROF, bs) + require.NoError(t, err) + } else { + profileBytes, err := os.ReadFile(expectedPPROF) + require.NoError(t, err) + expectedProfile, err := pprof.RawFromBytes(profileBytes) + require.NoError(t, err) + + actualStacktraces := bench.StackCollapseProto(resp.Msg, 0, 1) + expectedStacktraces := bench.StackCollapseProto(expectedProfile.Profile, metric.valueIDX, 1) + + require.Equal(t, expectedStacktraces, actualStacktraces) + } +} diff --git a/pkg/test/integration/ingest_pprof_test.go b/pkg/test/integration/ingest_pprof_test.go index 07ad536b07..7701c3e928 100644 --- a/pkg/test/integration/ingest_pprof_test.go +++ b/pkg/test/integration/ingest_pprof_test.go @@ -1,51 +1,33 @@ package integration import ( - "bytes" - "context" - "encoding/json" - "errors" "fmt" - "io" - "math/rand" - "mime/multipart" - "net/http" "os" - "path/filepath" - "strings" "testing" - "time" "connectrpc.com/connect" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/grafana/pyroscope/pkg/og/ingestion" - - querierv1 "github.com/grafana/pyroscope/api/gen/proto/go/querier/v1" - typesv1 "github.com/grafana/pyroscope/api/gen/proto/go/types/v1" + profilev1 "github.com/grafana/pyroscope/api/gen/proto/go/google/v1" + pprof2 "github.com/grafana/pyroscope/pkg/og/convert/pprof" "github.com/grafana/pyroscope/pkg/og/convert/pprof/bench" + "github.com/grafana/pyroscope/pkg/og/ingestion" "github.com/grafana/pyroscope/pkg/pprof" - "github.com/grafana/pyroscope/pkg/util/connectgrpc" - - pushv1 "github.com/grafana/pyroscope/api/gen/proto/go/push/v1" - - pprof2 "github.com/grafana/pyroscope/pkg/og/convert/pprof" - "github.com/grafana/pyroscope/pkg/og/structs/flamebearer" ) const repoRoot = "../../../" var ( golangHeap = []expectedMetric{ - {"memory:alloc_objects:count:space:bytes", 0}, - {"memory:alloc_space:bytes:space:bytes", 1}, - {"memory:inuse_objects:count:space:bytes", 2}, - {"memory:inuse_space:bytes:space:bytes", 3}, + {"memory:alloc_objects:count:space:bytes", nil, 0}, + {"memory:alloc_space:bytes:space:bytes", nil, 1}, + {"memory:inuse_objects:count:space:bytes", nil, 2}, + {"memory:inuse_space:bytes:space:bytes", nil, 3}, } golangCPU = []expectedMetric{ - {"process_cpu:samples:count:cpu:nanoseconds", 0}, - {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", 1}, + {"process_cpu:samples:count:cpu:nanoseconds", nil, 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 1}, } _ = golangHeap _ = golangCPU @@ -61,7 +43,7 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", 0}, + {"process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil, 0}, }, }, { @@ -101,8 +83,8 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"wall:sample:count:wall:microseconds", 0}, - {"wall:wall:microseconds:wall:microseconds", 1}, + {"wall:sample:count:wall:microseconds", nil, 0}, + {"wall:wall:microseconds:wall:microseconds", nil, 1}, }, }, { @@ -122,8 +104,8 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"memory:space:bytes:space:bytes", 1}, - {"memory:objects:count:space:bytes", 0}, + {"memory:space:bytes:space:bytes", nil, 1}, + {"memory:objects:count:space:bytes", nil, 0}, }, }, { @@ -131,8 +113,8 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"memory:inuse_space:bytes:inuse_space:bytes", 1}, - {"memory:inuse_objects:count:inuse_space:bytes", 0}, + {"memory:inuse_space:bytes:inuse_space:bytes", nil, 1}, + {"memory:inuse_objects:count:inuse_space:bytes", nil, 0}, }, }, { @@ -140,8 +122,8 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"wall:samples:count:wall:microseconds", 0}, - {"wall:wall:microseconds:wall:microseconds", 1}, + {"wall:samples:count:wall:microseconds", nil, 0}, + {"wall:wall:microseconds:wall:microseconds", nil, 1}, }, }, { @@ -150,7 +132,7 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"goroutines:goroutine:count:goroutine:count", 0}, + {"goroutines:goroutine:count:goroutine:count", nil, 0}, }, }, { @@ -159,8 +141,8 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"block:delay:nanoseconds:contentions:count", 1}, - {"block:contentions:count:contentions:count", 0}, + {"block:delay:nanoseconds:contentions:count", nil, 1}, + {"block:contentions:count:contentions:count", nil, 0}, }, }, { @@ -169,8 +151,8 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"mutex:contentions:count:contentions:count", 0}, - {"mutex:delay:nanoseconds:contentions:count", 1}, + {"mutex:contentions:count:contentions:count", nil, 0}, + {"mutex:delay:nanoseconds:contentions:count", nil, 1}, }, }, { @@ -179,8 +161,8 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"memory:alloc_objects:count:space:bytes", 0}, - {"memory:alloc_space:bytes:space:bytes", 1}, + {"memory:alloc_objects:count:space:bytes", nil, 0}, + {"memory:alloc_space:bytes:space:bytes", nil, 1}, }, }, { @@ -190,7 +172,7 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"process_cpu:samples:count::milliseconds", 0}, + {"process_cpu:samples:count::milliseconds", nil, 0}, }, spyName: pprof2.SpyNameForFunctionNameRewrite(), }, @@ -203,7 +185,7 @@ var ( expectStatusPush: 400, expectedError: "function id is 0", metrics: []expectedMetric{ - {"process_cpu:cpu:nanoseconds::nanoseconds", 0}, + {"process_cpu:cpu:nanoseconds::nanoseconds", nil, 0}, }, needFunctionIDFix: true, spyName: "dotnetspy", @@ -219,9 +201,9 @@ var ( expectedError: "function id is 0", metrics: []expectedMetric{ // notice how they all use process_cpu metric - {"process_cpu:cpu:nanoseconds::nanoseconds", 0}, - {"process_cpu:alloc_samples:count::nanoseconds", 2}, // this was rewriten by ingest handler to replace - - {"process_cpu:alloc_size:bytes::nanoseconds", 3}, // this was rewriten by ingest handler to replace - + {"process_cpu:cpu:nanoseconds::nanoseconds", nil, 0}, + {"process_cpu:alloc_samples:count::nanoseconds", nil, 2}, // this was rewriten by ingest handler to replace - + {"process_cpu:alloc_size:bytes::nanoseconds", nil, 3}, // this was rewriten by ingest handler to replace - }, needFunctionIDFix: true, spyName: "dotnetspy", @@ -233,10 +215,10 @@ var ( expectStatusIngest: 200, expectStatusPush: 200, metrics: []expectedMetric{ - {"process_cpu:cpu:nanoseconds::nanoseconds", 0}, - {"process_cpu:alloc_samples:count::nanoseconds", 2}, - {"process_cpu:alloc_size:bytes::nanoseconds", 3}, - {"process_cpu:alloc_size:bytes::nanoseconds", 3}, + {"process_cpu:cpu:nanoseconds::nanoseconds", nil, 0}, + {"process_cpu:alloc_samples:count::nanoseconds", nil, 2}, + {"process_cpu:alloc_size:bytes::nanoseconds", nil, 3}, + {"process_cpu:alloc_size:bytes::nanoseconds", nil, 3}, }, spyName: "dotnetspy", }, @@ -246,7 +228,7 @@ var ( expectStatusPush: 400, expectStatusIngest: 422, metrics: []expectedMetric{ - {"process_cpu:cpu:nanoseconds::nanoseconds", 0}, + {"process_cpu:cpu:nanoseconds::nanoseconds", nil, 0}, }, }, } @@ -257,15 +239,18 @@ func TestIngest(t *testing.T) { p.Start(t) defer p.Stop(t) - for _, testdatum := range testdata { - t.Run(testdatum.profile, func(t *testing.T) { - - appName := ingest(t, &p, testdatum) - - if testdatum.expectStatusIngest == 200 { - for _, metric := range testdatum.metrics { - render(t, &p, metric, appName, testdatum) - selectMerge(t, &p, metric, appName, testdatum, true) + for _, td := range testdata { + t.Run(td.profile, func(t *testing.T) { + rb := p.NewRequestBuilder(t). + Spy(td.spyName) + req := rb.IngestPPROFRequest(td.profile, td.prevProfile, td.sampleTypeConfig) + p.Ingest(t, req, td.expectStatusIngest) + + if td.expectStatusIngest == 200 { + for _, metric := range td.metrics { + rb.Render(metric.name) + profile := rb.SelectMergeProfile(metric.name, metric.query) + assertPPROF(t, profile, metric, td, true) } } }) @@ -273,39 +258,34 @@ func TestIngest(t *testing.T) { } func TestPush(t *testing.T) { - p := PyroscopeTest{} + p := new(PyroscopeTest) p.Start(t) defer p.Stop(t) - for _, testdatum := range testdata { - if testdatum.prevProfile != "" { + for _, td := range testdata { + if td.prevProfile != "" { continue } - t.Run(testdatum.profile, func(t *testing.T) { + t.Run(td.profile, func(t *testing.T) { + rb := p.NewRequestBuilder(t) + + req := rb.PushPPROFRequest(td.profile, td.metrics[0].name) + rb.Push(req, td.expectStatusPush, td.expectedError) - appName := push(t, &p, testdatum) + if td.expectStatusPush == 200 { + for _, metric := range td.metrics { + rb.Render(metric.name) + profile := rb.SelectMergeProfile(metric.name, metric.query) - if testdatum.expectStatusPush == 200 { - for _, metric := range testdatum.metrics { - render(t, &p, metric, appName, testdatum) - selectMerge(t, &p, metric, appName, testdatum, false) + assertPPROF(t, profile, metric, td, false) } } }) } - //time.Sleep(10 * time.Hour) } -func selectMerge(t *testing.T, p *PyroscopeTest, metric expectedMetric, name string, testdatum pprofTestData, fixes bool) { - qc := p.queryClient() - resp, err := qc.SelectMergeProfile(context.Background(), connect.NewRequest(&querierv1.SelectMergeProfileRequest{ - ProfileTypeID: metric.name, - Start: time.Unix(0, 0).UnixMilli(), - End: time.Now().UnixMilli(), - LabelSelector: fmt.Sprintf("{service_name=\"%s\"}", name), - })) +func assertPPROF(t *testing.T, resp *connect.Response[profilev1.Profile], metric expectedMetric, testdatum pprofTestData, fixes bool) { - require.NoError(t, err) assert.Equal(t, 1, len(resp.Msg.SampleType)) profileBytes, err := os.ReadFile(testdatum.profile) @@ -332,22 +312,6 @@ func selectMerge(t *testing.T, p *PyroscopeTest, metric expectedMetric, name str require.Equal(t, expectedStacktraces, actualStacktraces) } -func render(t *testing.T, p *PyroscopeTest, metric expectedMetric, appName string, testdatum pprofTestData) { - queryURL := p.URL() + "/pyroscope/render?query=" + metric.name + "{service_name=\"" + appName + "\"}&from=946656000&until=now&format=collapsed" - fmt.Println(queryURL) - queryRes, err := http.Get(queryURL) - require.NoError(t, err) - body := bytes.NewBuffer(nil) - _, err = io.Copy(body, queryRes.Body) - assert.NoError(t, err) - fb := new(flamebearer.FlamebearerProfile) - err = json.Unmarshal(body.Bytes(), fb) - assert.NoError(t, err, testdatum.profile, body.String(), queryURL) - assert.Greater(t, len(fb.Flamebearer.Names), 1, testdatum.profile, body.String(), queryRes) - assert.Greater(t, fb.Flamebearer.NumTicks, 1, testdatum.profile, body.String(), queryRes) - // todo check actual stacktrace contents -} - type pprofTestData struct { profile string prevProfile string @@ -362,132 +326,6 @@ type pprofTestData struct { type expectedMetric struct { name string + query map[string]string valueIDX int } - -func push(t *testing.T, p *PyroscopeTest, testdatum pprofTestData) string { - appName := createAppname(testdatum) - cl := p.pushClient() - - rawProfile, err := os.ReadFile(testdatum.profile) - require.NoError(t, err) - - rawProfile = updateTimestamp(t, rawProfile) - - metricName := strings.Split(testdatum.metrics[0].name, ":")[0] - - _, err = cl.Push(context.TODO(), connect.NewRequest(&pushv1.PushRequest{ - Series: []*pushv1.RawProfileSeries{{ - Labels: []*typesv1.LabelPair{ - {Name: "__name__", Value: metricName}, - {Name: "__delta__", Value: "false"}, - {Name: "service_name", Value: appName}, - }, - Samples: []*pushv1.RawSample{{RawProfile: rawProfile}}, - }}, - })) - if testdatum.expectStatusPush == 200 { - assert.NoError(t, err) - } else { - assert.Error(t, err) - var connectErr *connect.Error - if ok := errors.As(err, &connectErr); ok { - toHTTP := connectgrpc.CodeToHTTP(connectErr.Code()) - assert.Equal(t, testdatum.expectStatusPush, int(toHTTP)) - if testdatum.expectedError != "" { - assert.Contains(t, connectErr.Error(), testdatum.expectedError) - } - } else { - assert.Fail(t, "unexpected error type", err) - } - } - - return appName -} - -func updateTimestamp(t *testing.T, rawProfile []byte) []byte { - expectedProfile, err := pprof.RawFromBytes(rawProfile) - require.NoError(t, err) - expectedProfile.Profile.TimeNanos = time.Now().Add(-time.Minute).UnixNano() - buf := bytes.NewBuffer(nil) - _, err = expectedProfile.WriteTo(buf) - require.NoError(t, err) - rawProfile = buf.Bytes() - return rawProfile -} - -func ingest(t *testing.T, p *PyroscopeTest, testdatum pprofTestData) string { - var ( - profile, prevProfile, sampleTypeConfig []byte - err error - ) - profile, err = os.ReadFile(testdatum.profile) - assert.NoError(t, err) - if testdatum.prevProfile != "" { - prevProfile, err = os.ReadFile(testdatum.prevProfile) - assert.NoError(t, err) - } - if testdatum.sampleTypeConfig != "" { - sampleTypeConfig, err = os.ReadFile(testdatum.sampleTypeConfig) - assert.NoError(t, err) - } - bs, ct := createPProfRequest(t, profile, prevProfile, sampleTypeConfig) - - spyName := "foo239" - if testdatum.spyName != "" { - spyName = testdatum.spyName - } - - appName := createAppname(testdatum) - url := p.URL() + "/ingest?name=" + appName + "&spyName=" + spyName - req, err := http.NewRequest("POST", url, bytes.NewReader(bs)) - require.NoError(t, err) - req.Header.Set("Content-Type", ct) - - res, err := http.DefaultClient.Do(req) - require.NoError(t, err) - require.Equal(t, testdatum.expectStatusIngest, res.StatusCode, testdatum.profile) - fmt.Printf("%+v %+v\n", testdatum, res) - return appName -} - -func createAppname(testdatum pprofTestData) string { - return fmt.Sprintf("pprof.integration.%s.%d", - strings.ReplaceAll(filepath.Base(testdatum.profile), "-", "_"), - rand.Uint64()) -} - -func createPProfRequest(t *testing.T, profile, prevProfile, sampleTypeConfig []byte) ([]byte, string) { - const ( - formFieldProfile = "profile" - formFieldPreviousProfile = "prev_profile" - formFieldSampleTypeConfig = "sample_type_config" - ) - - var b bytes.Buffer - w := multipart.NewWriter(&b) - - profileW, err := w.CreateFormFile(formFieldProfile, "not used") - require.NoError(t, err) - _, err = profileW.Write(profile) - require.NoError(t, err) - - if sampleTypeConfig != nil { - - sampleTypeConfigW, err := w.CreateFormFile(formFieldSampleTypeConfig, "not used") - require.NoError(t, err) - _, err = sampleTypeConfigW.Write(sampleTypeConfig) - require.NoError(t, err) - } - - if prevProfile != nil { - prevProfileW, err := w.CreateFormFile(formFieldPreviousProfile, "not used") - require.NoError(t, err) - _, err = prevProfileW.Write(prevProfile) - require.NoError(t, err) - } - err = w.Close() - require.NoError(t, err) - - return b.Bytes(), w.FormDataContentType() -}