Merge pull request #352 from bobrik/ivan/usdt-tracing

Switch from uprobes to usdt for tracing
cloudflare · Feb 27, 2024 · 9be354d · 9be354d
2 parents 85ad56e + 717a627
commit 9be354d
Show file tree

Hide file tree

Showing 8 changed files with 60 additions and 61 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -268,6 +268,9 @@ jobs:
         with:
           go-version: ^1.22
 
+      - name: Install systemtap-sdt-dev
+        run: sudo apt-get install -y systemtap-sdt-dev
+
       - name: Download libbpf.tar.gz
         uses: actions/download-artifact@v4
         with:
@@ -300,6 +303,9 @@ jobs:
         with:
           go-version: ^1.22
 
+      - name: Install systemtap-sdt-dev
+        run: sudo apt-get install -y systemtap-sdt-dev
+
       - name: Install libelf-dev
         run: sudo apt-get install -y libelf-dev
 
@@ -325,6 +331,9 @@ jobs:
           go-version: ^1.22
           cache: false # https://github.com/golangci/golangci-lint-action/issues/807
 
+      - name: Install systemtap-sdt-dev
+        run: sudo apt-get install -y systemtap-sdt-dev
+
       - name: Install libelf-dev
         run: sudo apt-get install -y libelf-dev
 
@@ -396,6 +405,9 @@ jobs:
         with:
           go-version: ^1.22
 
+      - name: Install systemtap-sdt-dev
+        run: sudo apt-get install -y systemtap-sdt-dev
+
       - name: Build
         run: make -j $(nproc) tracing-demos
 

diff --git a/examples/cfs-throttling-trace.bpf.c b/examples/cfs-throttling-trace.bpf.c
@@ -1,6 +1,7 @@
 #include <vmlinux.h>
 #include <bpf/bpf_tracing.h>
 #include <bpf/bpf_core_read.h>
+#include <bpf/usdt.bpf.h>
 #include "tracing.bpf.h"
 
 // Skipping 3 frames off the top as they are just bpf trampoline
@@ -25,22 +26,19 @@ struct {
     __type(value, struct span_parent_t);
 } traced_cgroups SEC(".maps");
 
-SEC("uprobe/./tracing/demos/cfs-throttling/demo:cfs_set_parent_span")
-int sock_set_parent_span(struct pt_regs *ctx)
+SEC("usdt/./tracing/demos/cfs-throttling/demo:ebpf_exporter:cfs_set_parent_span")
+int BPF_USDT(cfs_set_parent_span, u64 trace_id_hi, u64 trace_id_lo, u64 span_id)
 {
     u32 cgroup = bpf_get_current_cgroup_id();
-    u64 trace_id_hi = PT_REGS_PARM1(ctx);
-    u64 trace_id_lo = PT_REGS_PARM2(ctx);
-    u64 span_id = PT_REGS_PARM3(ctx);
     struct span_parent_t parent = { .trace_id_hi = trace_id_hi, .trace_id_lo = trace_id_lo, .span_id = span_id };
 
     bpf_map_update_elem(&traced_cgroups, &cgroup, &parent, BPF_ANY);
 
     return 0;
 }
 
-SEC("uprobe/./tracing/demos/cfs-throttling/demo:cfs_clear_parent_span")
-int sched_clear_parent_span(struct pt_regs *ctx)
+SEC("usdt/./tracing/demos/cfs-throttling/demo:ebpf_exporter:cfs_clear_parent_span")
+int BPF_USDT(cfs_clear_parent_span)
 {
     u32 cgroup = bpf_get_current_cgroup_id();
 

diff --git a/examples/sched-trace.bpf.c b/examples/sched-trace.bpf.c
@@ -1,6 +1,7 @@
 #include <vmlinux.h>
 #include <bpf/bpf_tracing.h>
 #include <bpf/bpf_core_read.h>
+#include <bpf/usdt.bpf.h>
 #include "tracing.bpf.h"
 
 // https://github.com/torvalds/linux/blob/v6.1/include/linux/sched.h#L84
@@ -68,22 +69,19 @@ static int task_state(unsigned int state)
     return STATE_TASK_PROBABLY_RUNNING;
 }
 
-SEC("uprobe/./tracing/demos/sched/demo:sched_set_parent_span")
-int sock_set_parent_span(struct pt_regs *ctx)
+SEC("usdt/./tracing/demos/sched/demo:ebpf_exporter:sched_set_parent_span")
+int BPF_USDT(sock_set_parent_span, u64 trace_id_hi, u64 trace_id_lo, u64 span_id)
 {
     u32 tgid = bpf_get_current_pid_tgid() >> 32;
-    u64 trace_id_hi = PT_REGS_PARM1(ctx);
-    u64 trace_id_lo = PT_REGS_PARM2(ctx);
-    u64 span_id = PT_REGS_PARM3(ctx);
     struct span_parent_t parent = { .trace_id_hi = trace_id_hi, .trace_id_lo = trace_id_lo, .span_id = span_id };
 
     bpf_map_update_elem(&traced_tgids, &tgid, &parent, BPF_ANY);
 
     return 0;
 }
 
-SEC("uprobe/./tracing/demos/sched/demo:sched_clear_parent_span")
-int sched_clear_parent_span(struct pt_regs *ctx)
+SEC("usdt/./tracing/demos/sched/demo:ebpf_exporter:sched_clear_parent_span")
+int BPF_USDT(sched_clear_parent_span)
 {
     u32 tgid = bpf_get_current_pid_tgid() >> 32;
 

diff --git a/examples/sock-trace.bpf.c b/examples/sock-trace.bpf.c
@@ -1,6 +1,7 @@
 #include <vmlinux.h>
 #include <bpf/bpf_tracing.h>
 #include <bpf/bpf_core_read.h>
+#include <bpf/usdt.bpf.h>
 #include "tracing.bpf.h"
 
 u32 yes = true;
@@ -92,8 +93,8 @@ int BPF_PROG(close_fd, unsigned int fd)
     return 0;
 }
 
-SEC("uprobe/./tracing/demos/sock/demo:enable_kernel_tracing")
-int enable_tracing()
+SEC("usdt/./tracing/demos/sock/demo:ebpf_exporter:enable_kernel_tracing")
+int BPF_USDT(enable_kernel_tracing)
 {
     u32 tgid = bpf_get_current_pid_tgid() >> 32;
 
@@ -116,14 +117,10 @@ int BPF_PROG(sched_process_exit, struct task_struct *p)
     return 0;
 }
 
-SEC("uprobe/./tracing/demos/sock/demo:sock_set_parent_span")
-int sock_set_parent_span(struct pt_regs *ctx)
+SEC("usdt/./tracing/demos/sock/demo:ebpf_exporter:sock_set_parent_span")
+int BPF_USDT(sock_set_parent_span, int fd, u64 trace_id_hi, u64 trace_id_lo, u64 span_id)
 {
     u32 tgid = bpf_get_current_pid_tgid() >> 32;
-    int fd = PT_REGS_PARM1(ctx);
-    u64 trace_id_hi = PT_REGS_PARM2(ctx);
-    u64 trace_id_lo = PT_REGS_PARM3(ctx);
-    u64 span_id = PT_REGS_PARM4(ctx);
     struct span_parent_t parent = { .trace_id_hi = trace_id_hi, .trace_id_lo = trace_id_lo, .span_id = span_id };
     struct file_key_t key = { .tgid = tgid, .fd = fd };
     struct sock **sk = bpf_map_lookup_elem(&fd_to_sock, &key);

diff --git a/tracing/demos/cfs-throttling/stitch.go b/tracing/demos/cfs-throttling/stitch.go
@@ -2,18 +2,17 @@ package main
 
 /*
 #include <stdint.h>
+#include <sys/sdt.h>
 
-void
-#if defined(__clang__)
-__attribute__ ((optnone))
-#endif
-cfs_set_parent_span(uint64_t trace_id_hi, uint64_t trace_id_lo, uint64_t span_id) { }
+void cfs_set_parent_span(uint64_t trace_id_hi, uint64_t trace_id_lo, uint64_t span_id)
+{
+	DTRACE_PROBE3(ebpf_exporter, cfs_set_parent_span, trace_id_hi, trace_id_lo, span_id);
+}
 
-void
-#if defined(__clang__)
-__attribute__ ((optnone))
-#endif
-cfs_clear_parent_span() { }
+void cfs_clear_parent_span()
+{
+	DTRACE_PROBE(ebpf_exporter, cfs_clear_parent_span);
+}
 */
 import "C"
 import (

diff --git a/tracing/demos/sched/stitch.go b/tracing/demos/sched/stitch.go
@@ -2,18 +2,17 @@ package main
 
 /*
 #include <stdint.h>
+#include <sys/sdt.h>
 
-void
-#if defined(__clang__)
-__attribute__ ((optnone))
-#endif
-sched_set_parent_span(uint64_t trace_id_hi, uint64_t trace_id_lo, uint64_t span_id) { }
+void sched_set_parent_span(uint64_t trace_id_hi, uint64_t trace_id_lo, uint64_t span_id)
+{
+	DTRACE_PROBE3(ebpf_exporter, sched_set_parent_span, trace_id_hi, trace_id_lo, span_id);
+}
 
-void
-#if defined(__clang__)
-__attribute__ ((optnone))
-#endif
-sched_clear_parent_span() { }
+void sched_clear_parent_span()
+{
+	DTRACE_PROBE(ebpf_exporter, sched_clear_parent_span);
+}
 */
 import "C"
 import (

diff --git a/tracing/demos/sock/enable.go b/tracing/demos/sock/enable.go
diff --git a/tracing/demos/sock/stitch.go b/tracing/demos/sock/stitch.go
@@ -2,18 +2,28 @@ package main
 
 /*
 #include <stdint.h>
-void
-#if defined(__clang__)
-__attribute__ ((optnone))
-#endif
-sock_set_parent_span(int sock_fd, uint64_t trace_id_hi, uint64_t trace_id_lo, uint64_t span_id) { }
+#include <sys/sdt.h>
+
+void enable_kernel_tracing()
+{
+	DTRACE_PROBE(ebpf_exporter, enable_kernel_tracing);
+}
+
+void sock_set_parent_span(int sock_fd, uint64_t trace_id_hi, uint64_t trace_id_lo, uint64_t span_id)
+{
+	DTRACE_PROBE4(ebpf_exporter, sock_set_parent_span, sock_fd, trace_id_hi, trace_id_lo, span_id);
+}
 */
 import "C"
 import (
 	"github.com/cloudflare/ebpf_exporter/v2/util"
 	"go.opentelemetry.io/otel/trace"
 )
 
+func enableKernelTracing() {
+	C.enable_kernel_tracing()
+}
+
 func sockSentParentSpan(fd uintptr, span trace.Span) {
 	byteOrder := util.GetHostByteOrder()