Merge branch 'main' into add-alloy-helm-tutorial

.drone/drone.jsonnet

@@ -77,7 +77,6 @@ local beyla() = pipeline('beyla') {
   ],
 };
 
-// TODO: don't create images if unit tests nor integration tests pass
 [
   beyla(),
 ] + [

.github/configs/cr.yml

@@ -0,0 +1,3 @@
+owner: grafana
+git-repo: helm-charts
+skip-existing: true

.github/configs/ct.yml

@@ -0,0 +1,14 @@
+## Reference: https://github.com/helm/chart-testing/blob/master/doc/ct_lint-and-install.md
+remote: origin
+target-branch: main
+chart-dirs:
+  - charts
+chart-repos:
+  - grafana=https://grafana.github.io/helm-charts
+validate-chart-schema: true
+validate-maintainers: true
+validate-yaml: true
+exclude-deprecated: true
+excluded-charts: []
+# TODO: disable after publicly releasing 1.0.0
+check-version-increment: false

.github/workflows/check_ebpf_integrity.yml

@@ -0,0 +1,17 @@
+name: Check eBPF code integrity
+
+on:
+  push:
+    branches: [ 'main', 'release-*' ]
+  pull_request:
+    branches: [ 'main', 'release-*' ]
+
+
+jobs:
+  test:
+    name: "Check eBPF code integrity"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: "Check eBPF code integrity"
+        run: make check-ebpf-integrity

.github/workflows/generator-image.yml

@@ -0,0 +1,48 @@
+name: Publish Beyla Docker Generator Image
+on:
+  workflow_dispatch:
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: grafana/beyla-generator
+
+jobs:
+  build-and-push-image:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3.1.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5.5.1
+        with:
+          images: "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}"
+
+      - name: Get current timestamp
+        id: timestamp
+        run: echo "::set-output name=ts::$(date +'%Y%m%d%H%M')"
+      - name: Build and push Docker image
+        id: push
+        uses: docker/build-push-action@v5.3.0
+        with:
+          context: .
+          file: ./generator.Dockerfile
+          push: true
+          platforms: linux/amd64,linux/arm64
+          labels: ${{ steps.meta.outputs.labels }}
+          tags: |
+            "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.timestamp.outputs.ts }}"
+            "${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:main"

.github/workflows/helm-release.yml

@@ -0,0 +1,16 @@
+name: Release Helm chart
+on:
+  workflow_dispatch:
+
+jobs:
+  release-beyla-helm-chart:
+    uses: grafana/helm-charts/.github/workflows/update-helm-repo.yaml@main
+    with:
+      charts_dir: charts/beyla
+      cr_configfile: .github/configs/cr.yml
+      ct_configfile: .github/configs/ct.yml
+    secrets:
+      # values are created in https://github.com/organizations/grafana/settings/apps/grafana-beyla-release
+      # and copied to a repository secret
+      github_app_id: ${{ secrets.BEYLA_RELEASE_APP_ID }}
+      github_app_pem: ${{ secrets.BEYLA_RELEASE_APP_PEM }}

.github/workflows/helm-test.yml

@@ -0,0 +1,54 @@
+name: Test
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+
+  workflow_call:
+    inputs:
+      filter_regex_include:
+        description: this sets the scope of the super linter
+        default: "charts/beyla/"
+        required: false
+        type: string
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+env:
+  CT_CONFIGFILE: "${{ github.workspace }}/.github/configs/ct.yml"
+
+jobs:
+  lint-helm-chart:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # required for chart-testing to work
+
+      - name: Regenerate docs
+        run: |
+          docker run --rm \
+            -v "$(pwd)/deployments/helm:/helm-docs" \
+            -u "$(id -u)" \
+            jnorwood/helm-docs
+          if ! git diff --exit-code; then
+            echo "Helm chart documentation is not up to date. Please run 'helm-docs' and commit changes!" >&2
+            exit 1
+          fi
+
+      - name: Set up Linting with chart-testing
+        uses: helm/chart-testing-action@v2.1.0
+
+      - name: Run chart-testing (list-changed)
+        id: list-changed
+        run: |
+          changed=$(ct list-changed --config "${CT_CONFIGFILE}")
+          if [[ -n "$changed" ]]; then
+            echo "::set-output name=changed::true"
+          fi
+
+      - name: Linting with chart-testing
+        run: ct lint --config "${CT_CONFIGFILE}"

.github/workflows/release-binaries.yml

@@ -18,6 +18,8 @@ jobs:
         uses: actions/setup-go@v3
         with:
           go-version: ${{ matrix.go }}
+      - name: Check eBPF integrity
+        run: make check-ebpf-integrity
       - name: Build Go release for linux/${{ matrix.arch }}
         run: make artifact
         env:

Makefile

@@ -18,10 +18,9 @@ IMG_NAME ?= beyla
 VERSION ?= dev
 IMG = $(IMG_REGISTRY)/$(IMG_ORG)/$(IMG_NAME):$(VERSION)
 
-# The generator is a local container image that provides a reproducible environment for
+# The generator is a container image that provides a reproducible environment for
 # building eBPF binaries
-GEN_IMG_NAME ?= ebpf-generator
-GEN_IMG ?= $(GEN_IMG_NAME):$(VERSION)
+GEN_IMG ?= ghcr.io/grafana/beyla-generator:main
 
 COMPOSE_ARGS ?= -f test/integration/docker-compose.yml
 
@@ -33,7 +32,7 @@ CLANG ?= clang
 CFLAGS := -O2 -g -Wall -Werror $(CFLAGS)
 
 # regular expressions for excluded file patterns
-EXCLUDE_COVERAGE_FILES="(bpfel_)|(/pingserver/)|(/test/collector/)|(integration/components)|(test/cmd)"
+EXCLUDE_COVERAGE_FILES="(bpfel_)|(/pingserver/)|(/test/collector/)|(integration/components)|(test/cmd)|(/grafana/beyla/docs/)|(/grafana/beyla/configs/)|(/grafana/beyla/examples/)"
 
 .DEFAULT_GOAL := all
 
@@ -289,7 +288,11 @@ artifact: compile
 	cp third_party_licenses.csv ./bin
 	tar -C ./bin -cvzf bin/beyla.tar.gz beyla LICENSE NOTICE third_party_licenses.csv
 
-.PHONE: clean-testoutput
+.PHONY: clean-testoutput
 clean-testoutput:
 	@echo "### Cleaning ${TEST_OUTPUT} folder"
 	rm -rf ${TEST_OUTPUT}/*
+
+.PHONY: check-ebpf-integrity
+check-ebpf-integrity: docker-generate
+	git diff --name-status --exit-code || (echo "Run make docker-generate locally and commit the code changes" && false)

README.md

@@ -6,8 +6,6 @@ Open source zero-code automatic instrumentation with eBPF and OpenTelemetry.
 
 [![Build Status](https://drone.grafana.net/api/badges/grafana/beyla/status.svg?ref=refs/heads/main)](https://drone.grafana.net/grafana/beyla)
 
-:green_circle: **We are hiring!** :green_circle: If you want to become a Beyla engineer, find our job post [here](https://boards.greenhouse.io/grafanalabs/jobs/5019971004).
-
 ## Introduction
 
 Beyla is a vendor agnostic, eBPF-based, OpenTelemetry/Prometheus application auto-instrumentation tool, which lets you easily get started with Application Observability. 

bpf/go_traceparent.h

@@ -136,7 +136,16 @@ static __always_inline void *extract_traceparent_from_req_headers(void *headers_
         }
         for (u64 i = 0; i < 8; i++)
         {
+            // break the bucket iteration when tophash is zero
+            // since "there are no more non-empty cells at higher indexes or overflows"
+            // ref: https://github.com/golang/go/blob/9050ce9b334419066c364e747499a2faf4425dad/src/runtime/map.go#L86
             if (map_value->tophash[i] == 0)
+            {
+                break;
+            }
+            // skip the cell if tophash is empty
+            // ref: https://github.com/golang/go/blob/9050ce9b334419066c364e747499a2faf4425dad/src/runtime/map.go#L87
+            if (map_value->tophash[i] == 1)
             {
                 continue;
             }

bpf/http_sock.c

@@ -304,6 +304,10 @@ int BPF_KPROBE(kprobe_tcp_sendmsg, struct sock *sk, struct msghdr *msg, size_t s
             return 0;
         }
         bpf_dbg_printk("=== kprobe SSL tcp_sendmsg=%d sock=%llx ssl=%llx ===", id, sk, ssl);
+        pid_connection_info_t *conn = bpf_map_lookup_elem(&ssl_to_conn, &ssl);
+        if (conn) {
+            finish_possible_delayed_http_request(conn);
+        }
         bpf_map_update_elem(&ssl_to_conn, &ssl, &s_args.p_conn, BPF_ANY);
     }
 

bpf/http_sock.h

@@ -78,6 +78,13 @@ struct {
     __uint(max_entries, 1);
 } http2_info_mem SEC(".maps");
 
+struct {
+    __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
+    __type(key, int);
+    __type(value, http_connection_metadata_t);
+    __uint(max_entries, 1);
+} connection_meta_mem SEC(".maps");
+
 static __always_inline u8 is_http(unsigned char *p, u32 len, u8 *packet_type) {
     if (len < MIN_HTTP_SIZE) {
         return 0;
@@ -207,6 +214,11 @@ static __always_inline http2_grpc_request_t* empty_http2_info() {
     return value;
 }
 
+static __always_inline http_connection_metadata_t* empty_connection_meta() {
+    int zero = 0;
+    return bpf_map_lookup_elem(&connection_meta_mem, &zero);
+}
+
 static __always_inline void finish_http(http_info_t *info) {
     if (info->start_monotime_ns != 0 && info->status != 0 && info->pid.host_pid != 0) {
         http_info_t *trace = bpf_ringbuf_reserve(&events, sizeof(http_info_t), 0);        
@@ -292,51 +304,64 @@ static __always_inline void process_http_response(http_info_t *info, unsigned ch
     info->status += (buf[RESPONSE_STATUS_POS + 2] - '0');
 }
 
-static __always_inline void handle_http_response(unsigned char *small_buf, pid_connection_info_t *pid_conn, http_info_t *info, int orig_len, u8 direction, u8 ssl) {
+static __always_inline http_connection_metadata_t *connection_meta(pid_connection_info_t *pid_conn, u8 direction, u8 packet_type) {
     http_connection_metadata_t *meta = bpf_map_lookup_elem(&filtered_connections, pid_conn);
-    http_connection_metadata_t dummy_meta = {};
+    if (meta) {
+        return meta;
+    }
 
+    meta = empty_connection_meta();
     if (!meta) {
-        // In case we can't find metadata stored by accept4 or sys_connect, we guess the
-        // HTTP type. If we are receiving a response, we should be a client, otherwise 
-        // guess a server.
+        return 0;
+    }
+
+    if (packet_type == PACKET_TYPE_RESPONSE) {
+        if (direction == TCP_RECV) {
+            meta->type = EVENT_HTTP_CLIENT;
+        } else {
+            meta->type = EVENT_HTTP_REQUEST;
+        }
+    } else {
         if (direction == TCP_RECV) {
-            dummy_meta.type = EVENT_HTTP_CLIENT;
+            meta->type = EVENT_HTTP_REQUEST;
         } else {
-            dummy_meta.type = EVENT_HTTP_REQUEST;
+            meta->type = EVENT_HTTP_CLIENT;
         }
+    }
+
+    task_pid(&meta->pid);
+
+    return meta;
+}
 
-        task_pid(&dummy_meta.pid);
-        meta = &dummy_meta;
+static __always_inline void handle_http_response(unsigned char *small_buf, pid_connection_info_t *pid_conn, http_info_t *info, int orig_len, u8 direction, u8 ssl) {
+    http_connection_metadata_t *meta = connection_meta(pid_conn, direction, PACKET_TYPE_RESPONSE);
+    if (!meta) {
+        bpf_dbg_printk("Can't get meta memory or connection not found");
+        return;
     }
 
     process_http_response(info, small_buf, meta, orig_len);
 
-    if ((direction != TCP_SEND) || (ssl != NO_SSL) /*|| (orig_len < KPROBES_LARGE_RESPONSE_LEN)*/) {
+    if ((direction != TCP_SEND) /*|| (ssl != NO_SSL) || (orig_len < KPROBES_LARGE_RESPONSE_LEN)*/) {
         finish_http(info);
     } else {
-        bpf_dbg_printk("Delaying finish http for large request, orig_len %d", orig_len);
+        if (ssl && (pid_conn->conn.s_port == 0) && (pid_conn->conn.d_port == 0)) {
+            bpf_dbg_printk("Fake connection info, finishing request");
+            finish_http(info);
+        } else {
+            bpf_dbg_printk("Delaying finish http for large request, orig_len %d", orig_len);
+        }
     }
 }
 
 static __always_inline void http2_grpc_start(http2_conn_stream_t *s_key, void *u_buf, int len, u8 direction) {
     http2_grpc_request_t *h2g_info = empty_http2_info();
     if (h2g_info) {
-        http_connection_metadata_t *meta = bpf_map_lookup_elem(&filtered_connections, &s_key->pid_conn);
-        http_connection_metadata_t dummy_meta = {};
-
+        http_connection_metadata_t *meta = connection_meta(&s_key->pid_conn, direction, PACKET_TYPE_REQUEST);
         if (!meta) {
-            // In case we can't find metadata stored by accept4 or sys_connect, we guess the
-            // HTTP type. If we are making a request, we should be a client, otherwise 
-            // guess a server.
-            if (direction == TCP_SEND) {
-                dummy_meta.type = EVENT_HTTP_CLIENT;
-            } else {
-                dummy_meta.type = EVENT_HTTP_REQUEST;
-            }
-
-            task_pid(&dummy_meta.pid);
-            meta = &dummy_meta;
+            bpf_dbg_printk("Can't get meta memory or connection not found");
+            return;
         }
 
         h2g_info->flags = EVENT_K_HTTP2_REQUEST;
@@ -478,7 +503,8 @@ static __always_inline void handle_buf_with_connection(pid_connection_info_t *pi
         bpf_dbg_printk("=== http_buffer_event len=%d pid=%d still_reading=%d ===", bytes_len, pid_from_pid_tgid(bpf_get_current_pid_tgid()), still_reading(info));
 
         if (packet_type == PACKET_TYPE_REQUEST && (info->status == 0)) {    
-            http_connection_metadata_t *meta = bpf_map_lookup_elem(&filtered_connections, pid_conn);
+            http_connection_metadata_t *meta = connection_meta(pid_conn, direction, PACKET_TYPE_REQUEST);
+
             get_or_create_trace_info(meta, pid_conn->pid, &pid_conn->conn, u_buf, bytes_len, capture_header_buffer);
 
             if (meta) {            
@@ -518,8 +544,7 @@ static __always_inline void handle_buf_with_connection(pid_connection_info_t *pi
     } else if (is_http2_or_grpc(small_buf, MIN_HTTP2_SIZE)) {
         bpf_dbg_printk("Found HTTP2 or gRPC connection");
         u8 is_ssl = ssl;
-        bpf_map_update_elem(&ongoing_http2_connections, pid_conn, &is_ssl, BPF_ANY);
-        bpf_map_delete_elem(&ongoing_http2_grpc, pid_conn);
+        bpf_map_update_elem(&ongoing_http2_connections, pid_conn, &is_ssl, BPF_ANY);        
     } else {
         u8 *h2g = bpf_map_lookup_elem(&ongoing_http2_connections, pid_conn);
         if (h2g && *h2g == ssl) {