Add tests for recycling services (#787)

test/integration/components/kube/kind.go

@@ -228,6 +228,29 @@ func deployManifest(cfg *envconf.Config, manifest string) error {
 	})
 }
 
+func deleteManifestFile(
+	manifestFile string,
+	cfg *envconf.Config,
+) error {
+	log := log()
+	log.With("file", manifestFile).Info("deleting manifest file")
+
+	b, err := os.ReadFile(manifestFile)
+	if err != nil {
+		return fmt.Errorf("reading manifest file %q: %w", manifestFile, err)
+	}
+
+	return deleteManifest(cfg, string(b))
+}
+
+func DeleteExistingManifestFile(cfg *envconf.Config, manifest string) error {
+	return deleteManifestFile(manifest, cfg)
+}
+
+func DeployManifestFile(cfg *envconf.Config, manifest string) error {
+	return deployManifestFile(manifest, cfg)
+}
+
 func deleteManifest(cfg *envconf.Config, manifest string) error {
 	return applyManifest(cfg, manifest, func(dri dynamic.ResourceInterface, obj *unstructured.Unstructured) error {
 		if err := dri.Delete(context.Background(), obj.GetName(), metav1.DeleteOptions{}); err != nil {

test/integration/k8s/common/k8s_common.go

@@ -19,6 +19,7 @@ var (
 	PingerManifest               = path.Join(PathManifests, "/06-instrumented-client.template.yml")
 	GrpcPingerManifest           = path.Join(PathManifests, "/06-instrumented-grpc-client.template.yml")
 	UninstrumentedPingerManifest = path.Join(PathManifests, "/06-uninstrumented-client.template.yml")
+	UninstrumentedAppManifest    = path.Join(PathManifests, "/05-uninstrumented-service.yml")
 )
 
 // Pinger stores the configuration data of a local pod that will be used to

test/integration/k8s/daemonset/k8s_daemonset_main_test.go

@@ -54,3 +54,5 @@ func TestMain(m *testing.M) {
 
 	cluster.Run(m)
 }
+
+type UninstrumentedApp struct{}

test/integration/k8s/daemonset/k8s_daemonset_traces_test.go

@@ -16,6 +16,7 @@ import (
 	"sigs.k8s.io/e2e-framework/pkg/features"
 
 	"github.com/grafana/beyla/test/integration/components/jaeger"
+	"github.com/grafana/beyla/test/integration/components/kube"
 	k8s "github.com/grafana/beyla/test/integration/k8s/common"
 )
 
@@ -25,7 +26,8 @@ import (
 func TestBasicTracing(t *testing.T) {
 	feat := features.New("Beyla is able to instrument an arbitrary process").
 		Assess("it sends traces for that service",
-			func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
+			func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
+				var podID string
 				test.Eventually(t, testTimeout, func(t require.TestingT) {
 					// Invoking both service instances, but we will expect that only one
 					// is instrumented, according to the discovery mechanisms
@@ -73,6 +75,12 @@ func TestBasicTracing(t *testing.T) {
 					}, trace.Processes[parent.ProcessID].Tags)
 					require.Empty(t, sd)
 
+					// Extract the pod id, so we can later check on restart of the pod that we have a different id
+					tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid")
+					assert.True(t, found)
+
+					podID = tag.Value.(string)
+					assert.NotEqual(t, "", podID)
 				}, test.Interval(100*time.Millisecond))
 
 				// Check that the "testserver" service is never instrumented
@@ -82,6 +90,70 @@ func TestBasicTracing(t *testing.T) {
 				var tq jaeger.TracesQuery
 				require.NoError(t, json.NewDecoder(resp.Body).Decode(&tq))
 				assert.Empty(t, tq.Data)
+
+				// Let's take down our services, keeping Beyla alive and then redeploy them
+				err = kube.DeleteExistingManifestFile(cfg, k8s.PathManifests+"/05-uninstrumented-service.yml")
+				assert.NoError(t, err, "we should see no error when deleting the uninstrumented service manifest file")
+
+				err = kube.DeployManifestFile(cfg, k8s.PathManifests+"/05-uninstrumented-service.yml")
+				assert.NoError(t, err, "we should see no error when re-deploying the uninstrumented service manifest file")
+
+				// We now use a different API, this ensures that after undeploying and redeploying the application we
+				// can still monitor its data
+				test.Eventually(t, testTimeout, func(t require.TestingT) {
+					// Invoking both service instances, but we will expect that only one
+					// is instrumented, according to the discovery mechanisms
+					resp, err := http.Get("http://localhost:38080/pingpongtoo")
+					require.NoError(t, err)
+					require.Equal(t, http.StatusOK, resp.StatusCode)
+
+					resp, err = http.Get("http://localhost:38081/pingpongtoo")
+					require.NoError(t, err)
+					require.Equal(t, http.StatusOK, resp.StatusCode)
+
+					resp, err = http.Get(jaegerQueryURL + "?service=otherinstance")
+					require.NoError(t, err)
+					if resp == nil {
+						return
+					}
+					require.Equal(t, http.StatusOK, resp.StatusCode)
+					var tq jaeger.TracesQuery
+					require.NoError(t, json.NewDecoder(resp.Body).Decode(&tq))
+					traces := tq.FindBySpan(jaeger.Tag{Key: "url.path", Type: "string", Value: "/pingpongtoo"})
+					require.NotEmpty(t, traces)
+					trace := traces[0]
+					require.NotEmpty(t, trace.Spans)
+
+					// Check that the service.namespace is set from the K8s namespace
+					assert.Len(t, trace.Processes, 1)
+					for _, proc := range trace.Processes {
+						sd := jaeger.DiffAsRegexp([]jaeger.Tag{
+							{Key: "service.namespace", Type: "string", Value: "^default$"},
+						}, proc.Tags)
+						require.Empty(t, sd)
+					}
+
+					// Check the information of the parent span
+					res := trace.FindByOperationName("GET /pingpongtoo")
+					require.Len(t, res, 1)
+					parent := res[0]
+					sd := jaeger.DiffAsRegexp([]jaeger.Tag{
+						{Key: "k8s.pod.name", Type: "string", Value: "^otherinstance-.*"},
+						{Key: "k8s.node.name", Type: "string", Value: ".+-control-plane$"},
+						{Key: "k8s.pod.uid", Type: "string", Value: k8s.UUIDRegex},
+						{Key: "k8s.pod.start_time", Type: "string", Value: k8s.TimeRegex},
+						{Key: "k8s.deployment.name", Type: "string", Value: "^otherinstance"},
+						{Key: "k8s.namespace.name", Type: "string", Value: "^default$"},
+					}, trace.Processes[parent.ProcessID].Tags)
+					require.Empty(t, sd)
+
+					// ensure the pod really restarted
+					tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid")
+					assert.True(t, found)
+
+					assert.NotEqual(t, podID, tag.Value.(string))
+				}, test.Interval(100*time.Millisecond))
+
 				return ctx
 			},
 		).Feature()

test/integration/k8s/daemonset_python/k8s_daemonset_traces_test.go

@@ -10,11 +10,13 @@ import (
 	"time"
 
 	"github.com/mariomac/guara/pkg/test"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"sigs.k8s.io/e2e-framework/pkg/envconf"
 	"sigs.k8s.io/e2e-framework/pkg/features"
 
 	"github.com/grafana/beyla/test/integration/components/jaeger"
+	"github.com/grafana/beyla/test/integration/components/kube"
 	k8s "github.com/grafana/beyla/test/integration/k8s/common"
 )
 
@@ -24,8 +26,9 @@ import (
 func TestPythonBasicTracing(t *testing.T) {
 	feat := features.New("Beyla is able to instrument an arbitrary process").
 		Assess("it sends traces for that service",
-			func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context {
+			func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context {
 				var trace jaeger.Trace
+				var podID string
 				test.Eventually(t, testTimeout, func(t require.TestingT) {
 					resp, err := http.Get("http://localhost:38083/greeting")
 					require.NoError(t, err)
@@ -56,14 +59,75 @@ func TestPythonBasicTracing(t *testing.T) {
 
 					// check the process information
 					sd = jaeger.DiffAsRegexp([]jaeger.Tag{
-						{Key: "k8s.pod.name", Type: "string", Value: "^pytestserver$"},
+						{Key: "k8s.pod.name", Type: "string", Value: "^pytestserver-.*"},
 						{Key: "k8s.node.name", Type: "string", Value: ".+-control-plane$"},
 						{Key: "k8s.pod.uid", Type: "string", Value: k8s.UUIDRegex},
 						{Key: "k8s.pod.start_time", Type: "string", Value: k8s.TimeRegex},
 						{Key: "k8s.namespace.name", Type: "string", Value: "^default$"},
 					}, trace.Processes[parent.ProcessID].Tags)
 					require.Empty(t, sd, sd.String())
+
+					// Extract the pod id, so we can later check on restart of the pod that we have a different id
+					tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid")
+					assert.True(t, found)
+
+					podID = tag.Value.(string)
+					assert.NotEqual(t, "", podID)
 				}, test.Interval(100*time.Millisecond))
+
+				// Let's take down our services, keeping Beyla alive and then redeploy them
+				err := kube.DeleteExistingManifestFile(cfg, k8s.PathManifests+"/05-uninstrumented-service-python.yml")
+				assert.NoError(t, err, "we should see no error when deleting the uninstrumented service manifest file")
+
+				err = kube.DeployManifestFile(cfg, k8s.PathManifests+"/05-uninstrumented-service-python.yml")
+				assert.NoError(t, err, "we should see no error when re-deploying the uninstrumented service manifest file")
+
+				// We now use /smoke instead of /greeting to ensure we see those APIs after a restart
+				test.Eventually(t, testTimeout, func(t require.TestingT) {
+					resp, err := http.Get("http://localhost:38083/smoke")
+					require.NoError(t, err)
+					require.Equal(t, http.StatusOK, resp.StatusCode)
+
+					resp, err = http.Get(jaegerQueryURL + "?service=pytestserver&operation=GET%20%2Fsmoke")
+					require.NoError(t, err)
+					if resp == nil {
+						return
+					}
+					require.Equal(t, http.StatusOK, resp.StatusCode)
+					var tq jaeger.TracesQuery
+					require.NoError(t, json.NewDecoder(resp.Body).Decode(&tq))
+					traces := tq.FindBySpan(jaeger.Tag{Key: "url.path", Type: "string", Value: "/smoke"})
+					require.NotEmpty(t, traces)
+					trace = traces[0]
+					require.NotEmpty(t, trace.Spans)
+
+					// Check the information of the parent span
+					res := trace.FindByOperationName("GET /smoke")
+					require.Len(t, res, 1)
+					parent := res[0]
+					sd := jaeger.Diff([]jaeger.Tag{
+						{Key: "service.namespace", Type: "string", Value: "integration-test"},
+						{Key: "telemetry.sdk.language", Type: "string", Value: "python"},
+					}, trace.Processes[parent.ProcessID].Tags)
+					require.Empty(t, sd, sd.String())
+
+					// check the process information
+					sd = jaeger.DiffAsRegexp([]jaeger.Tag{
+						{Key: "k8s.pod.name", Type: "string", Value: "^pytestserver-.*"},
+						{Key: "k8s.node.name", Type: "string", Value: ".+-control-plane$"},
+						{Key: "k8s.pod.uid", Type: "string", Value: k8s.UUIDRegex},
+						{Key: "k8s.pod.start_time", Type: "string", Value: k8s.TimeRegex},
+						{Key: "k8s.namespace.name", Type: "string", Value: "^default$"},
+					}, trace.Processes[parent.ProcessID].Tags)
+					require.Empty(t, sd, sd.String())
+
+					// ensure the pod really restarted
+					tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid")
+					assert.True(t, found)
+
+					assert.NotEqual(t, podID, tag.Value.(string))
+				}, test.Interval(100*time.Millisecond))
+
 				return ctx
 			},
 		).Feature()

test/integration/k8s/manifests/05-uninstrumented-service-python.yml

@@ -10,22 +10,31 @@ spec:
       name: http0
       targetPort: http0
 ---
-apiVersion: v1
-kind: Pod
+apiVersion: apps/v1
+kind: Deployment
 metadata:
   name: pytestserver
   labels:
     app: pytestserver
 spec:
-  containers:
-    - name: pytestserver
-      image: pythontestserver:dev
-      imagePullPolicy: Never # loaded into Kind from localhost
-      ports:
-        # exposing hostports to enable operation from tests
-        - containerPort: 8083
-          hostPort: 8083
-          name: http0
-      env:
-        - name: LOG_LEVEL
-          value: "DEBUG"
+  replicas: 1
+  selector:
+    matchLabels:
+      app: pytestserver
+  template:
+    metadata:
+      name: pytestserver
+      labels:
+        app: pytestserver
+    spec:
+      containers:
+        - name: pytestserver
+          image: pythontestserver:dev
+          imagePullPolicy: Never # loaded into Kind from localhost
+          ports:
+            - containerPort: 8083
+              hostPort: 8083
+              name: http0
+          env:
+            - name: LOG_LEVEL
+              value: "DEBUG"