From 0f5c6abfb2e697c1aa42dd89870fafe97dd48399 Mon Sep 17 00:00:00 2001 From: Nikola Grcevski <6207777+grcevski@users.noreply.github.com> Date: Fri, 26 Apr 2024 14:15:21 -0400 Subject: [PATCH] Add tests for recycling services (#787) --- test/integration/components/kube/kind.go | 23 ++++++ test/integration/k8s/common/k8s_common.go | 1 + .../k8s/daemonset/k8s_daemonset_main_test.go | 2 + .../daemonset/k8s_daemonset_traces_test.go | 74 ++++++++++++++++++- .../k8s_daemonset_traces_test.go | 68 ++++++++++++++++- .../05-uninstrumented-service-python.yml | 37 ++++++---- 6 files changed, 188 insertions(+), 17 deletions(-) diff --git a/test/integration/components/kube/kind.go b/test/integration/components/kube/kind.go index 71fc80412..63a6830ea 100644 --- a/test/integration/components/kube/kind.go +++ b/test/integration/components/kube/kind.go @@ -228,6 +228,29 @@ func deployManifest(cfg *envconf.Config, manifest string) error { }) } +func deleteManifestFile( + manifestFile string, + cfg *envconf.Config, +) error { + log := log() + log.With("file", manifestFile).Info("deleting manifest file") + + b, err := os.ReadFile(manifestFile) + if err != nil { + return fmt.Errorf("reading manifest file %q: %w", manifestFile, err) + } + + return deleteManifest(cfg, string(b)) +} + +func DeleteExistingManifestFile(cfg *envconf.Config, manifest string) error { + return deleteManifestFile(manifest, cfg) +} + +func DeployManifestFile(cfg *envconf.Config, manifest string) error { + return deployManifestFile(manifest, cfg) +} + func deleteManifest(cfg *envconf.Config, manifest string) error { return applyManifest(cfg, manifest, func(dri dynamic.ResourceInterface, obj *unstructured.Unstructured) error { if err := dri.Delete(context.Background(), obj.GetName(), metav1.DeleteOptions{}); err != nil { diff --git a/test/integration/k8s/common/k8s_common.go b/test/integration/k8s/common/k8s_common.go index cdf2e60ad..99d2a9271 100644 --- a/test/integration/k8s/common/k8s_common.go +++ b/test/integration/k8s/common/k8s_common.go @@ -19,6 +19,7 @@ var ( PingerManifest = path.Join(PathManifests, "/06-instrumented-client.template.yml") GrpcPingerManifest = path.Join(PathManifests, "/06-instrumented-grpc-client.template.yml") UninstrumentedPingerManifest = path.Join(PathManifests, "/06-uninstrumented-client.template.yml") + UninstrumentedAppManifest = path.Join(PathManifests, "/05-uninstrumented-service.yml") ) // Pinger stores the configuration data of a local pod that will be used to diff --git a/test/integration/k8s/daemonset/k8s_daemonset_main_test.go b/test/integration/k8s/daemonset/k8s_daemonset_main_test.go index 6a29fd33b..81b249b9d 100644 --- a/test/integration/k8s/daemonset/k8s_daemonset_main_test.go +++ b/test/integration/k8s/daemonset/k8s_daemonset_main_test.go @@ -54,3 +54,5 @@ func TestMain(m *testing.M) { cluster.Run(m) } + +type UninstrumentedApp struct{} diff --git a/test/integration/k8s/daemonset/k8s_daemonset_traces_test.go b/test/integration/k8s/daemonset/k8s_daemonset_traces_test.go index 253e269b8..08be19f60 100644 --- a/test/integration/k8s/daemonset/k8s_daemonset_traces_test.go +++ b/test/integration/k8s/daemonset/k8s_daemonset_traces_test.go @@ -16,6 +16,7 @@ import ( "sigs.k8s.io/e2e-framework/pkg/features" "github.com/grafana/beyla/test/integration/components/jaeger" + "github.com/grafana/beyla/test/integration/components/kube" k8s "github.com/grafana/beyla/test/integration/k8s/common" ) @@ -25,7 +26,8 @@ import ( func TestBasicTracing(t *testing.T) { feat := features.New("Beyla is able to instrument an arbitrary process"). Assess("it sends traces for that service", - func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { + var podID string test.Eventually(t, testTimeout, func(t require.TestingT) { // Invoking both service instances, but we will expect that only one // is instrumented, according to the discovery mechanisms @@ -73,6 +75,12 @@ func TestBasicTracing(t *testing.T) { }, trace.Processes[parent.ProcessID].Tags) require.Empty(t, sd) + // Extract the pod id, so we can later check on restart of the pod that we have a different id + tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid") + assert.True(t, found) + + podID = tag.Value.(string) + assert.NotEqual(t, "", podID) }, test.Interval(100*time.Millisecond)) // Check that the "testserver" service is never instrumented @@ -82,6 +90,70 @@ func TestBasicTracing(t *testing.T) { var tq jaeger.TracesQuery require.NoError(t, json.NewDecoder(resp.Body).Decode(&tq)) assert.Empty(t, tq.Data) + + // Let's take down our services, keeping Beyla alive and then redeploy them + err = kube.DeleteExistingManifestFile(cfg, k8s.PathManifests+"/05-uninstrumented-service.yml") + assert.NoError(t, err, "we should see no error when deleting the uninstrumented service manifest file") + + err = kube.DeployManifestFile(cfg, k8s.PathManifests+"/05-uninstrumented-service.yml") + assert.NoError(t, err, "we should see no error when re-deploying the uninstrumented service manifest file") + + // We now use a different API, this ensures that after undeploying and redeploying the application we + // can still monitor its data + test.Eventually(t, testTimeout, func(t require.TestingT) { + // Invoking both service instances, but we will expect that only one + // is instrumented, according to the discovery mechanisms + resp, err := http.Get("http://localhost:38080/pingpongtoo") + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + resp, err = http.Get("http://localhost:38081/pingpongtoo") + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + resp, err = http.Get(jaegerQueryURL + "?service=otherinstance") + require.NoError(t, err) + if resp == nil { + return + } + require.Equal(t, http.StatusOK, resp.StatusCode) + var tq jaeger.TracesQuery + require.NoError(t, json.NewDecoder(resp.Body).Decode(&tq)) + traces := tq.FindBySpan(jaeger.Tag{Key: "url.path", Type: "string", Value: "/pingpongtoo"}) + require.NotEmpty(t, traces) + trace := traces[0] + require.NotEmpty(t, trace.Spans) + + // Check that the service.namespace is set from the K8s namespace + assert.Len(t, trace.Processes, 1) + for _, proc := range trace.Processes { + sd := jaeger.DiffAsRegexp([]jaeger.Tag{ + {Key: "service.namespace", Type: "string", Value: "^default$"}, + }, proc.Tags) + require.Empty(t, sd) + } + + // Check the information of the parent span + res := trace.FindByOperationName("GET /pingpongtoo") + require.Len(t, res, 1) + parent := res[0] + sd := jaeger.DiffAsRegexp([]jaeger.Tag{ + {Key: "k8s.pod.name", Type: "string", Value: "^otherinstance-.*"}, + {Key: "k8s.node.name", Type: "string", Value: ".+-control-plane$"}, + {Key: "k8s.pod.uid", Type: "string", Value: k8s.UUIDRegex}, + {Key: "k8s.pod.start_time", Type: "string", Value: k8s.TimeRegex}, + {Key: "k8s.deployment.name", Type: "string", Value: "^otherinstance"}, + {Key: "k8s.namespace.name", Type: "string", Value: "^default$"}, + }, trace.Processes[parent.ProcessID].Tags) + require.Empty(t, sd) + + // ensure the pod really restarted + tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid") + assert.True(t, found) + + assert.NotEqual(t, podID, tag.Value.(string)) + }, test.Interval(100*time.Millisecond)) + return ctx }, ).Feature() diff --git a/test/integration/k8s/daemonset_python/k8s_daemonset_traces_test.go b/test/integration/k8s/daemonset_python/k8s_daemonset_traces_test.go index 04c650427..3944ad34d 100644 --- a/test/integration/k8s/daemonset_python/k8s_daemonset_traces_test.go +++ b/test/integration/k8s/daemonset_python/k8s_daemonset_traces_test.go @@ -10,11 +10,13 @@ import ( "time" "github.com/mariomac/guara/pkg/test" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "sigs.k8s.io/e2e-framework/pkg/envconf" "sigs.k8s.io/e2e-framework/pkg/features" "github.com/grafana/beyla/test/integration/components/jaeger" + "github.com/grafana/beyla/test/integration/components/kube" k8s "github.com/grafana/beyla/test/integration/k8s/common" ) @@ -24,8 +26,9 @@ import ( func TestPythonBasicTracing(t *testing.T) { feat := features.New("Beyla is able to instrument an arbitrary process"). Assess("it sends traces for that service", - func(ctx context.Context, t *testing.T, _ *envconf.Config) context.Context { + func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { var trace jaeger.Trace + var podID string test.Eventually(t, testTimeout, func(t require.TestingT) { resp, err := http.Get("http://localhost:38083/greeting") require.NoError(t, err) @@ -56,14 +59,75 @@ func TestPythonBasicTracing(t *testing.T) { // check the process information sd = jaeger.DiffAsRegexp([]jaeger.Tag{ - {Key: "k8s.pod.name", Type: "string", Value: "^pytestserver$"}, + {Key: "k8s.pod.name", Type: "string", Value: "^pytestserver-.*"}, {Key: "k8s.node.name", Type: "string", Value: ".+-control-plane$"}, {Key: "k8s.pod.uid", Type: "string", Value: k8s.UUIDRegex}, {Key: "k8s.pod.start_time", Type: "string", Value: k8s.TimeRegex}, {Key: "k8s.namespace.name", Type: "string", Value: "^default$"}, }, trace.Processes[parent.ProcessID].Tags) require.Empty(t, sd, sd.String()) + + // Extract the pod id, so we can later check on restart of the pod that we have a different id + tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid") + assert.True(t, found) + + podID = tag.Value.(string) + assert.NotEqual(t, "", podID) }, test.Interval(100*time.Millisecond)) + + // Let's take down our services, keeping Beyla alive and then redeploy them + err := kube.DeleteExistingManifestFile(cfg, k8s.PathManifests+"/05-uninstrumented-service-python.yml") + assert.NoError(t, err, "we should see no error when deleting the uninstrumented service manifest file") + + err = kube.DeployManifestFile(cfg, k8s.PathManifests+"/05-uninstrumented-service-python.yml") + assert.NoError(t, err, "we should see no error when re-deploying the uninstrumented service manifest file") + + // We now use /smoke instead of /greeting to ensure we see those APIs after a restart + test.Eventually(t, testTimeout, func(t require.TestingT) { + resp, err := http.Get("http://localhost:38083/smoke") + require.NoError(t, err) + require.Equal(t, http.StatusOK, resp.StatusCode) + + resp, err = http.Get(jaegerQueryURL + "?service=pytestserver&operation=GET%20%2Fsmoke") + require.NoError(t, err) + if resp == nil { + return + } + require.Equal(t, http.StatusOK, resp.StatusCode) + var tq jaeger.TracesQuery + require.NoError(t, json.NewDecoder(resp.Body).Decode(&tq)) + traces := tq.FindBySpan(jaeger.Tag{Key: "url.path", Type: "string", Value: "/smoke"}) + require.NotEmpty(t, traces) + trace = traces[0] + require.NotEmpty(t, trace.Spans) + + // Check the information of the parent span + res := trace.FindByOperationName("GET /smoke") + require.Len(t, res, 1) + parent := res[0] + sd := jaeger.Diff([]jaeger.Tag{ + {Key: "service.namespace", Type: "string", Value: "integration-test"}, + {Key: "telemetry.sdk.language", Type: "string", Value: "python"}, + }, trace.Processes[parent.ProcessID].Tags) + require.Empty(t, sd, sd.String()) + + // check the process information + sd = jaeger.DiffAsRegexp([]jaeger.Tag{ + {Key: "k8s.pod.name", Type: "string", Value: "^pytestserver-.*"}, + {Key: "k8s.node.name", Type: "string", Value: ".+-control-plane$"}, + {Key: "k8s.pod.uid", Type: "string", Value: k8s.UUIDRegex}, + {Key: "k8s.pod.start_time", Type: "string", Value: k8s.TimeRegex}, + {Key: "k8s.namespace.name", Type: "string", Value: "^default$"}, + }, trace.Processes[parent.ProcessID].Tags) + require.Empty(t, sd, sd.String()) + + // ensure the pod really restarted + tag, found := jaeger.FindIn(trace.Processes[parent.ProcessID].Tags, "k8s.pod.uid") + assert.True(t, found) + + assert.NotEqual(t, podID, tag.Value.(string)) + }, test.Interval(100*time.Millisecond)) + return ctx }, ).Feature() diff --git a/test/integration/k8s/manifests/05-uninstrumented-service-python.yml b/test/integration/k8s/manifests/05-uninstrumented-service-python.yml index 5bc5366c8..1bb721670 100644 --- a/test/integration/k8s/manifests/05-uninstrumented-service-python.yml +++ b/test/integration/k8s/manifests/05-uninstrumented-service-python.yml @@ -10,22 +10,31 @@ spec: name: http0 targetPort: http0 --- -apiVersion: v1 -kind: Pod +apiVersion: apps/v1 +kind: Deployment metadata: name: pytestserver labels: app: pytestserver spec: - containers: - - name: pytestserver - image: pythontestserver:dev - imagePullPolicy: Never # loaded into Kind from localhost - ports: - # exposing hostports to enable operation from tests - - containerPort: 8083 - hostPort: 8083 - name: http0 - env: - - name: LOG_LEVEL - value: "DEBUG" + replicas: 1 + selector: + matchLabels: + app: pytestserver + template: + metadata: + name: pytestserver + labels: + app: pytestserver + spec: + containers: + - name: pytestserver + image: pythontestserver:dev + imagePullPolicy: Never # loaded into Kind from localhost + ports: + - containerPort: 8083 + hostPort: 8083 + name: http0 + env: + - name: LOG_LEVEL + value: "DEBUG" \ No newline at end of file