From 68210094cf00ed2beb97e6b293cf0eb5ba5bfccd Mon Sep 17 00:00:00 2001 From: Aviral Takkar <39969667+avtakkar@users.noreply.github.com> Date: Thu, 18 Jul 2024 16:07:35 -0700 Subject: [PATCH] fix, test: fix SAS parser, scrape prom metrics from scanner (#54) * test: scrape prom metrics from scanner * fix: handle error properly --- build/ci/k8s/scanner.yml | 88 ++++++++----------- build/package/Dockerfile | 2 +- .../ama-metrics-prometheus-config.yml | 4 +- pkg/urlparser/azure.go | 2 +- pkg/urlparser/azure_test.go | 5 ++ tests/dockerfiles/scanner.Dockerfile | 6 +- tests/scanner/scanner.go | 52 ++++++++++- 7 files changed, 100 insertions(+), 59 deletions(-) diff --git a/build/ci/k8s/scanner.yml b/build/ci/k8s/scanner.yml index fda7011..1696529 100644 --- a/build/ci/k8s/scanner.yml +++ b/build/ci/k8s/scanner.yml @@ -1,63 +1,49 @@ -# Runs the 'scanner' benchmark scenario on all cluster nodes. -apiVersion: v1 -kind: ConfigMap -metadata: - name: tests-scanner-actions - namespace: peerd-ns - labels: - app: tests-scanner -data: - wasm: | - #!/usr/bin/env bash - set -xe - - nerdctl run \ - --hosts-dir "/etc/containerd/certs.d" \ - --snapshotter=overlaybd \ - --net host -i --rm $TESTS_SCANNER_IMAGE ---- apiVersion: apps/v1 kind: DaemonSet metadata: - name: &name tests-scanner + name: scanner-streaming-ds namespace: peerd-ns - labels: - app: *name + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics/prometheus' + prometheus.io/port: '5004' spec: selector: matchLabels: - app: *name + app: peerd-test template: metadata: labels: - app: *name + app: peerd-test spec: - hostNetwork: true - hostPID: true + initContainers: + - name: sleep + image: busybox + command: ["sh", "-c", "sleep $(shuf -i 1-10 -n 1)"] containers: - - image: docker.io/alexeldeib/nsenter:latest # https://github.com/alexeldeib/azbench/blob/main/images/nsenter/entrypoint.sh - imagePullPolicy: Always - name: *name - args: ["wasm"] - resources: - # requests: - # cpu: 0.5 - # memory: 2000Mi - # limits: - # cpu: 0.5 - # memory: 2000Mi - securityContext: - privileged: true - volumeMounts: - - name: actions - mountPath: "/opt/actions" - - name: hostmount - mountPath: "/mnt/actions" - volumes: - - name: hostmount - hostPath: - path: /opt/actions - type: DirectoryOrCreate - - name: actions - configMap: - name: tests-scanner-actions + - name: scanner-streaming + image: $TESTS_SCANNER_IMAGE + imagePullPolicy: Always + ports: + - containerPort: 5004 + name: metrics +--- +apiVersion: v1 +kind: Service +metadata: + name: scanner-streaming-svc + namespace: peerd-ns + annotations: + prometheus.io/scrape: 'true' + prometheus.io/path: '/metrics/prometheus' + prometheus.io/port: '5004' +spec: + type: NodePort + selector: + app: peerd-test + ports: + - name: metrics + protocol: TCP + port: 5004 + nodePort: 30008 + targetPort: metrics diff --git a/build/package/Dockerfile b/build/package/Dockerfile index 26565c2..982b1d8 100644 --- a/build/package/Dockerfile +++ b/build/package/Dockerfile @@ -1,4 +1,4 @@ -FROM mcr.microsoft.com/oss/go/microsoft/golang:1.22-fips-cbl-mariner2.0 as builder +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.22-fips-cbl-mariner2.0 AS builder COPY ./ / diff --git a/build/package/peerd-helm/templates/ama-metrics-prometheus-config.yml b/build/package/peerd-helm/templates/ama-metrics-prometheus-config.yml index 0304475..5e2a58a 100644 --- a/build/package/peerd-helm/templates/ama-metrics-prometheus-config.yml +++ b/build/package/peerd-helm/templates/ama-metrics-prometheus-config.yml @@ -6,13 +6,13 @@ data: global: scrape_interval: 15s scrape_configs: - - job_name: peerd + - job_name: peerd-exps kubernetes_sd_configs: - role: pod relabel_configs: - source_labels: [__meta_kubernetes_pod_label_app] action: keep - regex: peerd + regex: "peerd|peerd-test" - source_labels: [__meta_kubernetes_pod_container_port_number] action: keep regex: "5004" diff --git a/pkg/urlparser/azure.go b/pkg/urlparser/azure.go index a5dca32..f2c7bf0 100644 --- a/pkg/urlparser/azure.go +++ b/pkg/urlparser/azure.go @@ -12,7 +12,7 @@ import ( var ( regexes = []*regexp.Regexp{ // Azure Container Registry public cloud data endpoints. - regexp.MustCompile(`https:\/\/[a-zA-Z0-9\.]+\.azurecr\.[a-z\.]+\?[a-zA-Z0-9\.\&\=\-]+\&d=sha256:([a-zA-Z0-9]{64})[.]*`), + regexp.MustCompile(`https:\/\/[a-zA-Z0-9\.]+\.azurecr\.[a-z\.]+\/?\?[a-zA-Z0-9\.\&\=\-]+\&d=sha256:([a-zA-Z0-9]{64})[.]*`), // Microsoft Artifact Registry public cloud data endpoints. regexp.MustCompile(`https:\/\/[a-zA-Z0-9]+\.data.mcr.microsoft.com\/[a-zA-Z0-9\-]+\/\/docker\/registry\/v2\/blobs\/sha256\/[a-z0-9]{2}\/([a-zA-Z0-9]{64})\/data.*`), diff --git a/pkg/urlparser/azure_test.go b/pkg/urlparser/azure_test.go index cb5eed0..10eab4c 100644 --- a/pkg/urlparser/azure_test.go +++ b/pkg/urlparser/azure_test.go @@ -49,6 +49,11 @@ var ( "", false, }, + { + "https://aviral26.eastus.data.azurecr.io/?t=63F0223FE7717ED2EF4FD672251A1E3A62E8CD88&h=aviral26.azurecr.io&c=114c2ff3-318c-4f00-86b0-91ea33a781a4&r=scanner&d=sha256:526e51f1c889e6e9ef66e398aac57c388d1305c2490f0d235ff9c8346ab42ec1&p=-qVpMgk5_1AOKyTBbUmjdL4jTWrEGDMjmIJC60c8PuIDY7rIDOa9mkmP-b5OrHfnkNf9E6dDMycKey89jmOA5n3og-O6sd0fckVKJHLm6EoeAHi8qWNKA6AjshSRFN3CvcM9dXdmVpCmR1cR0Q-Cn9tPwyYpRNwQYdkbC5K17RdFUNVTfSETnrKGp5q0fM5WJRqJjB1D2XCSABvyw4Xp6Urp9fj-CcL1qI7DOef_hyHoktZyz47FQyRCQNaNpctoZJcQHjauC2i3zb_V5UIV4Bvy8ag6dfltxT7Z_tM3jpfuyv7dd-l--j3vMsvSp2KB_80WNdJPOpy_wMBSkOns3K_6DpmHdl7bO7keY6-UUYnx6c9nhYRDw7YZ6TsU5kFXndIJAzqfb87DGAmVEPPDJqvWYViL2NiPn_hmeBtre8ti26gVjITkdrrrkAt0TlIzG1fQeYsKKml49Dn-ftpkqzGWTT3soTO8TpGesx6sooCnpAkOsFWLd_pKBIzq-UvQwvJzLVVtf_tiB74ZmLrTCgXgm61q0Lqtq2lgc8zMY6kCQJl-BeycSWlxoEaNMgiLD4UaOOkoeuyOsJGQ-LxPJtVjxu0cuE41lppK_sijGpNylKnECs_cKKhIARlVrrY6kn63qEjQaW6CVMAbylVI8w&s=pBKH0rivCRPRzKcJhLm_D1xApjOwME1BsWEgATUSlL67R_rSbR218vQtoeca4nvuVMNlVGTxhElYhKuWYeVvx57wwHQ-3byGRhhM5FAKozixJpTli2O7oCSou8DRK1NoDATKLNGQ_RPgpZsXlrqQEIZNTn2U1rh5m6jwiKcyGBUX-R3BnymIt-kESrY48Z5u2Ey8orP6zXPupfb8S_pW3p5rfHk0fK9d6VzfZrVXTZbbQyIKXCRupZgVm7Uqvz3j0UTagYR-sy0-aec5ex639-GoHjTmzgwDVXVJkFV11fTw4CBHZtkjeFvXPiKV8ZlxlA0E-lBbaSRgU15ftSf04Q&v=1&l=false", + "sha256:526e51f1c889e6e9ef66e398aac57c388d1305c2490f0d235ff9c8346ab42ec1", + true, + }, } ) diff --git a/tests/dockerfiles/scanner.Dockerfile b/tests/dockerfiles/scanner.Dockerfile index ef78079..dca94db 100644 --- a/tests/dockerfiles/scanner.Dockerfile +++ b/tests/dockerfiles/scanner.Dockerfile @@ -1,10 +1,10 @@ -FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 as scannerbase +FROM mcr.microsoft.com/cbl-mariner/base/core:2.0 AS scannerbase ARG FILE_PATH=/usr/local/bin/scannerbase RUN dd if=/dev/urandom of=$FILE_PATH bs=1 count=$((600 * 1024 * 1024)) -FROM mcr.microsoft.com/oss/go/microsoft/golang:1.21-fips-cbl-mariner2.0 as builder +FROM mcr.microsoft.com/oss/go/microsoft/golang:1.22-fips-cbl-mariner2.0 as builder COPY ./ /src/ @@ -29,4 +29,6 @@ RUN groupadd -g $USER_ID scanner && \ COPY --from=scannerbase --chown=scanner:root /usr/local/bin/scannerbase /usr/local/bin/scannerbase COPY --from=builder --chown=scanner:root /src/bin/tests/tests /src/bin/tests/tests +EXPOSE 5004 + ENTRYPOINT ["/src/bin/tests/tests", "scanner"] diff --git a/tests/scanner/scanner.go b/tests/scanner/scanner.go index e53c26a..7a508d0 100644 --- a/tests/scanner/scanner.go +++ b/tests/scanner/scanner.go @@ -6,10 +6,17 @@ import ( "context" "crypto/rand" "encoding/binary" + "errors" + "fmt" "io" + "net/http" "os" "time" + pcontext "github.com/azure/peerd/pkg/context" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/rs/zerolog" "github.com/schollz/progressbar/v3" ) @@ -21,6 +28,22 @@ const ( func Scanner(ctx context.Context) error { l := zerolog.Ctx(ctx) + byteThroughputHist := prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "scanner_bytes_throughput_mib_per_second", + Help: "Speed of scan in Mib per second.", + Buckets: prometheus.ExponentialBuckets(1, 1.1, 100), + }, []string{"self", "op"}) + prometheus.DefaultRegisterer.MustRegister(byteThroughputHist) + + errChan := make(chan error, 1) + + go func() { + http.Handle("/metrics/prometheus", promhttp.Handler()) + if err := http.ListenAndServe("0.0.0.0:5004", nil); err != nil && !errors.Is(err, http.ErrServerClosed) { + errChan <- err + } + }() + sleep(l) l.Info().Str("path", path).Msg("starting scanner") @@ -37,7 +60,7 @@ func Scanner(ctx context.Context) error { size := info.Size() bar := progressbar.DefaultBytes(size, "reading") - + measure(bar, time.Now(), l, byteThroughputHist, float64(size)) w, err := io.Copy(io.MultiWriter(io.Discard, bar), f) if err != nil { l.Fatal().Err(err).Msg("failed to read file") @@ -46,7 +69,32 @@ func Scanner(ctx context.Context) error { l.Info().Int64("size", size).Int64("read", w).Msg("complete") } - return nil + if err := <-errChan; err != nil { + l.Error().Msg(fmt.Sprintf("prom error: %v", err)) + } + + l.Info().Msg("sleeping to allow metrics scraping") + time.Sleep(24 * 365 * time.Hour) + + return err +} + +func measure(bar *progressbar.ProgressBar, startTime time.Time, l *zerolog.Logger, byteThroughputHist *prometheus.HistogramVec, size float64) { + go func() { + for { + time.Sleep(500 * time.Microsecond) + count := bar.State().CurrentBytes + + if count >= size { + break + } + + elapsed := time.Since(startTime) + speed := float64(count) / elapsed.Seconds() + l.Debug().Float64("speed", speed).Float64("bytes", count).Dur("elapsed", elapsed).Msg("speed") + byteThroughputHist.WithLabelValues(pcontext.NodeName, "read").Observe(speed / float64(1024*1024)) + } + }() } func sleep(l *zerolog.Logger) {