From f390cf2eac1cd58916c316cb1411e819e9ee9da1 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Thu, 22 Jan 2026 15:51:58 +0100 Subject: [PATCH 1/5] feat: add Prometheus metrics with OTel sidecar - Add metrics package exposing token request counter on :9990 - Add OTel Collector sidecar to scrape and push metrics to Prometheus - Update cloudbuild.yaml to build/push OTel image and deploy via service.yaml - Parallelize Docker builds for faster CI --- cloudbuild.yaml | 55 +++++++++++++------- go.mod | 14 ++++- go.sum | 104 +++++++++++++++++++++++++++++++++++++ handler/handler.go | 10 ++++ main.go | 16 +++++- metrics/metrics.go | 47 +++++++++++++++++ metrics/metrics_test.go | 11 ++++ otel-collector/Dockerfile | 2 + otel-collector/config.yaml | 30 +++++++++++ service.yaml | 51 ++++++++++++++++++ 10 files changed, 319 insertions(+), 21 deletions(-) create mode 100644 metrics/metrics.go create mode 100644 metrics/metrics_test.go create mode 100644 otel-collector/Dockerfile create mode 100644 otel-collector/config.yaml create mode 100644 service.yaml diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 5502224..9cd193b 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -1,39 +1,56 @@ substitutions: _REGION: us-east1 _SERVICE_NAME: speed-proxy - _ALLOWED_ORIGIN: https://speed.measurementlab.net - _TOKEN_EXCHANGE_URL: https://auth.mlab-sandbox.measurementlab.net/v0/token/integration + _MLAB_PROJECT: mlab-sandbox steps: - # 1. Build the container image + # 1. Build the speed-proxy container image - name: "gcr.io/cloud-builders/docker" id: Build + waitFor: ['-'] args: ["build", "-t", "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/${_SERVICE_NAME}:$BUILD_ID", "-t", "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/${_SERVICE_NAME}:latest", "."] - # 2. Push to Artifact Registry + # 2. Build the otel-collector sidecar image + - name: "gcr.io/cloud-builders/docker" + id: BuildOtel + waitFor: ['-'] + args: ["build", "-t", "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/otel-collector:$BUILD_ID", "-t", "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/otel-collector:latest", "otel-collector"] + + # 3. Push speed-proxy image to Artifact Registry - name: "gcr.io/cloud-builders/docker" id: Push args: ["push", "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/${_SERVICE_NAME}:${BUILD_ID}"] waitFor: ["Build"] - # 3. Deploy to Cloud Run + # 4. Push otel-collector image to Artifact Registry + - name: "gcr.io/cloud-builders/docker" + id: PushOtel + args: ["push", "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/otel-collector:${BUILD_ID}"] + waitFor: ["BuildOtel"] + + # 5. Substitute placeholders in service.yaml and deploy - name: "gcr.io/google.com/cloudsdktool/cloud-sdk" id: Deploy - entrypoint: gcloud - args: [ - "run", "deploy", "${_SERVICE_NAME}", - "--image", "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/${_SERVICE_NAME}:$BUILD_ID", - "--platform", "managed", - "--region", "${_REGION}", - "--allow-unauthenticated", - "--set-secrets=API_KEY=speed-proxy-api-key:latest", - "--set-env-vars=TOKEN_EXCHANGE_URL=${_TOKEN_EXCHANGE_URL},ALLOWED_ORIGIN=${_ALLOWED_ORIGIN}", - "--memory=256Mi", - "--cpu=1", - "--project=${PROJECT_ID}" - ] - waitFor: ["Push"] + entrypoint: bash + args: + - -c + - | + sed -e "s|OTEL_IMAGE_PLACEHOLDER|${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/otel-collector:${BUILD_ID}|g" \ + -e "s|IMAGE_PLACEHOLDER|${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/${_SERVICE_NAME}:${BUILD_ID}|g" \ + -e "s|MLAB_PROJECT_PLACEHOLDER|${_MLAB_PROJECT}|g" \ + service.yaml > service-deploy.yaml + gcloud run services replace service-deploy.yaml \ + --region=${_REGION} \ + --project=${PROJECT_ID} + gcloud run services set-iam-policy ${_SERVICE_NAME} \ + --region=${_REGION} \ + --project=${PROJECT_ID} \ + --quiet \ + <(echo '{"bindings":[{"role":"roles/run.invoker","members":["allUsers"]}]}') + waitFor: ["Push", "PushOtel"] images: - "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/${_SERVICE_NAME}:$BUILD_ID" - "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/${_SERVICE_NAME}:latest" + - "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/otel-collector:$BUILD_ID" + - "${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/otel-collector:latest" diff --git a/go.mod b/go.mod index 93755af..bbe127c 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,21 @@ module github.com/m-lab/speed-proxy go 1.25.5 -require github.com/m-lab/go v0.1.76 +require ( + github.com/m-lab/go v0.1.76 + github.com/prometheus/client_golang v1.7.1 +) require ( github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.1.1 // indirect + github.com/golang/protobuf v1.4.2 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.10.0 // indirect + github.com/prometheus/procfs v0.1.3 // indirect github.com/stretchr/testify v1.10.0 // indirect + golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect + google.golang.org/protobuf v1.23.0 // indirect ) diff --git a/go.sum b/go.sum index e0d504c..dc03a6b 100644 --- a/go.sum +++ b/go.sum @@ -1,14 +1,118 @@ +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1 h1:TEBmxO80TM04L8IuMWk77SGL1HomBmKTdzdJLLWznxI= github.com/araddon/dateparse v0.0.0-20200409225146-d820a6159ab1/go.mod h1:SLqhdZcd+dF3TEVL2RMoob5bBP5R1P1qkox+HtCBgGI= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8= github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3 h1:Iy7Ifq2ysilWU4QlCx/97OoI4xT1IV7i8byT/EyIT/M= +github.com/kabukky/httpscerts v0.0.0-20150320125433-617593d7dcb3/go.mod h1:BYpt4ufZiIGv2nXn4gMxnfKV306n3mWXgNu/d2TqdTU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/m-lab/go v0.1.76 h1:zdxI5k0AIZaf99Cyh7zjNyNBBZxTlhIJvWg7AX710bU= github.com/m-lab/go v0.1.76/go.mod h1:BirARfHWjjXHaCGNyWCm/CKW1OarjuEj8Yn6Z2rc0M4= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.7.1 h1:NTGy1Ja9pByO+xAeH/qiWnLrKtr3hJPNjaVUwnjpdpA= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handler/handler.go b/handler/handler.go index 5285d63..0d5071b 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -8,6 +8,10 @@ import ( "io" "log" "net/http" + "strconv" + "time" + + "github.com/m-lab/speed-proxy/metrics" ) // Config contains the configuration for the token handler. @@ -91,12 +95,18 @@ func (h *Handler) exchangeToken(ctx context.Context, apiKey string) (string, err } req.Header.Set("Content-Type", "application/json") + start := time.Now() resp, err := h.config.HTTPClient.Do(req) + metrics.UpstreamRequestDuration.Observe(time.Since(start).Seconds()) + if err != nil { + metrics.UpstreamRequestsTotal.WithLabelValues("error").Inc() return "", fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() + metrics.UpstreamRequestsTotal.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc() + if resp.StatusCode != http.StatusOK { body, _ := io.ReadAll(resp.Body) return "", fmt.Errorf("token exchange failed with status %d: %s", resp.StatusCode, string(body)) diff --git a/main.go b/main.go index 0f497c0..9bcc971 100644 --- a/main.go +++ b/main.go @@ -11,8 +11,11 @@ import ( "time" "github.com/m-lab/go/flagx" + "github.com/m-lab/go/prometheusx" "github.com/m-lab/go/rtx" "github.com/m-lab/speed-proxy/handler" + "github.com/m-lab/speed-proxy/metrics" + "github.com/prometheus/client_golang/prometheus/promhttp" ) var ( @@ -30,6 +33,8 @@ func main() { log.Fatal("-api-key is required") } + prometheusx.MustServeMetrics() + ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) defer stop() @@ -41,8 +46,17 @@ func main() { HTTPClient: &http.Client{Timeout: 10 * time.Second}, }) + // Wrap handler with promhttp instrumentation. + tokenHandler := promhttp.InstrumentHandlerDuration( + metrics.TokenRequestDuration, + promhttp.InstrumentHandlerCounter( + metrics.TokenRequestsTotal, + http.HandlerFunc(h.Token), + ), + ) + mux := http.NewServeMux() - mux.HandleFunc("/v0/token", h.Token) + mux.Handle("/v0/token", tokenHandler) mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) w.Write([]byte("ok")) diff --git a/metrics/metrics.go b/metrics/metrics.go new file mode 100644 index 0000000..41753d3 --- /dev/null +++ b/metrics/metrics.go @@ -0,0 +1,47 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + // TokenRequestsTotal counts incoming token requests. + // Used with promhttp.InstrumentHandlerCounter. + TokenRequestsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "speedproxy_token_requests_total", + Help: "Total number of token requests.", + }, + []string{"code", "method"}, + ) + + // TokenRequestDuration measures incoming token request latency. + // Used with promhttp.InstrumentHandlerDuration. + TokenRequestDuration = promauto.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "speedproxy_token_request_duration_seconds", + Help: "Duration of token requests.", + Buckets: prometheus.DefBuckets, + }, + []string{"code", "method"}, + ) + + // UpstreamRequestsTotal counts upstream token exchange requests. + UpstreamRequestsTotal = promauto.NewCounterVec( + prometheus.CounterOpts{ + Name: "speedproxy_upstream_requests_total", + Help: "Total number of upstream token exchange requests.", + }, + []string{"code"}, + ) + + // UpstreamRequestDuration measures upstream token exchange latency. + UpstreamRequestDuration = promauto.NewHistogram( + prometheus.HistogramOpts{ + Name: "speedproxy_upstream_request_duration_seconds", + Help: "Duration of upstream token exchange requests.", + Buckets: prometheus.DefBuckets, + }, + ) +) diff --git a/metrics/metrics_test.go b/metrics/metrics_test.go new file mode 100644 index 0000000..befd7bb --- /dev/null +++ b/metrics/metrics_test.go @@ -0,0 +1,11 @@ +package metrics + +import ( + "testing" + + "github.com/m-lab/go/prometheusx/promtest" +) + +func TestLintMetrics(t *testing.T) { + promtest.LintMetrics(t) +} diff --git a/otel-collector/Dockerfile b/otel-collector/Dockerfile new file mode 100644 index 0000000..d73bd22 --- /dev/null +++ b/otel-collector/Dockerfile @@ -0,0 +1,2 @@ +FROM otel/opentelemetry-collector-contrib:latest +COPY config.yaml /etc/otelcol-contrib/config.yaml diff --git a/otel-collector/config.yaml b/otel-collector/config.yaml new file mode 100644 index 0000000..0328f1c --- /dev/null +++ b/otel-collector/config.yaml @@ -0,0 +1,30 @@ +extensions: + basicauth/prometheus: + client_auth: + username: ${env:PROMETHEUS_USER} + password: ${env:PROMETHEUS_PASSWORD} + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + prometheus: + config: + scrape_configs: + - job_name: 'speed-proxy' + scrape_interval: 15s + static_configs: + - targets: ['localhost:9990'] + +exporters: + prometheusremotewrite: + endpoint: ${env:PROMETHEUS_ENDPOINT} + auth: + authenticator: basicauth/prometheus + timeout: 30s + +service: + extensions: [basicauth/prometheus, health_check] + pipelines: + metrics: + receivers: [prometheus] + exporters: [prometheusremotewrite] diff --git a/service.yaml b/service.yaml new file mode 100644 index 0000000..2b00ae3 --- /dev/null +++ b/service.yaml @@ -0,0 +1,51 @@ +apiVersion: serving.knative.dev/v1 +kind: Service +metadata: + name: speed-proxy +spec: + template: + metadata: + annotations: + run.googleapis.com/container-dependencies: '{"speed-proxy":["otel-collector"]}' + spec: + containers: + - name: speed-proxy + image: IMAGE_PLACEHOLDER + ports: + - containerPort: 8080 + env: + - name: API_KEY + valueFrom: + secretKeyRef: + name: speed-proxy-api-key + key: latest + - name: TOKEN_EXCHANGE_URL + value: "https://auth.mlab-sandbox.measurementlab.net/v0/token/integration" + - name: ALLOWED_ORIGIN + value: "https://speed.measurementlab.net" + resources: + limits: + memory: 256Mi + cpu: "1" + - name: otel-collector + image: OTEL_IMAGE_PLACEHOLDER + env: + - name: PROMETHEUS_ENDPOINT + value: "https://prometheus-basicauth.MLAB_PROJECT_PLACEHOLDER.measurementlab.net/api/v1/write" + - name: PROMETHEUS_USER + valueFrom: + secretKeyRef: + name: prometheus-support-build-prom-auth-user + key: latest + - name: PROMETHEUS_PASSWORD + valueFrom: + secretKeyRef: + name: prometheus-support-build-prom-auth-pass + key: latest + startupProbe: + httpGet: + path: / + port: 13133 + initialDelaySeconds: 0 + periodSeconds: 1 + failureThreshold: 10 From 9ad1b5a5d5ec1bc63bc0bd313ae1db1dca757f76 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Thu, 22 Jan 2026 16:17:28 +0100 Subject: [PATCH 2/5] fix(service): use MLAB_PROJECT_PLACEHOLDER for TOKEN_EXCHANGE_URL --- service.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service.yaml b/service.yaml index 2b00ae3..3a460e3 100644 --- a/service.yaml +++ b/service.yaml @@ -20,7 +20,7 @@ spec: name: speed-proxy-api-key key: latest - name: TOKEN_EXCHANGE_URL - value: "https://auth.mlab-sandbox.measurementlab.net/v0/token/integration" + value: "https://auth.MLAB_PROJECT_PLACEHOLDER.measurementlab.net/v0/token/integration" - name: ALLOWED_ORIGIN value: "https://speed.measurementlab.net" resources: From 9dcef6c221561fa4d8ee3ea6f934ca9af89c1c8d Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Thu, 22 Jan 2026 16:19:00 +0100 Subject: [PATCH 3/5] feat: make ALLOWED_ORIGIN configurable via substitution --- cloudbuild.yaml | 2 ++ service.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/cloudbuild.yaml b/cloudbuild.yaml index 9cd193b..8cfb0cc 100644 --- a/cloudbuild.yaml +++ b/cloudbuild.yaml @@ -2,6 +2,7 @@ substitutions: _REGION: us-east1 _SERVICE_NAME: speed-proxy _MLAB_PROJECT: mlab-sandbox + _ALLOWED_ORIGIN: https://speed.measurementlab.net steps: # 1. Build the speed-proxy container image @@ -38,6 +39,7 @@ steps: sed -e "s|OTEL_IMAGE_PLACEHOLDER|${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/otel-collector:${BUILD_ID}|g" \ -e "s|IMAGE_PLACEHOLDER|${_REGION}-docker.pkg.dev/${PROJECT_ID}/m-lab/${_SERVICE_NAME}:${BUILD_ID}|g" \ -e "s|MLAB_PROJECT_PLACEHOLDER|${_MLAB_PROJECT}|g" \ + -e "s|ALLOWED_ORIGIN_PLACEHOLDER|${_ALLOWED_ORIGIN}|g" \ service.yaml > service-deploy.yaml gcloud run services replace service-deploy.yaml \ --region=${_REGION} \ diff --git a/service.yaml b/service.yaml index 3a460e3..9893e13 100644 --- a/service.yaml +++ b/service.yaml @@ -22,7 +22,7 @@ spec: - name: TOKEN_EXCHANGE_URL value: "https://auth.MLAB_PROJECT_PLACEHOLDER.measurementlab.net/v0/token/integration" - name: ALLOWED_ORIGIN - value: "https://speed.measurementlab.net" + value: "ALLOWED_ORIGIN_PLACEHOLDER" resources: limits: memory: 256Mi From db0ed4779f6e0e941cb9fbf9cbd850b73e5c283c Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Thu, 22 Jan 2026 16:49:40 +0100 Subject: [PATCH 4/5] fix(metrics): separate duration histogram by success/error result --- handler/handler.go | 4 +++- metrics/metrics.go | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/handler/handler.go b/handler/handler.go index 0d5071b..4ce66f6 100644 --- a/handler/handler.go +++ b/handler/handler.go @@ -97,14 +97,16 @@ func (h *Handler) exchangeToken(ctx context.Context, apiKey string) (string, err start := time.Now() resp, err := h.config.HTTPClient.Do(req) - metrics.UpstreamRequestDuration.Observe(time.Since(start).Seconds()) + duration := time.Since(start).Seconds() if err != nil { + metrics.UpstreamRequestDuration.WithLabelValues("error").Observe(duration) metrics.UpstreamRequestsTotal.WithLabelValues("error").Inc() return "", fmt.Errorf("failed to send request: %w", err) } defer resp.Body.Close() + metrics.UpstreamRequestDuration.WithLabelValues("success").Observe(duration) metrics.UpstreamRequestsTotal.WithLabelValues(strconv.Itoa(resp.StatusCode)).Inc() if resp.StatusCode != http.StatusOK { diff --git a/metrics/metrics.go b/metrics/metrics.go index 41753d3..3fae15a 100644 --- a/metrics/metrics.go +++ b/metrics/metrics.go @@ -37,11 +37,12 @@ var ( ) // UpstreamRequestDuration measures upstream token exchange latency. - UpstreamRequestDuration = promauto.NewHistogram( + UpstreamRequestDuration = promauto.NewHistogramVec( prometheus.HistogramOpts{ Name: "speedproxy_upstream_request_duration_seconds", Help: "Duration of upstream token exchange requests.", Buckets: prometheus.DefBuckets, }, + []string{"result"}, ) ) From 611d2bb60eaf16983198a01269b8ecff8e3d76b3 Mon Sep 17 00:00:00 2001 From: Roberto D'Auria Date: Thu, 22 Jan 2026 18:28:47 +0100 Subject: [PATCH 5/5] fix(metrics): add GCP resource detection for unique instance labels Use resourcedetection/gcp processor to automatically add Cloud Run metadata as metric labels, including unique faas_instance ID. This prevents metric collisions when multiple instances report to Prometheus. --- otel-collector/config.yaml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/otel-collector/config.yaml b/otel-collector/config.yaml index 0328f1c..daa129c 100644 --- a/otel-collector/config.yaml +++ b/otel-collector/config.yaml @@ -15,16 +15,25 @@ receivers: static_configs: - targets: ['localhost:9990'] +processors: + resourcedetection/gcp: + detectors: [gcp] + timeout: 2s + override: false + exporters: prometheusremotewrite: endpoint: ${env:PROMETHEUS_ENDPOINT} auth: authenticator: basicauth/prometheus timeout: 30s + resource_to_telemetry_conversion: + enabled: true service: extensions: [basicauth/prometheus, health_check] pipelines: metrics: receivers: [prometheus] + processors: [resourcedetection/gcp] exporters: [prometheusremotewrite]