feat(prometheus): add workspace label to the real time metrics (#8920)

* feat(prometheus): add workspace label to the real time metrics (#12836)

Co-authored-by: Zachary Hu <6426329+outsinre@users.noreply.github.com>
(cherry picked from commit da87c42)

* fix: fix code

* fix: fix code

---------

Co-authored-by: Jun Ouyang <ouyangjun1999@gmail.com>

changelog/unreleased/kong/feat-add-workspace-label-to-prometheus.yml

@@ -0,0 +1,4 @@
+message: |
+  **Prometheus**: Added workspace label to Prometheus plugin metrics.
+type: feature
+scope: Plugin

kong/pdk/log.lua

@@ -26,6 +26,7 @@ local phase_checker = require "kong.pdk.private.phases"
 local utils = require "kong.tools.utils"
 local cycle_aware_deep_copy = utils.cycle_aware_deep_copy
 local constants = require "kong.constants"
+local workspace = require "kong.workspaces"
 
 local sub = string.sub
 local type = type

kong/plugins/prometheus/exporter.lua

@@ -110,48 +110,48 @@ local function init()
   if http_subsystem then
     metrics.status = prometheus:counter("http_requests_total",
                                         "HTTP status codes per consumer/service/route in Kong",
-                                        {"service", "route", "code", "source", "consumer"})
+                                        {"service", "route", "code", "source", "workspace", "consumer"})
   else
     metrics.status = prometheus:counter("stream_sessions_total",
                                         "Stream status codes per service/route in Kong",
-                                        {"service", "route", "code", "source"})
+                                        {"service", "route", "code", "source", "workspace"})
   end
   metrics.kong_latency = prometheus:histogram("kong_latency_ms",
                                               "Latency added by Kong and enabled plugins " ..
                                               "for each service/route in Kong",
-                                              {"service", "route"},
+                                              {"service", "route", "workspace"},
                                               KONG_LATENCY_BUCKETS)
   metrics.upstream_latency = prometheus:histogram("upstream_latency_ms",
                                                   "Latency added by upstream response " ..
                                                   "for each service/route in Kong",
-                                                  {"service", "route"},
+                                                  {"service", "route", "workspace"},
                                                   UPSTREAM_LATENCY_BUCKETS)
 
 
   if http_subsystem then
     metrics.total_latency = prometheus:histogram("request_latency_ms",
                                                  "Total latency incurred during requests " ..
                                                  "for each service/route in Kong",
-                                                 {"service", "route"},
+                                                 {"service", "route", "workspace"},
                                                  UPSTREAM_LATENCY_BUCKETS)
   else
     metrics.total_latency = prometheus:histogram("session_duration_ms",
                                                  "latency incurred in stream session " ..
                                                  "for each service/route in Kong",
-                                                 {"service", "route"},
+                                                 {"service", "route", "workspace"},
                                                  UPSTREAM_LATENCY_BUCKETS)
   end
 
   if http_subsystem then
     metrics.bandwidth = prometheus:counter("bandwidth_bytes",
                                           "Total bandwidth (ingress/egress) " ..
                                           "throughput in bytes",
-                                          {"service", "route", "direction", "consumer"})
+                                          {"service", "route", "direction", "workspace","consumer"})
   else -- stream has no consumer
     metrics.bandwidth = prometheus:counter("bandwidth_bytes",
                                           "Total bandwidth (ingress/egress) " ..
                                           "throughput in bytes",
-                                          {"service", "route", "direction"})
+                                          {"service", "route", "direction", "workspace"})
   end
 
   -- XXX EE
@@ -213,9 +213,9 @@ end
 
 -- Since in the prometheus library we create a new table for each diverged label
 -- so putting the "more dynamic" label at the end will save us some memory
-local labels_table_bandwidth = {0, 0, 0, 0}
-local labels_table_status = {0, 0, 0, 0, 0}
-local labels_table_latency = {0, 0}
+local labels_table_bandwidth = {0, 0, 0, 0, 0}
+local labels_table_status = {0, 0, 0, 0, 0, 0}
+local labels_table_latency = {0, 0, 0}
 local upstream_target_addr_health_table = {
   { value = 0, labels = { 0, 0, 0, "healthchecks_off", ngx.config.subsystem } },
   { value = 0, labels = { 0, 0, 0, "healthy", ngx.config.subsystem } },
@@ -263,10 +263,12 @@ local function log(message, serialized)
     consumer = nil -- no consumer in stream
   end
 
+  local workspace = message.workspace_name or ""
   if serialized.ingress_size or serialized.egress_size then
     labels_table_bandwidth[1] = service_name
     labels_table_bandwidth[2] = route_name
-    labels_table_bandwidth[4] = consumer
+    labels_table_bandwidth[4] = workspace
+    labels_table_bandwidth[5] = consumer
 
     local ingress_size = serialized.ingress_size
     if ingress_size and ingress_size > 0 then
@@ -292,14 +294,16 @@ local function log(message, serialized)
       labels_table_status[4] = "kong"
     end
 
-    labels_table_status[5] = consumer
+    labels_table_status[5] = workspace
+    labels_table_status[6] = consumer
 
     metrics.status:inc(1, labels_table_status)
   end
 
   if serialized.latencies then
     labels_table_latency[1] = service_name
     labels_table_latency[2] = route_name
+    labels_table_latency[3] = workspace
 
     if http_subsystem then
       local request_latency = serialized.latencies.request

spec/03-plugins/26-prometheus/02-access_spec.lua

@@ -123,7 +123,7 @@ describe("Plugin: prometheus (access)", function()
       local body = assert.res_status(200, res)
       assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
 
-      return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer=""} 1', nil, true)
+      return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true)
     end)
 
     res = assert(proxy_client:send {
@@ -143,7 +143,7 @@ describe("Plugin: prometheus (access)", function()
       local body = assert.res_status(200, res)
       assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
 
-      return body:find('http_requests_total{service="mock-service",route="http-route",code="400",source="service",consumer=""} 1', nil, true)
+      return body:find('http_requests_total{service="mock-service",route="http-route",code="400",source="service",workspace="default",consumer=""} 1', nil, true)
     end)
   end)
 
@@ -168,7 +168,7 @@ describe("Plugin: prometheus (access)", function()
       local body = assert.res_status(200, res)
       assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
 
-      return body:find('http_requests_total{service="mock-grpc-service",route="grpc-route",code="200",source="service",consumer=""} 1', nil, true)
+      return body:find('http_requests_total{service="mock-grpc-service",route="grpc-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true)
     end)
 
     ok, resp = proxy_client_grpcs({
@@ -191,7 +191,7 @@ describe("Plugin: prometheus (access)", function()
       local body = assert.res_status(200, res)
       assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
 
-      return body:find('http_requests_total{service="mock-grpcs-service",route="grpcs-route",code="200",source="service",consumer=""} 1', nil, true)
+      return body:find('http_requests_total{service="mock-grpcs-service",route="grpcs-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true)
     end)
   end)
 
@@ -213,10 +213,10 @@ describe("Plugin: prometheus (access)", function()
       })
       local body = assert.res_status(200, res)
       assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
-      assert.matches('kong_stream_sessions_total{service="tcp-service",route="tcp-route",code="200",source="service"} 1', body, nil, true)
-      assert.matches('kong_session_duration_ms_bucket{service="tcp%-service",route="tcp%-route",le="%+Inf"} %d+', body)
+      assert.matches('kong_stream_sessions_total{service="tcp-service",route="tcp-route",code="200",source="service",workspace="default"} 1', body, nil, true)
+      assert.matches('kong_session_duration_ms_bucket{service="tcp%-service",route="tcp%-route",workspace="default",le="%+Inf"} %d+', body)
 
-      return body:find('kong_stream_sessions_total{service="tcp-service",route="tcp-route",code="200",source="service"} 1', nil, true)
+      return body:find('kong_stream_sessions_total{service="tcp-service",route="tcp-route",code="200",source="service",workspace="default"} 1', nil, true)
     end)
 
     thread:join()
@@ -463,7 +463,7 @@ describe("Plugin: prometheus (access) per-consumer metrics", function()
       local body = assert.res_status(200, res)
       assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
 
-      return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer="alice"} 1', nil, true)
+      return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer="alice"} 1', nil, true)
     end)
 
     res = assert(proxy_client:send {
@@ -484,7 +484,7 @@ describe("Plugin: prometheus (access) per-consumer metrics", function()
       local body = assert.res_status(200, res)
       assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
 
-      return body:find('http_requests_total{service="mock-service",route="http-route",code="400",source="service",consumer="alice"} 1', nil, true)
+      return body:find('http_requests_total{service="mock-service",route="http-route",code="400",source="service",workspace="default",consumer="alice"} 1', nil, true)
     end)
   end)
 
@@ -505,10 +505,10 @@ describe("Plugin: prometheus (access) per-consumer metrics", function()
         path    = "/metrics",
       })
       body = assert.res_status(200, res)
-      return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer="alice"} 1', nil, true)
+      return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer="alice"} 1', nil, true)
     end)
 
-    assert.matches('http_requests_total{service="mock-service",route="http-route",code="401",source="kong",consumer=""} 1', body, nil, true)
+    assert.matches('http_requests_total{service="mock-service",route="http-route",code="401",source="kong",workspace="default",consumer=""} 1', body, nil, true)
 
     assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
   end)

spec/03-plugins/26-prometheus/03-custom-serve_spec.lua

@@ -70,7 +70,7 @@ describe("Plugin: prometheus (custom server)",function()
         path    = "/metrics",
       })
       local body = assert.res_status(200, res)
-      assert.matches('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer=""} 1', body, nil, true)
+      assert.matches('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer=""} 1', body, nil, true)
     end)
     it("custom port returns 404 for anything other than /metrics", function()
       local client = helpers.http_client("127.0.0.1", 9542)

spec/03-plugins/26-prometheus/04-status_api_spec.lua

@@ -207,7 +207,7 @@ describe("Plugin: prometheus (access via status API)", function()
 
     helpers.wait_until(function()
       local body = get_metrics()
-      return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer=""} 1', nil, true)
+      return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true)
     end)
 
     res = assert(proxy_client:send {
@@ -220,14 +220,14 @@ describe("Plugin: prometheus (access via status API)", function()
     assert.res_status(400, res)
     local body = get_metrics()
 
-    assert.matches('kong_kong_latency_ms_bucket{service="mock%-service",route="http%-route",le="%+Inf"} +%d', body)
-    assert.matches('kong_upstream_latency_ms_bucket{service="mock%-service",route="http%-route",le="%+Inf"} +%d', body)
-    assert.matches('kong_request_latency_ms_bucket{service="mock%-service",route="http%-route",le="%+Inf"} +%d', body)
+    assert.matches('kong_kong_latency_ms_bucket{service="mock%-service",route="http%-route",workspace="default",le="%+Inf"} +%d', body)
+    assert.matches('kong_upstream_latency_ms_bucket{service="mock%-service",route="http%-route",workspace="default",le="%+Inf"} +%d', body)
+    assert.matches('kong_request_latency_ms_bucket{service="mock%-service",route="http%-route",workspace="default",le="%+Inf"} +%d', body)
 
-    assert.matches('http_requests_total{service="mock-service",route="http-route",code="400",source="service",consumer=""} 1', body, nil, true)
-    assert.matches('kong_bandwidth_bytes{service="mock%-service",route="http%-route",direction="ingress",consumer=""} %d+', body)
+    assert.matches('http_requests_total{service="mock-service",route="http-route",code="400",source="service",workspace="default",consumer=""} 1', body, nil, true)
+    assert.matches('kong_bandwidth_bytes{service="mock%-service",route="http%-route",direction="ingress",workspace="default",consumer=""} %d+', body)
 
-    assert.matches('kong_bandwidth_bytes{service="mock%-service",route="http%-route",direction="egress",consumer=""} %d+', body)
+    assert.matches('kong_bandwidth_bytes{service="mock%-service",route="http%-route",direction="egress",workspace="default",consumer=""} %d+', body)
   end)
 
   it("increments the count for proxied grpc requests", function()
@@ -245,7 +245,7 @@ describe("Plugin: prometheus (access via status API)", function()
 
     helpers.wait_until(function()
       local body = get_metrics()
-      return body:find('http_requests_total{service="mock-grpc-service",route="grpc-route",code="200",source="service",consumer=""} 1', nil, true)
+      return body:find('http_requests_total{service="mock-grpc-service",route="grpc-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true)
     end)
 
     ok, resp = proxy_client_grpcs({
@@ -262,7 +262,7 @@ describe("Plugin: prometheus (access via status API)", function()
 
     helpers.wait_until(function()
       local body = get_metrics()
-      return body:find('http_requests_total{service="mock-grpcs-service",route="grpcs-route",code="200",source="service",consumer=""} 1', nil, true)
+      return body:find('http_requests_total{service="mock-grpcs-service",route="grpcs-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true)
     end)
   end)
 

spec/03-plugins/26-prometheus/05-metrics_spec.lua

@@ -165,7 +165,7 @@ for _, strategy in helpers.each_strategy() do
 
         assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
 
-        return body:find('http_requests_total{service="mock-ssl-service",route="mock-ssl-route",code="400",source="service",consumer=""} 1',
+        return body:find('http_requests_total{service="mock-ssl-service",route="mock-ssl-route",code="400",source="service",workspace="default",consumer=""} 1',
           nil, true)
       end)
     end)
@@ -212,7 +212,7 @@ for _, strategy in helpers.each_strategy() do
 
         assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
 
-        return body:find('kong_http_requests_total{service="",route="serverless",code="200",source="kong",consumer=""} 1',
+        return body:find('kong_http_requests_total{service="",route="serverless",code="200",source="kong",workspace="default",consumer=""} 1',
           nil, true)
       end)
     end)