From da87c4279b4d45078d5ea511050b4e4f1dd47122 Mon Sep 17 00:00:00 2001 From: Jun Ouyang Date: Thu, 25 Apr 2024 02:17:01 +0800 Subject: [PATCH] feat(prometheus): add workspace label to the real time metrics (#12836) Co-authored-by: Zachary Hu <6426329+outsinre@users.noreply.github.com> --- ...feat-add-workspace-label-to-prometheus.yml | 4 +++ kong/pdk/log.lua | 7 +++++ kong/plugins/prometheus/exporter.lua | 30 +++++++++++-------- .../26-prometheus/02-access_spec.lua | 22 +++++++------- .../26-prometheus/03-custom-serve_spec.lua | 2 +- .../26-prometheus/04-status_api_spec.lua | 18 +++++------ .../26-prometheus/05-metrics_spec.lua | 4 +-- 7 files changed, 51 insertions(+), 36 deletions(-) create mode 100644 changelog/unreleased/kong/feat-add-workspace-label-to-prometheus.yml diff --git a/changelog/unreleased/kong/feat-add-workspace-label-to-prometheus.yml b/changelog/unreleased/kong/feat-add-workspace-label-to-prometheus.yml new file mode 100644 index 000000000000..9760df0f9e7e --- /dev/null +++ b/changelog/unreleased/kong/feat-add-workspace-label-to-prometheus.yml @@ -0,0 +1,4 @@ +message: | + **Prometheus**: Added workspace label to Prometheus plugin metrics. +type: feature +scope: Plugin diff --git a/kong/pdk/log.lua b/kong/pdk/log.lua index 531abedb4305..5b98c18bef37 100644 --- a/kong/pdk/log.lua +++ b/kong/pdk/log.lua @@ -19,6 +19,7 @@ local phase_checker = require "kong.pdk.private.phases" local utils = require "kong.tools.utils" local cycle_aware_deep_copy = utils.cycle_aware_deep_copy local constants = require "kong.constants" +local workspace = require "kong.workspaces" local sub = string.sub local type = type @@ -861,6 +862,9 @@ do client_ip = var.remote_addr, started_at = okong.request.get_start_time(), source = response_source_name, + + workspace = ctx.workspace, + workspace_name = workspace.get_workspace_name(), } return edit_result(ctx, root) @@ -902,6 +906,9 @@ do consumer = cycle_aware_deep_copy(ctx.authenticated_consumer), client_ip = var.remote_addr, started_at = okong.request.get_start_time(), + + workspace = ctx.workspace, + workspace_name = workspace.get_workspace_name(), } return edit_result(ctx, root) diff --git a/kong/plugins/prometheus/exporter.lua b/kong/plugins/prometheus/exporter.lua index 02eb4ba3e969..d94d9a08e14b 100644 --- a/kong/plugins/prometheus/exporter.lua +++ b/kong/plugins/prometheus/exporter.lua @@ -101,21 +101,21 @@ local function init() if http_subsystem then metrics.status = prometheus:counter("http_requests_total", "HTTP status codes per consumer/service/route in Kong", - {"service", "route", "code", "source", "consumer"}) + {"service", "route", "code", "source", "workspace", "consumer"}) else metrics.status = prometheus:counter("stream_sessions_total", "Stream status codes per service/route in Kong", - {"service", "route", "code", "source"}) + {"service", "route", "code", "source", "workspace"}) end metrics.kong_latency = prometheus:histogram("kong_latency_ms", "Latency added by Kong and enabled plugins " .. "for each service/route in Kong", - {"service", "route"}, + {"service", "route", "workspace"}, KONG_LATENCY_BUCKETS) metrics.upstream_latency = prometheus:histogram("upstream_latency_ms", "Latency added by upstream response " .. "for each service/route in Kong", - {"service", "route"}, + {"service", "route", "workspace"}, UPSTREAM_LATENCY_BUCKETS) @@ -123,13 +123,13 @@ local function init() metrics.total_latency = prometheus:histogram("request_latency_ms", "Total latency incurred during requests " .. "for each service/route in Kong", - {"service", "route"}, + {"service", "route", "workspace"}, UPSTREAM_LATENCY_BUCKETS) else metrics.total_latency = prometheus:histogram("session_duration_ms", "latency incurred in stream session " .. "for each service/route in Kong", - {"service", "route"}, + {"service", "route", "workspace"}, UPSTREAM_LATENCY_BUCKETS) end @@ -137,12 +137,12 @@ local function init() metrics.bandwidth = prometheus:counter("bandwidth_bytes", "Total bandwidth (ingress/egress) " .. "throughput in bytes", - {"service", "route", "direction", "consumer"}) + {"service", "route", "direction", "workspace","consumer"}) else -- stream has no consumer metrics.bandwidth = prometheus:counter("bandwidth_bytes", "Total bandwidth (ingress/egress) " .. "throughput in bytes", - {"service", "route", "direction"}) + {"service", "route", "direction", "workspace"}) end -- Hybrid mode status @@ -198,9 +198,9 @@ end -- Since in the prometheus library we create a new table for each diverged label -- so putting the "more dynamic" label at the end will save us some memory -local labels_table_bandwidth = {0, 0, 0, 0} -local labels_table_status = {0, 0, 0, 0, 0} -local labels_table_latency = {0, 0} +local labels_table_bandwidth = {0, 0, 0, 0, 0} +local labels_table_status = {0, 0, 0, 0, 0, 0} +local labels_table_latency = {0, 0, 0} local upstream_target_addr_health_table = { { value = 0, labels = { 0, 0, 0, "healthchecks_off", ngx.config.subsystem } }, { value = 0, labels = { 0, 0, 0, "healthy", ngx.config.subsystem } }, @@ -248,10 +248,12 @@ local function log(message, serialized) consumer = nil -- no consumer in stream end + local workspace = message.workspace_name or "" if serialized.ingress_size or serialized.egress_size then labels_table_bandwidth[1] = service_name labels_table_bandwidth[2] = route_name - labels_table_bandwidth[4] = consumer + labels_table_bandwidth[4] = workspace + labels_table_bandwidth[5] = consumer local ingress_size = serialized.ingress_size if ingress_size and ingress_size > 0 then @@ -277,7 +279,8 @@ local function log(message, serialized) labels_table_status[4] = "kong" end - labels_table_status[5] = consumer + labels_table_status[5] = workspace + labels_table_status[6] = consumer metrics.status:inc(1, labels_table_status) end @@ -285,6 +288,7 @@ local function log(message, serialized) if serialized.latencies then labels_table_latency[1] = service_name labels_table_latency[2] = route_name + labels_table_latency[3] = workspace if http_subsystem then local request_latency = serialized.latencies.request diff --git a/spec/03-plugins/26-prometheus/02-access_spec.lua b/spec/03-plugins/26-prometheus/02-access_spec.lua index 36cd7933f554..f1478b558383 100644 --- a/spec/03-plugins/26-prometheus/02-access_spec.lua +++ b/spec/03-plugins/26-prometheus/02-access_spec.lua @@ -116,7 +116,7 @@ describe("Plugin: prometheus (access)", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer=""} 1', nil, true) + return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true) end) res = assert(proxy_client:send { @@ -136,7 +136,7 @@ describe("Plugin: prometheus (access)", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('http_requests_total{service="mock-service",route="http-route",code="400",source="service",consumer=""} 1', nil, true) + return body:find('http_requests_total{service="mock-service",route="http-route",code="400",source="service",workspace="default",consumer=""} 1', nil, true) end) end) @@ -161,7 +161,7 @@ describe("Plugin: prometheus (access)", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('http_requests_total{service="mock-grpc-service",route="grpc-route",code="200",source="service",consumer=""} 1', nil, true) + return body:find('http_requests_total{service="mock-grpc-service",route="grpc-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true) end) ok, resp = proxy_client_grpcs({ @@ -184,7 +184,7 @@ describe("Plugin: prometheus (access)", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('http_requests_total{service="mock-grpcs-service",route="grpcs-route",code="200",source="service",consumer=""} 1', nil, true) + return body:find('http_requests_total{service="mock-grpcs-service",route="grpcs-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true) end) end) @@ -206,10 +206,10 @@ describe("Plugin: prometheus (access)", function() }) local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - assert.matches('kong_stream_sessions_total{service="tcp-service",route="tcp-route",code="200",source="service"} 1', body, nil, true) - assert.matches('kong_session_duration_ms_bucket{service="tcp%-service",route="tcp%-route",le="%+Inf"} %d+', body) + assert.matches('kong_stream_sessions_total{service="tcp-service",route="tcp-route",code="200",source="service",workspace="default"} 1', body, nil, true) + assert.matches('kong_session_duration_ms_bucket{service="tcp%-service",route="tcp%-route",workspace="default",le="%+Inf"} %d+', body) - return body:find('kong_stream_sessions_total{service="tcp-service",route="tcp-route",code="200",source="service"} 1', nil, true) + return body:find('kong_stream_sessions_total{service="tcp-service",route="tcp-route",code="200",source="service",workspace="default"} 1', nil, true) end) thread:join() @@ -456,7 +456,7 @@ describe("Plugin: prometheus (access) per-consumer metrics", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer="alice"} 1', nil, true) + return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer="alice"} 1', nil, true) end) res = assert(proxy_client:send { @@ -477,7 +477,7 @@ describe("Plugin: prometheus (access) per-consumer metrics", function() local body = assert.res_status(200, res) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('http_requests_total{service="mock-service",route="http-route",code="400",source="service",consumer="alice"} 1', nil, true) + return body:find('http_requests_total{service="mock-service",route="http-route",code="400",source="service",workspace="default",consumer="alice"} 1', nil, true) end) end) @@ -498,10 +498,10 @@ describe("Plugin: prometheus (access) per-consumer metrics", function() path = "/metrics", }) body = assert.res_status(200, res) - return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer="alice"} 1', nil, true) + return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer="alice"} 1', nil, true) end) - assert.matches('http_requests_total{service="mock-service",route="http-route",code="401",source="kong",consumer=""} 1', body, nil, true) + assert.matches('http_requests_total{service="mock-service",route="http-route",code="401",source="kong",workspace="default",consumer=""} 1', body, nil, true) assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) end) diff --git a/spec/03-plugins/26-prometheus/03-custom-serve_spec.lua b/spec/03-plugins/26-prometheus/03-custom-serve_spec.lua index 357fea2507ae..4cefa165bbc3 100644 --- a/spec/03-plugins/26-prometheus/03-custom-serve_spec.lua +++ b/spec/03-plugins/26-prometheus/03-custom-serve_spec.lua @@ -63,7 +63,7 @@ describe("Plugin: prometheus (custom server)",function() path = "/metrics", }) local body = assert.res_status(200, res) - assert.matches('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer=""} 1', body, nil, true) + assert.matches('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer=""} 1', body, nil, true) end) it("custom port returns 404 for anything other than /metrics", function() local client = helpers.http_client("127.0.0.1", 9542) diff --git a/spec/03-plugins/26-prometheus/04-status_api_spec.lua b/spec/03-plugins/26-prometheus/04-status_api_spec.lua index a837ee39e693..2fec1a089b03 100644 --- a/spec/03-plugins/26-prometheus/04-status_api_spec.lua +++ b/spec/03-plugins/26-prometheus/04-status_api_spec.lua @@ -200,7 +200,7 @@ describe("Plugin: prometheus (access via status API)", function() helpers.wait_until(function() local body = get_metrics() - return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",consumer=""} 1', nil, true) + return body:find('http_requests_total{service="mock-service",route="http-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true) end) res = assert(proxy_client:send { @@ -213,14 +213,14 @@ describe("Plugin: prometheus (access via status API)", function() assert.res_status(400, res) local body = get_metrics() - assert.matches('kong_kong_latency_ms_bucket{service="mock%-service",route="http%-route",le="%+Inf"} +%d', body) - assert.matches('kong_upstream_latency_ms_bucket{service="mock%-service",route="http%-route",le="%+Inf"} +%d', body) - assert.matches('kong_request_latency_ms_bucket{service="mock%-service",route="http%-route",le="%+Inf"} +%d', body) + assert.matches('kong_kong_latency_ms_bucket{service="mock%-service",route="http%-route",workspace="default",le="%+Inf"} +%d', body) + assert.matches('kong_upstream_latency_ms_bucket{service="mock%-service",route="http%-route",workspace="default",le="%+Inf"} +%d', body) + assert.matches('kong_request_latency_ms_bucket{service="mock%-service",route="http%-route",workspace="default",le="%+Inf"} +%d', body) - assert.matches('http_requests_total{service="mock-service",route="http-route",code="400",source="service",consumer=""} 1', body, nil, true) - assert.matches('kong_bandwidth_bytes{service="mock%-service",route="http%-route",direction="ingress",consumer=""} %d+', body) + assert.matches('http_requests_total{service="mock-service",route="http-route",code="400",source="service",workspace="default",consumer=""} 1', body, nil, true) + assert.matches('kong_bandwidth_bytes{service="mock%-service",route="http%-route",direction="ingress",workspace="default",consumer=""} %d+', body) - assert.matches('kong_bandwidth_bytes{service="mock%-service",route="http%-route",direction="egress",consumer=""} %d+', body) + assert.matches('kong_bandwidth_bytes{service="mock%-service",route="http%-route",direction="egress",workspace="default",consumer=""} %d+', body) end) it("increments the count for proxied grpc requests", function() @@ -238,7 +238,7 @@ describe("Plugin: prometheus (access via status API)", function() helpers.wait_until(function() local body = get_metrics() - return body:find('http_requests_total{service="mock-grpc-service",route="grpc-route",code="200",source="service",consumer=""} 1', nil, true) + return body:find('http_requests_total{service="mock-grpc-service",route="grpc-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true) end) ok, resp = proxy_client_grpcs({ @@ -255,7 +255,7 @@ describe("Plugin: prometheus (access via status API)", function() helpers.wait_until(function() local body = get_metrics() - return body:find('http_requests_total{service="mock-grpcs-service",route="grpcs-route",code="200",source="service",consumer=""} 1', nil, true) + return body:find('http_requests_total{service="mock-grpcs-service",route="grpcs-route",code="200",source="service",workspace="default",consumer=""} 1', nil, true) end) end) diff --git a/spec/03-plugins/26-prometheus/05-metrics_spec.lua b/spec/03-plugins/26-prometheus/05-metrics_spec.lua index a6d56b808b01..69e506aa80d0 100644 --- a/spec/03-plugins/26-prometheus/05-metrics_spec.lua +++ b/spec/03-plugins/26-prometheus/05-metrics_spec.lua @@ -158,7 +158,7 @@ for _, strategy in helpers.each_strategy() do assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('http_requests_total{service="mock-ssl-service",route="mock-ssl-route",code="400",source="service",consumer=""} 1', + return body:find('http_requests_total{service="mock-ssl-service",route="mock-ssl-route",code="400",source="service",workspace="default",consumer=""} 1', nil, true) end) end) @@ -205,7 +205,7 @@ for _, strategy in helpers.each_strategy() do assert.matches('kong_nginx_metric_errors_total 0', body, nil, true) - return body:find('kong_http_requests_total{service="",route="serverless",code="200",source="kong",consumer=""} 1', + return body:find('kong_http_requests_total{service="",route="serverless",code="200",source="kong",workspace="default",consumer=""} 1', nil, true) end) end)