diff --git a/Makefile b/Makefile
index 25af2978..26b56327 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-VERSION=1.14.2
+VERSION=1.15.0-b1
default: versioncheck
diff --git a/README.md b/README.md
index 6dbd8e46..2fc9f2a5 100644
--- a/README.md
+++ b/README.md
@@ -112,8 +112,8 @@ scrape_configs:
The docker images are available via:
```bash
-docker pull pambrose/prometheus-proxy:1.14.2
-docker pull pambrose/prometheus-agent:1.14.2
+docker pull pambrose/prometheus-proxy:1.15.0-b1
+docker pull pambrose/prometheus-agent:1.15.0-b1
```
Start a proxy container with:
@@ -122,7 +122,7 @@ Start a proxy container with:
docker run --rm -p 8082:8082 -p 8092:8092 -p 50051:50051 -p 8080:8080 \
--env ADMIN_ENABLED=true \
--env METRICS_ENABLED=true \
- pambrose/prometheus-proxy:1.14.2
+ pambrose/prometheus-proxy:1.15.0-b1
```
Start an agent container with:
@@ -130,7 +130,7 @@ Start an agent container with:
```bash
docker run --rm -p 8083:8083 -p 8093:8093 \
--env AGENT_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \
- pambrose/prometheus-agent:1.14.2
+ pambrose/prometheus-agent:1.15.0-b1
```
Using the config
@@ -148,7 +148,7 @@ is in your current directory, run an agent container with:
docker run --rm -p 8083:8083 -p 8093:8093 \
--mount type=bind,source="$(pwd)"/prom-agent.conf,target=/app/prom-agent.conf \
--env AGENT_CONFIG=prom-agent.conf \
- pambrose/prometheus-agent:1.14.2
+ pambrose/prometheus-agent:1.15.0-b1
```
**Note:** The `WORKDIR` of the proxy and agent images is `/app`, so make sure to use `/app` as the base directory in the
@@ -193,6 +193,7 @@ argument is an agent config value, which should have an `agent.pathConfigs` valu
| --sd_enabled | SD_ENABLED
proxy.service.discovery.enabled | false | Service discovery endpoint enabled |
| --sd_path | SD_PATH
proxy.service.discovery.path | "discovery" | Service discovery endpoint path |
| --sd_target_prefix | SD_TARGET_PREFIX
proxy.service.discovery.targetPrefix | "http://localhost:8080/" | Service discovery target prefix |
+| --tf-disabled | TRANSPORT_FILTER_DISABLED
proxy.transportFilterDisabled | false | Transport filter disabled |
| --cert, -t | CERT_CHAIN_FILE_PATH
proxy.tls.certChainFilePath | | Certificate chain file path |
| --key, -k | PRIVATE_KEY_FILE_PATH
proxy.tls.privateKeyFilePath | | Private key file path |
| --trust, -s | TRUST_CERT_COLLECTION_FILE_PATH
proxy.tls.trustCertCollectionFilePath | | Trust certificate collection file path |
@@ -217,6 +218,7 @@ argument is an agent config value, which should have an `agent.pathConfigs` valu
| --max_retries | SCRAPE_MAX_RETRIES
agent.scrapeMaxRetries | 0 | Scrape maximum retries (0 disables scrape retries) |
| --chunk | CHUNK_CONTENT_SIZE_KBS
agent.chunkContentSizeKbs | 32 | Threshold for chunking data to Proxy and buffer size (KBs) |
| --gzip | MIN_GZIP_SIZE_BYTES
agent.minGzipSizeBytes | 1024 | Minimum size for content to be gzipped (bytes) |
+| --tf-disabled | TRANSPORT_FILTER_DISABLED
proxy.transportFilterDisabled | false | Transport filter disabled |
| --trust_all_x509 | TRUST_ALL_X509_CERTIFICATES
agent.http.enableTrustAllX509Certificates | false | Disable SSL verification for agent https endpoints |
| --cert, -t | CERT_CHAIN_FILE_PATH
agent.tls.certChainFilePath | | Certificate chain file path |
| --key, -k | PRIVATE_KEY_FILE_PATH
agent.tls.privateKeyFilePath | | Private key file path |
@@ -294,7 +296,7 @@ docker run --rm -p 8082:8082 -p 8092:8092 -p 50440:50440 -p 8080:8080 \
--env PROXY_CONFIG=tls-no-mutual-auth.conf \
--env ADMIN_ENABLED=true \
--env METRICS_ENABLED=true \
- pambrose/prometheus-proxy:1.14.2
+ pambrose/prometheus-proxy:1.15.0-b1
docker run --rm -p 8083:8083 -p 8093:8093 \
--mount type=bind,source="$(pwd)"/testing/certs,target=/app/testing/certs \
@@ -302,7 +304,7 @@ docker run --rm -p 8083:8083 -p 8093:8093 \
--env AGENT_CONFIG=tls-no-mutual-auth.conf \
--env PROXY_HOSTNAME=mymachine.lan:50440 \
--name docker-agent \
- pambrose/prometheus-agent:1.14.2
+ pambrose/prometheus-agent:1.15.0-b1
```
**Note:** The `WORKDIR` of the proxy and agent images is `/app`, so make sure to use `/app` as the base directory in the
@@ -321,15 +323,26 @@ This enables using the existing service discovery features already built into Pr
An example config can be found in
[federate.conf](https://github.com/pambrose/prometheus-proxy/blob/master/examples/federate.conf).
+## Nginx Support
+
+To use the prometheus_proxy with nginx as a reverse proxy, disable the transport filter with the
+`TRANSPORT_FILTER_DISABLED` environment var, the `--tf-disabled` CLI option, or the `agent.transportFilterDisabled`/
+`proxy.transportFilterDisabled` properties. Agents and the Proxy must run with the same `transporFilterDisabled` value.
+
+When using `transporFilterDisabled`, you will not see agent contexts immediately removed
+from the proxy when agents are terminated. Instead, agent contexts will be removed from the proxy
+after they age out from inactivity. The maximum age is controlled by the `proxy.internal.maxAgentInactivitySecs` value.
+The default value is 1 minute.
+
+An example nginx conf file is [here](https://github.com/pambrose/prometheus-proxy/tree/master/nginx/docker/nginx.conf)
+and an example agent/proxy conf file
+is [here](https://github.com/pambrose/prometheus-proxy/tree/master/nginx/nginx-proxy.conf)
+
## Grafana
[Grafana](https://grafana.com) dashboards for the proxy and agent
are [here](https://github.com/pambrose/prometheus-proxy/tree/master/grafana).
-## Nginx Support
-
-Nginx does not work as a reverse proxy for prometheus-proxy.
-
## Related Links
* [Prometheus.io](http://prometheus.io)
diff --git a/bin/docker-agent.sh b/bin/docker-agent.sh
index 8d7cd131..cbb01de5 100755
--- a/bin/docker-agent.sh
+++ b/bin/docker-agent.sh
@@ -3,4 +3,4 @@
docker run --rm -p 8083:8083 -p 8093:8093 \
--env AGENT_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \
--env PROXY_HOSTNAME=mymachine.lan \
- pambrose/prometheus-agent:1.14.2
+ pambrose/prometheus-agent:1.15.0-b1
diff --git a/bin/docker-proxy.sh b/bin/docker-proxy.sh
index b884ec6d..e245f8ce 100755
--- a/bin/docker-proxy.sh
+++ b/bin/docker-proxy.sh
@@ -2,4 +2,4 @@
docker run --rm -p 8082:8082 -p 8092:8092 -p 50051:50051 -p 8080:8080 \
--env PROXY_CONFIG='https://raw.githubusercontent.com/pambrose/prometheus-proxy/master/examples/simple.conf' \
- pambrose/prometheus-proxy:1.14.2
+ pambrose/prometheus-proxy:1.15.0-b1
diff --git a/build.gradle b/build.gradle
index 3da8bc0a..298939ef 100644
--- a/build.gradle
+++ b/build.gradle
@@ -15,13 +15,20 @@ plugins {
}
group = 'io.prometheus'
-version = '1.14.2'
+version = '1.15.0-b1'
sourceCompatibility = 11
targetCompatibility = 11
+buildConfig {
+ packageName("io.prometheus")
+
+ buildConfigField('String', 'APP_NAME', "\"${project.name}\"")
+ buildConfigField('String', 'APP_VERSION', "\"${project.version}\"")
+ buildConfigField('String', 'APP_RELEASE_DATE', "\"11/29/22\"")
+}
+
repositories {
- //maven { url "https://maven.pkg.jetbrains.space/public/p/ktor/eap" }
google()
mavenCentral()
maven { url = 'https://jitpack.io' }
@@ -85,14 +92,6 @@ dependencies {
testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine:$junit_version"
}
-buildConfig {
- packageName("io.prometheus")
-
- buildConfigField('String', 'APP_NAME', "\"${project.name}\"")
- buildConfigField('String', 'APP_VERSION', "\"${project.version}\"")
- buildConfigField('String', 'APP_RELEASE_DATE', "\"11/26/22\"")
-}
-
publishing {
publications {
mavenJava(MavenPublication) {
diff --git a/etc/compose/proxy.yml b/etc/compose/proxy.yml
index 6da760f0..d5e226d7 100644
--- a/etc/compose/proxy.yml
+++ b/etc/compose/proxy.yml
@@ -1,6 +1,6 @@
prometheus-proxy:
autoredeploy: true
- image: 'pambrose/prometheus-proxy:1.14.2'
+ image: 'pambrose/prometheus-proxy:1.15.0-b1'
ports:
- '8080:8080'
- '8082:8082'
diff --git a/etc/config/config.conf b/etc/config/config.conf
index f695122b..0c0f5952 100644
--- a/etc/config/config.conf
+++ b/etc/config/config.conf
@@ -3,6 +3,8 @@ proxy {
agent.port = 50051 // Listen port for agent connections
+ transportFilterDisabled = false // Assign to true if using nginx as a reverse proxy
+
service.discovery {
enabled = false // Enable service discovery
path = "discovery" // Service discovery path
@@ -83,6 +85,8 @@ proxy {
agent {
name = "" // Agent name used in metrics reporting
+ transportFilterDisabled = false // Assign to true if using nginx as a reverse proxy
+
consolidated = false
// See: https://github.com/grpc/grpc.github.io/issues/371
diff --git a/nginx/docker/Dockerfile b/nginx/docker/Dockerfile
new file mode 100644
index 00000000..6619805f
--- /dev/null
+++ b/nginx/docker/Dockerfile
@@ -0,0 +1,4 @@
+FROM nginx
+COPY ./nginx.conf /etc/nginx/conf.d/default.conf
+
+EXPOSE 50440
diff --git a/nginx/docker/nginx.conf b/nginx/docker/nginx.conf
new file mode 100644
index 00000000..3c57744a
--- /dev/null
+++ b/nginx/docker/nginx.conf
@@ -0,0 +1,16 @@
+server {
+ # 50440 is the agent.proxy.port value
+ listen 50440 http2;
+
+ # Prevent nginx from closing the gRPC connections (not working for me)
+ # https://stackoverflow.com/questions/67430437/grpc-send-timeout-doesnt-work-nginx-closes-grpc-streams-unexpectedly
+ client_header_timeout 1d;
+ client_body_timeout 1d;
+
+ location / {
+ # The nginx gRPX options: https://nginx.org/en/docs/http/ngx_http_grpc_module.html
+ # 50051 is the proxy.agent.port value
+ grpc_pass grpc://alta.lan:50051;
+ grpc_socket_keepalive on;
+ }
+}
diff --git a/nginx/docker/run.sh b/nginx/docker/run.sh
new file mode 100755
index 00000000..95dc9976
--- /dev/null
+++ b/nginx/docker/run.sh
@@ -0,0 +1,2 @@
+docker build -t pambrose/nginx2 .
+docker run --rm -p 50440:50440 pambrose/nginx2
\ No newline at end of file
diff --git a/nginx/nginx-proxy.conf b/nginx/nginx-proxy.conf
new file mode 100644
index 00000000..611b8e19
--- /dev/null
+++ b/nginx/nginx-proxy.conf
@@ -0,0 +1,32 @@
+proxy {
+ # Required for use with nginx reverse proxy
+ transportFilterDisabled = true
+}
+
+agent {
+ # Required for use with nginx reverse proxy
+ transportFilterDisabled = true
+
+ proxy {
+ # nginx http2 port specified in nginx.conf
+ port = 50440
+ }
+
+ pathConfigs: [
+ {
+ name: "App1 metrics"
+ path: app1_metrics
+ url: "http://localhost:8082/metrics"
+ },
+ {
+ name: "App2 metrics"
+ path: app2_metrics
+ url: "http://app2.local:9100/metrics"
+ },
+ {
+ name: "App3 metrics"
+ path: app3_metrics
+ url: "http://app3.local:9100/metrics"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/src/main/java/io/prometheus/common/ConfigVals.java b/src/main/java/io/prometheus/common/ConfigVals.java
index f695acc0..be536256 100644
--- a/src/main/java/io/prometheus/common/ConfigVals.java
+++ b/src/main/java/io/prometheus/common/ConfigVals.java
@@ -1,4 +1,4 @@
-// generated by tscfg 0.9.997 on Thu Oct 13 20:14:33 EDT 2022
+// generated by tscfg 0.9.997 on Mon Nov 28 23:31:57 PST 2022
// source: etc/config/config.conf
package io.prometheus.common;
@@ -28,6 +28,7 @@ public static class Agent {
public final int scrapeMaxRetries;
public final int scrapeTimeoutSecs;
public final Agent.Tls tls;
+ public final boolean transportFilterDisabled;
public Agent(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) {
this.admin = c.hasPathOrNull("admin") ? new Agent.Admin(c.getConfig("admin"), parentPath + "admin.", $tsCfgValidator) : new Agent.Admin(com.typesafe.config.ConfigFactory.parseString("admin{}"), parentPath + "admin.", $tsCfgValidator);
this.chunkContentSizeKbs = c.hasPathOrNull("chunkContentSizeKbs") ? c.getInt("chunkContentSizeKbs") : 32;
@@ -42,6 +43,7 @@ public Agent(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgVa
this.scrapeMaxRetries = c.hasPathOrNull("scrapeMaxRetries") ? c.getInt("scrapeMaxRetries") : 0;
this.scrapeTimeoutSecs = c.hasPathOrNull("scrapeTimeoutSecs") ? c.getInt("scrapeTimeoutSecs") : 15;
this.tls = c.hasPathOrNull("tls") ? new Agent.Tls(c.getConfig("tls"), parentPath + "tls.", $tsCfgValidator) : new Agent.Tls(com.typesafe.config.ConfigFactory.parseString("tls{}"), parentPath + "tls.", $tsCfgValidator);
+ this.transportFilterDisabled = c.hasPathOrNull("transportFilterDisabled") && c.getBoolean("transportFilterDisabled");
}
private static java.util.List $_LAgent_PathConfigs$Elm(com.typesafe.config.ConfigList cl, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) {
@@ -88,7 +90,6 @@ public static class Internal {
public final int reconnectPauseSecs;
public final int scrapeRequestBacklogUnhealthySize;
public final Internal.Zipkin zipkin;
-
public Internal(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) {
this.cioTimeoutSecs = c.hasPathOrNull("cioTimeoutSecs") ? c.getInt("cioTimeoutSecs") : 90;
this.heartbeatCheckPauseMillis = c.hasPathOrNull("heartbeatCheckPauseMillis") ? c.getInt("heartbeatCheckPauseMillis") : 500;
@@ -208,6 +209,7 @@ public static class Proxy2 {
public final Proxy2.Metrics2 metrics;
public final Proxy2.Service service;
public final Proxy2.Tls2 tls;
+ public final boolean transportFilterDisabled;
public Proxy2(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgValidator $tsCfgValidator) {
this.admin = c.hasPathOrNull("admin") ? new Proxy2.Admin2(c.getConfig("admin"), parentPath + "admin.", $tsCfgValidator) : new Proxy2.Admin2(com.typesafe.config.ConfigFactory.parseString("admin{}"), parentPath + "admin.", $tsCfgValidator);
this.agent = c.hasPathOrNull("agent") ? new Proxy2.Agent2(c.getConfig("agent"), parentPath + "agent.", $tsCfgValidator) : new Proxy2.Agent2(com.typesafe.config.ConfigFactory.parseString("agent{}"), parentPath + "agent.", $tsCfgValidator);
@@ -216,6 +218,7 @@ public Proxy2(com.typesafe.config.Config c, java.lang.String parentPath, $TsCfgV
this.metrics = c.hasPathOrNull("metrics") ? new Proxy2.Metrics2(c.getConfig("metrics"), parentPath + "metrics.", $tsCfgValidator) : new Proxy2.Metrics2(com.typesafe.config.ConfigFactory.parseString("metrics{}"), parentPath + "metrics.", $tsCfgValidator);
this.service = c.hasPathOrNull("service") ? new Proxy2.Service(c.getConfig("service"), parentPath + "service.", $tsCfgValidator) : new Proxy2.Service(com.typesafe.config.ConfigFactory.parseString("service{}"), parentPath + "service.", $tsCfgValidator);
this.tls = c.hasPathOrNull("tls") ? new Proxy2.Tls2(c.getConfig("tls"), parentPath + "tls.", $tsCfgValidator) : new Proxy2.Tls2(com.typesafe.config.ConfigFactory.parseString("tls{}"), parentPath + "tls.", $tsCfgValidator);
+ this.transportFilterDisabled = c.hasPathOrNull("transportFilterDisabled") && c.getBoolean("transportFilterDisabled");
}
public static class Admin2 {
diff --git a/src/main/kotlin/io/prometheus/Agent.kt b/src/main/kotlin/io/prometheus/Agent.kt
index fdfe65fc..0527a1de 100644
--- a/src/main/kotlin/io/prometheus/Agent.kt
+++ b/src/main/kotlin/io/prometheus/Agent.kt
@@ -153,7 +153,7 @@ class Agent(
scrapeRequestBacklogSize.set(0)
lastMsgSentMark = clock.markNow()
- if (grpcService.connectAgent()) {
+ if (grpcService.connectAgent(configVals.agent.transportFilterDisabled)) {
grpcService.registerAgent(initialConnectionLatch)
pathManager.registerPaths()
diff --git a/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt b/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt
index 73a5f6cf..34df0d51 100644
--- a/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt
+++ b/src/main/kotlin/io/prometheus/agent/AgentGrpcService.kt
@@ -25,7 +25,7 @@ import com.github.pambrose.common.util.simpleClassName
import com.github.pambrose.common.utils.TlsContext
import com.github.pambrose.common.utils.TlsContext.Companion.PLAINTEXT_CONTEXT
import com.github.pambrose.common.utils.TlsUtils.buildClientTlsContext
-import com.google.protobuf.Empty
+import io.grpc.ClientInterceptor
import io.grpc.ClientInterceptors
import io.grpc.ManagedChannel
import io.grpc.Status
@@ -34,6 +34,7 @@ import io.prometheus.Agent
import io.prometheus.common.BaseOptions.Companion.HTTPS_PREFIX
import io.prometheus.common.BaseOptions.Companion.HTTP_PREFIX
import io.prometheus.common.GrpcObjects
+import io.prometheus.common.GrpcObjects.EMPTY_INSTANCE
import io.prometheus.common.GrpcObjects.newAgentInfo
import io.prometheus.common.GrpcObjects.newRegisterAgentRequest
import io.prometheus.common.GrpcObjects.newScrapeResponseChunk
@@ -143,15 +144,23 @@ internal class AgentGrpcService(
intercept(grpcTracing.newClientInterceptor())
}
- val interceptors = listOf(AgentClientInterceptor(agent))
+ val interceptors =
+ buildList {
+ if (!options.transportFilterDisabled)
+ add(AgentClientInterceptor(agent))
+ }
stub = ProxyServiceGrpcKt.ProxyServiceCoroutineStub(ClientInterceptors.intercept(channel, interceptors))
}
// If successful, this will create an agentContext on the Proxy and an interceptor will add an agent_id to the headers`
- suspend fun connectAgent() =
+ suspend fun connectAgent(transportFilterDisabled: Boolean) =
try {
logger.info { "Connecting to proxy at ${agent.proxyHost} using ${tlsContext.desc()}..." }
- stub.connectAgent(Empty.getDefaultInstance())
+ if (transportFilterDisabled)
+ stub.connectAgentWithTransportFilterDisabled(EMPTY_INSTANCE).also { agent.agentId = it.agentId }
+ else
+ stub.connectAgent(EMPTY_INSTANCE)
+
logger.info { "Connected to proxy at ${agent.proxyHost} using ${tlsContext.desc()}" }
agent.metrics { connectCount.labels(agent.launchId, "success").inc() }
true
diff --git a/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt b/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt
index e03b46cc..9eaa8a30 100644
--- a/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt
+++ b/src/main/kotlin/io/prometheus/agent/AgentHttpService.kt
@@ -33,7 +33,6 @@ import io.ktor.client.plugins.auth.providers.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
import io.ktor.http.*
-import io.ktor.http.HttpStatusCode.Companion.NotFound
import io.ktor.network.sockets.*
import io.prometheus.Agent
import io.prometheus.common.ScrapeResults
@@ -109,7 +108,7 @@ internal class AgentHttpService(val agent: Agent) {
else {
retryOnException(maxRetries)
retryIf(maxRetries) { _, response ->
- !response.status.isSuccess() && response.status != NotFound
+ !response.status.isSuccess() && response.status != HttpStatusCode.NotFound
}
modifyRequest { it.headers.append("x-retry-count", retryCount.toString()) }
exponentialDelay()
diff --git a/src/main/kotlin/io/prometheus/agent/AgentOptions.kt b/src/main/kotlin/io/prometheus/agent/AgentOptions.kt
index 55ff82cf..bcce54f8 100644
--- a/src/main/kotlin/io/prometheus/agent/AgentOptions.kt
+++ b/src/main/kotlin/io/prometheus/agent/AgentOptions.kt
@@ -128,14 +128,17 @@ class AgentOptions(argv: Array, exitOnMissingConfig: Boolean) :
assignAdminPort(agentConfigVals.admin.port)
assignMetricsEnabled(agentConfigVals.metrics.enabled)
assignMetricsPort(agentConfigVals.metrics.port)
+ assignTransportFilterDisabled(agentConfigVals.transportFilterDisabled)
assignDebugEnabled(agentConfigVals.admin.debugEnabled)
assignCertChainFilePath(agentConfigVals.tls.certChainFilePath)
assignPrivateKeyFilePath(agentConfigVals.tls.privateKeyFilePath)
assignTrustCertCollectionFilePath(agentConfigVals.tls.trustCertCollectionFilePath)
- logger.info { "agent.internal.cioTimeoutSecs: ${agentConfigVals.internal.cioTimeoutSecs.seconds}" }
logger.info { "agent.scrapeTimeoutSecs: ${agentConfigVals.scrapeTimeoutSecs.seconds}" }
+ logger.info { "agent.internal.cioTimeoutSecs: ${agentConfigVals.internal.cioTimeoutSecs.seconds}" }
+ logger.info { "agent.internal.heartbeatCheckPauseMillis: ${agentConfigVals.internal.heartbeatCheckPauseMillis}" }
+ logger.info { "agent.internal.heartbeatMaxInactivitySecs: ${agentConfigVals.internal.heartbeatMaxInactivitySecs}" }
}
}
diff --git a/src/main/kotlin/io/prometheus/common/BaseOptions.kt b/src/main/kotlin/io/prometheus/common/BaseOptions.kt
index 1f950c56..88b54f89 100644
--- a/src/main/kotlin/io/prometheus/common/BaseOptions.kt
+++ b/src/main/kotlin/io/prometheus/common/BaseOptions.kt
@@ -28,6 +28,7 @@ import com.typesafe.config.ConfigFactory
import com.typesafe.config.ConfigParseOptions
import com.typesafe.config.ConfigResolveOptions
import com.typesafe.config.ConfigSyntax
+import io.prometheus.common.EnvVars.*
import mu.KLogging
import java.io.File
import java.io.FileNotFoundException
@@ -67,6 +68,10 @@ abstract class BaseOptions protected constructor(
var debugEnabled = false
private set
+ @Parameter(names = ["--tf-disabled"], description = "Transport filter disabled")
+ var transportFilterDisabled = false
+ private set
+
@Parameter(names = ["-t", "--cert"], description = "Certificate chain file path")
var certChainFilePath = ""
private set
@@ -130,49 +135,55 @@ abstract class BaseOptions protected constructor(
protected fun assignAdminEnabled(defaultVal: Boolean) {
if (!adminEnabled)
- adminEnabled = EnvVars.ADMIN_ENABLED.getEnv(defaultVal)
+ adminEnabled = ADMIN_ENABLED.getEnv(defaultVal)
logger.info { "adminEnabled: $adminEnabled" }
}
protected fun assignAdminPort(defaultVal: Int) {
if (adminPort == -1)
- adminPort = EnvVars.ADMIN_PORT.getEnv(defaultVal)
+ adminPort = ADMIN_PORT.getEnv(defaultVal)
logger.info { "adminPort: $adminPort" }
}
protected fun assignMetricsEnabled(defaultVal: Boolean) {
if (!metricsEnabled)
- metricsEnabled = EnvVars.METRICS_ENABLED.getEnv(defaultVal)
+ metricsEnabled = METRICS_ENABLED.getEnv(defaultVal)
logger.info { "metricsEnabled: $metricsEnabled" }
}
protected fun assignDebugEnabled(defaultVal: Boolean) {
if (!debugEnabled)
- debugEnabled = EnvVars.DEBUG_ENABLED.getEnv(defaultVal)
+ debugEnabled = DEBUG_ENABLED.getEnv(defaultVal)
logger.info { "debugEnabled: $debugEnabled" }
}
protected fun assignMetricsPort(defaultVal: Int) {
if (metricsPort == -1)
- metricsPort = EnvVars.METRICS_PORT.getEnv(defaultVal)
+ metricsPort = METRICS_PORT.getEnv(defaultVal)
logger.info { "metricsPort: $metricsPort" }
}
+ protected fun assignTransportFilterDisabled(defaultVal: Boolean) {
+ if (!transportFilterDisabled)
+ transportFilterDisabled = TRANSPORT_FILTER_DISABLED.getEnv(defaultVal)
+ logger.info { "transportFilterDisabled: $transportFilterDisabled" }
+ }
+
protected fun assignCertChainFilePath(defaultVal: String) {
if (certChainFilePath.isEmpty())
- certChainFilePath = EnvVars.CERT_CHAIN_FILE_PATH.getEnv(defaultVal)
+ certChainFilePath = CERT_CHAIN_FILE_PATH.getEnv(defaultVal)
logger.info { "certChainFilePath: $certChainFilePath" }
}
protected fun assignPrivateKeyFilePath(defaultVal: String) {
if (privateKeyFilePath.isEmpty())
- privateKeyFilePath = EnvVars.PRIVATE_KEY_FILE_PATH.getEnv(defaultVal)
+ privateKeyFilePath = PRIVATE_KEY_FILE_PATH.getEnv(defaultVal)
logger.info { "privateKeyFilePath: $privateKeyFilePath" }
}
protected fun assignTrustCertCollectionFilePath(defaultVal: String) {
if (trustCertCollectionFilePath.isEmpty())
- trustCertCollectionFilePath = EnvVars.TRUST_CERT_COLLECTION_FILE_PATH.getEnv(defaultVal)
+ trustCertCollectionFilePath = TRUST_CERT_COLLECTION_FILE_PATH.getEnv(defaultVal)
logger.info { "trustCertCollectionFilePath: $trustCertCollectionFilePath" }
}
diff --git a/src/main/kotlin/io/prometheus/common/EnvVars.kt b/src/main/kotlin/io/prometheus/common/EnvVars.kt
index 58e2daf2..25985acb 100644
--- a/src/main/kotlin/io/prometheus/common/EnvVars.kt
+++ b/src/main/kotlin/io/prometheus/common/EnvVars.kt
@@ -34,6 +34,7 @@ enum class EnvVars {
AGENT_CONFIG,
PROXY_HOSTNAME,
AGENT_NAME,
+ TRANSPORT_FILTER_DISABLED,
CONSOLIDATED,
SCRAPE_TIMEOUT_SECS,
SCRAPE_MAX_RETRIES,
diff --git a/src/main/kotlin/io/prometheus/common/GrpcObjects.kt b/src/main/kotlin/io/prometheus/common/GrpcObjects.kt
index 322621b8..d8840e93 100644
--- a/src/main/kotlin/io/prometheus/common/GrpcObjects.kt
+++ b/src/main/kotlin/io/prometheus/common/GrpcObjects.kt
@@ -19,6 +19,7 @@
package io.prometheus.common
import com.google.protobuf.ByteString
+import com.google.protobuf.Empty
import io.prometheus.grpc.AgentInfo
import io.prometheus.grpc.ChunkData
import io.prometheus.grpc.ChunkedScrapeResponse
@@ -346,4 +347,6 @@ internal object GrpcObjects {
block()
build()
}
+
+ internal val EMPTY_INSTANCE = Empty.getDefaultInstance()
}
diff --git a/src/main/kotlin/io/prometheus/proxy/AgentContextCleanupService.kt b/src/main/kotlin/io/prometheus/proxy/AgentContextCleanupService.kt
index e52cb8af..04a502cd 100644
--- a/src/main/kotlin/io/prometheus/proxy/AgentContextCleanupService.kt
+++ b/src/main/kotlin/io/prometheus/proxy/AgentContextCleanupService.kt
@@ -47,7 +47,7 @@ internal class AgentContextCleanupService(
.forEach { (agentId, agentContext) ->
val inactivityDuration = agentContext.inactivityDuration
if (inactivityDuration > maxAgentInactivityTime) {
- logger.info { "Evicting agent after $inactivityDuration of inactivity $agentContext" }
+ logger.info { "Evicting agentId ${agentContext.agentId} after $inactivityDuration (max $maxAgentInactivityTime) of inactivity: $agentContext" }
proxy.removeAgentContext(agentId, "Eviction")
proxy.metrics { agentEvictionCount.inc() }
}
diff --git a/src/main/kotlin/io/prometheus/proxy/AgentContextManager.kt b/src/main/kotlin/io/prometheus/proxy/AgentContextManager.kt
index 6dfa91e0..0dca4050 100644
--- a/src/main/kotlin/io/prometheus/proxy/AgentContextManager.kt
+++ b/src/main/kotlin/io/prometheus/proxy/AgentContextManager.kt
@@ -35,7 +35,7 @@ internal class AgentContextManager(private val isTestMode: Boolean) {
val totalAgentScrapeRequestBacklogSize: Int get() = agentContextMap.values.sumOf { it.scrapeRequestBacklogSize }
fun addAgentContext(agentContext: AgentContext): AgentContext? {
- logger.debug { "Registering agentId: ${agentContext.agentId}" }
+ logger.info { "Registering agentId: ${agentContext.agentId}" }
return agentContextMap.put(agentContext.agentId, agentContext)
}
@@ -48,7 +48,7 @@ internal class AgentContextManager(private val isTestMode: Boolean) {
logger.warn { "Missing AgentContext for agentId: $agentId ($reason)" }
else {
if (!isTestMode)
- logger.info { "Removed AgentContext $agentContext for agentId: $agentId ($reason)" }
+ logger.info { "Removed $agentContext for agentId: $agentId ($reason)" }
agentContext.invalidate()
}
agentContext
diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyGrpcService.kt b/src/main/kotlin/io/prometheus/proxy/ProxyGrpcService.kt
index 766ca08a..438b2b7a 100644
--- a/src/main/kotlin/io/prometheus/proxy/ProxyGrpcService.kt
+++ b/src/main/kotlin/io/prometheus/proxy/ProxyGrpcService.kt
@@ -75,11 +75,18 @@ internal class ProxyGrpcService(
inProcessServerName = inProcessName
) {
val proxyService = ProxyServiceImpl(proxy)
- val interceptors = mutableListOf(ProxyServerInterceptor())
- if (proxy.isZipkinEnabled)
- interceptors += grpcTracing.newServerInterceptor()
+ val interceptors =
+ buildList {
+ if (!options.transportFilterDisabled)
+ add(ProxyServerInterceptor())
+ if (proxy.isZipkinEnabled)
+ add(grpcTracing.newServerInterceptor())
+ }
+
addService(ServerInterceptors.intercept(proxyService.bindService(), interceptors))
- addTransportFilter(ProxyServerTransportFilter(proxy))
+
+ if (!options.transportFilterDisabled)
+ addTransportFilter(ProxyServerTransportFilter(proxy))
}
grpcServer.shutdownWithJvm(2.seconds)
diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt b/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt
index 205b1e94..83960f8c 100644
--- a/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt
+++ b/src/main/kotlin/io/prometheus/proxy/ProxyHttpConfig.kt
@@ -38,7 +38,6 @@ import io.ktor.server.response.*
import io.ktor.server.routing.*
import io.prometheus.Proxy
import kotlinx.coroutines.async
-import kotlinx.coroutines.delay
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonPrimitive
@@ -68,6 +67,7 @@ internal object ProxyHttpConfig : KLogging() {
HttpStatusCode.Found -> {
"$status: ${call.request.toLogString()} -> ${call.response.headers[HttpHeaders.Location]} - ${call.request.origin.remoteHost}"
}
+
else -> "$status: ${call.request.toLogString()} - ${call.request.origin.remoteHost}"
}
}
@@ -97,10 +97,10 @@ internal object ProxyHttpConfig : KLogging() {
}
routing {
- get("/__test__") {
- delay(30.seconds)
- call.respondWith("Test value", Plain, OK)
- }
+// get("/__test__") {
+// delay(30.seconds)
+// call.respondWith("Test value", Plain, OK)
+// }
if (proxy.options.sdEnabled) {
logger.info { "Adding /${proxy.options.sdPath} service discovery endpoint" }
@@ -117,8 +117,8 @@ internal object ProxyHttpConfig : KLogging() {
put("__metrics_path__", JsonPrimitive(path))
val agentContexts = proxy.pathManager.getAgentContextInfo(path)?.agentContexts
- put("agentName", JsonPrimitive(agentContexts?.map { it.agentName }?.joinToString()))
- put("hostName", JsonPrimitive(agentContexts?.map { it.hostName }?.joinToString()))
+ put("agentName", JsonPrimitive(agentContexts?.joinToString { it.agentName }))
+ put("hostName", JsonPrimitive(agentContexts?.joinToString { it.hostName }))
}
}
}
diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt b/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt
index d608622a..9e8f9d5b 100644
--- a/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt
+++ b/src/main/kotlin/io/prometheus/proxy/ProxyOptions.kt
@@ -22,7 +22,6 @@ import com.beust.jcommander.Parameter
import io.prometheus.Proxy
import io.prometheus.common.BaseOptions
import io.prometheus.common.EnvVars.*
-import kotlin.time.Duration.Companion.seconds
class ProxyOptions(argv: Array) : BaseOptions(Proxy::class.java.simpleName, argv, PROXY_CONFIG.name) {
@@ -86,13 +85,16 @@ class ProxyOptions(argv: Array) : BaseOptions(Proxy::class.java.simpleNa
assignAdminPort(proxyConfigVals.admin.port)
assignMetricsEnabled(proxyConfigVals.metrics.enabled)
assignMetricsPort(proxyConfigVals.metrics.port)
+ assignTransportFilterDisabled(proxyConfigVals.transportFilterDisabled)
assignDebugEnabled(proxyConfigVals.admin.debugEnabled)
assignCertChainFilePath(proxyConfigVals.tls.certChainFilePath)
assignPrivateKeyFilePath(proxyConfigVals.tls.privateKeyFilePath)
assignTrustCertCollectionFilePath(proxyConfigVals.tls.trustCertCollectionFilePath)
- logger.info { "proxy.internal.scrapeRequestTimeoutSecs: ${proxyConfigVals.internal.scrapeRequestTimeoutSecs.seconds}" }
+ logger.info { "proxy.internal.scrapeRequestTimeoutSecs: ${proxyConfigVals.internal.scrapeRequestTimeoutSecs}" }
+ logger.info { "proxy.internal.staleAgentCheckPauseSecs: ${proxyConfigVals.internal.staleAgentCheckPauseSecs}" }
+ logger.info { "proxy.internal.maxAgentInactivitySecs: ${proxyConfigVals.internal.maxAgentInactivitySecs}" }
}
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyServerTransportFilter.kt b/src/main/kotlin/io/prometheus/proxy/ProxyServerTransportFilter.kt
index 040fb968..210ea18f 100644
--- a/src/main/kotlin/io/prometheus/proxy/ProxyServerTransportFilter.kt
+++ b/src/main/kotlin/io/prometheus/proxy/ProxyServerTransportFilter.kt
@@ -23,12 +23,13 @@ import com.github.pambrose.common.util.isNotNull
import io.grpc.Attributes
import io.grpc.ServerTransportFilter
import io.prometheus.Proxy
+import io.prometheus.proxy.ProxyServiceImpl.Companion.UNKNOWN_ADDRESS
import mu.KLogging
internal class ProxyServerTransportFilter(private val proxy: Proxy) : ServerTransportFilter() {
override fun transportReady(attributes: Attributes): Attributes {
- val remoteAddress = attributes.get(REMOTE_ADDR_KEY)?.toString() ?: "Unknown"
+ val remoteAddress = attributes.get(REMOTE_ADDR_KEY)?.toString() ?: UNKNOWN_ADDRESS
val agentContext = AgentContext(remoteAddress)
proxy.agentContextManager.addAgentContext(agentContext)
diff --git a/src/main/kotlin/io/prometheus/proxy/ProxyServiceImpl.kt b/src/main/kotlin/io/prometheus/proxy/ProxyServiceImpl.kt
index dc68a24d..9c4f6671 100644
--- a/src/main/kotlin/io/prometheus/proxy/ProxyServiceImpl.kt
+++ b/src/main/kotlin/io/prometheus/proxy/ProxyServiceImpl.kt
@@ -23,6 +23,9 @@ import com.github.pambrose.common.util.isNull
import com.google.protobuf.Empty
import io.grpc.Status
import io.prometheus.Proxy
+import io.prometheus.agent.RequestFailureException
+import io.prometheus.common.GrpcObjects.EMPTY_INSTANCE
+import io.prometheus.common.GrpcObjects.newAgentInfo
import io.prometheus.common.GrpcObjects.newHeartBeatResponse
import io.prometheus.common.GrpcObjects.newPathMapSizeResponse
import io.prometheus.common.GrpcObjects.newRegisterAgentResponse
@@ -52,8 +55,31 @@ import java.util.concurrent.atomic.AtomicLong
internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.ProxyServiceCoroutineImplBase() {
- override suspend fun connectAgent(request: Empty): Empty =
- proxy.metrics { connectCount.inc() }.let { Empty.getDefaultInstance() }
+ override suspend fun connectAgent(request: Empty): Empty {
+ if (proxy.options.transportFilterDisabled) {
+ "Agent (false) and Proxy (true) do not have matching transportFilterDisabled config values".also { msg ->
+ logger.error { msg }
+ throw RequestFailureException(msg)
+ }
+ }
+
+ proxy.metrics { connectCount.inc() }
+ return EMPTY_INSTANCE
+ }
+
+ override suspend fun connectAgentWithTransportFilterDisabled(request: Empty): AgentInfo {
+ if (!proxy.options.transportFilterDisabled) {
+ "Agent (true) and Proxy (false) do not have matching transportFilterDisabled config values".also { msg ->
+ logger.error { msg }
+ throw RequestFailureException(msg)
+ }
+ }
+
+ proxy.metrics { connectCount.inc() }
+ val agentContext = AgentContext(UNKNOWN_ADDRESS)
+ proxy.agentContextManager.addAgentContext(agentContext)
+ return newAgentInfo(agentContext.agentId)
+ }
override suspend fun registerAgent(request: RegisterAgentRequest): RegisterAgentResponse {
val agentId = request.agentId
@@ -136,7 +162,7 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P
logger.error(throwable) { "Error in writeResponsesToProxy(): $arg" }
}
}
- return Empty.getDefaultInstance()
+ return EMPTY_INSTANCE
}
override suspend fun writeChunkedResponsesToProxy(requests: Flow): Empty {
@@ -150,6 +176,7 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P
logger.debug { "Reading header for scrapeId: $scrapeId}" }
chunkedContextMap[scrapeId] = ChunkedContext(response)
}
+
"chunk" -> {
response.chunk
.apply {
@@ -159,6 +186,7 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P
context.applyChunk(chunkBytes.toByteArray(), chunkByteCount, chunkCount, chunkChecksum)
}
}
+
"summary" -> {
response.summary
.apply {
@@ -169,6 +197,7 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P
proxy.scrapeRequestManager.assignScrapeResults(context.scrapeResults)
}
}
+
else -> throw IllegalStateException("Invalid field name in writeChunkedResponsesToProxy()")
}
}
@@ -180,10 +209,11 @@ internal class ProxyServiceImpl(private val proxy: Proxy) : ProxyServiceGrpcKt.P
logger.error(throwable) { "Error in writeChunkedResponsesToProxy(): $arg" }
}
}
- return Empty.getDefaultInstance()
+ return EMPTY_INSTANCE
}
companion object : KLogging() {
private val PATH_ID_GENERATOR = AtomicLong(0L)
+ internal const val UNKNOWN_ADDRESS = "Unknown"
}
}
\ No newline at end of file
diff --git a/src/main/proto/proxy_service.proto b/src/main/proto/proxy_service.proto
index f307902a..38b0425f 100644
--- a/src/main/proto/proxy_service.proto
+++ b/src/main/proto/proxy_service.proto
@@ -126,6 +126,9 @@ service ProxyService {
rpc connectAgent (google.protobuf.Empty) returns (google.protobuf.Empty) {
}
+ rpc connectAgentWithTransportFilterDisabled (google.protobuf.Empty) returns (AgentInfo) {
+ }
+
rpc registerAgent (RegisterAgentRequest) returns (RegisterAgentResponse) {
}