diff --git a/.chloggen/clickhouse-add-client-info.yaml b/.chloggen/clickhouse-add-client-info.yaml new file mode 100644 index 0000000000000..f21c1c2d8ac8f --- /dev/null +++ b/.chloggen/clickhouse-add-client-info.yaml @@ -0,0 +1,28 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: clickhouseexporter + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add client info to queries + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [34915, 37146] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: This change adds client product info to the system.query_log for more insight on where queries originate + + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/exporter/clickhouseexporter/README.md b/exporter/clickhouseexporter/README.md index 4beea98f17f47..3c61da556505f 100644 --- a/exporter/clickhouseexporter/README.md +++ b/exporter/clickhouseexporter/README.md @@ -290,6 +290,11 @@ Connection options: - `compress` (default = lz4): Controls the compression algorithm. Valid options: `none` (disabled), `zstd`, `lz4` (default), `gzip`, `deflate`, `br`, `true` (lz4). Ignored if `compress` is set in the `endpoint` or `connection_params`. - `async_insert` (default = true): Enables [async inserts](https://clickhouse.com/docs/en/optimize/asynchronous-inserts). Ignored if async inserts are configured in the `endpoint` or `connection_params`. Async inserts may still be overridden server-side. +Additional DSN features: + +The underlying `clickhouse-go` module offers additional configuration. These can be set in the exporter's `endpoint` or `connection_params` config values. +- `client_info_product` Must be in `productName/version` format with comma separated entries. By default the exporter will append its binary build information. You can use this information to track the origin of `INSERT` statements in the `system.query_log` table. + ClickHouse tables: - `logs_table_name` (default = otel_logs): The table name for logs. diff --git a/exporter/clickhouseexporter/config.go b/exporter/clickhouseexporter/config.go index cb2a2e4a504db..74e42f9a0bb2b 100644 --- a/exporter/clickhouseexporter/config.go +++ b/exporter/clickhouseexporter/config.go @@ -20,6 +20,9 @@ import ( // Config defines configuration for clickhouse exporter. type Config struct { + // collectorVersion is the build version of the collector. This is overridden when an exporter is initialized. + collectorVersion string + TimeoutSettings exporterhelper.TimeoutConfig `mapstructure:",squash"` configretry.BackOffConfig `mapstructure:"retry_on_failure"` QueueSettings exporterhelper.QueueConfig `mapstructure:"sending_queue"` @@ -147,6 +150,15 @@ func (cfg *Config) buildDSN() (string, error) { queryParams.Set("compress", cfg.Compress) } + productInfo := queryParams.Get("client_info_product") + collectorProductInfo := fmt.Sprintf("%s/%s", "otelcol", cfg.collectorVersion) + if productInfo == "" { + productInfo = collectorProductInfo + } else { + productInfo = fmt.Sprintf("%s,%s", productInfo, collectorProductInfo) + } + queryParams.Set("client_info_product", productInfo) + // Use database from config if not specified in path, or if config is not default. if dsnURL.Path == "" || cfg.Database != defaultDatabase { dsnURL.Path = cfg.Database diff --git a/exporter/clickhouseexporter/config_test.go b/exporter/clickhouseexporter/config_test.go index b3167ed52fc7d..eace48911f893 100644 --- a/exporter/clickhouseexporter/config_test.go +++ b/exporter/clickhouseexporter/config_test.go @@ -47,14 +47,15 @@ func TestLoadConfig(t *testing.T) { { id: component.NewIDWithName(metadata.Type, "full"), expected: &Config{ - Endpoint: defaultEndpoint, - Database: "otel", - Username: "foo", - Password: "bar", - TTL: 72 * time.Hour, - LogsTableName: "otel_logs", - TracesTableName: "otel_traces", - CreateSchema: true, + collectorVersion: "unknown", + Endpoint: defaultEndpoint, + Database: "otel", + Username: "foo", + Password: "bar", + TTL: 72 * time.Hour, + LogsTableName: "otel_logs", + TracesTableName: "otel_traces", + CreateSchema: true, TimeoutSettings: exporterhelper.TimeoutConfig{ Timeout: 5 * time.Second, }, @@ -282,7 +283,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: false, }, - want: "clickhouse://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "clickhouse://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "Support tcp scheme", @@ -292,7 +293,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: false, }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "prefers database name from config over from DSN", @@ -305,7 +306,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: false, }, - want: "clickhouse://foo:bar@127.0.0.1:9000/otel?async_insert=true&compress=lz4", + want: "clickhouse://foo:bar@127.0.0.1:9000/otel?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "use database name from DSN if not set in config", @@ -317,7 +318,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: false, }, - want: "clickhouse://foo:bar@127.0.0.1:9000/otel?async_insert=true&compress=lz4", + want: "clickhouse://foo:bar@127.0.0.1:9000/otel?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "invalid config", @@ -337,7 +338,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: true, }, - want: "https://127.0.0.1:9000/default?async_insert=true&compress=lz4&secure=true", + want: "https://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4&secure=true", }, { name: "Preserve query parameters", @@ -347,7 +348,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: true, }, - want: "clickhouse://127.0.0.1:9000/default?async_insert=true&compress=lz4&foo=bar&secure=true", + want: "clickhouse://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4&foo=bar&secure=true", }, { name: "Parse clickhouse settings", @@ -359,7 +360,7 @@ func TestConfig_buildDSN(t *testing.T) { DialTimeout: 30 * time.Second, Compress: clickhouse.CompressionBrotli, }, - want: "https://127.0.0.1:9000/default?async_insert=true&compress=br&dial_timeout=30s&secure=true", + want: "https://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=br&dial_timeout=30s&secure=true", }, { name: "Should respect connection parameters", @@ -370,7 +371,7 @@ func TestConfig_buildDSN(t *testing.T) { wantChOptions: ChOptions{ Secure: true, }, - want: "clickhouse://127.0.0.1:9000/default?async_insert=true&compress=lz4&foo=bar&secure=true", + want: "clickhouse://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4&foo=bar&secure=true", }, { name: "support replace database in DSN with config to override database", @@ -378,21 +379,21 @@ func TestConfig_buildDSN(t *testing.T) { Endpoint: "tcp://127.0.0.1:9000/otel", Database: "override", }, - want: "tcp://127.0.0.1:9000/override?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/override?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "when config option is missing, preserve async_insert false in DSN", fields: fields{ Endpoint: "tcp://127.0.0.1:9000?async_insert=false", }, - want: "tcp://127.0.0.1:9000/default?async_insert=false&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=false&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "when config option is missing, preserve async_insert true in DSN", fields: fields{ Endpoint: "tcp://127.0.0.1:9000?async_insert=true", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "ignore config option when async_insert is present in connection params as false", @@ -402,7 +403,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configTrue, }, - want: "tcp://127.0.0.1:9000/default?async_insert=false&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=false&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "ignore config option when async_insert is present in connection params as true", @@ -412,7 +413,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configFalse, }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "ignore config option when async_insert is present in DSN as false", @@ -421,7 +422,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configTrue, }, - want: "tcp://127.0.0.1:9000/default?async_insert=false&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=false&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "use async_insert true config option when it is not present in DSN", @@ -430,7 +431,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configTrue, }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "use async_insert false config option when it is not present in DSN", @@ -439,7 +440,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configFalse, }, - want: "tcp://127.0.0.1:9000/default?async_insert=false&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=false&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "set async_insert to true when not present in config or DSN", @@ -447,7 +448,7 @@ func TestConfig_buildDSN(t *testing.T) { Endpoint: "tcp://127.0.0.1:9000", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "connection_params takes priority over endpoint and async_insert option.", @@ -457,7 +458,7 @@ func TestConfig_buildDSN(t *testing.T) { AsyncInsert: &configFalse, }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "use compress br config option when it is not present in DSN", @@ -466,7 +467,7 @@ func TestConfig_buildDSN(t *testing.T) { Compress: "br", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=br", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=br", }, { name: "set compress to lz4 when not present in config or DSN", @@ -474,7 +475,7 @@ func TestConfig_buildDSN(t *testing.T) { Endpoint: "tcp://127.0.0.1:9000", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=lz4", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", }, { name: "connection_params takes priority over endpoint and compress option.", @@ -483,12 +484,29 @@ func TestConfig_buildDSN(t *testing.T) { ConnectionParams: map[string]string{"compress": "br"}, Compress: "lz4", }, - want: "tcp://127.0.0.1:9000/default?async_insert=true&compress=br", + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=br", + }, + { + name: "include default otel product info in DSN", + fields: fields{ + Endpoint: "tcp://127.0.0.1:9000", + }, + + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=otelcol%2Ftest&compress=lz4", + }, + { + name: "correctly append default product info when value is included in DSN", + fields: fields{ + Endpoint: "tcp://127.0.0.1:9000?client_info_product=customProductInfo%2Fv1.2.3", + }, + + want: "tcp://127.0.0.1:9000/default?async_insert=true&client_info_product=customProductInfo%2Fv1.2.3%2Cotelcol%2Ftest&compress=lz4", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := createDefaultConfig().(*Config) + cfg.collectorVersion = "test" mergeConfigWithFields(cfg, tt.fields) dsn, err := cfg.buildDSN() diff --git a/exporter/clickhouseexporter/factory.go b/exporter/clickhouseexporter/factory.go index 4c545d5a9fb86..7780295525346 100644 --- a/exporter/clickhouseexporter/factory.go +++ b/exporter/clickhouseexporter/factory.go @@ -32,6 +32,8 @@ func NewFactory() exporter.Factory { func createDefaultConfig() component.Config { return &Config{ + collectorVersion: "unknown", + TimeoutSettings: exporterhelper.NewDefaultTimeoutConfig(), QueueSettings: exporterhelper.NewDefaultQueueConfig(), BackOffConfig: configretry.NewDefaultBackOffConfig(), @@ -60,6 +62,7 @@ func createLogsExporter( cfg component.Config, ) (exporter.Logs, error) { c := cfg.(*Config) + c.collectorVersion = set.BuildInfo.Version exporter, err := newLogsExporter(set.Logger, c) if err != nil { return nil, fmt.Errorf("cannot configure clickhouse logs exporter: %w", err) @@ -86,6 +89,7 @@ func createTracesExporter( cfg component.Config, ) (exporter.Traces, error) { c := cfg.(*Config) + c.collectorVersion = set.BuildInfo.Version exporter, err := newTracesExporter(set.Logger, c) if err != nil { return nil, fmt.Errorf("cannot configure clickhouse traces exporter: %w", err) @@ -110,6 +114,7 @@ func createMetricExporter( cfg component.Config, ) (exporter.Metrics, error) { c := cfg.(*Config) + c.collectorVersion = set.BuildInfo.Version exporter, err := newMetricsExporter(set.Logger, c) if err != nil { return nil, fmt.Errorf("cannot configure clickhouse metrics exporter: %w", err)