-
Notifications
You must be signed in to change notification settings - Fork 113
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add apm clickhouse query api (#692)
* feat: add apm clickhouse query api * chore: add tests * chore: update target frameworks * chore: update * chore: update test timezone * refactor: recode service * chore: update test * Update pr_run_test_ci.yml java from 1.11 to 1.17 * relator: update * chore: update * feat: update to clickhouse query for kibana * chore: update * fix: restore * fix: fix clickhouse query init time * chore: update version sort * chore: update * chore: update * chore: update bad code and update tests * chore: update clickhouse tests * chore: update * chore: update * chore: update * chore: update tests * chore: update --------- Co-authored-by: qinyouzeng <qinyouzeng@masastack,com>
- Loading branch information
1 parent
1b2625e
commit 7012964
Showing
38 changed files
with
1,712 additions
and
109 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
77 changes: 77 additions & 0 deletions
77
...rib/StackSdks/Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse/ApmClickhouseServiceExtensions.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,77 @@ | ||
| // Copyright (c) MASA Stack All rights reserved. | ||
| // Licensed under the MIT License. See LICENSE.txt in the project root for license information. | ||
|
|
||
| namespace Microsoft.Extensions.DependencyInjection; | ||
|
|
||
| public static class ApmClickhouseServiceExtensions | ||
| { | ||
| internal static ILogger Logger { get; private set; } | ||
|
|
||
| public static IServiceCollection AddMASAStackApmClickhouse(this IServiceCollection services, string connectionStr, string logTable, string traceTable, string? logSourceTable = null, string? traceSourceTable = null, Action<IDbConnection>? configer = null) | ||
| { | ||
| services.AddMASAStackClickhouse(connectionStr, logTable, traceTable, logSourceTable, traceSourceTable, con => | ||
| { | ||
| Constants.Init(MasaStackClickhouseConnection.LogTable.Split('.')[0], MasaStackClickhouseConnection.LogTable.Split('.')[1], MasaStackClickhouseConnection.TraceTable.Split('.')[1], "otel_errors"); | ||
| Init(services, con); | ||
| configer?.Invoke(con); | ||
| }); | ||
| services.AddScoped<IApmService, ClickhouseApmService>(); | ||
| return services; | ||
| } | ||
|
|
||
| private static void Init(IServiceCollection services, IDbConnection connection) | ||
| { | ||
| var serviceProvider = services.BuildServiceProvider(); | ||
| var logfactory = serviceProvider.GetRequiredService<ILoggerFactory>(); | ||
| Logger = logfactory.CreateLogger("Masa.Contrib.StackSdks.Tsc.Apm.Clickhouse"); | ||
| InitTable(connection); | ||
| } | ||
|
|
||
| private static void InitTable(IDbConnection connection) | ||
| { | ||
| if (Convert.ToInt32(connection.ExecuteScalar($"select count() from system.tables where database ='{Constants.Database}' and name in ['{Constants.ErrorTable}','{Constants.ErrorTable}_v']")) > 0) | ||
| return; | ||
| var createTableSqls = new string[]{ | ||
| @$"CREATE TABLE {Constants.Database}.{Constants.ErrorTable} | ||
| ( | ||
| `Timestamp` DateTime64(9) CODEC(Delta(8), ZSTD(1)), | ||
| `TraceId` String CODEC(ZSTD(1)), | ||
| `SpanId` String CODEC(ZSTD(1)), | ||
| `Attributes.exception.message` String CODEC(ZSTD(1)), | ||
| `Attributes.exception.type` String CODEC(ZSTD(1)), | ||
| `ServiceName` String CODEC(ZSTD(1)), | ||
| `Resource.service.namespace` String CODEC(ZSTD(1)), | ||
| `Attributes.http.target` String CODEC(ZSTD(1)), | ||
| INDEX idx_log_id TraceId TYPE bloom_filter(0.001) GRANULARITY 1, | ||
| INDEX idx_log_spanid SpanId TYPE bloom_filter(0.001) GRANULARITY 1, | ||
| INDEX idx_log_environment `Resource.service.namespace` TYPE bloom_filter(0.001) GRANULARITY 1, | ||
| INDEX idx_log_servicename ServiceName TYPE bloom_filter(0.001) GRANULARITY 1, | ||
| INDEX idx_log_type `Attributes.exception.type` TYPE bloom_filter(0.001) GRANULARITY 1, | ||
| INDEX idx_log_endpoint `Attributes.http.target` TYPE bloom_filter(0.001) GRANULARITY 1, | ||
| INDEX idx_string_message `Attributes.exception.message` TYPE tokenbf_v1(30720, 2, 0) GRANULARITY 1 | ||
| ) | ||
| ENGINE = MergeTree | ||
| PARTITION BY toDate(Timestamp) | ||
| ORDER BY (Timestamp, | ||
| ServiceName, | ||
| `Resource.service.namespace`, | ||
| `Attributes.exception.type`, | ||
| `Attributes.http.target`) | ||
| TTL toDateTime(Timestamp) + toIntervalDay(30) | ||
| SETTINGS index_granularity = 8192, | ||
| ttl_only_drop_parts = 1; | ||
| ", | ||
| $@"CREATE MATERIALIZED VIEW {Constants.Database}.{Constants.ErrorTable}_v TO {Constants.ErrorTableFull} | ||
| AS | ||
| SELECT | ||
| Timestamp,TraceId,SpanId, Body AS `Attributes.exception.message`,LogAttributes['exception.type'] AS `Attributes.exception.type`, | ||
| ServiceName,ResourceAttributes['service.namespace'] AS `Resource.service.namespace`, LogAttributes['RequestPath'] AS `Attributes.http.target` | ||
| FROM {MasaStackClickhouseConnection.LogSourceTable} | ||
| WHERE mapContains(LogAttributes, 'exception.type') | ||
| "}; | ||
| foreach (var sql in createTableSqls) | ||
| { | ||
| connection.ExecuteSql(sql); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.