From 5067f1554cb5fc7f23442d5f9ab5d255e26a3b37 Mon Sep 17 00:00:00 2001 From: jennypavlova Date: Mon, 14 Oct 2024 14:33:22 +0200 Subject: [PATCH] [APM][Otel] Add Otel client based on PoC data (#192293) Closes [#192115](https://github.com/elastic/kibana/issues/192115) Closes [#192465](https://github.com/elastic/kibana/issues/192465) ## Summary This PR adds synthrace client for Otel native data and a simple scenario. This is the first step of adding it and in the future it will include more metrics and use cases. >[!NOTE] > To run ES the command needs "xpack.otel_data.registry.enabled=true" flag > `yarn es snapshot --license trial --E "xpack.otel_data.registry.enabled=true"` ## Next steps - We currently have only `service_destination` in the metrics indices we can include the other types in the future - After we have all the UI changes we can add more scenarios (also using the opentelemetry demo data and not only the e2e PoC example) ## Testing - Run ES: ```bash yarn es snapshot --license trial --E "xpack.otel_data.registry.enabled=true" ``` - Run Kibana: ```bash yarn start ``` >[!WARNING] If the e2e PoC is used the first 2 steps should be skipped - Run syntrace: ```bash node scripts/synthtrace otel_simple_trace.ts --clean ``` - Check indices in DevTools for the generated data: ```bash GET *metrics-generic.otel*/_search GET *traces-generic.otel*/_search GET *logs-generic.otel*/_search ``` - Check in the APM UI (all the tabs) >[!WARNING] Currently the UI changes done in APM are not merged so some errors are expected) https://github.com/user-attachments/assets/92f63610-82da-40f3-89bb-00be83c55377 --------- Co-authored-by: miriam.aparicio --- packages/kbn-apm-synthtrace-client/index.ts | 1 + .../src/lib/otel/error.ts | 33 +++ .../src/lib/otel/index.ts | 213 ++++++++++++++++++ .../src/lib/otel/metric.ts | 33 +++ .../src/lib/otel/transaction.ts | 44 ++++ packages/kbn-apm-synthtrace/index.ts | 1 + .../kbn-apm-synthtrace/src/cli/scenario.ts | 2 + .../src/cli/utils/bootstrap.ts | 8 + .../src/cli/utils/get_otel_es_client.ts | 34 +++ .../src/cli/utils/start_live_data_upload.ts | 20 +- .../src/cli/utils/synthtrace_worker.ts | 17 +- .../src/lib/otel/otel_synthtrace_es_client.ts | 96 ++++++++ .../src/scenarios/otel_simple_trace.ts | 48 ++++ 13 files changed, 546 insertions(+), 4 deletions(-) create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts create mode 100644 packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts create mode 100644 packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts create mode 100644 packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts create mode 100644 packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts diff --git a/packages/kbn-apm-synthtrace-client/index.ts b/packages/kbn-apm-synthtrace-client/index.ts index 6ac3b6525ec000..d3d24a8940a3b6 100644 --- a/packages/kbn-apm-synthtrace-client/index.ts +++ b/packages/kbn-apm-synthtrace-client/index.ts @@ -37,3 +37,4 @@ export type { ESDocumentWithOperation, SynthtraceESAction, SynthtraceGenerator } export { log, type LogDocument, LONG_FIELD_NAME } from './src/lib/logs'; export { type AssetDocument } from './src/lib/assets'; export { syntheticsMonitor, type SyntheticsMonitorDocument } from './src/lib/synthetics'; +export { otel, type OtelDocument } from './src/lib/otel'; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts new file mode 100644 index 00000000000000..63265d45fe8862 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/error.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import type { OtelDocument } from '../../..'; +import { Serializable } from '../serializable'; + +export interface OtelErrorDocument extends OtelDocument { + 'event.name'?: string; + attributes?: { + 'exception.message'?: string; + 'error.stack_trace'?: string; + 'exception.handled'?: boolean; + 'exception.type'?: string; + 'processor.event'?: string; + 'timestamp.us'?: number; + 'event.name'?: string; + 'error.id'?: string; + }; +} + +export class OtelError extends Serializable { + constructor(fields: OtelErrorDocument) { + super({ + ...fields, + }); + } +} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts new file mode 100644 index 00000000000000..86bb74dd94ff4b --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/index.ts @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Fields } from '../entity'; +import { Serializable } from '../serializable'; +import { OtelError } from './error'; +import { OtelMetric } from './metric'; +import { OtelTransaction } from './transaction'; + +interface OtelSharedResourceAttributes { + 'service.name'?: string; + 'agent.name'?: string; + 'agent.version'?: string; + 'metricset.interval'?: string; + 'service.instance.id'?: string; + 'telemetry.sdk.language'?: string; + 'telemetry.sdk.name'?: string; + 'telemetry.sdk.version'?: string; + 'some.resource.attribute'?: string; +} + +export interface OtelDocument extends Fields { + data_stream?: { + dataset: string; + namespace: string; + type: string; + }; + attributes?: { + 'timestamp.us'?: number; + 'metricset.name'?: string; + [key: string]: any; + }; + resource?: { + attributes?: OtelSharedResourceAttributes; + dropped_attributes_count?: number; + schema_url?: string; + }; + scope?: { + attributes?: { + 'service.framework.name'?: string; + 'service.framework.version'?: string; + }; + dropped_attributes_count?: number; + name?: string; + }; + name?: string; + trace_id?: string; + trace?: { id: string }; + span_id?: string; + span?: { id: string }; + dropped_attributes_count?: number; + dropped_events_count?: number; + dropped_links_count?: number; + timestamp_us?: number; +} + +class Otel extends Serializable { + constructor(fields: OtelDocument) { + super({ + ...fields, + }); + } + + error(spanId: string) { + return new OtelError({ + ...this.fields, + attributes: { + 'exception.message': 'boom', + 'exception.handled': false, + 'exception.type': '*errors.errorString', + 'error.stack_trace': 'Error: INTERNAL: Boom', + 'processor.event': 'error', + 'timestamp.us': 1726580752010657, + 'event.name': 'exception', + 'error.id': `error-${spanId}`, + }, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'logs', + }, + 'event.name': 'exception', + dropped_attributes_count: 0, + resource: { + attributes: { + 'agent.name': 'opentelemetry/go', + 'agent.version': '1.28.0', + 'service.name': 'sendotlp-synth', + 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', + }, + dropped_attributes_count: 0, + schema_url: 'https://opentelemetry.io/schemas/1.26.0', + }, + scope: { + attributes: { + 'service.framework.name': 'sendotlp-synth', + 'service.framework.version': '', + }, + dropped_attributes_count: 0, + name: 'sendotlp-synth', + }, + span_id: spanId, + }); + } + + metric() { + return new OtelMetric({ + ...this.fields, + attributes: { + 'metricset.name': 'service_destination', + 'processor.event': 'metric', + 'event.outcome': 'success', + 'service.target.name': 'foo_service', + 'service.target.type': 'http', + 'span.name': 'child1', + 'span.destination.service.resource': 'foo_service:8080', + }, + data_stream: { + dataset: 'service_destination.10m.otel', + namespace: 'default', + type: 'metrics', + }, + metrics: { + service_summary: 2, + }, + resource: { + attributes: { + 'agent.name': 'otlp', + 'agent.version': '1.28.0', + 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', + 'service.name': 'sendotlp-synth', + 'metricset.interval': '10m', + }, + dropped_attributes_count: 0, + }, + scope: { + dropped_attributes_count: 0, + name: 'github.com/elastic/opentelemetry-collector-components/connector/spanmetricsconnectorv2', + }, + }); + } + + // In Otel we have only spans (https://opentelemetry.io/docs/concepts/signals/traces/#spans) + // we call the root span a transaction to match our data model + transaction(id: string) { + return new OtelTransaction({ + ...this.fields, + attributes: { + 'event.outcome': 'success', + 'event.success_count': 1, + 'processor.event': 'transaction', + 'timestamp.us': 1726580752010657, + 'transaction.duration.us': 15202, + 'transaction.id': id, + 'transaction.name': 'parent-synth', + 'transaction.representative_count': 1, + 'transaction.result': 'HTTP 2xx', + 'transaction.root': true, + 'transaction.sampled': true, + 'transaction.type': 'unknown', + }, + data_stream: { + dataset: 'generic.otel', + namespace: 'default', + type: 'traces', + }, + duration: 11742370, + kind: 'Internal', + name: 'parent-synth', + resource: { + attributes: { + 'agent.name': 'otlp', + 'agent.version': '1.28.0', + 'service.instance.id': '89117ac1-0dbf-4488-9e17-4c2c3b76943a', + 'service.name': 'sendotlp-synth', + }, + dropped_attributes_count: 0, + schema_url: 'https://opentelemetry.io/schemas/1.26.0', + }, + scope: { + attributes: { + 'service.framework.name': 'sendotlp-synth', + 'service.framework.version': '', + }, + dropped_attributes_count: 0, + name: 'sendotlp-synth', + }, + span_id: id, + status: { + code: 'Unset', + }, + }); + } +} + +export function create(id: string): Otel { + return new Otel({ + trace_id: id, + dropped_attributes_count: 0, + dropped_events_count: 0, + dropped_links_count: 0, + }); +} + +export const otel = { + create, +}; diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts new file mode 100644 index 00000000000000..2f238b36c5acae --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/metric.ts @@ -0,0 +1,33 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { OtelDocument } from '.'; +import { Serializable } from '../serializable'; + +export interface OtelMetricDocument extends OtelDocument { + attributes?: { + 'metricset.name'?: string; + 'processor.event'?: string; + 'event.outcome'?: string; + 'service.target.name'?: string; + 'service.target.type'?: string; + 'span.name'?: string; + 'span.destination.service.resource'?: string; + }; + metrics?: { + service_summary?: number; + }; +} +export class OtelMetric extends Serializable { + constructor(fields: OtelMetricDocument) { + super({ + ...fields, + }); + } +} diff --git a/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts new file mode 100644 index 00000000000000..d6f7c7111b1872 --- /dev/null +++ b/packages/kbn-apm-synthtrace-client/src/lib/otel/transaction.ts @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { OtelDocument } from '.'; +import { Serializable } from '../serializable'; + +export interface OtelTransactionDocument extends OtelDocument { + attributes?: { + 'event.outcome'?: string; + 'event.success_count'?: number; + 'processor.event'?: string; + 'timestamp.us'?: number; + 'transaction.duration.us'?: number; + 'transaction.id'?: string; + 'transaction.name'?: string; + 'transaction.representative_count'?: number; + 'transaction.result'?: string; + 'transaction.root'?: boolean; + 'transaction.sampled'?: boolean; + 'transaction.type'?: string; + }; + status?: { + code?: string; + }; + dropped_events_count?: number; + dropped_links_count?: number; + duration?: number; + kind?: string; + name?: string; +} + +export class OtelTransaction extends Serializable { + constructor(fields: OtelTransactionDocument) { + super({ + ...fields, + }); + } +} diff --git a/packages/kbn-apm-synthtrace/index.ts b/packages/kbn-apm-synthtrace/index.ts index b717b9a45af998..ebd35da3aa19e5 100644 --- a/packages/kbn-apm-synthtrace/index.ts +++ b/packages/kbn-apm-synthtrace/index.ts @@ -17,6 +17,7 @@ export { MonitoringSynthtraceEsClient } from './src/lib/monitoring/monitoring_sy export { LogsSynthtraceEsClient } from './src/lib/logs/logs_synthtrace_es_client'; export { AssetsSynthtraceEsClient } from './src/lib/assets/assets_synthtrace_es_client'; export { SyntheticsSynthtraceEsClient } from './src/lib/synthetics/synthetics_synthtrace_es_client'; +export { OtelSynthtraceEsClient } from './src/lib/otel/otel_synthtrace_es_client'; export { addObserverVersionTransform, deleteSummaryFieldTransform, diff --git a/packages/kbn-apm-synthtrace/src/cli/scenario.ts b/packages/kbn-apm-synthtrace/src/cli/scenario.ts index 85b0ee56fc9e80..4f1550b8bdbc8a 100644 --- a/packages/kbn-apm-synthtrace/src/cli/scenario.ts +++ b/packages/kbn-apm-synthtrace/src/cli/scenario.ts @@ -13,6 +13,7 @@ import { InfraSynthtraceEsClient, LogsSynthtraceEsClient, SyntheticsSynthtraceEsClient, + OtelSynthtraceEsClient, } from '../..'; import { AssetsSynthtraceEsClient } from '../lib/assets/assets_synthtrace_es_client'; import { Logger } from '../lib/utils/create_logger'; @@ -25,6 +26,7 @@ interface EsClients { infraEsClient: InfraSynthtraceEsClient; assetsEsClient: AssetsSynthtraceEsClient; syntheticsEsClient: SyntheticsSynthtraceEsClient; + otelEsClient: OtelSynthtraceEsClient; } type Generate = (options: { diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts index 6c6b065dabfc7c..22d07f73c56cbc 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/bootstrap.ts @@ -16,6 +16,7 @@ import { getServiceUrls } from './get_service_urls'; import { RunOptions } from './parse_run_cli_flags'; import { getAssetsEsClient } from './get_assets_es_client'; import { getSyntheticsEsClient } from './get_synthetics_es_client'; +import { getOtelSynthtraceEsClient } from './get_otel_es_client'; export async function bootstrap(runOptions: RunOptions) { const logger = createLogger(runOptions.logLevel); @@ -68,6 +69,11 @@ export async function bootstrap(runOptions: RunOptions) { logger, concurrency: runOptions.concurrency, }); + const otelEsClient = getOtelSynthtraceEsClient({ + target: esUrl, + logger, + concurrency: runOptions.concurrency, + }); if (runOptions.clean) { await apmEsClient.clean(); @@ -75,6 +81,7 @@ export async function bootstrap(runOptions: RunOptions) { await infraEsClient.clean(); await assetsEsClient.clean(); await syntheticsEsClient.clean(); + await otelEsClient.clean(); } return { @@ -84,6 +91,7 @@ export async function bootstrap(runOptions: RunOptions) { infraEsClient, assetsEsClient, syntheticsEsClient, + otelEsClient, version, kibanaUrl, esUrl, diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts new file mode 100644 index 00000000000000..0671ea66c472f7 --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/cli/utils/get_otel_es_client.ts @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Client } from '@elastic/elasticsearch'; +import { Logger } from '../../lib/utils/create_logger'; +import { RunOptions } from './parse_run_cli_flags'; +import { getEsClientTlsSettings } from './ssl'; +import { OtelSynthtraceEsClient } from '../../lib/otel/otel_synthtrace_es_client'; + +export function getOtelSynthtraceEsClient({ + target, + logger, + concurrency, +}: Pick & { + target: string; + logger: Logger; +}) { + const client = new Client({ + node: target, + tls: getEsClientTlsSettings(target), + }); + + return new OtelSynthtraceEsClient({ + client, + logger, + concurrency, + }); +} diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts b/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts index 90fa0189469add..79c9907dc13d12 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/start_live_data_upload.ts @@ -26,8 +26,15 @@ export async function startLiveDataUpload({ }) { const file = runOptions.file; - const { logger, apmEsClient, logsEsClient, infraEsClient, assetsEsClient, syntheticsEsClient } = - await bootstrap(runOptions); + const { + logger, + apmEsClient, + logsEsClient, + infraEsClient, + assetsEsClient, + syntheticsEsClient, + otelEsClient, + } = await bootstrap(runOptions); const scenario = await getScenario({ file, logger }); const { generate } = await scenario({ ...runOptions, logger }); @@ -65,7 +72,14 @@ export async function startLiveDataUpload({ const generatorsAndClients = generate({ range: timerange(bucketFrom.getTime(), bucketTo.getTime()), - clients: { logsEsClient, apmEsClient, infraEsClient, assetsEsClient, syntheticsEsClient }, + clients: { + logsEsClient, + apmEsClient, + infraEsClient, + assetsEsClient, + syntheticsEsClient, + otelEsClient, + }, }); const generatorsAndClientsArray = castArray(generatorsAndClients); diff --git a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts index a5defe4b6e1b48..78c89d110c8921 100644 --- a/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts +++ b/packages/kbn-apm-synthtrace/src/cli/utils/synthtrace_worker.ts @@ -20,6 +20,7 @@ import { getLogsEsClient } from './get_logs_es_client'; import { getInfraEsClient } from './get_infra_es_client'; import { getAssetsEsClient } from './get_assets_es_client'; import { getSyntheticsEsClient } from './get_synthetics_es_client'; +import { getOtelSynthtraceEsClient } from './get_otel_es_client'; export interface WorkerData { bucketFrom: Date; @@ -65,6 +66,12 @@ async function start() { logger, }); + const otelEsClient = getOtelSynthtraceEsClient({ + concurrency: runOptions.concurrency, + target: esUrl, + logger, + }); + const file = runOptions.file; const scenario = await logger.perf('get_scenario', () => getScenario({ file, logger })); @@ -80,6 +87,7 @@ async function start() { infraEsClient, assetsEsClient, syntheticsEsClient, + otelEsClient, }); } @@ -88,7 +96,14 @@ async function start() { const generatorsAndClients = logger.perf('generate_scenario', () => generate({ range: timerange(bucketFrom, bucketTo), - clients: { logsEsClient, apmEsClient, infraEsClient, assetsEsClient, syntheticsEsClient }, + clients: { + logsEsClient, + apmEsClient, + infraEsClient, + assetsEsClient, + syntheticsEsClient, + otelEsClient, + }, }) ); diff --git a/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts new file mode 100644 index 00000000000000..e2162925e3c72c --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/lib/otel/otel_synthtrace_es_client.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { Client } from '@elastic/elasticsearch'; +import { ESDocumentWithOperation } from '@kbn/apm-synthtrace-client'; +import { OtelDocument } from '@kbn/apm-synthtrace-client'; +import { pipeline, Readable, Transform } from 'stream'; +import { SynthtraceEsClient, SynthtraceEsClientOptions } from '../shared/base_client'; +import { getDedotTransform } from '../shared/get_dedot_transform'; +import { getSerializeTransform } from '../shared/get_serialize_transform'; +import { Logger } from '../utils/create_logger'; + +export type OtelSynthtraceEsClientOptions = Omit; + +export class OtelSynthtraceEsClient extends SynthtraceEsClient { + constructor(options: { client: Client; logger: Logger } & OtelSynthtraceEsClientOptions) { + super({ + ...options, + pipeline: otelPipeline(), + }); + this.dataStreams = ['metrics-generic.otel*', 'traces-generic.otel*', 'logs-generic.otel*']; + } +} + +function otelPipeline() { + return (base: Readable) => { + return pipeline( + base, + getSerializeTransform(), + getRoutingTransform(), + getDedotTransform(), + (err: unknown) => { + if (err) { + throw err; + } + } + ); + }; +} + +export function getRoutingTransform() { + return new Transform({ + objectMode: true, + transform(document: ESDocumentWithOperation, encoding, callback) { + const namespace = 'default'; + let index: string | undefined; + + switch (document?.attributes?.['processor.event']) { + case 'transaction': + case 'span': + index = `traces-generic.otel-${namespace}-synth`; + break; + + case 'error': + index = `logs-generic.otel-${namespace}-synth`; + break; + + case 'metric': + const metricsetName = document?.attributes?.['metricset.name']; + if ( + metricsetName === 'transaction' || + metricsetName === 'service_transaction' || + metricsetName === 'service_destination' || + metricsetName === 'service_summary' + ) { + index = `metrics-generic.otel.${metricsetName}.${document.attributes[ + 'metricset.interval' + ]!}-${namespace}-synth`; + } else { + index = `metrics-generic.otel.internal-${namespace}-synth`; + } + break; + default: + if (document?.attributes?.['event.action'] != null) { + index = `logs-generic.otel-${namespace}-synth`; + } + break; + } + + if (!index) { + const error = new Error('Cannot determine index for event'); + Object.assign(error, { document }); + } + + document._index = index; + + callback(null, document); + }, + }); +} diff --git a/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts new file mode 100644 index 00000000000000..7721a8651905fa --- /dev/null +++ b/packages/kbn-apm-synthtrace/src/scenarios/otel_simple_trace.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the "Elastic License + * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side + * Public License v 1"; you may not use this file except in compliance with, at + * your election, the "Elastic License 2.0", the "GNU Affero General Public + * License v3.0 only", or the "Server Side Public License, v 1". + */ + +import { otel, generateShortId, OtelDocument } from '@kbn/apm-synthtrace-client'; +import { times } from 'lodash'; +import { Scenario } from '../cli/scenario'; +import { withClient } from '../lib/utils/with_client'; + +const scenario: Scenario = async (runOptions) => { + return { + generate: ({ range, clients: { otelEsClient } }) => { + const { numOtelTraces = 5 } = runOptions.scenarioOpts || {}; + const { logger } = runOptions; + const traceId = generateShortId(); + const spanId = generateShortId(); + + const otelDocs = times(numOtelTraces / 2).map((index) => otel.create(traceId)); + + const otelWithMetricsAndErrors = range + .interval('30s') + .rate(1) + .generator((timestamp) => + otelDocs.flatMap((oteld) => { + return [ + oteld.metric().timestamp(timestamp), + oteld.transaction(spanId).timestamp(timestamp), + oteld.error(spanId).timestamp(timestamp), + ]; + }) + ); + + return [ + withClient( + otelEsClient, + logger.perf('generating_otel_trace', () => otelWithMetricsAndErrors) + ), + ]; + }, + }; +}; + +export default scenario;