diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/outlier-detection-processor.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/outlier-detection-processor.js new file mode 100644 index 0000000000..7611a24a89 --- /dev/null +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/outlier-detection-processor.js @@ -0,0 +1,61 @@ +'use strict'; + +const { BatchSpanProcessor } = require('@opentelemetry/sdk-trace-base'); +const { diag } = require('@opentelemetry/api'); + +class OutlierDetectionBatchSpanProcessor extends BatchSpanProcessor { + constructor(exporter, config) { + super(exporter, config); + this._traces = new Map(); + } + + onEnd(span) { + if (span.instrumentationLibrary.name === 'artillery-playwright') { + super.onEnd(span); + } else { + const traceId = span.spanContext().traceId; + + // When an outlier span is recognised the whole trace it belongs to is exported, so all the spans that belong to the trace need to be grouped and held until the trace finishes. + if (!this._traces.has(traceId)) { + this._traces.set(traceId, { + spans: [], + hasOutlier: false + }); + } + const traceData = this._traces.get(traceId); + traceData.spans.push(span); + + // Since only request level spans are screened for outliers, the outlier check is performed only if the span is a request level span - has 'http.url' attribute + if (span.attributes['http.url'] && this._isOutlier(span)) { + traceData.hasOutlier = true; + } + + // The trace ends when the root span ends, so we only filter and send data when the span that ended is the root span + // The traces that do not have outlier spans are dropped and the rest is sent to buffer/export + if (!span.parentSpanId) { + if (traceData.hasOutlier) { + traceData.spans.forEach(super.onEnd, this); + } + this._traces.delete(traceId); + } + } + } + // Export only outliers on shut down as well for http engine + onShutdown() { + this._traces.forEach((traceData, traceId) => { + if (traceData.hasOutlier) { + traceData.spans.forEach(super.onEnd, this); + } + }); + this._traces.clear(); + // By here all the available HTTP engine traces are processed and sent to buffer, the parent onShutDown will cover for possible playwright engine traces and the shutdown process + super.onShutdown(); + } + + // The outlier detection logic based on the provided criteria + _isOutlier(span) { + return !!span.attributes.outlier; + } +} + +module.exports = { OutlierDetectionBatchSpanProcessor }; diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js index 6b2b77d2c3..08c580aacb 100644 --- a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js @@ -3,6 +3,9 @@ const debug = require('debug')('plugin:publish-metrics:open-telemetry'); const grpc = require('@grpc/grpc-js'); const { traceExporters, validateExporter } = require('../exporters'); +const { + OutlierDetectionBatchSpanProcessor +} = require('../outlier-detection-processor'); const { SemanticAttributes } = require('@opentelemetry/semantic-conventions'); const { @@ -57,8 +60,12 @@ class OTelTraceConfig { this.exporterOpts ); + const Processor = this.config.smartSampling + ? OutlierDetectionBatchSpanProcessor + : BatchSpanProcessor; + this.tracerProvider.addSpanProcessor( - new BatchSpanProcessor(this.exporter, { + new Processor(this.exporter, { scheduledDelayMillis: 1000 }) ); diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/http.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/http.js index f55a74d914..cbf07b898a 100644 --- a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/http.js +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/http.js @@ -15,6 +15,8 @@ const { class OTelHTTPTraceReporter extends OTelTraceBase { constructor(config, script) { super(config, script); + this.outlierCriteria = config.smartSampling; + this.statusAsErrorThreshold = 400; } run() { this.setTracer('http'); @@ -86,10 +88,14 @@ class OTelHTTPTraceReporter extends OTelTraceBase { if (!userContext.vars['__otlpHTTPRequestSpan']) { return done(); } - const span = userContext.vars['__otlpHTTPRequestSpan']; let endTime; + const scenarioSpan = userContext.vars['__httpScenarioSpan']; + if (this.config.smartSampling) { + this.tagResponseOutliers(span, scenarioSpan, res, this.outlierCriteria); + } + if (res.timings && res.timings.phases) { span.setAttribute('response.time.ms', res.timings.phases.firstByte); @@ -134,7 +140,7 @@ class OTelHTTPTraceReporter extends OTelTraceBase { res.request.options.headers['user-agent'] }); - if (res.statusCode >= 400) { + if (res.statusCode >= this.statusAsErrorThreshold) { span.setStatus({ code: SpanStatusCode.ERROR, message: res.statusMessage @@ -161,6 +167,14 @@ class OTelHTTPTraceReporter extends OTelTraceBase { code: SpanStatusCode.ERROR, message: err.message || err }); + + if (this.config.smartSampling) { + requestSpan.setAttributes({ + outlier: 'true', + 'outlier.type.error': true + }); + } + requestSpan.end(); this.pendingRequestSpans--; } else { @@ -171,10 +185,50 @@ class OTelHTTPTraceReporter extends OTelTraceBase { code: SpanStatusCode.ERROR, message: err.message || err }); + + if (this.config.smartSampling) { + scenarioSpan.setAttributes({ + outlier: 'true', + 'outlier.type.error': true + }); + } + scenarioSpan.end(); this.pendingScenarioSpans--; return done(); } + + tagResponseOutliers(span, scenarioSpan, res, criteria) { + const types = {}; + const details = []; + if (res.statusCode >= this.statusAsErrorThreshold) { + types['outlier.type.status_code'] = true; + details.push(`HTTP Status Code >= ${this.statusAsErrorThreshold}`); + } + if (criteria.thresholds && res.timings?.phases) { + Object.entries(criteria.thresholds).forEach(([name, value]) => { + if (res.timings.phases[name] >= value) { + types[`outlier.type.${name}`] = true; + details.push(`'${name}' >= ${value}`); + } + }); + } + + if (!details.length) { + return; + } + + span.setAttributes({ + outlier: 'true', + 'outlier.details': details.join(', '), + ...types + }); + + scenarioSpan.setAttributes({ + outlier: 'true', + ...types + }); + } } module.exports = { diff --git a/packages/types/schema/plugins/publish-metrics.js b/packages/types/schema/plugins/publish-metrics.js index ec0ee75054..71e2fc1865 100644 --- a/packages/types/schema/plugins/publish-metrics.js +++ b/packages/types/schema/plugins/publish-metrics.js @@ -181,7 +181,13 @@ const OpenTelemetryReporterSchema = Joi.object({ .default('otlp-http'), sampleRate: artilleryNumberOrString, useRequestNames: artilleryBooleanOrString, - attributes: Joi.object().unknown() + attributes: Joi.object().unknown(), + smartSampling: Joi.object({ + thresholds: Joi.object({ + firstByte: artilleryNumberOrString, + total: artilleryNumberOrString + }) + }) }) }) .unknown(false)