From 2c4c368009c88863491a0dbfe45b3782027dab81 Mon Sep 17 00:00:00 2001 From: David Luna Date: Thu, 2 May 2024 13:46:12 +0200 Subject: [PATCH 1/7] chore(mock-server): make ui printer optional --- package-lock.json | 4 +++- packages/mockotlpserver/db/README.md | 2 +- packages/mockotlpserver/lib/cli.js | 15 ++++++++++++++- packages/mockotlpserver/lib/ui.js | 10 ++++++++-- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/package-lock.json b/package-lock.json index 17e3b14e..f2bba7c2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,10 +1,12 @@ { "name": "elastic-otel-node", + "version": "0.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "elastic-otel-node", + "version": "0.1.0", "license": "Apache-2.0", "workspaces": [ "packages/*", @@ -7807,7 +7809,7 @@ }, "packages/opentelemetry-node": { "name": "@elastic/opentelemetry-node", - "version": "0.1.2", + "version": "0.1.3", "license": "Apache-2.0", "dependencies": { "@opentelemetry/exporter-logs-otlp-proto": "^0.51.0", diff --git a/packages/mockotlpserver/db/README.md b/packages/mockotlpserver/db/README.md index 66ad2fb2..1d8cd94c 100644 --- a/packages/mockotlpserver/db/README.md +++ b/packages/mockotlpserver/db/README.md @@ -1,3 +1,3 @@ # UI traces folder -this folder is to place all the traces data we want to plt in the web ui. \ No newline at end of file +this folder is to place all the traces data we want to plot in the web ui. \ No newline at end of file diff --git a/packages/mockotlpserver/lib/cli.js b/packages/mockotlpserver/lib/cli.js index a0523d89..b5f1dd53 100755 --- a/packages/mockotlpserver/lib/cli.js +++ b/packages/mockotlpserver/lib/cli.js @@ -50,6 +50,8 @@ const PRINTER_NAMES = [ 'metrics-summary', 'logs-summary', 'summary', + + 'trace-web', // printers to be seen in a web page ]; // This adds a custom cli option type to dashdash, to support `-o json,waterfall` @@ -121,9 +123,20 @@ async function main() { process.exit(0); } + + const hasWebPrinter = opts.o.some((printerName) => printerName.endsWith('web')); + console.log('hasWebPrinter', hasWebPrinter) + console.log(opts.o) + const otlpServer = new MockOtlpServer({ log, - services: ['http', 'grpc', 'ui'], + // TODO: this is a shortcut to enable/disable the UI printer but + // the service is not started at all. The dev cann't check UI for previous traces. + // Another option could be to promote the UIPrinter to a new type `FilePrinter` + // which saves traces, metrics & logs into files. This would: + // - allow the UI work without printers + // - have more control on what's stored on disk + services: hasWebPrinter ? ['http', 'grpc', 'ui'] : ['http', 'grpc'], grpcHostname: opts.hostname || DEFAULT_HOSTNAME, httpHostname: opts.hostname || DEFAULT_HOSTNAME, uiHostname: opts.hostname || DEFAULT_HOSTNAME, diff --git a/packages/mockotlpserver/lib/ui.js b/packages/mockotlpserver/lib/ui.js index f3ab29cd..26d5354b 100644 --- a/packages/mockotlpserver/lib/ui.js +++ b/packages/mockotlpserver/lib/ui.js @@ -47,7 +47,7 @@ class UiPrinter extends Printer { } /** - * Prints into files the spns belonging to a trace + * Prints into files the spans belonging to a trace * @param {import('./types').ExportTraceServiceRequest} traceReq */ printTrace(traceReq) { @@ -170,9 +170,15 @@ class UiService extends Service { const traceFiles = fs.readdirSync(dataPath).filter((f) => { return f.startsWith('trace-'); }); + const sortedFiles = traceFiles.sort((fileA, fileB) => { + const statA = fs.statSync(`${dataPath}/${fileA}`); + const statB = fs.statSync(`${dataPath}/${fileB}`); + + return new Date(statB.birthtime).getTime() - new Date(statA.birthtime).getTime(); + }); res.writeHead(200, {'Content-Type': 'application/json'}); - res.end(JSON.stringify(traceFiles)); + res.end(JSON.stringify(sortedFiles)); return; } else if (req.url.startsWith('/api/traces/')) { const traceId = req.url.replace('/api/traces/', ''); From 7585492616bc9498fb56f5945c8a2498c7989693 Mon Sep 17 00:00:00 2001 From: David Luna Date: Thu, 2 May 2024 15:20:16 +0200 Subject: [PATCH 2/7] chore(mock-server): create file printer --- packages/mockotlpserver/lib/cli.js | 22 +++--- packages/mockotlpserver/lib/printers.js | 53 ++++++++++++++ packages/mockotlpserver/lib/ui.js | 96 +------------------------ 3 files changed, 63 insertions(+), 108 deletions(-) diff --git a/packages/mockotlpserver/lib/cli.js b/packages/mockotlpserver/lib/cli.js index b5f1dd53..cfe38c96 100755 --- a/packages/mockotlpserver/lib/cli.js +++ b/packages/mockotlpserver/lib/cli.js @@ -24,7 +24,7 @@ const dashdash = require('dashdash'); const luggite = require('./luggite'); -const {JSONPrinter, InspectPrinter} = require('./printers'); +const {JSONPrinter, InspectPrinter, FilePrinter} = require('./printers'); const {TraceWaterfallPrinter} = require('./waterfall'); const {MetricsSummaryPrinter} = require('./metrics-summary'); const {LogsSummaryPrinter} = require('./logs-summary'); @@ -123,20 +123,9 @@ async function main() { process.exit(0); } - - const hasWebPrinter = opts.o.some((printerName) => printerName.endsWith('web')); - console.log('hasWebPrinter', hasWebPrinter) - console.log(opts.o) - const otlpServer = new MockOtlpServer({ log, - // TODO: this is a shortcut to enable/disable the UI printer but - // the service is not started at all. The dev cann't check UI for previous traces. - // Another option could be to promote the UIPrinter to a new type `FilePrinter` - // which saves traces, metrics & logs into files. This would: - // - allow the UI work without printers - // - have more control on what's stored on disk - services: hasWebPrinter ? ['http', 'grpc', 'ui'] : ['http', 'grpc'], + services: ['http', 'grpc', 'ui'], grpcHostname: opts.hostname || DEFAULT_HOSTNAME, httpHostname: opts.hostname || DEFAULT_HOSTNAME, uiHostname: opts.hostname || DEFAULT_HOSTNAME, @@ -199,6 +188,13 @@ async function main() { case 'logs-summary': printers.push(new LogsSummaryPrinter(log)); break; + case 'trace-summary': + printers.push(new TraceWaterfallPrinter(log)); + break; + + case 'trace-web': + printers.push(new FilePrinter(log)); + break; } }); printers.forEach((p) => p.subscribe()); diff --git a/packages/mockotlpserver/lib/printers.js b/packages/mockotlpserver/lib/printers.js index 46e6bb31..d6eaf91d 100644 --- a/packages/mockotlpserver/lib/printers.js +++ b/packages/mockotlpserver/lib/printers.js @@ -26,6 +26,9 @@ * printer.subscribe(); */ +const fs = require('fs'); +const path = require('path'); + const { diagchSub, CH_OTLP_V1_LOGS, @@ -153,8 +156,58 @@ class JSONPrinter extends Printer { } } +/** + * This printer converts to a possible JSON representation of each service + * request and saves it to a file. **Warning**: Converting OTLP service requests to JSON is fraught. + */ +class FilePrinter extends Printer { + constructor(log, indent, signals = ['trace'], dbDir = path.resolve(__dirname, '../db')) { + super(log); + this._indent = indent || 0; + this._signals = signals; + this._dbDir = dbDir; + } + printTrace(trace) { + if (!this._signals.includes('trace')) return; + const str = jsonStringifyTrace(trace, { + indent: this._indent, + normAttributes: true, + }); + const normTrace = JSON.parse(str); + const tracesMap = new Map(); + normTrace.resourceSpans.forEach((resSpan) => { + resSpan.scopeSpans.forEach((scopeSpan) => { + scopeSpan.spans.forEach((span) => { + let traceSpans = tracesMap.get(span.traceId); + + if (!traceSpans) { + traceSpans = []; + tracesMap.set(span.traceId, traceSpans); + } + traceSpans.push(span); + }); + }); + }); + + // Group als spans from the same trace into an ndjson file + for (const [traceId, traceSpans] of tracesMap.entries()) { + const filePath = path.join(this._dbDir, `trace-${traceId}.ndjson`); + const stream = fs.createWriteStream(filePath, { + flags: 'a', + encoding: 'utf-8', + }); + + for (const span of traceSpans) { + stream.write(JSON.stringify(span) + '\n'); + } + stream.close(); + } + } +} + module.exports = { Printer, JSONPrinter, InspectPrinter, + FilePrinter, }; diff --git a/packages/mockotlpserver/lib/ui.js b/packages/mockotlpserver/lib/ui.js index 26d5354b..960f9077 100644 --- a/packages/mockotlpserver/lib/ui.js +++ b/packages/mockotlpserver/lib/ui.js @@ -21,101 +21,10 @@ const fs = require('fs'); const path = require('path'); const http = require('http'); -const Long = require('long'); - -const {Printer} = require('./printers'); const {Service} = require('./service'); // helper functions -/** - * @typedef {Object} SpanTree - * @property {import('./types').Span} span - * @property {import('./types').Span[]} children - */ - -/** - * @typedef {Object} TraceTree - * @property {string} id - * @property {SpanTree[]} children - */ - -class UiPrinter extends Printer { - constructor(log) { - super(log); - this._dbDir = path.resolve(__dirname, '../db'); - } - - /** - * Prints into files the spans belonging to a trace - * @param {import('./types').ExportTraceServiceRequest} traceReq - */ - printTrace(traceReq) { - /** @type {Map} */ - const tracesMap = new Map(); - - // Group all spans by trace - traceReq.resourceSpans.forEach((resSpan) => { - resSpan.scopeSpans.forEach((scopeSpan) => { - scopeSpan.spans.forEach((span) => { - const traceId = span.traceId.toString('hex'); - let traceSpans = tracesMap.get(traceId); - - if (!traceSpans) { - traceSpans = []; - tracesMap.set(traceId, traceSpans); - } - traceSpans.push(span); - }); - }); - }); - - // Write into a file - // TODO: manage lifetime of old trace ndjson files. - for (const [traceId, traceSpans] of tracesMap.entries()) { - const filePath = path.join(this._dbDir, `trace-${traceId}.ndjson`); - const stream = fs.createWriteStream(filePath, { - flags: 'a', - encoding: 'utf-8', - }); - - for (const span of traceSpans) { - const formatted = this._formatSpan(span); - stream.write( - JSON.stringify(Object.assign({}, span, formatted)) + '\n' - ); - } - stream.close(); - } - } - - /** - * @param {import('./types').Span} span - */ - _formatSpan(span) { - const traceId = span.traceId.toString('hex'); - const spanId = span.spanId.toString('hex'); - const parentSpanId = span.parentSpanId?.toString('hex'); - const formatted = {traceId, spanId}; - - if (parentSpanId) { - formatted.parentSpanId = parentSpanId; - } - formatted.startTimeUnixNano = new Long( - span.startTimeUnixNano.low, - span.startTimeUnixNano.high, - span.startTimeUnixNano.unsigned - ).toString(); - formatted.endTimeUnixNano = new Long( - span.endTimeUnixNano.low, - span.endTimeUnixNano.high, - span.endTimeUnixNano.unsigned - ).toString(); - - return formatted; - } -} - /** * @param {http.ServerResponse} res */ @@ -152,7 +61,6 @@ class UiService extends Service { super(); this._opts = opts; this._server = null; - this._printer = new UiPrinter(opts.log); } async start() { @@ -174,7 +82,7 @@ class UiService extends Service { const statA = fs.statSync(`${dataPath}/${fileA}`); const statB = fs.statSync(`${dataPath}/${fileB}`); - return new Date(statB.birthtime).getTime() - new Date(statA.birthtime).getTime(); + return new Date(statA.birthtime).getTime() - new Date(statB.birthtime).getTime(); }); res.writeHead(200, {'Content-Type': 'application/json'}); @@ -217,8 +125,6 @@ class UiService extends Service { }); }); - this._printer.subscribe(); - return new Promise((resolve, reject) => { this._server.listen(port, hostname, () => { resolve(); From c527c7780c1cddaae08f96821668a2302ea94d4a Mon Sep 17 00:00:00 2001 From: David Luna Date: Thu, 2 May 2024 15:21:41 +0200 Subject: [PATCH 3/7] chore(mock-server): fix lint issues --- packages/mockotlpserver/lib/cli.js | 3 --- packages/mockotlpserver/lib/printers.js | 7 ++++++- packages/mockotlpserver/lib/ui.js | 5 ++++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/mockotlpserver/lib/cli.js b/packages/mockotlpserver/lib/cli.js index cfe38c96..a58f067b 100755 --- a/packages/mockotlpserver/lib/cli.js +++ b/packages/mockotlpserver/lib/cli.js @@ -188,9 +188,6 @@ async function main() { case 'logs-summary': printers.push(new LogsSummaryPrinter(log)); break; - case 'trace-summary': - printers.push(new TraceWaterfallPrinter(log)); - break; case 'trace-web': printers.push(new FilePrinter(log)); diff --git a/packages/mockotlpserver/lib/printers.js b/packages/mockotlpserver/lib/printers.js index d6eaf91d..baf33fed 100644 --- a/packages/mockotlpserver/lib/printers.js +++ b/packages/mockotlpserver/lib/printers.js @@ -161,7 +161,12 @@ class JSONPrinter extends Printer { * request and saves it to a file. **Warning**: Converting OTLP service requests to JSON is fraught. */ class FilePrinter extends Printer { - constructor(log, indent, signals = ['trace'], dbDir = path.resolve(__dirname, '../db')) { + constructor( + log, + indent, + signals = ['trace'], + dbDir = path.resolve(__dirname, '../db') + ) { super(log); this._indent = indent || 0; this._signals = signals; diff --git a/packages/mockotlpserver/lib/ui.js b/packages/mockotlpserver/lib/ui.js index 960f9077..af9f7fdb 100644 --- a/packages/mockotlpserver/lib/ui.js +++ b/packages/mockotlpserver/lib/ui.js @@ -82,7 +82,10 @@ class UiService extends Service { const statA = fs.statSync(`${dataPath}/${fileA}`); const statB = fs.statSync(`${dataPath}/${fileB}`); - return new Date(statA.birthtime).getTime() - new Date(statB.birthtime).getTime(); + return ( + new Date(statA.birthtime).getTime() - + new Date(statB.birthtime).getTime() + ); }); res.writeHead(200, {'Content-Type': 'application/json'}); From 944ec3c7de08f96f61b70d470fa12264a8cc0b19 Mon Sep 17 00:00:00 2001 From: David Luna Date: Thu, 2 May 2024 15:35:53 +0200 Subject: [PATCH 4/7] chore(mock-server): fix trace sorting in UI --- packages/mockotlpserver/lib/ui.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/mockotlpserver/lib/ui.js b/packages/mockotlpserver/lib/ui.js index af9f7fdb..41eedca7 100644 --- a/packages/mockotlpserver/lib/ui.js +++ b/packages/mockotlpserver/lib/ui.js @@ -83,8 +83,8 @@ class UiService extends Service { const statB = fs.statSync(`${dataPath}/${fileB}`); return ( - new Date(statA.birthtime).getTime() - - new Date(statB.birthtime).getTime() + new Date(statB.birthtime).getTime() - + new Date(statA.birthtime).getTime() ); }); From 56d46352ce005a1e7e4a75fdc8e2756f2f304070 Mon Sep 17 00:00:00 2001 From: David Luna Date: Fri, 3 May 2024 11:41:22 +0200 Subject: [PATCH 5/7] chore(mock-server): add boolean option for ui --- packages/mockotlpserver/lib/cli.js | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/packages/mockotlpserver/lib/cli.js b/packages/mockotlpserver/lib/cli.js index a58f067b..808a7534 100755 --- a/packages/mockotlpserver/lib/cli.js +++ b/packages/mockotlpserver/lib/cli.js @@ -51,7 +51,7 @@ const PRINTER_NAMES = [ 'logs-summary', 'summary', - 'trace-web', // printers to be seen in a web page + 'trace-file', // saving into fs for UI and other processing ]; // This adds a custom cli option type to dashdash, to support `-o json,waterfall` @@ -99,6 +99,11 @@ const OPTIONS = [ type: 'string', help: `The hostname on which servers should listen, by default this is "${DEFAULT_HOSTNAME}".`, }, + { + names: ['ui'], + type: 'bool', + help: `Start a web server to inspect traces with some charts.`, + }, ]; async function main() { @@ -123,9 +128,15 @@ async function main() { process.exit(0); } + /** @type {Array<'http'|'grpc'|'ui'>} */ + const services = ['http', 'grpc']; + if (opts.ui) { + services.push('ui'); + } + const otlpServer = new MockOtlpServer({ log, - services: ['http', 'grpc', 'ui'], + services, grpcHostname: opts.hostname || DEFAULT_HOSTNAME, httpHostname: opts.hostname || DEFAULT_HOSTNAME, uiHostname: opts.hostname || DEFAULT_HOSTNAME, @@ -189,7 +200,7 @@ async function main() { printers.push(new LogsSummaryPrinter(log)); break; - case 'trace-web': + case 'trace-file': printers.push(new FilePrinter(log)); break; } From 558321be6f4444c680d7085604fadf5986e64f8b Mon Sep 17 00:00:00 2001 From: David Luna Date: Tue, 7 May 2024 13:06:50 +0200 Subject: [PATCH 6/7] chore(mock-server): add file printer when UI is enabled --- packages/mockotlpserver/lib/cli.js | 21 ++++++++++++++++++--- packages/mockotlpserver/lib/printers.js | 2 +- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/mockotlpserver/lib/cli.js b/packages/mockotlpserver/lib/cli.js index 808a7534..97f17d61 100755 --- a/packages/mockotlpserver/lib/cli.js +++ b/packages/mockotlpserver/lib/cli.js @@ -54,8 +54,15 @@ const PRINTER_NAMES = [ 'trace-file', // saving into fs for UI and other processing ]; -// This adds a custom cli option type to dashdash, to support `-o json,waterfall` -// options for specifying multiple printers (aka output modes). +/** + * This adds a custom cli option type to dashdash, to support `-o json,waterfall` + * options for specifying multiple printers (aka output modes). + * + * @param {any} option + * @param {string} optstr + * @param {string} arg + * @returns {Array} + */ function parseCommaSepPrinters(option, optstr, arg) { const printers = arg .trim() @@ -130,8 +137,12 @@ async function main() { /** @type {Array<'http'|'grpc'|'ui'>} */ const services = ['http', 'grpc']; + /** @type {Array} */ + const outputs = opts.o; + if (opts.ui) { services.push('ui'); + outputs.push('trace-file'); } const otlpServer = new MockOtlpServer({ @@ -143,8 +154,12 @@ async function main() { }); await otlpServer.start(); + // Avoid duplication of printers + const printersSet = new Set(outputs); const printers = []; - opts.o.forEach((printerName) => { + + printersSet.forEach((printerName) => { + console.log(printerName) switch (printerName) { case 'trace-inspect': printers.push(new InspectPrinter(log, ['trace'])); diff --git a/packages/mockotlpserver/lib/printers.js b/packages/mockotlpserver/lib/printers.js index baf33fed..25048b12 100644 --- a/packages/mockotlpserver/lib/printers.js +++ b/packages/mockotlpserver/lib/printers.js @@ -194,7 +194,7 @@ class FilePrinter extends Printer { }); }); - // Group als spans from the same trace into an ndjson file + // Group all spans from the same trace into an ndjson file. for (const [traceId, traceSpans] of tracesMap.entries()) { const filePath = path.join(this._dbDir, `trace-${traceId}.ndjson`); const stream = fs.createWriteStream(filePath, { From a2ee3560531fe231fe0d90f0fc3bab36b59ccd52 Mon Sep 17 00:00:00 2001 From: David Luna Date: Tue, 7 May 2024 13:08:50 +0200 Subject: [PATCH 7/7] chore(mock-server): lint fix --- packages/mockotlpserver/lib/cli.js | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/mockotlpserver/lib/cli.js b/packages/mockotlpserver/lib/cli.js index 97f17d61..0b2ed54b 100755 --- a/packages/mockotlpserver/lib/cli.js +++ b/packages/mockotlpserver/lib/cli.js @@ -57,10 +57,10 @@ const PRINTER_NAMES = [ /** * This adds a custom cli option type to dashdash, to support `-o json,waterfall` * options for specifying multiple printers (aka output modes). - * - * @param {any} option - * @param {string} optstr - * @param {string} arg + * + * @param {any} option + * @param {string} optstr + * @param {string} arg * @returns {Array} */ function parseCommaSepPrinters(option, optstr, arg) { @@ -109,7 +109,7 @@ const OPTIONS = [ { names: ['ui'], type: 'bool', - help: `Start a web server to inspect traces with some charts.`, + help: 'Start a web server to inspect traces with some charts.', }, ]; @@ -159,7 +159,6 @@ async function main() { const printers = []; printersSet.forEach((printerName) => { - console.log(printerName) switch (printerName) { case 'trace-inspect': printers.push(new InspectPrinter(log, ['trace']));