From e896c29624bb6ec8feb4db224bd21c06712e94bf Mon Sep 17 00:00:00 2001 From: Simon Bordeyne Date: Sat, 15 Nov 2025 14:40:39 +0100 Subject: [PATCH 1/2] tooling: add log generator and collection to compose.yaml --- .alloy/config.alloy | 49 +++++++++++++++++++++++++++++++++++++++++++++ .editorconfig | 3 +++ compose.yaml | 27 +++++++++++++++++++++++++ 3 files changed, 79 insertions(+) create mode 100644 .alloy/config.alloy create mode 100644 .editorconfig diff --git a/.alloy/config.alloy b/.alloy/config.alloy new file mode 100644 index 00000000..865cc741 --- /dev/null +++ b/.alloy/config.alloy @@ -0,0 +1,49 @@ +livedebugging { + enabled = true +} + +discovery.docker "linux" { + host = "unix:///var/run/docker.sock" +} + +discovery.relabel "logs_integrations_docker" { + targets = [] + + rule { + source_labels = ["__meta_docker_container_name"] + regex = "/(.*)" + target_label = "service_name" + } +} + +loki.source.docker "default" { + host = "unix:///var/run/docker.sock" + targets = discovery.docker.linux.targets + labels = {"platform" = "docker"} + relabel_rules = discovery.relabel.logs_integrations_docker.rules + forward_to = [loki.process.logs.receiver] +} + +loki.process "logs" { + stage.static_labels { + values = { + env = "dev", + } + } + + forward_to = [loki.write.local.receiver] +} + +loki.write "local" { + endpoint { + url = string.format( + "http://victorialogs:9428/insert/loki/api/v1/push?disable_message_parsing=1&_stream_fields=%s", + string.join([ + "env", + "platform", + "service_name", + "level", + ], ","), + ) + } +} \ No newline at end of file diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 00000000..f7926408 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,3 @@ +[*.ts] +indent_style = space +indent_size = 2 \ No newline at end of file diff --git a/compose.yaml b/compose.yaml index 133d8f18..94256c49 100644 --- a/compose.yaml +++ b/compose.yaml @@ -1,7 +1,34 @@ services: + # Generate some logs for testing, in JSON format + flog-json: + image: mingrammer/flog:latest + command: ["flog", "-f", "json", "--number", "36000", "--sleep", "100ms"] + # Run Grafana Alloy to collect logs from Docker and forward them to VictoriaLogs + alloy: + image: grafana/alloy:latest + privileged: true + ports: + - 12345:12345 + - 4317:4317 + - 4318:4318 + volumes: + - ./.alloy/config.alloy:/etc/alloy/config.alloy:ro + - /proc:/rootproc:ro + - /var/run/docker.sock.raw:/var/run/docker.sock + - /sys:/sys:ro + - /:/rootfs:ro + - /dev/disk/:/dev/disk:ro + - /var/lib/docker/:/var/lib/docker:ro + command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy + extra_hosts: + - "host.docker.internal:host-gateway" + devices: + - /dev/kmsg socks-proxy: image: serjs/go-socks5-proxy restart: always + environment: + REQUIRE_AUTH: "false" victorialogs: image: victoriametrics/victoria-logs:v1.26.0 ports: From 268e67dac7052904d2909bc76a77e9fa1bdd0819 Mon Sep 17 00:00:00 2001 From: Simon Bordeyne Date: Sat, 15 Nov 2025 14:41:32 +0100 Subject: [PATCH 2/2] wip: add first draft of autocomplete engine for LogsQL Based on the code for the official Loki datasource. LogsQL and LogQL are actually pretty similar, making it a good base to get started. Currently implemented: - stream autocomplete - syntax highlighting for LogQL --- src/components/QueryEditor/QueryField.tsx | 2 + .../monaco-query-field/MonacoQueryField.tsx | 98 ++- .../MonacoQueryFieldProps.ts | 5 +- .../completion/CompletionDataProvider.ts | 124 +++ .../completion/NeverCaseError.ts | 5 + .../completion/completionUtils.ts | 172 +++++ .../completion/completions.ts | 452 +++++++++++ .../completion/situation.ts | 725 ++++++++++++++++++ src/language/index.ts | 221 ++++++ src/language/utils.ts | 166 ++++ src/languageUtils.ts | 38 + src/language_provider.ts | 91 ++- src/queryUtils.ts | 8 +- src/types.ts | 9 + 14 files changed, 2089 insertions(+), 27 deletions(-) create mode 100644 src/components/monaco-query-field/completion/CompletionDataProvider.ts create mode 100644 src/components/monaco-query-field/completion/NeverCaseError.ts create mode 100644 src/components/monaco-query-field/completion/completionUtils.ts create mode 100644 src/components/monaco-query-field/completion/completions.ts create mode 100644 src/components/monaco-query-field/completion/situation.ts create mode 100644 src/language/index.ts create mode 100644 src/language/utils.ts diff --git a/src/components/QueryEditor/QueryField.tsx b/src/components/QueryEditor/QueryField.tsx index 7d2d4a20..335faf73 100644 --- a/src/components/QueryEditor/QueryField.tsx +++ b/src/components/QueryEditor/QueryField.tsx @@ -18,6 +18,7 @@ const QueryField: React.FC = ( history, onRunQuery, onChange, + datasource, 'data-testid': dataTestId }) => { @@ -35,6 +36,7 @@ const QueryField: React.FC = ( \/?\s]+)/g, + // Default: /(-?\d*\.\d\w*)|([^`~!#%^&*()\-=+\[{\]}\\|;:'",.<>\/?\s]+)/g + // Removed `"`, `=`, and `-`, from the exclusion list, so now the completion provider can decide to overwrite any matching words, or just insert text at the cursor + }); + } +} + const getStyles = (theme: GrafanaTheme2, placeholder: string) => { return { container: css` @@ -67,33 +91,66 @@ const getStyles = (theme: GrafanaTheme2, placeholder: string) => { const MonacoQueryField = (props: Props) => { // we need only one instance of `overrideServices` during the lifetime of the react component const containerRef = useRef(null); - const { onBlur, onRunQuery, initialValue, placeholder, readOnly } = props; + const { onBlur, onRunQuery, initialValue, placeholder, readOnly, history, timeRange, datasource } = props; const onRunQueryRef = useLatest(onRunQuery); const onBlurRef = useLatest(onBlur); + const historyRef = useLatest(history); + const langProviderRef = useLatest(datasource.languageProvider); + const completionDataProviderRef = useRef(null); + const autocompleteCleanupCallback = useRef<(() => void) | null>(null); const theme = useTheme2(); const styles = getStyles(theme, placeholder); + useEffect(() => { + // when we unmount, we unregister the autocomplete-function, if it was registered + return () => { + autocompleteCleanupCallback.current?.(); + }; + }, []); + return ( -
+
{ + ensureVictoriaLogsLogsQL(monaco); + }} onMount={(editor, monaco) => { // we setup on-blur editor.onDidBlurEditorWidget(() => { onBlurRef.current(editor.getValue()); }); + const dataProvider = new CompletionDataProvider(langProviderRef.current!, historyRef, timeRange); + completionDataProviderRef.current = dataProvider; + const completionProvider = getCompletionProvider(monaco, dataProvider); + + // completion-providers in monaco are not registered directly to editor-instances, + // they are registered to languages. this makes it hard for us to have + // separate completion-providers for every query-field-instance + // (but we need that, because they might connect to different datasources). + // the trick we do is, we wrap the callback in a "proxy", + // and in the proxy, the first thing is, we check if we are called from + // "our editor instance", and if not, we just return nothing. if yes, + // we call the completion-provider. + const filteringCompletionProvider: monacoTypes.languages.CompletionItemProvider = { + ...completionProvider, + provideCompletionItems: (model, position, context, token) => { + // if the model-id does not match, then this call is from a different editor-instance, + // not "our instance", so return nothing + if (editor.getModel()?.id !== model.id) { + return { suggestions: [] }; + } + return completionProvider.provideCompletionItems(model, position, context, token); + }, + }; + const { dispose } = monaco.languages.registerCompletionItemProvider(LANG_ID, filteringCompletionProvider); + + autocompleteCleanupCallback.current = dispose; const updateElementHeight = () => { const containerDiv = containerRef.current; if (containerDiv !== null) { @@ -110,10 +167,10 @@ const MonacoQueryField = (props: Props) => { // handle: shift + enter editor.addAction({ - id: "execute-shift-enter", - label: "Execute", + id: 'execute-shift-enter', + label: 'Execute', keybindings: [monaco.KeyMod.Shift | monaco.KeyCode.Enter], - run: () => onRunQueryRef.current(editor.getValue() || "") + run: () => onRunQueryRef.current(editor.getValue() || ''), }); /* Something in this configuration of monaco doesn't bubble up [mod]+K, which the @@ -127,13 +184,8 @@ const MonacoQueryField = (props: Props) => { const placeholderDecorators = [ { range: new monaco.Range(1, 1, 1, 1), - contents: [ - { value: "**bold** _italics_ regular `code`" } - ], - options: { - className: styles.placeholder, - isWholeLine: true, - }, + contents: [{ value: '**bold** _italics_ regular `code`' }], + options: { className: styles.placeholder, isWholeLine: true }, }, ]; diff --git a/src/components/monaco-query-field/MonacoQueryFieldProps.ts b/src/components/monaco-query-field/MonacoQueryFieldProps.ts index 5ec6678e..5fda4a25 100755 --- a/src/components/monaco-query-field/MonacoQueryFieldProps.ts +++ b/src/components/monaco-query-field/MonacoQueryFieldProps.ts @@ -1,5 +1,6 @@ -import { HistoryItem } from '@grafana/data'; +import { HistoryItem, TimeRange } from '@grafana/data'; +import { VictoriaLogsDatasource } from '../../datasource'; import { Query } from '../../types'; export type Props = { @@ -7,6 +8,8 @@ export type Props = { history: Array>; placeholder: string; readOnly?: boolean; + timeRange?: TimeRange; + datasource: VictoriaLogsDatasource; onRunQuery: (value: string) => void; onBlur: (value: string) => void; }; diff --git a/src/components/monaco-query-field/completion/CompletionDataProvider.ts b/src/components/monaco-query-field/completion/CompletionDataProvider.ts new file mode 100644 index 00000000..996dc8f5 --- /dev/null +++ b/src/components/monaco-query-field/completion/CompletionDataProvider.ts @@ -0,0 +1,124 @@ +import { chain } from 'lodash'; + +import { HistoryItem, TimeRange } from '@grafana/data'; + +// import { escapeLabelValueInExactSelector } from '../../../languageUtils'; +import LanguageProvider from '../../../language_provider'; +import { FilterFieldType, ParserAndLabelKeysResult, Query } from '../../../types'; + +import { Label } from './situation'; + + +interface HistoryRef { + current: Array>; +} + +export class CompletionDataProvider { + constructor( + public languageProvider: LanguageProvider, + private historyRef: HistoryRef = { current: [] }, + public timeRange: TimeRange | undefined + ) { + this.queryToLabelKeysCache = new Map(); + } + private queryToLabelKeysCache: Map; + +// private buildSelector(labels: Label[]): string { +// const allLabelTexts = labels.map( +// (label) => `${label.name}${label.op}"${escapeLabelValueInExactSelector(label.value)}"` +// ); + +// return `{${allLabelTexts.join(',')}}`; +// } + + setTimeRange(timeRange: TimeRange) { + this.timeRange = timeRange; + this.queryToLabelKeysCache.clear(); + } + + getHistory() { + return chain(this.historyRef.current) + .orderBy('ts', 'desc') + .map((history: HistoryItem) => history.query.expr.trim()) + .filter() + .uniq() + .value(); + } + + public buildQuery(otherLabels: Label[]): string { + if (otherLabels.length === 0) { + return '*'; + } + return otherLabels.map( + (label) => `${label.name}:=${label.value}` + ).join(' AND '); + } + + async getLabelNames(query: string): Promise { + const hits = await this.languageProvider.getFieldList({ + timeRange: this.timeRange, + type: FilterFieldType.FieldName, + query, + }) + return hits + .filter((hit) => !hit.value.startsWith("_")) + .map((hit) => hit.value); + } +// if (otherLabels.length === 0) { +// // If there is no filtering, we use getLabelKeys because it has better caching +// // and all labels should already be fetched +// await this.languageProvider.start(); +// return this.languageProvider.getFieldList(); +// } +// const possibleLabelNames = await this.languageProvider.fetchLabels({ +// streamSelector: this.buildSelector(otherLabels), +// timeRange: this.timeRange, +// }); +// const usedLabelNames = new Set(otherLabels.map((l) => l.name)); // names used in the query +// return possibleLabelNames.filter((label) => !usedLabelNames.has(label)); +// } + + async getLabelValues(labelName: string, query: string): Promise { + const hits = await this.languageProvider.getFieldList({ + timeRange: this.timeRange, + type: FilterFieldType.FieldValue, + field: labelName, + query, + }); + return hits + .map((hit) => hit.value); + } + + /** + * Runs a Loki query to extract label keys from the result. + * The result is cached for the query string. + * + * Since various "situations" in the monaco code editor trigger this function, it is prone to being called multiple times for the same query + * Here is a lightweight and simple cache to avoid calling the backend multiple times for the same query. + * + * @param logQuery + */ + async getParserAndLabelKeys(logQuery: string): Promise { + const EXTRACTED_LABEL_KEYS_MAX_CACHE_SIZE = 2; + const cachedLabelKeys = this.queryToLabelKeysCache.has(logQuery) ? this.queryToLabelKeysCache.get(logQuery) : null; + if (cachedLabelKeys) { + // cache hit! Serve stale result from cache + return cachedLabelKeys; + } else { + // If cache is larger than max size, delete the first (oldest) index + if (this.queryToLabelKeysCache.size >= EXTRACTED_LABEL_KEYS_MAX_CACHE_SIZE) { + // Make room in the cache for the fresh result by deleting the "first" index + const keys = this.queryToLabelKeysCache.keys(); + const firstKey = keys.next().value; + if (firstKey !== undefined) { + this.queryToLabelKeysCache.delete(firstKey); + } + } + // Fetch a fresh result from the backend + const labelKeys = await this.languageProvider.getParserAndLabelKeys(logQuery, { timeRange: this.timeRange }); + // Add the result to the cache + this.queryToLabelKeysCache.set(logQuery, labelKeys); + return labelKeys; + } + } +} diff --git a/src/components/monaco-query-field/completion/NeverCaseError.ts b/src/components/monaco-query-field/completion/NeverCaseError.ts new file mode 100644 index 00000000..12d563f7 --- /dev/null +++ b/src/components/monaco-query-field/completion/NeverCaseError.ts @@ -0,0 +1,5 @@ +export class NeverCaseError extends Error { + constructor(value: never) { + super(`Unexpected case in switch statement: ${JSON.stringify(value)}`); + } +} diff --git a/src/components/monaco-query-field/completion/completionUtils.ts b/src/components/monaco-query-field/completion/completionUtils.ts new file mode 100644 index 00000000..4cacc113 --- /dev/null +++ b/src/components/monaco-query-field/completion/completionUtils.ts @@ -0,0 +1,172 @@ +import type { Monaco, monacoTypes } from '@grafana/ui'; + +import { CompletionDataProvider } from './CompletionDataProvider'; +import { NeverCaseError } from './NeverCaseError'; +import { CompletionType, getCompletions } from './completions'; +import { getSituation, Situation } from './situation'; + +// from: monacoTypes.languages.CompletionItemInsertTextRule.InsertAsSnippet +const INSERT_AS_SNIPPET_ENUM_VALUE = 4; + +export function getSuggestOptions(): monacoTypes.editor.ISuggestOptions { + return { + // monaco-editor sometimes provides suggestions automatically, i am not + // sure based on what, seems to be by analyzing the words already + // written. + // to try it out: + // - enter `go_goroutines{job~` + // - have the cursor at the end of the string + // - press ctrl-enter + // - you will get two suggestions + // those were not provided by grafana, they are offered automatically. + // i want to remove those. the only way i found is: + // - every suggestion-item has a `kind` attribute, + // that controls the icon to the left of the suggestion. + // - items auto-generated by monaco have `kind` set to `text`. + // - we make sure grafana-provided suggestions do not have `kind` set to `text`. + // - and then we tell monaco not to show suggestions of kind `text` + showWords: false, + }; +} + +function getMonacoCompletionItemKind(type: CompletionType, monaco: Monaco): monacoTypes.languages.CompletionItemKind { + switch (type) { + case 'DURATION': + return monaco.languages.CompletionItemKind.Unit; + case 'FUNCTION': + return monaco.languages.CompletionItemKind.Variable; + case 'HISTORY': + return monaco.languages.CompletionItemKind.Snippet; + case 'LABEL_NAME': + return monaco.languages.CompletionItemKind.Enum; + case 'LABEL_VALUE': + return monaco.languages.CompletionItemKind.EnumMember; + case 'PATTERN': + return monaco.languages.CompletionItemKind.Constructor; + case 'PARSER': + return monaco.languages.CompletionItemKind.Class; + case 'LINE_FILTER': + return monaco.languages.CompletionItemKind.TypeParameter; + case 'PIPE_OPERATION': + return monaco.languages.CompletionItemKind.Interface; + default: + throw new NeverCaseError(type); + } +} + +export function getCompletionProvider( + monaco: Monaco, + dataProvider: CompletionDataProvider +): monacoTypes.languages.CompletionItemProvider { + const provideCompletionItems = ( + model: monacoTypes.editor.ITextModel, + position: monacoTypes.Position + ): monacoTypes.languages.ProviderResult => { + const word = model.getWordAtPosition(position); + const wordUntil = model.getWordUntilPosition(position); + + // documentation says `position` will be "adjusted" in `getOffsetAt` + // i don't know what that means, to be sure i clone it + const positionClone = { + column: position.column, + lineNumber: position.lineNumber, + }; + const offset = model.getOffsetAt(positionClone); + const situation = getSituation(model.getValue(), offset); + const range = calculateRange(situation, word, wordUntil, monaco, position); + const completionsPromise = situation != null ? getCompletions(situation, dataProvider) : Promise.resolve([]); + return completionsPromise.then((items) => { + // monaco by default alphabetically orders the items. + // to stop it, we use a number-as-string sortkey, + // so that monaco keeps the order we use + const maxIndexDigits = items.length.toString().length; + const suggestions: monacoTypes.languages.CompletionItem[] = items.map((item, index) => ({ + kind: getMonacoCompletionItemKind(item.type, monaco), + label: item.label, + insertText: item.insertText, + insertTextRules: item.isSnippet ? INSERT_AS_SNIPPET_ENUM_VALUE : undefined, + detail: item.detail, + documentation: item.documentation, + sortText: index.toString().padStart(maxIndexDigits, '0'), // to force the order we have + range: range, + command: item.triggerOnInsert + ? { + id: 'editor.action.triggerSuggest', + title: '', + } + : undefined, + })); + return { suggestions }; + }); + }; + + return { + triggerCharacters: ['{', ',', '[', '(', '=', '~', ' ', '"', '|'], + provideCompletionItems, + }; +} + +export const calculateRange = ( + situation: Situation | null, + word: monacoTypes.editor.IWordAtPosition | null, + wordUntil: monacoTypes.editor.IWordAtPosition, + monaco: Monaco, + position: monacoTypes.Position +): monacoTypes.Range => { + if ( + situation && + situation?.type === 'IN_LABEL_SELECTOR_WITH_LABEL_NAME' && + 'betweenQuotes' in situation && + situation.betweenQuotes + ) { + // Word until won't have second quote if they are between quotes + const indexOfFirstQuote = wordUntil?.word?.indexOf('"') ?? 0; + + const indexOfLastQuote = word?.word?.lastIndexOf('"') ?? 0; + + const indexOfEquals = word?.word.indexOf('='); + const indexOfLastEquals = word?.word.lastIndexOf('='); + + // Just one equals "=" the cursor is somewhere within a label value + // e.g. value="labe^l-value" or value="^label-value" etc + // We want the word to include everything within the quotes, so the result from autocomplete overwrites the existing label value + if ( + indexOfLastEquals === indexOfEquals && + indexOfFirstQuote !== -1 && + indexOfLastQuote !== -1 && + indexOfLastEquals !== -1 + ) { + return word != null + ? monaco.Range.lift({ + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: wordUntil.startColumn + indexOfFirstQuote + 1, + endColumn: wordUntil.startColumn + indexOfLastQuote, + }) + : monaco.Range.fromPositions(position); + } + } + + if (situation && situation.type === 'IN_LABEL_SELECTOR_WITH_LABEL_NAME') { + // Otherwise we want the range to be calculated as the cursor position, as we want to insert the autocomplete, instead of overwriting existing text + // The cursor position is the length of the wordUntil + return word != null + ? monaco.Range.lift({ + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: wordUntil.endColumn, + endColumn: wordUntil.endColumn, + }) + : monaco.Range.fromPositions(position); + } + + // And for all other non-label cases, we want to use the word start and end column + return word != null + ? monaco.Range.lift({ + startLineNumber: position.lineNumber, + endLineNumber: position.lineNumber, + startColumn: word.startColumn, + endColumn: word.endColumn, + }) + : monaco.Range.fromPositions(position); +}; diff --git a/src/components/monaco-query-field/completion/completions.ts b/src/components/monaco-query-field/completion/completions.ts new file mode 100644 index 00000000..54a78ae6 --- /dev/null +++ b/src/components/monaco-query-field/completion/completions.ts @@ -0,0 +1,452 @@ +import { trimEnd } from 'lodash'; + +import { escapeLabelValueInExactSelector } from '../../../language/utils'; + +import { CompletionDataProvider } from './CompletionDataProvider'; +// import { NeverCaseError } from './NeverCaseError'; +import type { Situation, Label } from './situation'; + + +export type CompletionType = + | 'HISTORY' + | 'FUNCTION' + | 'DURATION' + | 'LABEL_NAME' + | 'LABEL_VALUE' + | 'PATTERN' + | 'PARSER' + | 'LINE_FILTER' + | 'PIPE_OPERATION'; + +type Completion = { + type: CompletionType; + label: string; + insertText: string; + detail?: string; + documentation?: string; + triggerOnInsert?: boolean; + isSnippet?: boolean; +}; + +const LOG_COMPLETIONS: Completion[] = [ + { + type: 'PATTERN', + label: '{}', + insertText: '{$0}', + isSnippet: true, + triggerOnInsert: true, + }, +]; + +// const AGGREGATION_COMPLETIONS: Completion[] = AGGREGATION_OPERATORS.map((f) => ({ +// type: 'FUNCTION', +// label: f.label, +// insertText: `${f.insertText ?? ''}($0)`, // i don't know what to do when this is nullish. it should not be. +// isSnippet: true, +// triggerOnInsert: true, +// detail: f.detail, +// documentation: f.documentation, +// })); + +// const FUNCTION_COMPLETIONS: Completion[] = RANGE_VEC_FUNCTIONS.map((f) => ({ +// type: 'FUNCTION', +// label: f.label, +// insertText: `${f.insertText ?? ''}({$0}[\\$__auto])`, // i don't know what to do when this is nullish. it should not be. +// isSnippet: true, +// triggerOnInsert: true, +// detail: f.detail, +// documentation: f.documentation, +// })); + +// const BUILT_IN_FUNCTIONS_COMPLETIONS: Completion[] = BUILT_IN_FUNCTIONS.map((f) => ({ +// type: 'FUNCTION', +// label: f.label, +// insertText: `${f.insertText ?? ''}($0)`, +// isSnippet: true, +// triggerOnInsert: true, +// detail: f.detail, +// documentation: f.documentation, +// })); + +// const DURATION_COMPLETIONS: Completion[] = ['$__auto', '1m', '5m', '10m', '30m', '1h', '1d'].map((text) => ({ +// type: 'DURATION', +// label: text, +// insertText: text, +// })); + +// const UNWRAP_FUNCTION_COMPLETIONS: Completion[] = [ +// { +// type: 'FUNCTION', +// label: 'duration_seconds', +// documentation: 'Will convert the label value in seconds from the go duration format (e.g 5m, 24s30ms).', +// insertText: 'duration_seconds()', +// }, +// { +// type: 'FUNCTION', +// label: 'duration', +// documentation: 'Short version of duration_seconds().', +// insertText: 'duration()', +// }, +// { +// type: 'FUNCTION', +// label: 'bytes', +// documentation: 'Will convert the label value to raw bytes applying the bytes unit (e.g. 5 MiB, 3k, 1G).', +// insertText: 'bytes()', +// }, +// ]; + +// const LOGFMT_ARGUMENT_COMPLETIONS: Completion[] = [ +// { +// type: 'FUNCTION', +// label: '--strict', +// documentation: +// 'Strict parsing. The logfmt parser stops scanning the log line and returns early with an error when it encounters any poorly formatted key/value pair.', +// insertText: '--strict', +// }, +// { +// type: 'FUNCTION', +// label: '--keep-empty', +// documentation: +// 'Retain standalone keys with empty value. The logfmt parser retains standalone keys (keys without a value) as labels with value set to empty string.', +// insertText: '--keep-empty', +// }, +// ]; + +// function getPipeOperationsCompletions(prefix = ''): Completion[] { +// const completions: Completion[] = []; +// completions.push({ +// type: 'PIPE_OPERATION', +// label: 'line_format', +// insertText: `${prefix}line_format "{{.$0}}"`, +// isSnippet: true, +// documentation: explainOperator(LokiOperationId.LineFormat), +// }); + +// completions.push({ +// type: 'PIPE_OPERATION', +// label: 'label_format', +// insertText: `${prefix}label_format`, +// isSnippet: true, +// documentation: explainOperator(LokiOperationId.LabelFormat), +// }); + +// completions.push({ +// type: 'PIPE_OPERATION', +// label: 'unwrap', +// insertText: `${prefix}unwrap`, +// documentation: explainOperator(LokiOperationId.Unwrap), +// }); + +// completions.push({ +// type: 'PIPE_OPERATION', +// label: 'decolorize', +// insertText: `${prefix}decolorize`, +// documentation: explainOperator(LokiOperationId.Decolorize), +// }); + +// completions.push({ +// type: 'PIPE_OPERATION', +// label: 'drop', +// insertText: `${prefix}drop`, +// documentation: explainOperator(LokiOperationId.Drop), +// }); + +// completions.push({ +// type: 'PIPE_OPERATION', +// label: 'keep', +// insertText: `${prefix}keep`, +// documentation: explainOperator(LokiOperationId.Keep), +// }); + +// return completions; +// } + +async function getAllHistoryCompletions(dataProvider: CompletionDataProvider): Promise { + const history = await dataProvider.getHistory(); + + return history.map((expr) => ({ + type: 'HISTORY', + label: expr, + insertText: expr, + })); +} + +// async function getLabelNamesForSelectorCompletions( +// otherLabels: Label[], +// dataProvider: CompletionDataProvider +// ): Promise { +// const labelNames = await dataProvider.getLabelNames(otherLabels); + +// return labelNames.map((label) => ({ +// type: 'LABEL_NAME', +// label, +// insertText: `${label}=`, +// triggerOnInsert: true, +// })); +// } + +async function getLabelNamesForSelectorCompletions( + otherLabels: Label[], + dataProvider: CompletionDataProvider +): Promise { + const labelNames = await dataProvider.getLabelNames(dataProvider.buildQuery(otherLabels)); + return labelNames.map((label) => ({ + type: 'LABEL_NAME', + label, + insertText: `${label}="$0"`, + isSnippet: true, + triggerOnInsert: true, + })); +} + +// async function getInGroupingCompletions(logQuery: string, dataProvider: CompletionDataProvider): Promise { +// const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery); + +// return extractedLabelKeys.map((label) => ({ +// type: 'LABEL_NAME', +// label, +// insertText: label, +// triggerOnInsert: false, +// })); +// } + +const PARSERS = ['json', 'logfmt', 'pattern', 'regexp', 'unpack']; + +async function getParserCompletions( + prefix: string, + hasJSON: boolean, + hasLogfmt: boolean, +) { + const allParsers = new Set(PARSERS); + const completions: Completion[] = []; + + const remainingParsers = Array.from(allParsers).sort(); + remainingParsers.forEach((parser) => { + completions.push({ + type: 'PARSER', + label: `${prefix}unpack_${parser} from _msg`, + insertText: `${prefix}unpack_${parser} from _msg`, + }); + }); + + return completions; +} + +export async function getLabelFilterCompletions(query: string, dataProvider: CompletionDataProvider): Promise { + const labelNames = await dataProvider.getLabelNames(query); + + return labelNames.map((label) => ({ + type: 'LABEL_NAME', + label, + insertText: `${label}:="$0"`, + isSnippet: true, + triggerOnInsert: true, + })); +} + +export async function getAfterSelectorCompletions( + logQuery: string, + afterPipe: boolean, + hasSpace: boolean, + dataProvider: CompletionDataProvider +): Promise { + let query = logQuery; + if (afterPipe) { + query = trimEnd(logQuery, '| '); + } + + const { hasJSON, hasLogfmt } = await dataProvider.getParserAndLabelKeys(query); + + const prefix = `${hasSpace ? '' : ' '}${afterPipe ? '' : '| '}`; + + const parserCompletions = await getParserCompletions(prefix, hasJSON, hasLogfmt); + const labelFilterCompletions = hasSpace ? await getLabelFilterCompletions(query, dataProvider) : []; + const pipeOperations: Completion[] = []; + + return [ + { + type: 'LINE_FILTER', + label: `""`, + insertText: `"$0"`, + isSnippet: true, + }, + ...parserCompletions, + ...pipeOperations, + ...labelFilterCompletions, + ]; +} + +// export async function getLogfmtCompletions( +// logQuery: string, +// flags: boolean, +// trailingComma: boolean | undefined, +// trailingSpace: boolean | undefined, +// otherLabels: string[], +// dataProvider: CompletionDataProvider +// ): Promise { +// let completions: Completion[] = []; + +// if (trailingComma) { +// // Remove the trailing comma, otherwise the sample query will fail. +// logQuery = trimEnd(logQuery, ', '); +// } +// const { extractedLabelKeys, hasJSON, hasLogfmt, hasPack } = await dataProvider.getParserAndLabelKeys(logQuery); +// const pipeOperations = getPipeOperationsCompletions('| '); + +// /** +// * The user is not in the process of writing another label, and has not specified 2 flags. +// * The current grammar doesn't allow us to know which flags were used (by node name), so we consider flags = true +// * when 2 have been used. +// * For example: +// * - {label="value"} | logfmt ^ +// * - {label="value"} | logfmt --strict ^ +// * - {label="value"} | logfmt --strict --keep-empty ^ +// */ +// if (!trailingComma && !flags) { +// completions = [...LOGFMT_ARGUMENT_COMPLETIONS]; +// } + +// /** +// * If the user has no trailing comma and has a trailing space it can mean that they finished writing the logfmt +// * part and want to move on, for example, with other parsers or pipe operations. +// * For example: +// * - {label="value"} | logfmt --flag ^ +// * - {label="value"} | logfmt label, label2 ^ +// */ +// if (!trailingComma && trailingSpace) { +// /** +// * Don't offer parsers if there is no label argument: {label="value"} | logfmt ^ +// * The reason is that it would be unusual that they would want to use another parser just after logfmt, and +// * more likely that they would want a flag, labels, or continue with pipe operations. +// * +// * Offer parsers with at least one label argument: {label="value"} | logfmt label ^ +// * The rationale here is to offer the same completions as getAfterSelectorCompletions(). +// */ +// const parserCompletions = +// otherLabels.length > 0 +// ? await getParserCompletions('| ', hasJSON, hasLogfmt, hasPack, extractedLabelKeys, true) +// : []; +// completions = [...completions, ...parserCompletions, ...pipeOperations]; +// } + +// const labels = extractedLabelKeys.filter((label) => !otherLabels.includes(label)); + +// /** +// * We want to decide whether to use a trailing comma or not based on the data we have of the current +// * situation. In particular, the following scenarios will not lead to a trailing comma: +// * {label="value"} | logfmt ^ +// * - trailingSpace: true, trailingComma: false, otherLabels: [] +// * {label="value"} | logfmt lab^ +// * trailingSpace: false, trailignComma: false, otherLabels: [lab] +// * {label="value"} | logfmt label,^ +// * trailingSpace: false, trailingComma: true, otherLabels: [label] +// * {label="value"} | logfmt label, ^ +// * trailingSpace: true, trailingComma: true, otherLabels: [label] +// */ +// let labelPrefix = ''; +// if (otherLabels.length > 0 && trailingSpace) { +// labelPrefix = trailingComma ? '' : ', '; +// } + +// const labelCompletions: Completion[] = labels.map((label) => ({ +// type: 'LABEL_NAME', +// label, +// insertText: labelPrefix + label, +// triggerOnInsert: false, +// })); + +// completions = [...completions, ...labelCompletions]; + +// return completions; +// } + +async function getLabelValuesForMetricCompletions( + labelName: string, + betweenQuotes: boolean, + otherLabels: Label[], + dataProvider: CompletionDataProvider +): Promise { + const values = await dataProvider.getLabelValues(labelName, dataProvider.buildQuery(otherLabels)); + return values.map((text) => ({ + type: 'LABEL_VALUE', + label: text, + insertText: betweenQuotes ? escapeLabelValueInExactSelector(text) : `"${escapeLabelValueInExactSelector(text)}"`, + })); +} + +// async function getAfterUnwrapCompletions( +// logQuery: string, +// dataProvider: CompletionDataProvider +// ): Promise { +// const { unwrapLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery); + +// const labelCompletions: Completion[] = unwrapLabelKeys.map((label) => ({ +// type: 'LABEL_NAME', +// label, +// insertText: label, +// triggerOnInsert: false, +// })); + +// return [...labelCompletions, ...UNWRAP_FUNCTION_COMPLETIONS]; +// } + +// async function getAfterKeepAndDropCompletions(logQuery: string, dataProvider: CompletionDataProvider) { +// const { extractedLabelKeys } = await dataProvider.getParserAndLabelKeys(logQuery); +// const labelCompletions: Completion[] = extractedLabelKeys.map((label) => ({ +// type: 'LABEL_NAME', +// label, +// insertText: label, +// triggerOnInsert: false, +// })); + +// return [...labelCompletions]; +// } + +export async function getCompletions( + situation: Situation, + dataProvider: CompletionDataProvider +): Promise { + switch (situation.type) { + case 'EMPTY': + case 'AT_ROOT': + const historyCompletions = await getAllHistoryCompletions(dataProvider); + return [ + ...historyCompletions, + ...LOG_COMPLETIONS, + ]; + case 'IN_LABEL_SELECTOR_NO_LABEL_NAME': + return getLabelNamesForSelectorCompletions(situation.otherLabels, dataProvider); + case 'IN_LABEL_SELECTOR_WITH_LABEL_NAME': + return getLabelValuesForMetricCompletions( + situation.labelName, + situation.betweenQuotes, + situation.otherLabels, + dataProvider + ); + case 'AFTER_SELECTOR': + return getAfterSelectorCompletions(situation.logQuery, situation.afterPipe, situation.hasSpace, dataProvider); + // case 'IN_RANGE': + // return DURATION_COMPLETIONS; + // case 'IN_GROUPING': + // return getInGroupingCompletions(situation.logQuery, dataProvider); + // case 'AFTER_UNWRAP': + // return getAfterUnwrapCompletions(situation.logQuery, dataProvider); + // case 'IN_AGGREGATION': + // return [...FUNCTION_COMPLETIONS, ...AGGREGATION_COMPLETIONS]; + // case 'AFTER_KEEP_AND_DROP': + // return getAfterKeepAndDropCompletions(situation.logQuery, dataProvider); + // case 'IN_LOGFMT': + // return getLogfmtCompletions( + // situation.logQuery, + // situation.flags, + // situation.trailingComma, + // situation.trailingSpace, + // situation.otherLabels, + // dataProvider + // ); + default: + console.warn('Unhandled situation type in getCompletions:', situation.type); + throw new Error(`Unhandled situation type in getCompletions: ${situation.type}`); + // throw new NeverCaseError(situation); + } +} diff --git a/src/components/monaco-query-field/completion/situation.ts b/src/components/monaco-query-field/completion/situation.ts new file mode 100644 index 00000000..46446697 --- /dev/null +++ b/src/components/monaco-query-field/completion/situation.ts @@ -0,0 +1,725 @@ +import type { SyntaxNode, TreeCursor } from '@lezer/common'; + +import { + parser, + VectorAggregationExpr, + String, + Selector, + RangeAggregationExpr, + Range, + PipelineExpr, + PipelineStage, + Matchers, + Matcher, + LogQL, + LogRangeExpr, + LogExpr, + Logfmt, + Identifier, + Grouping, + Expr, + LiteralExpr, + MetricExpr, + UnwrapExpr, + DropLabelsExpr, + KeepLabelsExpr, + DropLabels, + KeepLabels, + ParserFlag, + LabelExtractionExpression, + LabelExtractionExpressionList, + LogfmtExpressionParser, +} from '@grafana/lezer-logql'; + +import { getNodesFromQuery } from '../../../queryUtils'; + +type Direction = 'parent' | 'firstChild' | 'lastChild' | 'nextSibling'; +type NodeType = number; + +type Path = Array<[Direction, NodeType]>; + +function move(node: SyntaxNode, direction: Direction): SyntaxNode | null { + return node[direction]; +} + +// TODO: Implement properly +function getLogQueryFromMetricsQueryAtPosition(text: string, pos: number): string { + return text; +} + +/** + * Iteratively calls walk with given path until it returns null, then we return the last non-null node. + * @param node + * @param path + */ +function traverse(node: SyntaxNode, path: Path): SyntaxNode | null { + let current: SyntaxNode | null = node; + let next = walk(current, path); + while (next) { + let nextTmp = walk(next, path); + if (nextTmp) { + next = nextTmp; + } else { + return next; + } + } + return null; +} + +/** + * Walks a single step from the provided node, following the path. + * @param node + * @param path + */ +function walk(node: SyntaxNode, path: Path): SyntaxNode | null { + let current: SyntaxNode | null = node; + for (const [direction, expectedNode] of path) { + current = move(current, direction); + if (current === null) { + // we could not move in the direction, we stop + return null; + } + if (current.type.id !== expectedNode) { + // the reached node has wrong type, we stop + return null; + } + } + return current; +} + +function getNodeText(node: SyntaxNode, text: string): string { + return text.slice(node.from, node.to); +} + +function parseStringLiteral(text: string): string { + // If it is a string-literal, it is inside quotes of some kind + const inside = text.slice(1, text.length - 1); + + // Very simple un-escaping: + + // Double quotes + if (text.startsWith('"') && text.endsWith('"')) { + // NOTE: this is not 100% perfect, we only unescape the double-quote, + // there might be other characters too + return inside.replace(/\\"/gm, '"'); + } + + // Single quotes + if (text.startsWith("'") && text.endsWith("'")) { + // NOTE: this is not 100% perfect, we only unescape the single-quote, + // there might be other characters too + return inside.replace(/\\'/gm, "'"); + } + + // Backticks + if (text.startsWith('`') && text.endsWith('`')) { + return inside; + } + + throw new Error(`Invalid string literal: ${text}`); +} + +export type LabelOperator = '=' | '!=' | '=~' | '!~'; + +export type Label = { + name: string; + value: string; + op: LabelOperator; +}; + +export type Situation = + | { + type: 'EMPTY'; + } + | { + type: 'AT_ROOT'; + } + | { + type: 'IN_LOGFMT'; + otherLabels: string[]; + flags: boolean; + trailingSpace: boolean; + trailingComma: boolean; + logQuery: string; + } + | { + type: 'IN_RANGE'; + } + | { + type: 'IN_AGGREGATION'; + } + | { + type: 'IN_GROUPING'; + logQuery: string; + } + | { + type: 'IN_LABEL_SELECTOR_NO_LABEL_NAME'; + otherLabels: Label[]; + } + | { + type: 'IN_LABEL_SELECTOR_WITH_LABEL_NAME'; + labelName: string; + betweenQuotes: boolean; + otherLabels: Label[]; + } + | { + type: 'AFTER_SELECTOR'; + afterPipe: boolean; + hasSpace: boolean; + logQuery: string; + } + | { + type: 'AFTER_UNWRAP'; + logQuery: string; + } + | { + type: 'AFTER_KEEP_AND_DROP'; + logQuery: string; + }; + +type Resolver = { + paths: NodeType[][]; + fun: (node: SyntaxNode, text: string, pos: number) => Situation | null; +}; + +function isPathMatch(resolverPath: NodeType[], cursorPath: number[]): boolean { + return resolverPath.every((item, index) => item === cursorPath[index]); +} + +const ERROR_NODE_ID = 0; + +const RESOLVERS: Resolver[] = [ + { + paths: [ + [Selector], + [Selector, Matchers], + [Matchers], + [ERROR_NODE_ID, Matchers], + [ERROR_NODE_ID, Matchers, Selector], + ], + fun: resolveSelector, + }, + { + paths: [ + [LogQL], + [RangeAggregationExpr], + [ERROR_NODE_ID, LogRangeExpr, RangeAggregationExpr], + [ERROR_NODE_ID, LabelExtractionExpressionList], + [LogRangeExpr], + [ERROR_NODE_ID, LabelExtractionExpressionList], + [LabelExtractionExpressionList], + [LogfmtExpressionParser], + ], + fun: resolveLogfmtParser, + }, + { + paths: [[LogQL], [ERROR_NODE_ID, Selector]], + fun: resolveTopLevel, + }, + { + paths: [[String, Matcher]], + fun: resolveMatcher, + }, + { + paths: [[Grouping]], + fun: resolveLabelsForGrouping, + }, + { + paths: [[LogRangeExpr]], + fun: resolveLogRange, + }, + { + paths: [ + [ERROR_NODE_ID, Matcher], + [ERROR_NODE_ID, Matchers, Selector], + ], + fun: resolveMatcher, + }, + { + paths: [[ERROR_NODE_ID, Range]], + fun: resolveDurations, + }, + { + paths: [[ERROR_NODE_ID, LogRangeExpr]], + fun: resolveLogRangeFromError, + }, + { + paths: [[ERROR_NODE_ID, LiteralExpr, MetricExpr, VectorAggregationExpr]], + fun: () => ({ type: 'IN_AGGREGATION' }), + }, + { + paths: [ + [ERROR_NODE_ID, PipelineStage, PipelineExpr], + [PipelineStage, PipelineExpr], + ], + fun: resolvePipeError, + }, + { + paths: [[ERROR_NODE_ID, UnwrapExpr], [UnwrapExpr]], + fun: resolveAfterUnwrap, + }, + { + paths: [ + [ERROR_NODE_ID, DropLabelsExpr], + [ERROR_NODE_ID, DropLabels], + [ERROR_NODE_ID, KeepLabelsExpr], + [ERROR_NODE_ID, KeepLabels], + ], + fun: resolveAfterKeepAndDrop, + }, +]; + +const LABEL_OP_MAP = new Map([ + ['Eq', '='], + ['Re', '=~'], + ['Neq', '!='], + ['Nre', '!~'], +]); + +function getLabelOp(opNode: SyntaxNode): LabelOperator | null { + return LABEL_OP_MAP.get(opNode.name) ?? null; +} + +function getLabel(matcherNode: SyntaxNode, text: string): Label | null { + if (matcherNode.type.id !== Matcher) { + return null; + } + + const nameNode = walk(matcherNode, [['firstChild', Identifier]]); + + if (nameNode === null) { + return null; + } + + const opNode = nameNode.nextSibling; + if (opNode === null) { + return null; + } + + const op = getLabelOp(opNode); + if (op === null) { + return null; + } + + const valueNode = walk(matcherNode, [['lastChild', String]]); + + if (valueNode === null) { + return null; + } + + const name = getNodeText(nameNode, text); + const value = parseStringLiteral(getNodeText(valueNode, text)); + + return { name, value, op }; +} + +function getLabels(selectorNode: SyntaxNode, text: string): Label[] { + if (selectorNode.type.id !== Selector && selectorNode.type.id !== Matchers) { + return []; + } + + let listNode: SyntaxNode | null = null; + + // If parent node is selector, we want to start with the current Matcher node + if (selectorNode?.parent?.type.id === Selector) { + listNode = selectorNode; + } else { + // Parent node needs to be returned first because otherwise both of the other walks will return a non-null node and this function will return the labels on the left side of the current node, the other two walks should be mutually exclusive when the parent is null + listNode = + // Node in-between labels + traverse(selectorNode, [['parent', Matchers]]) ?? + // Node after all other labels + walk(selectorNode, [['firstChild', Matchers]]) ?? + // Node before all other labels + walk(selectorNode, [['lastChild', Matchers]]); + } + + const labels: Label[] = []; + + while (listNode !== null) { + const matcherNode = walk(listNode, [['lastChild', Matcher]]); + if (matcherNode !== null) { + const label = getLabel(matcherNode, text); + if (label !== null) { + labels.push(label); + } + } + + // there might be more labels + listNode = walk(listNode, [['firstChild', Matchers]]); + } + + // our labels-list is last-first, so we reverse it + labels.reverse(); + + return labels; +} + +function resolveAfterUnwrap(node: SyntaxNode, text: string, pos: number): Situation | null { + return { + type: 'AFTER_UNWRAP', + logQuery: getLogQueryFromMetricsQueryAtPosition(text, pos).trim(), + }; +} + +function resolvePipeError(node: SyntaxNode, text: string, pos: number): Situation | null { + /** + * Examples: + * - {level="info"} |^ + * - count_over_time({level="info"} |^ [4m]) + */ + let exprNode: SyntaxNode | null = null; + if (node.type.id === ERROR_NODE_ID) { + exprNode = walk(node, [ + ['parent', PipelineStage], + ['parent', PipelineExpr], + ]); + } else if (node.type.id === PipelineStage) { + exprNode = walk(node, [['parent', PipelineExpr]]); + } + + if (exprNode?.parent?.type.id === LogExpr || exprNode?.parent?.type.id === LogRangeExpr) { + return resolveLogOrLogRange(exprNode.parent, text, pos, true); + } + + return null; +} + +function resolveLabelsForGrouping(node: SyntaxNode, text: string, pos: number): Situation | null { + const aggrExpNode = walk(node, [['parent', VectorAggregationExpr]]); + if (aggrExpNode === null) { + return null; + } + const bodyNode = aggrExpNode.getChild('MetricExpr'); + if (bodyNode === null) { + return null; + } + + const selectorNode = walk(bodyNode, [ + ['firstChild', RangeAggregationExpr], + ['lastChild', LogRangeExpr], + ['firstChild', Selector], + ]); + + if (selectorNode === null) { + return null; + } + + return { + type: 'IN_GROUPING', + logQuery: getLogQueryFromMetricsQueryAtPosition(text, pos).trim(), + }; +} + +function resolveMatcher(node: SyntaxNode, text: string, pos: number): Situation | null { + // we can arrive here for two reasons. `node` is either: + // - a StringNode (like in `{job="^"}`) + // - or an error node (like in `{job=^}`) + const inStringNode = !node.type.isError; + + const parent = walk(node, [['parent', Matcher]]); + if (parent === null) { + return null; + } + + const labelNameNode = walk(parent, [['firstChild', Identifier]]); + if (labelNameNode === null) { + return null; + } + + const labelName = getNodeText(labelNameNode, text); + + // now we need to go up, to the parent of Matcher, + // there can be one or many `Matchers` parents, we have + // to go through all of them + + const firstListNode = walk(parent, [['parent', Matchers]]); + if (firstListNode === null) { + return null; + } + + let listNode = firstListNode; + + // we keep going through the parent-nodes as long as they are Matchers. + // as soon as we reach Selector, we stop + let selectorNode: SyntaxNode | null = null; + while (selectorNode === null) { + const parent = listNode.parent; + if (parent === null) { + return null; + } + + switch (parent.type.id) { + case Matchers: + //we keep looping + listNode = parent; + continue; + case Selector: + // we reached the end, we can stop the loop + selectorNode = parent; + continue; + default: + // we reached some other node, we stop + return null; + } + } + + // now we need to find the other names + const allLabels = getLabels(selectorNode, text); + + // we need to remove "our" label from all-labels, if it is in there + const otherLabels = allLabels.filter((label) => label.name !== labelName); + + return { + type: 'IN_LABEL_SELECTOR_WITH_LABEL_NAME', + labelName, + betweenQuotes: inStringNode, + otherLabels, + }; +} + +function resolveLogfmtParser(_: SyntaxNode, text: string, cursorPosition: number): Situation | null { + // We want to know if the cursor if after a log query with logfmt parser. + // E.g. `{x="y"} | logfmt ^` + /** + * Wait until the user adds a space to be sure of what the last identifier is. Otherwise + * it creates suggestion bugs with queries like {label="value"} | parser^ suggest "parser" + * and it can be inserted with extra pipes or commas. + */ + const tree = parser.parse(text); + + // Adjust the cursor position if there are spaces at the end of the text. + const trimRightTextLen = text.substring(0, cursorPosition).trimEnd().length; + const position = trimRightTextLen < cursorPosition ? trimRightTextLen : cursorPosition; + + const cursor = tree.cursorAt(position); + + // Check if the user cursor is in any node that requires logfmt suggestions. + const expectedNodes = [Logfmt, ParserFlag, LabelExtractionExpression, LabelExtractionExpressionList]; + let inLogfmt = false; + do { + const { node } = cursor; + if (!expectedNodes.includes(node.type.id)) { + continue; + } + if (cursor.from <= position && cursor.to >= position) { + inLogfmt = true; + break; + } + } while (cursor.next()); + + if (!inLogfmt) { + return null; + } + + const flags = getNodesFromQuery(text, [ParserFlag]).length > 1; + const labelNodes = getNodesFromQuery(text, [LabelExtractionExpression]); + const otherLabels = labelNodes + .map((label: SyntaxNode) => label.getChild(Identifier)) + .filter((label: SyntaxNode | null): label is SyntaxNode => label !== null) + .map((label: SyntaxNode) => getNodeText(label, text)); + + const logQuery = getLogQueryFromMetricsQueryAtPosition(text, position).trim(); + const trailingSpace = text.charAt(cursorPosition - 1) === ' '; + const trailingComma = text.trimEnd().charAt(position - 1) === ','; + + return { + type: 'IN_LOGFMT', + otherLabels, + flags, + trailingSpace, + trailingComma, + logQuery, + }; +} + +function resolveTopLevel(node: SyntaxNode, text: string, pos: number): Situation | null { + /** + * The following queries trigger resolveTopLevel(). + * - Empty query + * - {label="value"} ^ + * - {label="value"} | parser ^ + * From here, we need to determine if the user is in a resolveLogOrLogRange() or simply at the root. + */ + const logExprNode = walk(node, [ + ['lastChild', Expr], + ['lastChild', LogExpr], + ]); + + /** + * Wait until the user adds a space to be sure of what the last identifier is. Otherwise + * it creates suggestion bugs with queries like {label="value"} | parser^ suggest "parser" + * and it can be inserted with extra pipes. + */ + if (logExprNode != null && text.endsWith(' ')) { + return resolveLogOrLogRange(logExprNode, text, pos, false); + } + + // `s` situation, with the cursor at the end. + // (basically, user enters a non-special characters as first + // character in query field) + const idNode = walk(node, [ + ['firstChild', ERROR_NODE_ID], + ['firstChild', Identifier], + ]); + + if (idNode != null) { + return { + type: 'AT_ROOT', + }; + } + + // no patterns match + return null; +} + +function resolveDurations(node: SyntaxNode, text: string, pos: number): Situation { + return { + type: 'IN_RANGE', + }; +} + +function resolveLogRange(node: SyntaxNode, text: string, pos: number): Situation | null { + const partialQuery = text.substring(0, pos).trimEnd(); + const afterPipe = partialQuery.endsWith('|'); + + return resolveLogOrLogRange(node, text, pos, afterPipe); +} + +function resolveLogRangeFromError(node: SyntaxNode, text: string, pos: number): Situation | null { + const parent = walk(node, [['parent', LogRangeExpr]]); + if (parent === null) { + return null; + } + + const partialQuery = text.substring(0, pos).trimEnd(); + const afterPipe = partialQuery.endsWith('|'); + + return resolveLogOrLogRange(parent, text, pos, afterPipe); +} + +function resolveLogOrLogRange(node: SyntaxNode, text: string, pos: number, afterPipe: boolean): Situation | null { + // Here the `node` is either a LogExpr or a LogRangeExpr + // We want to handle the case where we are next to a selector + const selectorNode = walk(node, [['firstChild', Selector]]); + + // Check that the selector is before the cursor, not after it + if (!selectorNode || selectorNode.to > pos) { + return null; + } + + return { + type: 'AFTER_SELECTOR', + afterPipe, + hasSpace: text.charAt(pos - 1) === ' ', + logQuery: getLogQueryFromMetricsQueryAtPosition(text, pos).trim(), + }; +} + +function resolveSelector(node: SyntaxNode, text: string, pos: number): Situation | null { + // for example `{^}` + + // false positive: + // `{a="1"^}` + const child = walk(node, [['firstChild', Matchers]]); + if (child !== null) { + // means the label-matching part contains at least one label already. + // + // in this case, we will need to have a `,` character at the end, + // to be able to suggest adding the next label. + // the area between the end-of-the-child-node and the cursor-pos + // must contain a `,` in this case. + const textToCheck = text.slice(child.from, pos); + if (!textToCheck.trim().endsWith(',')) { + return null; + } + } + + const selectorNode = node.type.id === ERROR_NODE_ID ? walk(node, [['parent', Matchers]]) : node; + if (!selectorNode) { + return null; + } + + const otherLabels = getLabels(selectorNode, text); + + return { + type: 'IN_LABEL_SELECTOR_NO_LABEL_NAME', + otherLabels, + }; +} + +function resolveAfterKeepAndDrop(node: SyntaxNode, text: string, pos: number): Situation | null { + let logQuery = getLogQueryFromMetricsQueryAtPosition(text, pos).trim(); + let keepAndDropParent: SyntaxNode | null = null; + let parent = node.parent; + while (parent !== null) { + if (parent.type.id === PipelineStage) { + keepAndDropParent = parent; + break; + } + parent = parent.parent; + } + + if (keepAndDropParent?.type.id === PipelineStage) { + logQuery = logQuery.slice(0, keepAndDropParent.from); + } + + return { + type: 'AFTER_KEEP_AND_DROP', + logQuery, + }; +} + +// If there is an error in the current cursor position, it's likely that the user is +// in the middle of writing a query. If we can't find an error node, we use the node +// at the cursor position to identify the situation. +function resolveCursor(text: string, cursorPos: number): TreeCursor { + // Sometimes the cursor is a couple spaces after the end of the expression. + // To account for this situation, we "move" the cursor position back to the real end + // of the expression. + const trimRightTextLen = text.trimEnd().length; + const pos = trimRightTextLen < cursorPos ? trimRightTextLen : cursorPos; + + const tree = parser.parse(text); + const cursor = tree.cursorAt(pos); + + do { + if (cursor.from === pos && cursor.to === pos && cursor.node.type.isError) { + return cursor; + } + } while (cursor.next()); + + return tree.cursorAt(pos); +} + +export function getSituation(text: string, pos: number): Situation | null { + // there is a special case when we are at the start of writing text, + // so we handle that case first + + if (text === '') { + return { + type: 'EMPTY', + }; + } + + const cursor = resolveCursor(text, pos); + const currentNode = cursor.node; + + const ids = [cursor.type.id]; + while (cursor.parent()) { + ids.push(cursor.type.id); + } + + for (let resolver of RESOLVERS) { + for (let path of resolver.paths) { + if (isPathMatch(path, ids)) { + const situation = resolver.fun(currentNode, text, pos); + if (situation) { + return situation; + } + } + } + } + + return null; +} diff --git a/src/language/index.ts b/src/language/index.ts new file mode 100644 index 00000000..2c87bbdc --- /dev/null +++ b/src/language/index.ts @@ -0,0 +1,221 @@ +import { languages } from "monaco-editor"; + +export const languageConfiguration: languages.LanguageConfiguration = { + // the default separators except `@$` + wordPattern: /(-?\d*\.\d\w*)|([^`~!#%^&*()\-=+\[{\]}\\|;:'",.<>\/?\s]+)/g, + comments: { + lineComment: "#", + }, + brackets: [ + ["{", "}"], + ["[", "]"], + ["(", ")"], + ], + autoClosingPairs: [ + { open: "{", close: "}" }, + { open: "[", close: "]" }, + { open: "(", close: ")" }, + { open: '"', close: '"' }, + { open: "'", close: "'" }, + { open: "`", close: "`" }, + ], + surroundingPairs: [ + { open: "{", close: "}" }, + { open: "[", close: "]" }, + { open: "(", close: ")" }, + { open: '"', close: '"' }, + { open: "'", close: "'" }, + { open: "`", close: "`" }, + { open: "<", close: ">" }, + ], + folding: {}, +}; + +// LogQL built-in aggregation operators +// https://grafana.com/docs/loki/latest/logql/metric_queries/#built-in-aggregation-operators +const aggregations = [ + "sum", + "avg", + "min", + "max", + "stddev", + "stdvar", + "count", + "topk", + "bottomk", +]; + +// LogQL parser expressions +// https://grafana.com/docs/loki/latest/logql/log_queries/#parser-expression +const parsers = ["json", "logfmt", "regexp", "unpack", "pattern"]; + +// LogQL format expressions +// https://grafana.com/docs/loki/latest/logql/log_queries/#parser-expression + +const format_expressions = ["line_format", "label_format"]; + +// LogQL vector aggregations +// https://grafana.com/docs/loki/latest/logql/metric_queries/#range-vector-aggregation +const vector_aggregations = [ + "count_over_time", + "rate", + "bytes_over_time", + "bytes_rate", + "avg_over_time", + "sum_over_time", + "min_over_time", + "max_over_time", + "stdvar_over_time", + "stddev_over_time", + "quantile_over_time", + "first_over_time", + "last_over_time", + "absent_over_time", +]; + +// LogQL by and without clauses +const vector_matching = ["by", "without"]; +// Produce a regex matching elements : (by|without) +const vectorMatchingRegex = `(${vector_matching.reduce( + (prev, curr) => `${prev}|${curr}` +)})`; + +// LogQL Operators +const operators = [ + "+", + "-", + "*", + "/", + "%", + "^", + "==", + "!=", + ">", + "<", + ">=", + "<=", + "|=", + "!=", + "|~", + "!~", + "and", + "or", + "unless", + "|", +]; + +// Merging all the keywords in one list +const keywords = aggregations + .concat(parsers) + .concat(format_expressions) + .concat(vector_aggregations) + .concat(vector_matching); + +export const monarchlanguage: languages.IMonarchLanguage = { + ignoreCase: false, + defaultToken: "", + tokenPostfix: ".vlogsql", + keywords: keywords, + operators: operators, + vectorMatching: vectorMatchingRegex, + + // we include these common regular expressions + symbols: /[=>](?!@symbols)/, "@brackets"], + [ + /@symbols/, + { + cases: { + "@operators": "delimiter", + "@default": "", + }, + }, + ], + + // numbers + [/\d+(?:\.\d)?(?:ms|ns|us|µs|[smhdwy])/, "number"], // durations + [/\d+(?:\.\d)?(?:b|kib|Kib|kb|KB|mib|Mib|mb|MB|gib|Gib|gb|GB|tib|Tib|tb|TB|pib|Pib|pb|PB|eib|Eib|eb|EB])/, "number"], // bytes + [/\d*\d+[eE]([\-+]?\d+)?(@floatsuffix)/, "number.float"], + [/\d*\.\d+([eE][\-+]?\d+)?(@floatsuffix)/, "number.float"], + [/0[xX][0-9a-fA-F']*[0-9a-fA-F](@integersuffix)/, "number.hex"], + [/0[0-7']*[0-7](@integersuffix)/, "number.octal"], + [/0[bB][0-1']*[0-1](@integersuffix)/, "number.binary"], + [/\d[\d']*\d(@integersuffix)/, "number"], + [/\d(@integersuffix)/, "number"], + ], + + string_double: [ + // Set to token: number to differentiate color + [/\{\{(.*?)\}\}/, { token: 'number' }], + [/[^\\"]/, "string"], + [/@escapes/, "string.escape"], + [/\\./, "string.escape.invalid"], + [/"/, "string", "@pop"], + ], + + string_single: [ + [/[^\\']+/, "string"], + [/@escapes/, "string.escape"], + [/\\./, "string.escape.invalid"], + [/'/, "string", "@pop"], + ], + + string_backtick: [ + // Set to token: number to differentiate color + [/\{\{(.*?)\}\}/, { token: 'number' }], + [/[^\\`]/, "string"], + [/@escapes/, "string.escape"], + [/\\./, "string.escape.invalid"], + [/`/, "string", "@pop"], + ], + + clauses: [ + [/[^(,)]/, "tag"], + [/\)/, "identifier", "@pop"], + ], + + whitespace: [[/[ \t\r\n]+/, "white"]], + }, +}; diff --git a/src/language/utils.ts b/src/language/utils.ts new file mode 100644 index 00000000..cd93aed6 --- /dev/null +++ b/src/language/utils.ts @@ -0,0 +1,166 @@ +import { invert } from 'lodash'; + +import { AbstractLabelMatcher, AbstractLabelOperator, AbstractQuery, TimeRange } from '@grafana/data'; + +function roundMsToMin(milliseconds: number): number { + return roundSecToMin(milliseconds / 1000); +} + +function roundSecToMin(seconds: number): number { + return Math.floor(seconds / 60); +} + +export function shouldRefreshLabels(range?: TimeRange, prevRange?: TimeRange): boolean { + if (range && prevRange) { + const sameMinuteFrom = roundMsToMin(range.from.valueOf()) === roundMsToMin(prevRange.from.valueOf()); + const sameMinuteTo = roundMsToMin(range.to.valueOf()) === roundMsToMin(prevRange.to.valueOf()); + // If both are same, don't need to refresh + return !(sameMinuteFrom && sameMinuteTo); + } + return false; +} + +// Loki regular-expressions use the RE2 syntax (https://github.com/google/re2/wiki/Syntax), +// so every character that matches something in that list has to be escaped. +// the list of meta characters is: *+?()|\.[]{}^$ +// we make a javascript regular expression that matches those characters: +const RE2_METACHARACTERS = /[*+?()|\\.\[\]{}^$]/g; +function escapeLokiRegexp(value: string): string { + return value.replace(RE2_METACHARACTERS, '\\$&'); +} + +// based on the openmetrics-documentation, the 3 symbols we have to handle are: +// - \n ... the newline character +// - \ ... the backslash character +// - " ... the double-quote character +export function escapeLabelValueInExactSelector(labelValue: string): string { + return labelValue.replace(/\\/g, '\\\\').replace(/\n/g, '\\n').replace(/"/g, '\\"'); +} + +export function unescapeLabelValue(labelValue: string): string { + return labelValue.replace(/\\n/g, '\n').replace(/\\"/g, '"').replace(/\\\\/g, '\\'); +} + +export function escapeLabelValueInRegexSelector(labelValue: string): string { + return escapeLabelValueInExactSelector(escapeLokiRegexp(labelValue)); +} + +export function escapeLabelValueInSelector(labelValue: string, selector?: string): string { + return isRegexSelector(selector) + ? escapeLabelValueInRegexSelector(labelValue) + : escapeLabelValueInExactSelector(labelValue); +} + +export function isRegexSelector(selector?: string) { + if (selector && (selector.includes('=~') || selector.includes('!~'))) { + return true; + } + return false; +} + +export function isBytesString(string: string) { + const BYTES_KEYWORDS = [ + 'b', + 'kib', + 'Kib', + 'kb', + 'KB', + 'mib', + 'Mib', + 'mb', + 'MB', + 'gib', + 'Gib', + 'gb', + 'GB', + 'tib', + 'Tib', + 'tb', + 'TB', + 'pib', + 'Pib', + 'pb', + 'PB', + 'eib', + 'Eib', + 'eb', + 'EB', + ]; + const regex = new RegExp(`^(?:-?\\d+(?:\\.\\d+)?)(?:${BYTES_KEYWORDS.join('|')})$`); + const match = string.match(regex); + return !!match; +} + +// export function getLabelTypeFromFrame(labelKey: string, frame?: DataFrame, index?: number): null | LabelType { +// if (!frame || index === undefined) { +// return null; +// } + +// const typeField = frame.fields.find((field) => field.name === 'labelTypes')?.values[index]; +// if (!typeField) { +// return null; +// } +// switch (typeField[labelKey]) { +// case 'I': +// return LabelType.Indexed; +// case 'S': +// return LabelType.StructuredMetadata; +// case 'P': +// return LabelType.Parsed; +// default: +// return null; +// } +// } + +export const mapOpToAbstractOp: Record = { + [AbstractLabelOperator.Equal]: '=', + [AbstractLabelOperator.NotEqual]: '!=', + [AbstractLabelOperator.EqualRegEx]: '=~', + [AbstractLabelOperator.NotEqualRegEx]: '!~', +}; + +// eslint-disable-next-line @typescript-eslint/consistent-type-assertions +export const mapAbstractOperatorsToOp = invert(mapOpToAbstractOp) as Record; + +export function abstractQueryToExpr(labelBasedQuery: AbstractQuery): string { + const expr = labelBasedQuery.labelMatchers + .map((selector: AbstractLabelMatcher) => { + const operator = mapOpToAbstractOp[selector.operator]; + if (operator) { + return `${selector.name}${operator}"${selector.value}"`; + } else { + return ''; + } + }) + .filter((e: string) => e !== '') + .join(', '); + + return expr ? `{${expr}}` : ''; +} + +export function processLabels(labels: Array<{ [key: string]: string }>) { + const valueSet: { [key: string]: Set } = {}; + labels.forEach((label) => { + Object.keys(label).forEach((key) => { + if (!valueSet[key]) { + valueSet[key] = new Set(); + } + if (!valueSet[key].has(label[key])) { + valueSet[key].add(label[key]); + } + }); + }); + + const valueArray: { [key: string]: string[] } = {}; + limitSuggestions(Object.keys(valueSet)).forEach((key) => { + valueArray[key] = limitSuggestions(Array.from(valueSet[key])); + }); + + return { values: valueArray, keys: Object.keys(valueArray) }; +} + +// Max number of items (metrics, labels, values) that we display as suggestions. Prevents from running out of memory. +export const SUGGESTIONS_LIMIT = 10000; +export function limitSuggestions(items: string[]) { + return items.slice(0, SUGGESTIONS_LIMIT); +} diff --git a/src/languageUtils.ts b/src/languageUtils.ts index 0a00682b..1803177f 100644 --- a/src/languageUtils.ts +++ b/src/languageUtils.ts @@ -25,3 +25,41 @@ export function escapeLabelValueInSelector(labelValue: string, selector?: string ? escapeLabelValueInRegexSelector(labelValue) : escapeLabelValueInExactSelector(labelValue); } + +function isLogLineJSON(msg: string): boolean { + try { + const parsed = JSON.parse(msg); + return typeof parsed === 'object' && parsed !== null; + } catch (e) { + return false; + } +} +const LOGFMT_REGEXP = /(?:^|\s)([\w\(\)\[\]\{\}]+)=(""|(?:".*?[^\\]"|[^"\s]\S*))/; + +function isLogLineLogfmt(msg: string): boolean { + return LOGFMT_REGEXP.test(msg); +} + +export function extractLogParserFromSample(sample: Record[]): { + hasLogfmt: boolean; + hasJSON: boolean; +} { + if (sample.length === 0) { + return { hasJSON: false, hasLogfmt: false }; + } + + let hasJSON = false; + let hasLogfmt = false; + + sample.forEach((line) => { + const msg: string = line['_msg'] as string; + if (isLogLineJSON(msg)) { + hasJSON = true; + } + if (isLogLineLogfmt(msg)) { + hasLogfmt = true; + } + }); + + return { hasLogfmt, hasJSON }; +} diff --git a/src/language_provider.ts b/src/language_provider.ts index 6efa4ad4..68eefed3 100644 --- a/src/language_provider.ts +++ b/src/language_provider.ts @@ -1,7 +1,8 @@ import { getDefaultTimeRange, LanguageProvider, TimeRange, } from '@grafana/data'; import { VictoriaLogsDatasource } from './datasource'; -import { FieldHits, FieldHitsResponse, FilterFieldType } from './types'; +import { extractLogParserFromSample } from './languageUtils'; +import { FieldHits, FieldHitsResponse, FilterFieldType, ParserAndLabelKeysResult } from './types'; interface FetchFieldsOptions { type: FilterFieldType; @@ -11,6 +12,12 @@ interface FetchFieldsOptions { limit?: number; } +interface LogQueryOptions { + timeRange?: TimeRange; + query?: string; + limit?: number; +} + enum HitsValueType { NUMBER = 'number', DATE = 'date', @@ -38,6 +45,7 @@ export default class LogsQlLanguageProvider extends LanguageProvider { return Promise.all([]); }; + async getFieldList(options: FetchFieldsOptions, customParams?: URLSearchParams): Promise { if (options.type === FilterFieldType.FieldValue && !options.field) { console.warn('getFieldList: field is required for FieldValue type'); @@ -75,7 +83,9 @@ export default class LogsQlLanguageProvider extends LanguageProvider { if (this.cacheValues.size >= this.cacheSize) { const firstKey = this.cacheValues.keys().next().value; - this.cacheValues.delete(firstKey); + if (firstKey) { + this.cacheValues.delete(firstKey); + } } try { @@ -89,6 +99,83 @@ export default class LogsQlLanguageProvider extends LanguageProvider { } } + async query(options: LogQueryOptions): Promise[]> { + const url = 'select/logsql/query'; + const timeRange = this.getTimeRangeParams(options.timeRange); + const params = new URLSearchParams({ + query: options.query ?? '*', + start: timeRange.start.toString(), + end: timeRange.end.toString(), + limit: (options.limit ?? 10).toString(), + }); + try { + const res = (await this.datasource.postResource(url, params, { responseType: 'text' })); + const lines = res.split('\n').filter(line => line.trim() !== ''); + return lines.map(line => { + try { + return JSON.parse(line); + } catch (e) { + return { message: line }; + } + }); + } catch (error) { + throw error; + } + } + + /** + * Get parser and label keys for a selector + * + * This asynchronous function is used to fetch parsers and label keys for a selected log stream based on sampled lines. + * It returns a promise that resolves to an object with the following properties: + * + * - `extractedLabelKeys`: An array of available label keys associated with the log stream. + * - `hasJSON`: A boolean indicating whether JSON parsing is available for the stream. + * - `hasLogfmt`: A boolean indicating whether Logfmt parsing is available for the stream. + * - `hasPack`: A boolean indicating whether Pack parsing is available for the stream. + * - `unwrapLabelKeys`: An array of label keys that can be used for unwrapping log data. + * + * @param streamSelector - The selector for the log stream you want to analyze. + * @param options - (Optional) An object containing additional options. + * @param options.maxLines - (Optional) The number of log lines requested when determining parsers and label keys. + * @param options.timeRange - (Optional) The time range for which you want to retrieve label keys. If not provided, the default time range is used. + * Smaller maxLines is recommended for improved query performance. The default count is 10. + * @returns A promise containing an object with parser and label key information. + * @throws An error if the fetch operation fails. + */ + async getParserAndLabelKeys( + streamSelector: string, + options?: { maxLines?: number; timeRange?: TimeRange } + ): Promise { + const empty = { + extractedLabelKeys: [], + structuredMetadataKeys: [], + unwrapLabelKeys: [], + hasJSON: false, + hasLogfmt: false, + }; + + const sample = await this.query({ + query: streamSelector, + limit: options?.maxLines || 10, + timeRange: options?.timeRange, + }); + + if (!sample.length) { + return empty; + } + + const { hasLogfmt, hasJSON } = extractLogParserFromSample(sample); + + return { + extractedLabelKeys: [], + structuredMetadataKeys: [], + unwrapLabelKeys: [], + hasJSON, + hasLogfmt, + }; + } + getTimeRangeParams(timeRange?: TimeRange) { const range = timeRange ?? getDefaultTimeRange(); diff --git a/src/queryUtils.ts b/src/queryUtils.ts index 1b9bf48b..3660a36b 100644 --- a/src/queryUtils.ts +++ b/src/queryUtils.ts @@ -1,7 +1,7 @@ import { SyntaxNode } from "@lezer/common"; import { escapeRegExp } from 'lodash'; -import { Filter, FilterOp, LineFilter, OrFilter, parser, PipeExact, PipeMatch, String } from "@grafana/lezer-logql" +import { Filter, FilterOp, JsonExpressionParser, LabelParser, LineFilter, Logfmt, OrFilter, parser, PipeExact, PipeMatch, String } from "@grafana/lezer-logql" export function getNodesFromQuery(query: string, nodeTypes?: number[]): SyntaxNode[] { const nodes: SyntaxNode[] = []; @@ -70,3 +70,9 @@ export function getHighlighterExpressionsFromQuery(input = ''): string[] { } return results; } + +export function isQueryWithParser(query: string): { queryWithParser: boolean; parserCount: number } { + const nodes = getNodesFromQuery(query, [LabelParser, JsonExpressionParser, Logfmt]); + const parserCount = nodes.length; + return { queryWithParser: parserCount > 0, parserCount }; +} diff --git a/src/types.ts b/src/types.ts index 08f76ddf..147c0050 100644 --- a/src/types.ts +++ b/src/types.ts @@ -135,3 +135,12 @@ export enum TenantHeaderNames { } export type MultitenancyHeaders = Partial> + + +export interface ParserAndLabelKeysResult { + extractedLabelKeys: string[]; + structuredMetadataKeys: string[]; + hasJSON: boolean; + hasLogfmt: boolean; + unwrapLabelKeys: string[]; +}