wip: add first draft of autocomplete engine for LogsQL

Based on the code for the official Loki datasource. LogsQL and LogQL
are actually pretty similar, making it a good base to get started.

Currently implemented:
- stream autocomplete
- syntax highlighting for LogQL

Author: Simon Bordeyne

src/components/QueryEditor/QueryField.tsx

@@ -18,6 +18,7 @@ const QueryField: React.FC<QueryFieldProps> = (
     history,
     onRunQuery,
     onChange,
+    datasource,
     'data-testid': dataTestId
   }) => {
 
@@ -35,6 +36,7 @@ const QueryField: React.FC<QueryFieldProps> = (
           <MonacoQueryFieldWrapper
             runQueryOnBlur={false}
             history={history ?? []}
+            datasource={datasource}
             onChange={onChangeQuery}
             onRunQuery={onRunQuery}
             initialValue={query.expr ?? ''}

src/components/monaco-query-field/MonacoQueryField.tsx

@@ -1,12 +1,16 @@
 import { css } from '@emotion/css';
-import React, { useRef } from 'react';
+import React, { useEffect, useRef } from 'react';
 import { useLatest } from 'react-use';
 
 import { GrafanaTheme2 } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
-import { useTheme2, ReactMonacoEditor, monacoTypes } from '@grafana/ui';
+import { useTheme2, ReactMonacoEditor, monacoTypes, Monaco } from '@grafana/ui';
+
+import { languageConfiguration, monarchlanguage } from '../../language';
 
 import { Props } from './MonacoQueryFieldProps';
+import { CompletionDataProvider } from './completion/CompletionDataProvider';
+import { getCompletionProvider, getSuggestOptions } from './completion/completionUtils';
 
 const options: monacoTypes.editor.IStandaloneEditorConstructionOptions = {
   codeLens: false,
@@ -35,6 +39,7 @@ const options: monacoTypes.editor.IStandaloneEditorConstructionOptions = {
     horizontalScrollbarSize: 0,
   },
   scrollBeyondLastLine: false,
+  suggest: getSuggestOptions(),
   suggestFontSize: 12,
   wordWrap: 'on',
 };
@@ -48,6 +53,25 @@ const options: monacoTypes.editor.IStandaloneEditorConstructionOptions = {
 //    up & down. this we want to avoid)
 const EDITOR_HEIGHT_OFFSET = 2;
 
+// we must only run the lang-setup code once
+let LANGUAGE_SETUP_STARTED = false;
+const LANG_ID = 'victorialogs-logsql';
+
+function ensureVictoriaLogsLogsQL(monaco: Monaco) {
+  if (LANGUAGE_SETUP_STARTED === false) {
+    LANGUAGE_SETUP_STARTED = true;
+    monaco.languages.register({ id: LANG_ID });
+
+    monaco.languages.setMonarchTokensProvider(LANG_ID, monarchlanguage);
+    monaco.languages.setLanguageConfiguration(LANG_ID, {
+      ...languageConfiguration,
+      wordPattern: /(-?\d*\.\d\w*)|([^`~!#%^&*()+\[{\]}\\|;:',.<>\/?\s]+)/g,
+      // Default:  /(-?\d*\.\d\w*)|([^`~!#%^&*()\-=+\[{\]}\\|;:'",.<>\/?\s]+)/g
+      // Removed `"`, `=`, and `-`, from the exclusion list, so now the completion provider can decide to overwrite any matching words, or just insert text at the cursor
+    });
+  }
+}
+
 const getStyles = (theme: GrafanaTheme2, placeholder: string) => {
   return {
     container: css`
@@ -67,33 +91,66 @@ const getStyles = (theme: GrafanaTheme2, placeholder: string) => {
 const MonacoQueryField = (props: Props) => {
   // we need only one instance of `overrideServices` during the lifetime of the react component
   const containerRef = useRef<HTMLDivElement>(null);
-  const { onBlur, onRunQuery, initialValue, placeholder, readOnly } = props;
+  const { onBlur, onRunQuery, initialValue, placeholder, readOnly, history, timeRange, datasource } = props;
 
   const onRunQueryRef = useLatest(onRunQuery);
   const onBlurRef = useLatest(onBlur);
+  const historyRef = useLatest(history);
+  const langProviderRef = useLatest(datasource.languageProvider);
+  const completionDataProviderRef = useRef<CompletionDataProvider | null>(null);
+  const autocompleteCleanupCallback = useRef<(() => void) | null>(null);
 
   const theme = useTheme2();
   const styles = getStyles(theme, placeholder);
 
+  useEffect(() => {
+    // when we unmount, we unregister the autocomplete-function, if it was registered
+    return () => {
+      autocompleteCleanupCallback.current?.();
+    };
+  }, []);
+
   return (
-    <div
-      aria-label={selectors.components.QueryField.container}
-      className={styles.container}
-      ref={containerRef}
-    >
+    <div aria-label={selectors.components.QueryField.container} className={styles.container} ref={containerRef}>
       <ReactMonacoEditor
-        options={{
-          ...options,
-          readOnly
-        }}
-        language="promql"
+        options={{ ...options, readOnly }}
+        language={LANG_ID}
         value={initialValue}
+        beforeMount={(monaco) => {
+          ensureVictoriaLogsLogsQL(monaco);
+        }}
         onMount={(editor, monaco) => {
           // we setup on-blur
           editor.onDidBlurEditorWidget(() => {
             onBlurRef.current(editor.getValue());
           });
 
+          const dataProvider = new CompletionDataProvider(langProviderRef.current!, historyRef, timeRange);
+          completionDataProviderRef.current = dataProvider;
+          const completionProvider = getCompletionProvider(monaco, dataProvider);
+
+          // completion-providers in monaco are not registered directly to editor-instances,
+          // they are registered to languages. this makes it hard for us to have
+          // separate completion-providers for every query-field-instance
+          // (but we need that, because they might connect to different datasources).
+          // the trick we do is, we wrap the callback in a "proxy",
+          // and in the proxy, the first thing is, we check if we are called from
+          // "our editor instance", and if not, we just return nothing. if yes,
+          // we call the completion-provider.
+          const filteringCompletionProvider: monacoTypes.languages.CompletionItemProvider = {
+            ...completionProvider,
+            provideCompletionItems: (model, position, context, token) => {
+              // if the model-id does not match, then this call is from a different editor-instance,
+              // not "our instance", so return nothing
+              if (editor.getModel()?.id !== model.id) {
+                return { suggestions: [] };
+              }
+              return completionProvider.provideCompletionItems(model, position, context, token);
+            },
+          };
+          const { dispose } = monaco.languages.registerCompletionItemProvider(LANG_ID, filteringCompletionProvider);
+
+          autocompleteCleanupCallback.current = dispose;
           const updateElementHeight = () => {
             const containerDiv = containerRef.current;
             if (containerDiv !== null) {
@@ -110,10 +167,10 @@ const MonacoQueryField = (props: Props) => {
 
           // handle: shift + enter
           editor.addAction({
-            id: "execute-shift-enter",
-            label: "Execute",
+            id: 'execute-shift-enter',
+            label: 'Execute',
             keybindings: [monaco.KeyMod.Shift | monaco.KeyCode.Enter],
-            run: () => onRunQueryRef.current(editor.getValue() || "")
+            run: () => onRunQueryRef.current(editor.getValue() || ''),
           });
 
           /* Something in this configuration of monaco doesn't bubble up [mod]+K, which the
@@ -127,13 +184,8 @@ const MonacoQueryField = (props: Props) => {
             const placeholderDecorators = [
               {
                 range: new monaco.Range(1, 1, 1, 1),
-                contents: [
-                  { value: "**bold** _italics_ regular `code`" }
-                ],
-                options: {
-                  className: styles.placeholder,
-                  isWholeLine: true,
-                },
+                contents: [{ value: '**bold** _italics_ regular `code`' }],
+                options: { className: styles.placeholder, isWholeLine: true },
               },
             ];
 

src/components/monaco-query-field/MonacoQueryFieldProps.ts

@@ -1,12 +1,15 @@
-import { HistoryItem } from '@grafana/data';
+import { HistoryItem, TimeRange } from '@grafana/data';
 
+import { VictoriaLogsDatasource } from '../../datasource';
 import { Query } from '../../types';
 
 export type Props = {
   initialValue: string;
   history: Array<HistoryItem<Query>>;
   placeholder: string;
   readOnly?: boolean;
+  timeRange?: TimeRange;
+  datasource: VictoriaLogsDatasource;
   onRunQuery: (value: string) => void;
   onBlur: (value: string) => void;
 };

src/components/monaco-query-field/completion/CompletionDataProvider.ts

@@ -0,0 +1,124 @@
+import { chain } from 'lodash';
+
+import { HistoryItem, TimeRange } from '@grafana/data';
+
+// import { escapeLabelValueInExactSelector } from '../../../languageUtils';
+import LanguageProvider from '../../../language_provider';
+import { FilterFieldType, ParserAndLabelKeysResult, Query } from '../../../types';
+
+import { Label } from './situation';
+
+
+interface HistoryRef {
+  current: Array<HistoryItem<Query>>;
+}
+
+export class CompletionDataProvider {
+  constructor(
+    public languageProvider: LanguageProvider,
+    private historyRef: HistoryRef = { current: [] },
+    public timeRange: TimeRange | undefined
+  ) {
+    this.queryToLabelKeysCache = new Map();
+  }
+  private queryToLabelKeysCache: Map<string, any>;
+
+//   private buildSelector(labels: Label[]): string {
+//     const allLabelTexts = labels.map(
+//       (label) => `${label.name}${label.op}"${escapeLabelValueInExactSelector(label.value)}"`
+//     );
+
+//     return `{${allLabelTexts.join(',')}}`;
+//   }
+
+  setTimeRange(timeRange: TimeRange) {
+    this.timeRange = timeRange;
+    this.queryToLabelKeysCache.clear();
+  }
+
+  getHistory() {
+    return chain(this.historyRef.current)
+      .orderBy('ts', 'desc')
+      .map((history: HistoryItem<Query>) => history.query.expr.trim())
+      .filter()
+      .uniq()
+      .value();
+  }
+
+  public buildQuery(otherLabels: Label[]): string {
+    if (otherLabels.length === 0) {
+        return '*';
+    }
+    return otherLabels.map(
+        (label) => `${label.name}:=${label.value}`
+    ).join(' AND ');
+  }
+
+  async getLabelNames(query: string): Promise<string[]> {
+    const hits = await this.languageProvider.getFieldList({
+        timeRange: this.timeRange,
+        type: FilterFieldType.FieldName,
+        query,
+    })
+    return hits
+      .filter((hit) => !hit.value.startsWith("_"))
+      .map((hit) => hit.value);
+  }
+//     if (otherLabels.length === 0) {
+//       // If there is no filtering, we use getLabelKeys because it has better caching
+//       // and all labels should already be fetched
+//       await this.languageProvider.start();
+//       return this.languageProvider.getFieldList();
+//     }
+//     const possibleLabelNames = await this.languageProvider.fetchLabels({
+//       streamSelector: this.buildSelector(otherLabels),
+//       timeRange: this.timeRange,
+//     });
+//     const usedLabelNames = new Set(otherLabels.map((l) => l.name)); // names used in the query
+//     return possibleLabelNames.filter((label) => !usedLabelNames.has(label));
+//   }
+
+  async getLabelValues(labelName: string, query: string): Promise<string[]> {
+    const hits = await this.languageProvider.getFieldList({
+        timeRange: this.timeRange,
+        type: FilterFieldType.FieldValue,
+        field: labelName,
+        query,
+    });
+    return hits
+      .map((hit) => hit.value);
+  }
+
+  /**
+   * Runs a Loki query to extract label keys from the result.
+   * The result is cached for the query string.
+   *
+   * Since various "situations" in the monaco code editor trigger this function, it is prone to being called multiple times for the same query
+   * Here is a lightweight and simple cache to avoid calling the backend multiple times for the same query.
+   *
+   * @param logQuery
+   */
+  async getParserAndLabelKeys(logQuery: string): Promise<ParserAndLabelKeysResult> {
+    const EXTRACTED_LABEL_KEYS_MAX_CACHE_SIZE = 2;
+    const cachedLabelKeys = this.queryToLabelKeysCache.has(logQuery) ? this.queryToLabelKeysCache.get(logQuery) : null;
+    if (cachedLabelKeys) {
+      // cache hit! Serve stale result from cache
+      return cachedLabelKeys;
+    } else {
+      // If cache is larger than max size, delete the first (oldest) index
+      if (this.queryToLabelKeysCache.size >= EXTRACTED_LABEL_KEYS_MAX_CACHE_SIZE) {
+        // Make room in the cache for the fresh result by deleting the "first" index
+        const keys = this.queryToLabelKeysCache.keys();
+        const firstKey = keys.next().value;
+        if (firstKey !== undefined) {
+          this.queryToLabelKeysCache.delete(firstKey);
+        }
+      }
+      // Fetch a fresh result from the backend
+      const labelKeys = await this.languageProvider.getParserAndLabelKeys(logQuery, { timeRange: this.timeRange });
+      // Add the result to the cache
+      this.queryToLabelKeysCache.set(logQuery, labelKeys);
+      return labelKeys;
+    }
+  }
+}

src/components/monaco-query-field/completion/NeverCaseError.ts

@@ -0,0 +1,5 @@
+export class NeverCaseError extends Error {
+  constructor(value: never) {
+    super(`Unexpected case in switch statement: ${JSON.stringify(value)}`);
+  }
+}