WIP: Add query autocomplete in code builder

.alloy/config.alloy

@@ -0,0 +1,49 @@
+livedebugging {
+    enabled = true
+}
+
+discovery.docker "linux" {
+  host = "unix:///var/run/docker.sock"
+}
+
+discovery.relabel "logs_integrations_docker" {
+  targets = []
+
+  rule {
+    source_labels = ["__meta_docker_container_name"]
+    regex = "/(.*)"
+    target_label = "service_name"
+  }
+}
+
+loki.source.docker "default" {
+  host       = "unix:///var/run/docker.sock"
+  targets    = discovery.docker.linux.targets
+  labels     = {"platform" = "docker"}
+  relabel_rules = discovery.relabel.logs_integrations_docker.rules
+  forward_to = [loki.process.logs.receiver]
+}
+
+loki.process "logs" {
+    stage.static_labels {
+        values = {
+            env = "dev",
+        }
+    }
+
+    forward_to = [loki.write.local.receiver]
+}
+
+loki.write "local" {
+    endpoint {
+        url = string.format(
+            "http://victorialogs:9428/insert/loki/api/v1/push?disable_message_parsing=1&_stream_fields=%s",
+            string.join([
+                "env",
+                "platform",
+                "service_name",
+                "level",
+            ], ","),
+        )
+    }
+}

.editorconfig

@@ -0,0 +1,3 @@
+[*.ts]
+indent_style = space
+indent_size = 2

compose.yaml

@@ -1,7 +1,34 @@
 services:
+  # Generate some logs for testing, in JSON format
+  flog-json:
+    image: mingrammer/flog:latest
+    command: ["flog", "-f", "json", "--number", "36000", "--sleep", "100ms"]
+  # Run Grafana Alloy to collect logs from Docker and forward them to VictoriaLogs
+  alloy:
+    image: grafana/alloy:latest
+    privileged: true
+    ports:
+      - 12345:12345
+      - 4317:4317
+      - 4318:4318
+    volumes:
+      - ./.alloy/config.alloy:/etc/alloy/config.alloy:ro
+      - /proc:/rootproc:ro
+      - /var/run/docker.sock.raw:/var/run/docker.sock
+      - /sys:/sys:ro
+      - /:/rootfs:ro
+      - /dev/disk/:/dev/disk:ro
+      - /var/lib/docker/:/var/lib/docker:ro
+    command: run --server.http.listen-addr=0.0.0.0:12345 --storage.path=/var/lib/alloy/data /etc/alloy/config.alloy
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    devices:
+      - /dev/kmsg
   socks-proxy:
     image: serjs/go-socks5-proxy
     restart: always
+    environment:
+      REQUIRE_AUTH: "false"
   victorialogs:
     image: victoriametrics/victoria-logs:v1.26.0
     ports:

src/components/QueryEditor/QueryField.tsx

@@ -18,6 +18,7 @@ const QueryField: React.FC<QueryFieldProps> = (
     history,
     onRunQuery,
     onChange,
+    datasource,
     'data-testid': dataTestId
   }) => {
 
@@ -35,6 +36,7 @@ const QueryField: React.FC<QueryFieldProps> = (
           <MonacoQueryFieldWrapper
             runQueryOnBlur={false}
             history={history ?? []}
+            datasource={datasource}
             onChange={onChangeQuery}
             onRunQuery={onRunQuery}
             initialValue={query.expr ?? ''}

src/components/monaco-query-field/MonacoQueryField.tsx

@@ -1,12 +1,16 @@
 import { css } from '@emotion/css';
-import React, { useRef } from 'react';
+import React, { useEffect, useRef } from 'react';
 import { useLatest } from 'react-use';
 
 import { GrafanaTheme2 } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
-import { useTheme2, ReactMonacoEditor, monacoTypes } from '@grafana/ui';
+import { useTheme2, ReactMonacoEditor, monacoTypes, Monaco } from '@grafana/ui';
+
+import { languageConfiguration, monarchlanguage } from '../../language';
 
 import { Props } from './MonacoQueryFieldProps';
+import { CompletionDataProvider } from './completion/CompletionDataProvider';
+import { getCompletionProvider, getSuggestOptions } from './completion/completionUtils';
 
 const options: monacoTypes.editor.IStandaloneEditorConstructionOptions = {
   codeLens: false,
@@ -35,6 +39,7 @@ const options: monacoTypes.editor.IStandaloneEditorConstructionOptions = {
     horizontalScrollbarSize: 0,
   },
   scrollBeyondLastLine: false,
+  suggest: getSuggestOptions(),
   suggestFontSize: 12,
   wordWrap: 'on',
 };
@@ -48,6 +53,25 @@ const options: monacoTypes.editor.IStandaloneEditorConstructionOptions = {
 //    up & down. this we want to avoid)
 const EDITOR_HEIGHT_OFFSET = 2;
 
+// we must only run the lang-setup code once
+let LANGUAGE_SETUP_STARTED = false;
+const LANG_ID = 'victorialogs-logsql';
+
+function ensureVictoriaLogsLogsQL(monaco: Monaco) {
+  if (LANGUAGE_SETUP_STARTED === false) {
+    LANGUAGE_SETUP_STARTED = true;
+    monaco.languages.register({ id: LANG_ID });
+
+    monaco.languages.setMonarchTokensProvider(LANG_ID, monarchlanguage);
+    monaco.languages.setLanguageConfiguration(LANG_ID, {
+      ...languageConfiguration,
+      wordPattern: /(-?\d*\.\d\w*)|([^`~!#%^&*()+\[{\]}\\|;:',.<>\/?\s]+)/g,
+      // Default:  /(-?\d*\.\d\w*)|([^`~!#%^&*()\-=+\[{\]}\\|;:'",.<>\/?\s]+)/g
+      // Removed `"`, `=`, and `-`, from the exclusion list, so now the completion provider can decide to overwrite any matching words, or just insert text at the cursor
+    });
+  }
+}
+
 const getStyles = (theme: GrafanaTheme2, placeholder: string) => {
   return {
     container: css`
@@ -67,33 +91,66 @@ const getStyles = (theme: GrafanaTheme2, placeholder: string) => {
 const MonacoQueryField = (props: Props) => {
   // we need only one instance of `overrideServices` during the lifetime of the react component
   const containerRef = useRef<HTMLDivElement>(null);
-  const { onBlur, onRunQuery, initialValue, placeholder, readOnly } = props;
+  const { onBlur, onRunQuery, initialValue, placeholder, readOnly, history, timeRange, datasource } = props;
 
   const onRunQueryRef = useLatest(onRunQuery);
   const onBlurRef = useLatest(onBlur);
+  const historyRef = useLatest(history);
+  const langProviderRef = useLatest(datasource.languageProvider);
+  const completionDataProviderRef = useRef<CompletionDataProvider | null>(null);
+  const autocompleteCleanupCallback = useRef<(() => void) | null>(null);
 
   const theme = useTheme2();
   const styles = getStyles(theme, placeholder);
 
+  useEffect(() => {
+    // when we unmount, we unregister the autocomplete-function, if it was registered
+    return () => {
+      autocompleteCleanupCallback.current?.();
+    };
+  }, []);
+
   return (
-    <div
-      aria-label={selectors.components.QueryField.container}
-      className={styles.container}
-      ref={containerRef}
-    >
+    <div aria-label={selectors.components.QueryField.container} className={styles.container} ref={containerRef}>
       <ReactMonacoEditor
-        options={{
-          ...options,
-          readOnly
-        }}
-        language="promql"
+        options={{ ...options, readOnly }}
+        language={LANG_ID}
         value={initialValue}
+        beforeMount={(monaco) => {
+          ensureVictoriaLogsLogsQL(monaco);
+        }}
         onMount={(editor, monaco) => {
           // we setup on-blur
           editor.onDidBlurEditorWidget(() => {
             onBlurRef.current(editor.getValue());
           });
 
+          const dataProvider = new CompletionDataProvider(langProviderRef.current!, historyRef, timeRange);
+          completionDataProviderRef.current = dataProvider;
+          const completionProvider = getCompletionProvider(monaco, dataProvider);
+
+          // completion-providers in monaco are not registered directly to editor-instances,
+          // they are registered to languages. this makes it hard for us to have
+          // separate completion-providers for every query-field-instance
+          // (but we need that, because they might connect to different datasources).
+          // the trick we do is, we wrap the callback in a "proxy",
+          // and in the proxy, the first thing is, we check if we are called from
+          // "our editor instance", and if not, we just return nothing. if yes,
+          // we call the completion-provider.
+          const filteringCompletionProvider: monacoTypes.languages.CompletionItemProvider = {
+            ...completionProvider,
+            provideCompletionItems: (model, position, context, token) => {
+              // if the model-id does not match, then this call is from a different editor-instance,
+              // not "our instance", so return nothing
+              if (editor.getModel()?.id !== model.id) {
+                return { suggestions: [] };
+              }
+              return completionProvider.provideCompletionItems(model, position, context, token);
+            },
+          };
+          const { dispose } = monaco.languages.registerCompletionItemProvider(LANG_ID, filteringCompletionProvider);
+
+          autocompleteCleanupCallback.current = dispose;
           const updateElementHeight = () => {
             const containerDiv = containerRef.current;
             if (containerDiv !== null) {
@@ -110,10 +167,10 @@ const MonacoQueryField = (props: Props) => {
 
           // handle: shift + enter
           editor.addAction({
-            id: "execute-shift-enter",
-            label: "Execute",
+            id: 'execute-shift-enter',
+            label: 'Execute',
             keybindings: [monaco.KeyMod.Shift | monaco.KeyCode.Enter],
-            run: () => onRunQueryRef.current(editor.getValue() || "")
+            run: () => onRunQueryRef.current(editor.getValue() || ''),
           });
 
           /* Something in this configuration of monaco doesn't bubble up [mod]+K, which the
@@ -127,13 +184,8 @@ const MonacoQueryField = (props: Props) => {
             const placeholderDecorators = [
               {
                 range: new monaco.Range(1, 1, 1, 1),
-                contents: [
-                  { value: "**bold** _italics_ regular `code`" }
-                ],
-                options: {
-                  className: styles.placeholder,
-                  isWholeLine: true,
-                },
+                contents: [{ value: '**bold** _italics_ regular `code`' }],
+                options: { className: styles.placeholder, isWholeLine: true },
               },
             ];
 

src/components/monaco-query-field/MonacoQueryFieldProps.ts

@@ -1,12 +1,15 @@
-import { HistoryItem } from '@grafana/data';
+import { HistoryItem, TimeRange } from '@grafana/data';
 
+import { VictoriaLogsDatasource } from '../../datasource';
 import { Query } from '../../types';
 
 export type Props = {
   initialValue: string;
   history: Array<HistoryItem<Query>>;
   placeholder: string;
   readOnly?: boolean;
+  timeRange?: TimeRange;
+  datasource: VictoriaLogsDatasource;
   onRunQuery: (value: string) => void;
   onBlur: (value: string) => void;
 };