Add whitespace to searchText when EOF tokens extend beyond the previo…

…us token
guardian · Sep 24, 2024 · 3798591 · 3798591
1 parent d586836
commit 3798591
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 6 deletions.
diff --git a/prosemirror-client/src/cqlInput/CqlInput.spec.ts b/prosemirror-client/src/cqlInput/CqlInput.spec.ts
@@ -119,6 +119,14 @@ describe("CqlInput", () => {
 
       await waitFor("example");
     });
+
+    it("accepts whitespace after non-string tokens", async () => {
+      const { editor, waitFor } = await createCqlEditor("a AND");
+
+      await editor.insertText(" ");
+
+      await waitFor("a AND ");
+    });
   });
 
   describe("typeahead", () => {

diff --git a/prosemirror-client/src/cqlInput/utils.ts b/prosemirror-client/src/cqlInput/utils.ts
@@ -148,7 +148,7 @@ export const tokensToDoc = (_tokens: ProseMirrorToken[]): Node => {
   const nodes = joinSearchTextTokens(_tokens).reduce<Node[]>(
     (acc, token, index, tokens): Node[] => {
       switch (token.tokenType) {
-        case "QUERY_FIELD_KEY":
+        case "QUERY_FIELD_KEY": {
           const tokenKey = token.literal;
           const tokenValue = tokens[index + 1]?.literal;
           const previousToken = tokens[index - 1];
@@ -168,13 +168,23 @@ export const tokensToDoc = (_tokens: ProseMirrorToken[]): Node => {
               ]),
             ])
           );
+        }
         case "QUERY_VALUE":
-        case "EOF":
+        case "EOF": {
+          const previousToken = tokens[index - 1];
+          const previousNode = acc[acc.length - 1];
+          if (previousToken?.to < token.from && previousNode.type === searchText) {
+            return acc.slice(0, acc.length - 1).concat(
+              searchText.create(undefined, schema.text(previousNode.textContent + " "))
+            )
+          }
           return acc;
-        default:
+        }
+        default: {
           return acc.concat(
             searchText.create(undefined, schema.text(token.lexeme))
           );
+        }
       }
     },
     [] as Node[]

diff --git a/prosemirror-client/src/lang/scanner.ts b/prosemirror-client/src/lang/scanner.ts
@@ -132,9 +132,7 @@ export class Scanner {
   private addUnquotedString = () => {
     while (
       // Consume whitespace up until the last whitespace char
-      (!isWhitespace(this.peek()) ||
-        isWhitespace(this.peek(1)) ||
-        this.isAtEnd(1)) &&
+      (!isWhitespace(this.peek()) || (isWhitespace(this.peek(1)) || this.isAtEnd(1))) &&
       this.peek() != ")" &&
       !this.isAtEnd()
     ) {