feat: klipper config-file language

.gitignore

@@ -20,3 +20,6 @@ package-lock.json
 cypress/screenshots/
 cypress/videos/
 components.d.ts
+
+src/plugins/Codemirror/KlipperCfgLang/parser/klipperCfgParser.js
+src/plugins/Codemirror/KlipperCfgLang/parser/klipperCfgParser.terms.js

package.json

@@ -8,7 +8,10 @@
     },
     "scripts": {
         "serve": "vite serve",
-        "build": "vite build && npm run build.zip",
+        "build": "npm run build:lang && vite build && npm run build.zip",
+        "build:lang": "npm run build:parser:klipperCfg && npm run build:lang:klipperCfg",
+        "build:lang:klipperCfg": "rollup --config src/plugins/Codemirror/KlipperCfgLang/rollup.config.js",
+        "build:parser:klipperCfg": "lezer-generator src/plugins/Codemirror/KlipperCfgLang/parser/klipperCfg.grammar -o src/plugins/Codemirror/KlipperCfgLang/parser/klipperCfgParser.js",
         "format": "npm run format:base -- --write",
         "format:base": "prettier .",
         "format:check": "npm run format:base -- --check",
@@ -20,6 +23,7 @@
         "start": "vite build && vite preview",
         "test": "start-server-and-test preview http://127.0.0.1:4173/ 'cypress run'",
         "test:ui": "cypress open",
+        "test:parser:klipperCfg": "mocha --config src/plugins/Codemirror/.mocharc.json src/plugins/Codemirror/KlipperCfgLang/test/test-klipperCfg.js",
         "changelog": "git cliff v0.0.4..$(git describe --tags $(git rev-list --tags --max-count=1)) --output CHANGELOG.md"
     },
     "dependencies": {
@@ -45,6 +49,7 @@
         "hls.js": "^1.3.3",
         "jmuxer": "^2.0.5",
         "js-sha256": "^0.9.0",
+        "lezer": "^0.13.5",
         "lodash.kebabcase": "^4.1.1",
         "lodash.throttle": "^4.1.1",
         "overlayscrollbars": "^1.13.1",
@@ -70,7 +75,9 @@
     },
     "devDependencies": {
         "@intlify/vite-plugin-vue-i18n": "^2.5.0",
+        "@lezer/generator": "^1.3.0",
         "@mdi/js": "^7.0.0",
+        "@rollup/plugin-node-resolve": "^15.1.0",
         "@types/file-saver": "^2.0.5",
         "@types/jmuxer": "^2.0.3",
         "@types/lodash.kebabcase": "^4.1.6",
@@ -86,9 +93,12 @@
         "eslint-config-prettier": "^8.3.0",
         "eslint-plugin-jsonc": "^2.2.1",
         "eslint-plugin-vue": "^9.0.0",
+        "mocha": "^10.2.0",
         "prettier": "^2.5.1",
+        "rollup": "^2.79.1",
         "sass": "~1.32",
         "start-server-and-test": "^1.14.0",
+        "ts-node": "^10.9.1",
         "typescript": "^4.5.5",
         "unplugin-vue-components": "^0.22.12",
         "vite": "^3.2.7",

src/components/inputs/Codemirror.vue

@@ -14,13 +14,19 @@ import { EditorView, keymap } from '@codemirror/view'
 import { EditorState } from '@codemirror/state'
 import { vscodeDark } from '@uiw/codemirror-theme-vscode'
 import { StreamLanguage } from '@codemirror/language'
-import { klipper_config } from '@/plugins/StreamParserKlipperConfig'
 import { gcode } from '@/plugins/StreamParserGcode'
 import { indentWithTab } from '@codemirror/commands'
 import { json } from '@codemirror/lang-json'
 import { css } from '@codemirror/lang-css'
+import { klipperCfg } from '../../plugins/Codemirror/KlipperCfgLang/lang/klipperCfg'
+import { parseErrorLint } from '../../plugins/Codemirror/parseErrorLint'
 import { indentUnit } from '@codemirror/language'
 
+// for lezer grammar debugging
+/* import { logTree } from '../../plugins/Codemirror/printLezerTree'
+import { syntaxTree } from '@codemirror/language'
+import { parser } from '../../plugins/Codemirror/KlipperCfgLang/dist/klipperCfgParser.es.js' */
+
 @Component
 export default class Codemirror extends Mixins(BaseMixin) {
     private content = ''
@@ -49,6 +55,11 @@ export default class Codemirror extends Mixins(BaseMixin) {
         if (newVal !== cm_value) {
             this.setCmValue(newVal)
         }
+        // for lezer grammar debugging
+        /* const state = this.cminstance?.state ?? EditorState.create({})
+        logTree(syntaxTree(state), state.doc.toString())
+        const text = state.doc.toString()
+        console.log(parser.parse(text) + '') */
     }
 
     mounted(): void {
@@ -86,6 +97,7 @@ export default class Codemirror extends Mixins(BaseMixin) {
             basicSetup,
             vscodeDark,
             indentUnit.of(' '.repeat(this.tabSize)),
+            parseErrorLint,
             keymap.of([indentWithTab]),
             EditorView.updateListener.of((update) => {
                 this.content = update.state?.doc.toString()
@@ -95,7 +107,7 @@ export default class Codemirror extends Mixins(BaseMixin) {
             }),
         ]
 
-        if (['cfg', 'conf'].includes(this.fileExtension)) extensions.push(StreamLanguage.define(klipper_config))
+        if (['cfg', 'conf'].includes(this.fileExtension)) extensions.push(klipperCfg())
         else if (['gcode'].includes(this.fileExtension)) extensions.push(StreamLanguage.define(gcode))
         else if (['json'].includes(this.fileExtension)) extensions.push(json())
         else if (['css', 'scss', 'sass'].includes(this.fileExtension)) extensions.push(css())

src/plugins/Codemirror/.mocharc.json

@@ -0,0 +1,4 @@
+{
+    "extensions": ["ts"],
+    "node-option": ["loader=ts-node/esm"]
+}

src/plugins/Codemirror/KlipperCfgLang/lang/klipperCfg.ts

@@ -0,0 +1,41 @@
+import { parser } from '../dist/klipperCfgParser.es.js'
+import { LRLanguage, LanguageSupport, StreamLanguage, foldNodeProp } from '@codemirror/language'
+import { parseMixed } from '@lezer/common'
+import { klipper_config } from '../../../StreamParserKlipperConfig.js'
+
+const jinja2Parser = StreamLanguage.define(klipper_config).parser
+
+export const klipperCfgLang = LRLanguage.define({
+    parser: parser.configure({
+        props: [
+            foldNodeProp.add({
+                ConfigBlock(tree) {
+                    const body = tree.lastChild
+                    if (body == null) return null
+                    let lastOption = body.lastChild
+                    if (lastOption == null) return null
+                    while (lastOption.name == 'Comment') {
+                        lastOption = lastOption.prevSibling
+                        if (lastOption == null) return null
+                    }
+                    return { from: body.from - 1, to: lastOption.to - 1 }
+                },
+            }),
+        ],
+        wrap: parseMixed((node) => {
+            return node.name == 'Jinja2' ? { parser: jinja2Parser } : null
+        }),
+    }),
+    languageData: {
+        commentTokens: { line: '#' },
+    },
+})
+
+export function klipperCfg() {
+    return new LanguageSupport(klipperCfgLang)
+}
+
+/* 
+to generate the parser run:
+npx lezer-generator klipperCfg.grammar -o klipperCfgParser.js
+ */

src/plugins/Codemirror/KlipperCfgLang/parser/highlight.js

@@ -0,0 +1,27 @@
+import { styleTags, tags as t } from '@lezer/highlight'
+
+export const klipperConfigHighlighting = styleTags({
+    Import: t.keyword,
+    ImportKeyword: t.keyword,
+    Parameter: t.variableName,
+    ConfigBlock: t.keyword,
+    BlockType: t.keyword,
+    Identifier: t.regexp,
+
+    String: t.string,
+    Boolean: t.bool,
+    Number: t.number,
+    Cords: t.number,
+    Resolution: t.number,
+    Ratio: t.number,
+    Pin: t.namespace,
+    VirtualPin: t.namespace,
+    Path: t.string,
+    File: t.string,
+    FilePath: t.string,
+    Ipv4: t.number,
+    Ipv6: t.number,
+
+    AutoGenerated: t.regexp,
+    Comment: t.lineComment,
+})

src/plugins/Codemirror/KlipperCfgLang/parser/klipperCfg.grammar

@@ -0,0 +1,73 @@
+@top Program { (Import | ConfigBlock )+ }
+
+@skip { AutoGenerated (newline | eof) | space | blankLine | Comment newline? } 
+
+valueBlock<content> { indent (content (newline | eof) | valueBlock<content>)+ (dedent | eof) }
+sep<content, seperator> { content (seperator content)+ }
+
+
+Import { "[" ImportKeyword FilePath "]" (newline | eof) }
+ConfigBlock {"[" BlockType Identifier* "]" ( newline Body? | newline? eof ) }
+
+Body { Option+ eof? }
+Option { Parameter ":" Value | GcodeKeyword Jinja2 }
+
+Value { value (newline | eof) | value? newline valueBlock<value> }
+Jinja2 { jinja2 (newline | eof) | jinja2? newline valueBlock<jinja2> }
+
+value { Pin | pins | VirtualPin | Cords | Number | String | Boolean | Path | FilePath | Resolution | Ratio | Ipv4 | Ipv6 }
+pins { sep<(Pin | VirtualPin), ","> }
+Cords { sep<number, ","> }
+Number { number }
+
+
+@context trackIndent from "./tokens.js"
+@external tokens indentation from "./tokens.js" { indent, dedent }
+@external tokens newlines from "./tokens.js" { newline, blankLine, eof }
+
+@tokens {
+  extAscii { $[a-zA-Z0-9_\-.] }
+  unixPath { ![/<>|:&{}\t\f #;\[\]\n\r] }
+
+  ImportKeyword{ "include" }
+  GcodeKeyword{ extAscii* "gcode:" }
+  BlockType { extAscii+ }
+  Identifier { extAscii+ }
+  Parameter { extAscii+ }
+  Path { ("/"|"~/") unixPath+ ("/" unixPath+)* }
+  FilePath { (Path | unixPath+) "." unixPath+ }
+  Resolution { $[0-9]+ "x" $[0-9]+ }
+  Ratio { $[0-9]+ ":" $[0-9]+ }
+  Ipv4 { $[0-9]+ "." $[0-9]+ "." $[0-9]+ "." $[0-9]+ ((":"|"/") $[0-9]+)? }
+  hex { $[0-9a-fA-F] }
+  Ipv6 { (("::" (hex+ ":")* hex+) 
+        | (hex+ (":" hex+)* "::")
+        | ((hex+ ":")+ (":" hex+)+)
+        | ((hex+ ":" hex+ ":" hex+ ":" hex+ ":" hex+ ":" hex+))) 
+        ("/" $[0-9]+)?}
+
+  String { ![#;\n\r]+ }
+  jinja2 { ![#;\n\r]+ }
+  number { "-"? $[0-9]+ ("." $[0-9]*)? }
+  Boolean { "True" | "False" | "true" | "false" } 
+  Pin { ("^" | "~")? "!"? "P" $[A-Z]? $[0-9.]+ } 
+  VirtualPin { ("^" | "~")? "!"? extAscii+ ":" extAscii+ }
+
+  AutoGenerated { "#*#" ![\n\r]* }
+  Comment { ("#"|";") ![\n\r]* }
+
+  space { $[ \t\f]+ }
+
+  @precedence { space, jinja2, String }   // because spaces are allowed in string/jinja2
+  @precedence { AutoGenerated, Comment }  // AutoGenerated also starts with a #
+  @precedence { Resolution, Ipv4, Ipv6, Ratio, number, Pin, VirtualPin, Boolean, ImportKeyword, FilePath, Path, String }
+  @precedence { ImportKeyword, BlockType }// because the ImportKeyword can be canerated with extAscii
+  @precedence { GcodeKeyword, Parameter } // because the GcodeKeyword can be canerated with extAscii
+}
+
+@external propSource klipperConfigHighlighting from "./highlight"
+
+@detectDelim
+
+// to generate the parser run:
+// npx lezer-generator klipperCfg.grammar -o klipperCfgParser.js

src/plugins/Codemirror/KlipperCfgLang/parser/tokens.js

@@ -0,0 +1,82 @@
+/* ref: https://github.com/lezer-parser/python/blob/main/src/tokens.js */
+import { ExternalTokenizer, ContextTracker } from '@lezer/lr'
+
+import { newline as newlineToken, eof, blankLine, indent, dedent } from '../parser/klipperCfgParser.terms.js'
+
+const newline = 10,
+    carriageReturn = 13,
+    space = 32,
+    tab = 9
+
+function isLineBreak(ch) {
+    return ch == newline || ch == carriageReturn
+}
+
+export const newlines = new ExternalTokenizer(
+    (input, stack) => {
+        let prev
+        if (input.next < 0) {
+            input.acceptToken(eof)
+        } else if ((prev = input.peek(-1)) < 0 || isLineBreak(prev)) {
+            while (input.next == space || input.next == tab) {
+                input.advance()
+            }
+            if (isLineBreak(input.next)) input.acceptToken(blankLine, 1)
+        } else if (isLineBreak(input.next)) {
+            input.acceptToken(newlineToken, 1)
+        }
+    },
+    { contextual: true }
+)
+
+export const indentation = new ExternalTokenizer((input, stack) => {
+    let cDepth = stack.context.depth
+    if (cDepth < 0) return
+    let prev = input.peek(-1),
+        depth
+    if (prev == newline || prev == carriageReturn) {
+        let depth = 0,
+            chars = 0
+        for (;;) {
+            if (input.next == space) depth++
+            else if (input.next == tab) depth += 8 - (depth % 8)
+            else break
+            input.advance()
+            chars++
+        }
+        if (depth != cDepth && !isLineBreak(input.next)) {
+            if (depth < cDepth) input.acceptToken(dedent, -chars)
+            else input.acceptToken(indent)
+        }
+    }
+})
+
+function IndentLevel(parent, depth) {
+    this.parent = parent
+    // -1 means this is not an actual indent level but a set of brackets
+    this.depth = depth
+    this.hash = (parent ? (parent.hash + parent.hash) << 8 : 0) + depth + (depth << 4)
+}
+
+const topIndent = new IndentLevel(null, 0)
+
+function countIndent(space) {
+    let depth = 0
+    for (let i = 0; i < space.length; i++) depth += space.charCodeAt(i) == tab ? 8 - (depth % 8) : 1
+    return depth
+}
+
+export const trackIndent = new ContextTracker({
+    start: topIndent,
+    reduce(context) {
+        return context.depth < 0 ? context.parent : context
+    },
+    shift(context, term, stack, input) {
+        if (term == indent) return new IndentLevel(context, countIndent(input.read(input.pos, stack.pos)) >= 1 ? 1 : 0)
+        if (term == dedent) return context.parent
+        return context
+    },
+    hash(context) {
+        return context.hash
+    },
+})

src/plugins/Codemirror/KlipperCfgLang/rollup.config.js

@@ -0,0 +1,19 @@
+import { nodeResolve } from '@rollup/plugin-node-resolve'
+
+export default {
+    input: 'src/plugins/Codemirror/KlipperCfgLang/parser/klipperCfgParser.js',
+    output: [
+        {
+            format: 'cjs',
+            file: 'src/plugins/Codemirror/KlipperCfgLang/dist/klipperCfgParser.cjs',
+        },
+        {
+            format: 'es',
+            file: 'src/plugins/Codemirror/KlipperCfgLang/dist/klipperCfgParser.es.js',
+        },
+    ],
+    external(id) {
+        return !/^[\.\/]/.test(id)
+    },
+    plugins: [nodeResolve()],
+}

src/plugins/Codemirror/KlipperCfgLang/test/test-klipperCfg.js

@@ -0,0 +1,27 @@
+import { parser } from '../dist/klipperCfgParser.cjs'
+import { fileTests } from '../../mochaFileTests.js'
+import * as fs from 'fs'
+import * as path from 'path'
+
+const caseDir = 'src/plugins/Codemirror/KlipperCfgLang/test/testCases'
+const testConfigsDir = 'src/plugins/Codemirror/KlipperCfgLang/test/testConfigs'
+
+for (let file of fs.readdirSync(caseDir)) {
+    if (!/\.txt$/.test(file)) continue
+
+    let result = /^[^.]+/.exec(file)
+    let name = result ? result[0] : 'default-name'
+    describe(name, () => {
+        for (let { name, run } of fileTests(fs.readFileSync(path.join(caseDir, file), 'utf8'), file))
+            it(name, () => run(parser))
+    })
+}
+
+for (let file of fs.readdirSync(testConfigsDir)) {
+    let result = /^[^.]+/.exec(file)
+    let name = result ? result[0] : 'default-name'
+    describe(name, () => {
+        for (let { name, run } of fileTests(fs.readFileSync(path.join(testConfigsDir, file), 'utf8'), file, true))
+            it(name, () => run(parser))
+    })
+}