LDW initial integration

.cspell.json

@@ -7,21 +7,32 @@
     // Alphabetically sorted list of "allowed" words to ignore:
     "abicoder",
     "codegen",
+    "combinator",
+    "combinators",
     "contentful",
+    "devbox",
     "devcontainer",
+    "devcontainers",
+    "direnv",
     "doxygen",
     "ebnf",
+    "envrc",
     "inheritdoc",
     "instanceof",
     "ipfs",
     "jumpi",
     "metaslang",
     "mkdocs",
     "napi",
+    "nanopass",
+    "nanopasses",
     "nomic",
     "nomicfoundation",
     "nonterminal",
     "nonterminals",
+    "recognise",
+    "recognises",
+    "recognising",
     "rustup",
     "struct",
     "structs",

crates/codegen/ldw/.prettierrc.json

@@ -0,0 +1,7 @@
+{
+    "trailingComma": "none",
+    "tabWidth": 4,
+    "semi": true,
+    "singleQuote": true,
+    "printWidth": 120
+}

crates/codegen/ldw/Cargo.toml

@@ -0,0 +1,9 @@
+[package]
+name = "language-design-workbench"
+version = "1.0.0"
+edition = "2021"
+
+[lib]
+path = "src-rs/lib.rs"
+
+[dependencies]

crates/codegen/ldw/Installation.md

@@ -0,0 +1,66 @@
+# Installation
+
+There are three main methods to install and set up the Language Design Workbench:
+
+## 1. Using Devbox with direnv (Recommended)
+
+If you have [devbox](https://www.jetpack.io/devbox/), [Nix](https://nixos.org/),
+and [direnv](https://direnv.net/) installed on your system:
+
+1. Clone this repository:
+
+    ```sh
+    git clone <repository-url>
+    cd language-design-workbench
+    ```
+
+2. Allow direnv to load the environment:
+
+    ```sh
+    direnv allow
+    ```
+
+3. Install dependencies:
+
+    ```sh
+    npm install
+    ```
+
+Note: We recommend installing the [direnv extension for
+VSCode](https://marketplace.visualstudio.com/items?itemName=mkhl.direnv) for
+seamless integration. The project already includes a `.envrc` file for direnv
+configuration.
+
+## 2. Manual Installation
+
+If you prefer to use your local Node.js and npm installation:
+
+1. Ensure you have [Node.js](https://nodejs.org/) and [npm](https://www.npmjs.com/) installed on your system.
+2. Clone this repository:
+
+    ```sh
+    git clone <repository-url>
+    cd language-design-workbench
+    ```
+
+3. Install dependencies:
+
+    ```sh
+    npm install
+    ```
+
+## 3. Using a Devcontainer
+
+If you're using Visual Studio Code with the Remote - Containers extension:
+
+1. Use the "Clone Repository in Container Volume" option when cloning the
+   repository. This approach is recommended over checking out the source and then
+   reopening in the container, as it avoids potential performance issues associated
+   with mounting local volumes.
+
+2. VSCode will automatically set up the development environment and install
+   dependencies.
+
+Note: Be aware that using a local volume mount with devcontainers can have
+performance implications, especially on non-Linux hosts. The "Clone Repository
+in Container Volume" approach helps mitigate these issues.

crates/codegen/ldw/README.md

@@ -0,0 +1,41 @@
+# Language Design Workbench (LDW)
+
+## Overview
+
+The Language Design Workbench (LDW) is a tool for working with language
+grammars and generating tooling and documentation for them. It allows you to
+define grammars using a specific syntax and generates various TypeScript modules
+to support language processing, parsing, formatting, and testing.
+
+There is further documentation in the [docs/Overview.md](docs/Overview.md) file,
+which describes the system in more detail, and in particular is used as an input
+to AI prompts.
+
+This project is largely generated by AI, and forms a testbed for using AI for
+coding, especially in the field of programming language design and
+implementation.
+
+## [Installation](./Installation.md)
+
+## Usage
+
+### Using the LDW CLI
+
+The Language Design Workbench CLI is built with TypeScript and runs directly
+using ts-node, which allows for a smoother development experience and more
+accurate error messages.
+
+To use the Language Design Workbench CLI, run:
+
+```sh
+npm run ldw
+```
+
+For more information about available commands and options, run:
+
+```sh
+npm run ldw -- --help
+```
+
+Note: There's no need for a separate build step as the project uses ts-node to
+run TypeScript files directly.

crates/codegen/ldw/jest.config.js

@@ -0,0 +1,10 @@
+module.exports = {
+    preset: 'ts-jest',
+    testEnvironment: 'node',
+    roots: ['<rootDir>/src-ts', '<rootDir>/tests-ts'],
+    testMatch: ['**/__tests__/**/*.ts?(x)', '**/?(*.)+(spec|test).ts?(x)'],
+    transform: {
+        '^.+\\.tsx?$': 'ts-jest'
+    },
+    moduleFileExtensions: ['ts', 'tsx', 'js', 'jsx', 'json', 'node']
+};

crates/codegen/ldw/json-grammar-out/public/json/ldw.model

@@ -0,0 +1,55 @@
+// Generated on 2025-01-17T15:12:29.806Z
+model public::json {
+
+    value = {
+        boolean: boolean,
+        number: number,
+        string: string,
+        null: null,
+        list: list,
+        map: map
+    };
+
+    boolean = {
+          "true"
+        | "false"
+    };
+
+    number = string;
+
+    ureal = string;
+
+    uinteger = string;
+
+    digit = {  };
+
+    sign = {  };
+
+    string = string;
+
+    string_character = string;
+
+    null = {  };
+
+    list = seq<value>;
+
+    map = seq<map_entry>;
+
+    map_entry = {
+        key: string,
+        value: value
+    };
+
+    trivia = {
+        whitespace: whitespace,
+        line_comment: line_comment,
+        block_comment: block_comment
+    };
+
+    line_comment = string;
+
+    block_comment = string;
+
+    whitespace = string;
+
+}

crates/codegen/ldw/models.md

@@ -0,0 +1,113 @@
+# Models
+
+The following diagram shows an example structure involving a 4 pass
+transformation. In fact, this is the exact structure used by the Model Pipeline,
+which as you can see is recursively defined. There are multiple L4’s in that
+case, one for each output. The Grammar Pipeline is defined in exactly the same
+meta-circular manner, but with two L4’s - one that outputs the `parser.ts` code
+and one that outputs the `ldw.model` defining the output of the parser. Each of
+the `.ts` files need to be `.rs` for the Slang nanopasses. The `.ts` outputs
+would continue to be used for the Model and Grammar pipelines, recognising that
+the Grammar facilities are likely not needed at this stage, because the parser
+and `L0` model definition can be manually maintained.
+
+```mermaid
+%%{init: {"flowchart": {"htmlLabels": false}} }%%
+graph  LR
+
+    subgraph Grammar
+
+        GrammarDSL@{ shape: doc, label: "ldw.grammar" }
+        GrammarPipeline@{ shape: processes, label: "Grammar Pipeline<br/>(Nanopasses)" }
+        Parser@{ shape: doc, label: "parser.ts" }
+
+        GrammarDSL --> |transformed by| GrammarPipeline
+        GrammarPipeline --> |generates| Parser
+
+    end
+
+    subgraph L0
+
+        ModelDSL_0@{ shape: doc, label: "ldw.model" }
+        ModelPipeline_0@{ shape: processes, label: "Model Pipeline<br/>(Nanopasses)" }
+
+        subgraph ModelOutputs_0[Outputs]
+            direction LR
+            Model_0@{ shape: doc, label: "model.ts" }
+            Builder_0@{ shape: doc, label: "builder.ts" }
+            Visitor_0@{ shape: doc, label: "visitor.ts" }
+            Transformer_0@{ shape: doc, label: "transformer.ts" }
+        end
+
+        GrammarPipeline --> |generates| ModelDSL_0
+        ModelDSL_0 --> |transformed by| ModelPipeline_0 --> |generates| ModelOutputs_0
+        ModelDSL_0 ---> note
+
+        note@{ shape: comment, label: "This could be manually created or come from e.g. Slang rather than a grammar DSL" }
+
+    end
+
+    subgraph L1
+
+        ModelDSL_1@{ shape: doc, label: "ldw.model" }
+        ModelPipeline_1@{ shape: processes, label: "Model Pipeline<br/>(Nanopasses)" }
+
+        subgraph ModelOutputs_1[Outputs]
+            direction LR
+            Model_1@{ shape: doc, label: "model.ts" }
+            Builder_1@{ shape: doc, label: "builder.ts" }
+            Visitor_1@{ shape: doc, label: "visitor.ts" }
+            Transformer_1@{ shape: doc, label: "transformer.ts" }
+        end
+
+        ModelDSL_1 --> |modifies| ModelDSL_0
+        ModelDSL_1 --> |transformed by| ModelPipeline_1 --> |generates| ModelOutputs_1
+
+    end
+
+    subgraph L2
+
+        ModelDSL_2@{ shape: doc, label: "ldw.model" }
+        ModelPipeline_2@{ shape: processes, label: "Model Pipeline<br/>(Nanopasses)" }
+
+        subgraph ModelOutputs_2[Outputs]
+            direction LR
+            Model_2@{ shape: doc, label: "model.ts" }
+            Builder_2@{ shape: doc, label: "builder.ts" }
+            Visitor_2@{ shape: doc, label: "visitor.ts" }
+            Transformer_2@{ shape: doc, label: "transformer.ts" }
+        end
+
+        ModelDSL_2 --> |modifies| ModelDSL_1
+        ModelDSL_2 --> |transformed by| ModelPipeline_2 --> |generates| ModelOutputs_2
+
+    end
+
+    Pass_0@{ shape: doc, label: "pass0.ts<br/>{ transform(string): L0 }<br/>*Parser*" }
+    Pass_0 --> |uses| ModelOutputs_0
+    Pass_0 --> |uses| Parser
+
+    Pass_1@{ shape: doc, label: "pass1.ts<br/>{ transform(L0): L1 }" }
+    Pass_1 --> |uses| ModelOutputs_0
+    Pass_1 --> |uses| ModelOutputs_1
+
+    Pass_2@{ shape: doc, label: "pass2.ts<br/>{ transform(L1): L2 }" }
+    Pass_2 --> |uses| ModelOutputs_1
+    Pass_2 --> |uses| ModelOutputs_2
+
+    Pass_3@{ shape: doc, label: "pass3.ts<br/>{ transform(L2): string }<br/>*Generator*" }
+    Pass_3 --> |uses| ModelOutputs_2
+
+    subgraph Pipeline["{ transform(string): string }"]
+        direction TB
+
+        Combinators@{ shape: doc, label: "nanopass.ts<br/>(Combinators)" }
+        Program@{ shape: doc, label: "Program" }
+
+        Program --> |uses| Combinators
+        Program --> |uses| Pass_0
+        Program --> |uses| Pass_1
+        Program --> |uses| Pass_2
+        Program --> |uses| Pass_3
+    end
+```