site: output buffering & flushing

site/src/bash/grammar.js

@@ -55,7 +55,18 @@ export const semantics = grammar.createSemantics().addOperation("ast", {
     return v.ast();
   },
   Quoted_quoted(_lq, s, _rq) {
-    return { type: "quote", args: s.children.map((e) => e.ast()) };
+    const args = s.children.map((c) => {
+      let v = c.ast();
+      // Spaces are lost in QuotedText when appearing after InnerExpr
+      // As temporary workaround, include any preceding spaces into QuotedText value
+      if (c.ctorName === "QuotedText") {
+        let i = c.source.startIdx - 1;
+        for (i; s.source.sourceString[i] == " "; i--);
+        v = "".concat(" ".repeat(c.source.startIdx - 1 - i), v);
+      }
+      return v;
+    });
+    return { type: "quote", args };
   },
   QuotedText(q) {
     return q.children.map((c) => (c.ctorName !== "any" ? c.ast() : c.sourceString)).join("");

site/src/bash/interpreter.js

@@ -1,24 +1,28 @@
 import { grammar, semantics } from "./grammar.js";
 import ExecWorker from "./exec-worker?worker";
 import { serialize } from "./serializer.js";
+import * as renderer from "../renderer.js";
 import kernel from "../kernel.js";
 import fs from "../fs.js";
 import run from "./cli-runner.js";
 
 const execWorker = new ExecWorker();
 
-const capture = () => {
-  const ow = process.stdout.write;
-  let buffer = "";
-  process.stdout.write = (v) => (buffer += v);
-
-  return () => {
-    process.stdout.write = ow;
-    return buffer;
-  };
+// MAYBE simplify: handle fds in a local object, and leave filesystem just for FD=0 (required from webworker)
+/** Read from `opts.fds` and clear them. Render `opts.render` */
+const flush = async (opts = { fds: [1, 2], render: [1, 2] }) => {
+  const o = await Promise.all(opts.fds.map((fd) => fs.readFile(fs.getProcessFdPath(fd)).then((v) => [fd, v]))).then(
+    (o) => o.reduce((acc, [fd, v]) => ((acc[fd] = v), acc), {}),
+  );
+  // Empty contents
+  await Promise.all(opts.fds.map((fd) => fs.writeFile(fs.getProcessFdPath(fd), "")));
+  for (const r of opts.render || []) {
+    renderer.renderOutput(o[r], { error: r == 2 });
+  }
+  return o;
 };
 
-async function executeAst(node) {
+async function executeAst(node, opts = {}) {
   if (typeof node === "string") {
     return node;
   }
@@ -29,17 +33,17 @@ async function executeAst(node) {
         args.push(await executeAst(arg));
         continue;
       }
-      const free = capture();
-      await executeAst(arg);
-      args.push(free());
+      await executeAst(arg, { flush: false });
+      const { 1: o } = await flush({ fds: [1] });
+      args.push(o);
     }
     return args.join("");
   }
   if (node.type === "env") {
     const arg1 = node.args[1];
     // Wrap value with "quoted" node-type to reuse logic for capturing output
     const v = typeof arg1 !== "string" && arg1.type !== "quote" ? { type: "quote", args: [arg1] } : arg1;
-    const r = [node.args[0], await executeAst(v)];
+    const r = [node.args[0], await executeAst(v, opts)];
     if (node.cmd === true) {
       return r;
     }
@@ -58,32 +62,34 @@ async function executeAst(node) {
     const args = [];
     // Process arguments
     for (const arg of node.args) {
-      let av = await executeAst(arg);
+      let av = await executeAst(arg, opts);
       av !== undefined && args.push(av);
     }
     const env = { ...process.env };
     // Process environment variables
     for (const e of node.env) {
-      const [k, v] = await executeAst({ ...e, cmd: true });
+      const [k, v] = await executeAst({ ...e, cmd: true }, opts);
       env[k] = v;
     }
     console.log(`[execute::cmd] ${node.cmd} args=${JSON.stringify(args)} env=${JSON.stringify(env)}`);
 
     if (!cliSpec) {
       process.stderr.write(`cliersh: command not found: "${node.cmd}"\n`);
+      opts.flush !== false && (await flush());
       return process.exit(1);
     }
 
     // Handle "builtins" that need access to internals (e.g. renderer), and would not otherwise work from web-worker
     if (cliSpec.builtin) {
       await run({ name: node.cmd, cliSpec, args });
+      opts.flush !== false && (await flush());
       return process.exit(process.exitCode);
     }
 
     // Create and object containing required process properties for the worker
     const p = { env, stdout: { columns: process.stdout.columns }, stdin: { isTTY: process.stdin.isTTY } };
 
-    return new Promise((resolve) => {
+    await new Promise((resolve) => {
       execWorker.postMessage(serialize({ name: node.cmd, cliSpec, args, process: p, cliHandlerUrl }));
 
       execWorker.onmessage = ({ data }) => {
@@ -95,34 +101,40 @@ async function executeAst(node) {
         }
       };
     });
+    opts.flush !== false && (await flush());
+    return;
   }
   if (node.type === "and") {
     for (const child of node.args) {
-      await executeAst(child);
+      await executeAst(child, opts);
       if (process.lastExitCode !== 0) break;
     }
   }
   if (node.type === "or") {
     for (const child of node.args) {
-      await executeAst(child);
+      await executeAst(child, opts);
       if (process.lastExitCode === 0) break;
     }
   }
   if (node.type === "pipe") {
-    const free = capture();
-    await executeAst(node.args[0]);
-    // Write captured value into cpid's fd=0 (stdin)
-    await fs.writeFile(fs.getProcessFdPath(0), free());
+    await executeAst(node.args[0], { flush: false });
+    // Flush both stdout & stderr, only render stderr
+    const { 1: o } = await flush({ fds: [1, 2], render: [2] });
+    // Write captured value into fd=0 (stdin)
+    await fs.writeFile(fs.getProcessFdPath(0), o);
     process.stdin.isTTY = false;
-    await executeAst(node.args[1]);
+    await executeAst(node.args[1], opts);
+    // Empty fd=0 (stdin)
+    await fs.writeFile(fs.getProcessFdPath(0), "");
     process.stdin.isTTY = true;
   }
 }
 
 export default async function execute(input) {
   const r = grammar.match(input);
   if (!r.succeeded()) {
-    process.stderr.write(r.message);
+    // Render error directly
+    renderer.renderOutput(r.message, { error: true });
   }
   const ast = semantics(r).ast();
   return executeAst(ast);

site/src/fs.js

@@ -4,8 +4,10 @@ const root = await navigator.storage.getDirectory();
 
 class FileSystem {
   cwd = "/";
-  // Store a dedicated handle for stdin fd (`/proc/{PID}/fd/0`) to be used in `readFileSync`
+  // Store a dedicated handle for stdin fd (stdout and stderr are sent to main thread) to be used in `readFileSync`
   stdinHandle = null;
+  // Pending write operation
+  wp = Promise.resolve();
 
   getCwd() {
     return this.cwd;
@@ -34,16 +36,62 @@ class FileSystem {
     return `/proc/${kernel.getpid()}/fd/${fd}`;
   }
 
-  async writeFile(path, content) {
-    const handle = await this.#getFileHandle(path, true);
-    const writable = await handle.createWritable();
-    await writable.write(content);
-    await writable.close();
+  async writeFile(path, content, opts = {}) {
+    this.wp = this.wp.then(() => this.#writeFile(path, content, opts));
+    return this.wp;
   }
 
   async readFile(path) {
+    await this.wp;
+    return this.#readFile(path);
+  }
+
+  async #readFile(path) {
     const handle = await this.#getFileHandle(path);
-    return handle.getFile().then((f) => f.text());
+    return handle
+      .getFile()
+      .then((f) => f.text())
+      .then((r) => {
+        return r.slice(r.indexOf("\n") + 1);
+      });
+  }
+
+  async #writeFile(path, content, opts) {
+    let c;
+    // Check if file needs to be read (regenerating metadata)
+    if (opts.metadata || !opts.concat) {
+      const e = opts.concat ? await this.#readFile(path) : "";
+      c = `${JSON.stringify(opts.metadata || {})}\n${e}${content}`;
+    } else {
+      c = content;
+    }
+    return this.#writeRawFileContent(path, c, opts);
+  }
+
+  /** Get file metadata by reading stream until a "\n" is found */
+  async getFileMetadata(path) {
+    const reader = (await this.#getFileHandle(path).then((h) => h.getFile())).stream().getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+    while (true) {
+      const { value, done } = await reader.read();
+      if (done) return buffer;
+      buffer += decoder.decode(value, { stream: true });
+      if (buffer.indexOf("\n") !== -1) break;
+    }
+    try {
+      return JSON.parse(buffer.slice(0, buffer.indexOf("\n")));
+    } catch {
+      return {};
+    }
+  }
+
+  async #writeRawFileContent(path, content, opts) {
+    const handle = await this.#getFileHandle(path, true);
+    const position = opts.concat ? (await handle.getFile()).size : undefined;
+    const writable = await handle.createWritable({ keepExistingData: opts.concat });
+    await writable.write({ type: "write", data: content, position });
+    await writable.close();
   }
 
   /** Compatibility method with original `fs.readFileSync`
@@ -60,7 +108,8 @@ class FileSystem {
     const fileSize = this.stdinHandle.getSize();
     const buffer = new DataView(new ArrayBuffer(fileSize));
     this.stdinHandle.read(buffer, { at: 0 });
-    return new TextDecoder("utf-8").decode(new Uint8Array(buffer.buffer));
+    const content = new TextDecoder("utf-8").decode(new Uint8Array(buffer.buffer));
+    return content.slice(content.indexOf("\n"));
   }
 
   async readDir(path) {
@@ -78,8 +127,8 @@ class FileSystem {
     for (const f in fileMap) {
       await this.writeFile(f, fileMap[f]);
     }
-    // Create empty fd=0 file
-    await this.writeFile(this.getProcessFdPath(0), "");
+    // Create empty fd=0,1,2 file
+    await Promise.all([0, 1, 2].map((fd) => this.writeFile(this.getProcessFdPath(fd), "")));
   }
 
   // Create a stdin handle for web-worker

site/src/index.css

@@ -110,6 +110,9 @@ body.light {
   gap: 8px;
   margin-bottom: 4px;
 }
+.input-wrapper > span {
+  white-space: pre;
+}
 #sprompt.executing {
   display: none;
 }

site/src/main.js

@@ -26,21 +26,20 @@ lm.addEventListener("click", () =>
   document.body.classList[document.body.classList.contains("light") ? "remove" : "add"]("light"),
 );
 
-// Capture cli output
-const r = (...args) => renderer.renderOutput(...args);
-process.stdout.write = (v) => r(v);
-process.stderr.write = (v) => r(v, { error: true });
+// Initialize FS
+await fs.init({ "/README.md": "hello" });
+require("fs").readFileSync = fs.readFileSync.bind(fs);
+
+// Capture cli output, send it to the corresponding fd
+process.stdout.write = (v) => fs.writeFile(fs.getProcessFdPath(1), v, { concat: true });
+process.stderr.write = (v) => fs.writeFile(fs.getProcessFdPath(2), v, { concat: true });
 
 // Setup initial env values
 Object.assign(process.env, {
   SHELL: "cliersh",
   USER: "guest",
 });
 
-// Initialize FS
-await fs.init({ "/README.md": "hello" });
-require("fs").readFileSync = fs.readFileSync.bind(fs);
-
 handleKey(i, {
   Enter: () => {
     let inputValue = i.value;

site/test/grammar.test.js

@@ -96,21 +96,22 @@ assertResult('echo "1$(echo 2)"', {
   env: [],
 });
 // Quoted complex expr
-assertResult('echo "1$(echo 2 && echo 2)"', {
+assertResult('echo "1 $(echo 2 && echo 2)  3"', {
   type: "cmd",
   cmd: "echo",
   args: [
     {
       type: "quote",
       args: [
-        "1",
+        "1 ",
         {
           type: "and",
           args: [
             { type: "cmd", cmd: "echo", args: ["2"], env: [] },
             { type: "cmd", cmd: "echo", args: ["2"], env: [] },
           ],
         },
+        "  3",
       ],
     },
   ],