site: output buffering & flushing II

site/public/commands/echo.js

@@ -1,5 +1,5 @@
 export default {
   definition: { args: { type: "string", positional: true, stdin: true } },
   cliOptions: { help: { hidden: true } },
-  action: ({ args }) => args && Cli.logger.log(args.join(" ")),
+  action: ({ args }) => args && Cli.logger.log(args.join(" "), "\n"),
 };

site/src/bash/FDStack.js

@@ -0,0 +1,44 @@
+import kernel, { FileDescriptor } from "../kernel";
+import * as renderer from "../renderer";
+
+/**
+ * Object containing a stack for each FD.
+ * When a FD's value is updated, the new FD value is pushed into the stack. Once it has been used,
+ * is should popped, so previous FD value is restored
+ */
+class FileDescriptorStack {
+  value = { 0: [], 1: [], 2: [] };
+
+  init() {
+    for (const fd in this.value) {
+      this.value[fd] = [];
+      this.push(fd, new FileDescriptor("TTY"));
+    }
+  }
+
+  push(fd, value) {
+    kernel.setFD(fd, value);
+    this.value[fd].push(value);
+  }
+
+  pop(fd) {
+    const v = this.value[fd].pop();
+    const s = this.value[fd];
+    kernel.setFD(fd, s[s.length - 1]);
+    return v;
+  }
+
+  flush() {
+    for (const fd of [2, 1]) {
+      const d = kernel.getFD(fd);
+      if (d.type === "TTY") {
+        const buffer = d.flush();
+        buffer && renderer.renderOutput(buffer.replace(/\n?$/, "\n"), { error: fd == 2 });
+      }
+    }
+  }
+}
+
+const FDStack = new FileDescriptorStack();
+
+export default FDStack;

site/src/bash/cli-runner.js

@@ -4,7 +4,11 @@ export default async function run({ name, cliSpec, args }) {
   if (cliSpec.builtin !== undefined && cliSpec.cliOptions?.help?.hidden === undefined) {
     cliSpec.cliOptions.help = { ...cliSpec.cliOptions.help, hidden: true };
   }
-  const c = new Cli(cliSpec.definition || {}, { ...cliSpec.cliOptions, cliName: name });
+  const c = new Cli(cliSpec.definition || {}, {
+    logger: { error: (...m) => process.stderr.write("".concat(name, ": ", m.join(""))) },
+    ...cliSpec.cliOptions,
+    cliName: name,
+  });
   // Update default help template
   c.options.help.template =
     cliSpec.cliOptions.help?.template || "{usage}\n{description}\n{namespaces}\n{commands}\n{options}";

site/src/bash/grammar.js

@@ -22,7 +22,7 @@ export const grammar = ohm.grammar(String.raw`
     Expansion = "$" (word | "?")
     QuotedText = #( scaped | ~("\"" | "\\") ~("$" word) ~("$(") any )+
     Keyword = ~"-" ~digit word
-    arg = word_char+
+    arg = (word_char | "/" )+
     scaped = "\\" ("n" | "\"" | "\\" | "$")
     word = word_char+
     word_char = letter | digit | "-" | "." | "_" | ":" | "$"

site/src/bash/interpreter.js

@@ -2,131 +2,129 @@ import { grammar, semantics } from "./grammar.js";
 import ExecWorker from "./exec-worker?worker";
 import { serialize } from "./serializer.js";
 import * as renderer from "../renderer.js";
-import kernel from "../kernel.js";
+import kernel, { FileDescriptor } from "../kernel.js";
+import FDStack from "./FDStack.js";
 import fs from "../fs.js";
 import run from "./cli-runner.js";
 
-const execWorker = new ExecWorker();
+// Capture cli output, send it to the corresponding fd
+process.stdout.write = (v) => kernel.getFD(1).write(v);
+process.stderr.write = (v) => kernel.getFD(2).write(v);
 
-// MAYBE simplify: handle fds in a local object, and leave filesystem just for FD=0 (required from webworker)
-/** Read from `opts.fds` and clear them. Render `opts.render` */
-const flush = async (opts = { fds: [1, 2], render: [1, 2] }) => {
-  const o = await Promise.all(opts.fds.map((fd) => fs.readFile(fs.getProcessFdPath(fd)).then((v) => [fd, v]))).then(
-    (o) => o.reduce((acc, [fd, v]) => ((acc[fd] = v), acc), {}),
-  );
-  // Empty contents
-  await Promise.all(opts.fds.map((fd) => fs.writeFile(fs.getProcessFdPath(fd), "")));
-  for (const r of opts.render || []) {
-    renderer.renderOutput(o[r], { error: r == 2 });
-  }
-  return o;
-};
+const execWorker = new ExecWorker();
 
 async function executeAst(node, opts = {}) {
-  if (typeof node === "string") {
-    return node;
-  }
-  if (node.type === "quote") {
-    const args = [];
-    for (const arg of node.args) {
-      if (typeof arg === "string" || arg.type === "expansion") {
-        args.push(await executeAst(arg));
-        continue;
-      }
-      await executeAst(arg, { flush: false });
-      const { 1: o } = await flush({ fds: [1] });
-      args.push(o);
+  try {
+    if (typeof node === "string") {
+      return node;
     }
-    return args.join("");
-  }
-  if (node.type === "env") {
-    const arg1 = node.args[1];
-    // Wrap value with "quoted" node-type to reuse logic for capturing output
-    const v = typeof arg1 !== "string" && arg1.type !== "quote" ? { type: "quote", args: [arg1] } : arg1;
-    const r = [node.args[0], await executeAst(v, opts)];
-    if (node.cmd === true) {
-      return r;
+    if (node.type === "quote") {
+      FDStack.push(1, new FileDescriptor("PIPE"));
+      const args = [];
+      for (const arg of node.args) {
+        if (typeof arg === "string" || arg.type === "expansion") {
+          args.push(await executeAst(arg));
+          continue;
+        }
+        await executeAst(arg);
+        // Remove trailing new-lines (https://www.gnu.org/software/bash/manual/bash.html#Command-Substitution-1)
+        const buffer = kernel.getFD(1).flush().replace(/\n?$/, "");
+        args.push(buffer);
+      }
+      FDStack.pop(1);
+      return args.join("");
     }
-    process.env[r[0]] = r[1];
-  }
-  if (node.type === "expansion") {
-    const v = node.args[0];
-    // https://www.gnu.org/software/bash/manual/bash.html#Special-Parameters-1
-    if (v === "0") return process.env.SHELL;
-    if (v == "?") return process.lastExitCode.toString();
-    if (v == "$") return kernel.getpid();
-    return process.env[v];
-  }
-  if (node.type === "cmd") {
-    const cliSpec = CLI_COMMANDS[node.cmd];
-    const args = [];
-    // Process arguments
-    for (const arg of node.args) {
-      let av = await executeAst(arg, opts);
-      av !== undefined && args.push(av);
+    if (node.type === "env") {
+      const arg1 = node.args[1];
+      // Wrap value with "quoted" node-type to reuse logic for capturing output
+      const v = typeof arg1 !== "string" && arg1.type !== "quote" ? { type: "quote", args: [arg1] } : arg1;
+      const r = [node.args[0], await executeAst(v, opts)];
+      if (node.cmd === true) {
+        return r;
+      }
+      process.env[r[0]] = r[1];
     }
-    const env = { ...process.env };
-    // Process environment variables
-    for (const e of node.env) {
-      const [k, v] = await executeAst({ ...e, cmd: true }, opts);
-      env[k] = v;
+    if (node.type === "expansion") {
+      const v = node.args[0];
+      // https://www.gnu.org/software/bash/manual/bash.html#Special-Parameters-1
+      if (v === "0") return process.env.SHELL;
+      if (v == "?") return process.lastExitCode.toString();
+      if (v == "$") return kernel.getpid();
+      return process.env[v];
     }
-    console.log(`[execute::cmd] ${node.cmd} args=${JSON.stringify(args)} env=${JSON.stringify(env)}`);
+    if (node.type === "cmd") {
+      const cliSpec = CLI_COMMANDS[node.cmd];
+      const args = [];
+      // Process arguments
+      for (const arg of node.args) {
+        let av = await executeAst(arg, opts);
+        av !== undefined && args.push(av);
+      }
+      const env = { ...process.env };
+      // Process environment variables
+      for (const e of node.env) {
+        const [k, v] = await executeAst({ ...e, cmd: true }, opts);
+        env[k] = v;
+      }
+      console.log(`[execute::cmd] ${node.cmd} args=${JSON.stringify(args)} env=${JSON.stringify(env)}`);
 
-    if (!cliSpec) {
-      process.stderr.write(`cliersh: command not found: "${node.cmd}"\n`);
-      opts.flush !== false && (await flush());
-      return process.exit(1);
-    }
+      if (!cliSpec) {
+        process.stderr.write(`cliersh: command not found: "${node.cmd}"\n`);
+        return process.exit(1);
+      }
 
-    // Handle "builtins" that need access to internals (e.g. renderer), and would not otherwise work from web-worker
-    if (cliSpec.builtin) {
-      await run({ name: node.cmd, cliSpec, args });
-      opts.flush !== false && (await flush());
-      return process.exit(process.exitCode);
-    }
+      // Handle "builtins" that need access to internals (e.g. renderer), and would not otherwise work from web-worker
+      if (cliSpec.builtin) {
+        await run({ name: node.cmd, cliSpec, args });
+        return process.exit(process.exitCode);
+      }
 
-    // Create and object containing required process properties for the worker
-    const p = { env, stdout: { columns: process.stdout.columns }, stdin: { isTTY: process.stdin.isTTY } };
+      // Create and object containing required process properties for the worker
+      const p = { env, stdout: { columns: process.stdout.columns }, stdin: { isTTY: process.stdin.isTTY } };
 
-    await new Promise((resolve) => {
-      execWorker.postMessage(serialize({ name: node.cmd, cliSpec, args, process: p, cliHandlerUrl }));
+      return await new Promise((resolve) => {
+        execWorker.postMessage(serialize({ name: node.cmd, cliSpec, args, process: p, cliHandlerUrl }));
 
-      execWorker.onmessage = ({ data }) => {
-        if (data.type === "output") {
-          process[data.stream].write(data.value);
-        } else if (data.type === "exit") {
-          process.exit(data.exitCode);
-          resolve();
-        }
-      };
-    });
-    opts.flush !== false && (await flush());
-    return;
-  }
-  if (node.type === "and") {
-    for (const child of node.args) {
-      await executeAst(child, opts);
-      if (process.lastExitCode !== 0) break;
+        execWorker.onmessage = ({ data }) => {
+          if (data.type === "output") {
+            process[data.stream].write(data.value);
+          } else if (data.type === "exit") {
+            process.exit(data.exitCode);
+            resolve();
+          }
+        };
+      });
     }
-  }
-  if (node.type === "or") {
-    for (const child of node.args) {
-      await executeAst(child, opts);
-      if (process.lastExitCode === 0) break;
+    if (node.type === "and") {
+      for (const child of node.args) {
+        await executeAst(child, opts);
+        if (process.lastExitCode !== 0) break;
+      }
     }
-  }
-  if (node.type === "pipe") {
-    await executeAst(node.args[0], { flush: false });
-    // Flush both stdout & stderr, only render stderr
-    const { 1: o } = await flush({ fds: [1, 2], render: [2] });
-    // Write captured value into fd=0 (stdin)
-    await fs.writeFile(fs.getProcessFdPath(0), o);
-    process.stdin.isTTY = false;
-    await executeAst(node.args[1], opts);
-    // Empty fd=0 (stdin)
-    await fs.writeFile(fs.getProcessFdPath(0), "");
-    process.stdin.isTTY = true;
+    if (node.type === "or") {
+      for (const child of node.args) {
+        await executeAst(child, opts);
+        if (process.lastExitCode === 0) break;
+      }
+    }
+    if (node.type === "pipe") {
+      FDStack.push(1, new FileDescriptor("PIPE"));
+      await executeAst(node.args[0], { flush: false });
+      const pipe = FDStack.pop(1);
+      FDStack.flush();
+
+      // Write captured value (fd=1) into fd=0 (stdin)
+      FDStack.push(0, pipe);
+      await fs.writeFile(fs.getProcessFdPath(0), pipe.buffer);
+
+      await executeAst(node.args[1], opts);
+
+      // Remove fd=0 (stdin)
+      FDStack.pop(0);
+      await fs.deleteFile(fs.getProcessFdPath(0));
+    }
+  } finally {
+    FDStack.flush();
   }
 }
 
@@ -137,5 +135,8 @@ export default async function execute(input) {
     renderer.renderOutput(r.message, { error: true });
   }
   const ast = semantics(r).ast();
+
+  FDStack.init();
+
   return executeAst(ast);
 }

site/src/fs.js

@@ -46,6 +46,12 @@ class FileSystem {
     return this.#readFile(path);
   }
 
+  // https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system#deleting_a_file_or_folder
+  async deleteFile(path) {
+    await this.wp;
+    const h = await this.#getFileHandle(path);
+    return h.remove();
+  }
   async #readFile(path) {
     const handle = await this.#getFileHandle(path);
     return handle
@@ -127,13 +133,13 @@ class FileSystem {
     for (const f in fileMap) {
       await this.writeFile(f, fileMap[f]);
     }
-    // Create empty fd=0,1,2 file
-    await Promise.all([0, 1, 2].map((fd) => this.writeFile(this.getProcessFdPath(fd), "")));
   }
 
   // Create a stdin handle for web-worker
   async wwPrepareStdinHandle() {
-    this.stdinHandle = await this.#getFileHandle(this.getProcessFdPath(0)).then((h) => h.createSyncAccessHandle());
+    this.stdinHandle = await this.#getFileHandle(this.getProcessFdPath(0), true).then((h) =>
+      h.createSyncAccessHandle(),
+    );
     // Return cleanup function
     return () => this.stdinHandle.close();
   }

site/src/kernel.js

@@ -1,9 +1,37 @@
 class Kernel {
   cpid = 1;
+  fds = { 0: undefined, 1: undefined, 2: undefined };
 
   getpid() {
     return this.cpid;
   }
+
+  getFD(fd) {
+    return this.fds[fd];
+  }
+
+  setFD(fd, value) {
+    this.fds[fd] = value;
+  }
+}
+
+export class FileDescriptor {
+  buffer = "";
+  type = undefined;
+
+  constructor(type) {
+    this.type = type;
+  }
+
+  write(v) {
+    this.buffer += v;
+  }
+
+  flush() {
+    const v = this.buffer;
+    this.buffer = "";
+    return v;
+  }
 }
 
 const kernel = new Kernel();