Dockerfile

@@ -1,14 +1,14 @@
 # == base ======================
 FROM buildpack-deps:bookworm AS base
-ENV CACHEBUST=2024-09-06
-RUN apt update
+ENV CACHEBUST=2024-09-17
+RUN useradd -m -u 8000 observable-builder && mkdir /project && \
+    chown 8000:8000 /project && apt update
 
-# Rust envvars
 ENV RUSTUP_HOME=/usr/local/rustup \
     CARGO_HOME=/usr/local/cargo \
     RUST_VERSION=1.81.0 \
-    VIRTUAL_ENV=/var/local/python-venv
-ENV PATH=/usr/local/cargo/bin:$VIRTUAL_ENV/bin:$PATH
+    VIRTUAL_ENV=/home/observable-builder/.local/python-venv
+ENV PATH=/usr/local/cargo/bin:$VIRTUAL_ENV/bin:/home/observable-builder/.local/bin:$PATH
 
 # == node ======================
 FROM base AS node
@@ -19,8 +19,9 @@ EOF
 RUN --mount=type=cache,target=/var/cache/apt,id=framework-runtime-node \
     apt update \
     && apt install -y --no-install-recommends nodejs \
-    && npm install --global yarn
-RUN npm install --global svgo
+    && corepack enable \
+    && corepack enable pnpm \
+    && npm install --global svgo
 
 # == python ======================
 FROM base AS python
@@ -32,6 +33,9 @@ RUN --mount=type=cache,target=/var/cache/apt,id=framework-runtime-python \
       python3-wheel \
       python3-dev \
       python3-venv \
+      pipx
+USER 8000
+RUN pipx install poetry \
     && python3 -m venv $VIRTUAL_ENV
 
 # == R ===========================
@@ -123,3 +127,5 @@ COPY --from=python . .
 COPY --from=r . .
 COPY --from=duckdb . .
 COPY --from=rust . .
+USER 8000
+WORKDIR /project

bin/test.ts

@@ -5,6 +5,13 @@ import { dirname } from "node:path";
 import { run as runTests } from "node:test";
 import { spec } from "node:test/reporters";
 
+import { parseArgs } from "node:util";
+const { values: { "only": argOnly } } = parseArgs({
+  options: {
+    "only": { type: "boolean" },
+  }
+});
+
 export async function buildTestImage() {
   console.log("building image...");
   let stdio = new StringStream();
@@ -34,7 +41,7 @@ const files = await glob(["tests/**/*.test.ts"], {
 });
 
 await buildTestImage();
-runTests({ files, concurrency: true })
+runTests({ files, concurrency: true, only: argOnly })
   .on("test:fail", () => {
     process.exitCode = 1;
   })

package.json

@@ -12,9 +12,11 @@
   "devDependencies": {
     "@types/dockerode": "^3.3.28",
     "@types/node": "^20",
+    "@types/tar-fs": "^2.0.4",
     "dockerode": "^4.0.2",
     "glob": "^10.3.12",
     "semver": "^7.6.0",
+    "tar-fs": "^3.0.6",
     "tsx": "^4.7.1",
     "typescript": "^5.4.3"
   }

tests/archives.test.ts

@@ -1,3 +1,4 @@
+import { describe } from "node:test";
 import { binaryOnPathTest } from "./index.ts";
 
 const archiveTools = [
@@ -8,4 +9,6 @@ const archiveTools = [
   { binary: "zstd" },
 ];
 
-archiveTools.forEach(binaryOnPathTest);
+await describe("Archive tools", () => {
+  archiveTools.forEach(binaryOnPathTest);
+});

tests/data-manip.test.ts

@@ -1,3 +1,4 @@
+import { describe } from "node:test";
 import { binaryOnPathTest, binaryVersionTest } from "./index.ts";
 
 const dataManipTools = [
@@ -6,10 +7,12 @@ const dataManipTools = [
   { binary: "csv2parquet" },
 ];
 
-dataManipTools.forEach(binaryOnPathTest);
+describe("Data manipulation tools", () => {
+  dataManipTools.forEach(binaryOnPathTest);
 
-binaryVersionTest({
-  binary: "duckdb",
-  semver: "^1",
-  extract: /^v(.*) [0-9a-f]*$/,
+  binaryVersionTest({
+    binary: "duckdb",
+    semver: "^1",
+    extract: /^v(.*) [0-9a-f]*$/,
+  });
 });

tests/dataloader-languages.test.ts

@@ -1,41 +1,118 @@
-import { test } from "node:test";
-import { binaryVersionTest, runCommandInContainer } from "./index.ts";
-
-const dataLoaderLanguages = [
-  { binary: "node", semver: "^20.17" },
-  { binary: "npm", semver: "^10.5" },
-  { binary: "yarn", semver: "^1.22" },
-  {
-    binary: "python3",
-    semver: "^3.11",
-    prefix: "Python",
-  },
-  {
+import { test, describe } from "node:test";
+import os from "node:os";
+import assert from "node:assert/strict";
+import {
+  assertSemver,
+  binaryOnPathTest,
+  binaryVersionTest,
+  runCommandInContainer,
+} from "./index.ts";
+import { cp, mkdtemp, rm } from "node:fs/promises";
+import { join } from "node:path";
+
+describe("Dataloader languages", () => {
+  describe("JavaScript", () => {
+    binaryVersionTest({ binary: "node", semver: "^20.17" });
+    binaryVersionTest({ binary: "npm", semver: "^10.5" });
+
+    binaryVersionTest({
+      binary: "yarn",
+      semver: "^1.22",
+      expectStderr: /^! Corepack is about to download.*yarn/,
+    });
+    test("Yarn v2+ is available, and uses corepack", async () => {
+      const { stdout, stderr } = await runCommandInContainer([
+        "sh",
+        "-c",
+        "mkdir test && cd test && yarn init -2",
+      ]);
+      assert.ok(!stdout.includes("You don't seem to have Corepack enabled"));
+      assert.ok(!stderr.includes("You don't seem to have Corepack enabled"));
+    });
+    test("yarn ^4.4.1 is available via corepack", async () => {
+      const { stdout, stderr } = await runCommandInContainer([
+        "sh",
+        "-c",
+        "mkdir test && cd test && yarn init -2 > /dev/null && yarn --version",
+      ]);
+      assertSemver(stdout, "^4.4.1");
+    });
+
+    binaryVersionTest({
+      binary: "pnpm",
+      semver: "^9.10",
+      expectStderr: /^! Corepack is about to download.*pnpm/,
+    });
+  });
+
+  describe("Python", () => {
+    binaryVersionTest({
+      binary: "python3",
+      semver: "^3.11",
+      prefix: "Python",
+    });
+
+    binaryVersionTest({
+      binary: "pip",
+      semver: "^23.0.1",
+      extract: /^pip ([^ ]+) /,
+    });
+    binaryVersionTest({ binary: "pipx", semver: "^1.1.0" });
+    binaryVersionTest({
+      binary: "poetry",
+      semver: "^1.8.3",
+      prefix: "Poetry (version ",
+      suffix: ")",
+    });
+
+    test(`A Python virtual environment is activated`, async () => {
+      // should not throw
+      await runCommandInContainer(["pip", "install", "pip-install-test"]);
+    });
+
+    test(`Poetry can install dependencies in the virtualenv`, async () => {
+      // This will install dependencies using Poetry, and then try to run `ls`
+      // in the installed dependency's package. If the package is not
+      // installed here, the `ls` command will exit non-zero and
+      // `runCommandInContainer` will throw.
+      let res = await runCommandInContainer(
+        [
+          "sh",
+          "-c",
+          "poetry install; ls $(poetry env info --path)/lib/python3.11/site-packages/pip_install_test/__init__.py",
+        ],
+        {
+          workingDir: "/project/poetry-test",
+          hostContainerDirs: [{ host: "./tests/fixtures/poetry-test", container: "/project/poetry-test" }],
+        },
+      );
+    });
+  });
+
+  binaryVersionTest({
     binary: "Rscript",
     semver: "^4.4.1",
     extract: /^Rscript \(R\) version ([^\s]+)/,
-  },
-  {
-    name: "Rust",
-    binary: "cargo",
-    semver: "^1.81",
-    extract: /^cargo ([\d.]+)/,
-  },
-  {
-    binary: "rust-script",
-    semver: "^0.35",
-    prefix: "rust-script",
-  },
-  {
+  });
+
+  describe("Rust", () => {
+    binaryVersionTest({
+      name: "Rust",
+      binary: "cargo",
+      semver: "^1.81",
+      extract: /^cargo ([\d.]+)/,
+    });
+
+    binaryVersionTest({
+      binary: "rust-script",
+      semver: "^0.35",
+      prefix: "rust-script",
+    });
+  });
+
+  binaryVersionTest({
     binary: "perl",
     semver: "^5.36",
     extract: /^This is perl 5,[^(]* \(([^)]+)\)/,
-  },
-];
-
-dataLoaderLanguages.forEach(binaryVersionTest);
-
-await test(`A Python virtual environment is activated`, async () => {
-  // should not throw
-  await runCommandInContainer(["pip", "install", "requests"]);
+  });
 });

tests/editing-tools.test.ts

@@ -1,3 +1,4 @@
+import { describe } from "node:test";
 import { binaryOnPathTest } from "./index.ts";
 
 const textManipTools: { binary: string }[] = [
@@ -13,4 +14,6 @@ const textManipTools: { binary: string }[] = [
   { binary: "vim" },
 ];
 
-textManipTools.forEach(binaryOnPathTest);
+describe("editing tools", () => {
+  textManipTools.forEach(binaryOnPathTest);
+});

tests/fixtures/poetry-test/pyproject.toml

@@ -0,0 +1,6 @@
+[tool.poetry]
+package-mode = false
+
+[tool.poetry.dependencies]
+python = "^3.11"
+pip-install-test = "^0.5"

tests/general-cli.test.ts

@@ -1,3 +1,4 @@
+import { describe } from "node:test";
 import { binaryOnPathTest } from "./index.ts";
 
 const generalCliTools: { binary: string }[] = [
@@ -10,4 +11,6 @@ const generalCliTools: { binary: string }[] = [
   { binary: "vmstat" },
 ];
 
-generalCliTools.forEach(binaryOnPathTest);
+describe("General CLI tools", () => {
+  generalCliTools.forEach(binaryOnPathTest);
+});

tests/images.test.ts

@@ -1,3 +1,4 @@
+import { describe } from "node:test";
 import { binaryOnPathTest } from "./index.ts";
 
 const imageTools = [
@@ -6,4 +7,6 @@ const imageTools = [
   { binary: "convert", name: "imagemagick" },
 ];
 
-imageTools.forEach(binaryOnPathTest);
+describe("Image tools", () => {
+  imageTools.forEach(binaryOnPathTest);
+});