diff --git a/Dockerfile b/Dockerfile index 65ebe04..c25d6ab 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,14 @@ # == base ====================== FROM buildpack-deps:bookworm AS base -ENV CACHEBUST=2024-09-06 -RUN apt update +ENV CACHEBUST=2024-09-17 +RUN useradd -m -u 8000 observable-builder && mkdir /project && \ + chown 8000:8000 /project && apt update ENV RUSTUP_HOME=/usr/local/rustup \ CARGO_HOME=/usr/local/cargo \ RUST_VERSION=1.81.0 \ - VIRTUAL_ENV=/var/local/python-venv -ENV PATH=/usr/local/cargo/bin:$VIRTUAL_ENV/bin:/root/.local/bin:$PATH + VIRTUAL_ENV=/home/observable-builder/.local/python-venv +ENV PATH=/usr/local/cargo/bin:$VIRTUAL_ENV/bin:/home/observable-builder/.local/bin:$PATH # == node ====================== FROM base AS node @@ -32,8 +33,9 @@ RUN --mount=type=cache,target=/var/cache/apt,id=framework-runtime-python \ python3-wheel \ python3-dev \ python3-venv \ - pipx \ - && pipx install poetry \ + pipx +USER 8000 +RUN pipx install poetry \ && python3 -m venv $VIRTUAL_ENV # == R =========================== @@ -125,3 +127,5 @@ COPY --from=python . . COPY --from=r . . COPY --from=duckdb . . COPY --from=rust . . +USER 8000:8000 +WORKDIR /project diff --git a/package.json b/package.json index fd832c0..8371127 100644 --- a/package.json +++ b/package.json @@ -12,9 +12,11 @@ "devDependencies": { "@types/dockerode": "^3.3.28", "@types/node": "^20", + "@types/tar-fs": "^2.0.4", "dockerode": "^4.0.2", "glob": "^10.3.12", "semver": "^7.6.0", + "tar-fs": "^3.0.6", "tsx": "^4.7.1", "typescript": "^5.4.3" } diff --git a/tests/dataloader-languages.test.ts b/tests/dataloader-languages.test.ts index 9cfd7d6..80785fa 100644 --- a/tests/dataloader-languages.test.ts +++ b/tests/dataloader-languages.test.ts @@ -71,34 +71,21 @@ describe("Dataloader languages", () => { }); test(`Poetry can install dependencies in the virtualenv`, async () => { - let testDir = await mkdtemp(join(os.tmpdir(), "poetry-test-")); - try { - // This will install dependencies using Poetry, and then try to run `ls` - // in the installed dependency's package. If the package is not - // installed here, the `ls` command will exit non-zero and - // `runCommandInContainer` will throw. - await cp( - "./tests/fixtures/poetry-test/pyproject.toml", - `${testDir}/pyproject.toml`, - ); - let res = await runCommandInContainer( - [ - "sh", - "-c", - "poetry install; ls $(poetry env info --path)/lib/python3.11/site-packages/pip_install_test/__init__.py", - ], - { - workingDir: "/poetry-test", - mounts: [{ host: testDir, container: "/poetry-test" }], - }, - ); - } finally { - try { - await rm(testDir, { recursive: true }); - } catch { - /* ok */ - } - } + // This will install dependencies using Poetry, and then try to run `ls` + // in the installed dependency's package. If the package is not + // installed here, the `ls` command will exit non-zero and + // `runCommandInContainer` will throw. + let res = await runCommandInContainer( + [ + "sh", + "-c", + "poetry install; ls $(poetry env info --path)/lib/python3.11/site-packages/pip_install_test/__init__.py", + ], + { + workingDir: "/project/poetry-test", + hostContainerDirs: [{ host: "./tests/fixtures/poetry-test", container: "/project/poetry-test" }], + }, + ); }); }); diff --git a/tests/index.ts b/tests/index.ts index 4d0360b..cbc268d 100644 --- a/tests/index.ts +++ b/tests/index.ts @@ -1,10 +1,9 @@ import { test, before } from "node:test"; -import { resolve } from "node:path"; import assert from "node:assert"; import Dockerode from "dockerode"; import { Stream } from "node:stream"; import semverSatisfies from "semver/functions/satisfies"; -import { basename } from "node:path"; +import tar from "tar-fs"; export const IMAGE_TAG = "observablehq/framework-runtime:test"; @@ -87,13 +86,23 @@ function ensureDocker() { before(ensureDocker); +function copyFilesToContainer( + dockerContainer: Dockerode.Container, + directories: { host: string; container: string }[], +) { + for (const { host, container } of directories) { + const tarStream = tar.pack(host); + dockerContainer.putArchive(tarStream, { path: container }); + } +} + export async function runCommandInContainer( command: string[], { - mounts = [], - workingDir = "/", + hostContainerDirs = [], + workingDir = "/project", }: { - mounts?: { host: string; container: string}[]; + hostContainerDirs?: { host: string; container: string}[]; workingDir?: string; } = {}, ): Promise<{ stdout: string; stderr: string }> { @@ -102,13 +111,10 @@ export async function runCommandInContainer( WorkingDir: workingDir, Image: IMAGE_TAG, Cmd: command, - HostConfig: { - Binds: mounts.map( - ({ host, container }) => - `${resolve(host)}:${container}`, - ), - }, }); + + copyFilesToContainer(container, hostContainerDirs); + const stdout = new StringStream(); const stderr = new StringStream(); const attach = await container.attach({ diff --git a/yarn.lock b/yarn.lock index 15fab4c..7049026 100644 --- a/yarn.lock +++ b/yarn.lock @@ -215,9 +215,11 @@ __metadata: dependencies: "@types/dockerode": "npm:^3.3.28" "@types/node": "npm:^20" + "@types/tar-fs": "npm:^2.0.4" dockerode: "npm:^4.0.2" glob: "npm:^10.3.12" semver: "npm:^7.6.0" + tar-fs: "npm:^3.0.6" tsx: "npm:^4.7.1" typescript: "npm:^5.4.3" languageName: unknown @@ -287,6 +289,25 @@ __metadata: languageName: node linkType: hard +"@types/tar-fs@npm:^2.0.4": + version: 2.0.4 + resolution: "@types/tar-fs@npm:2.0.4" + dependencies: + "@types/node": "npm:*" + "@types/tar-stream": "npm:*" + checksum: 10c0/d1dd6944d0905debaabe5787af7f3aeb98f13a928d688d00fb7de0411040f8556c297d388abdd046f6b0646a374b53c198ade0484060b63ef36ad5ac585df138 + languageName: node + linkType: hard + +"@types/tar-stream@npm:*": + version: 3.1.3 + resolution: "@types/tar-stream@npm:3.1.3" + dependencies: + "@types/node": "npm:*" + checksum: 10c0/64f87d209bd2edf1a7d029a922a246ef0dcfb19e623b95714e2c074195a61ed4fe4d67d0c3c6dc33239ef7d18902fcb70df7f7e85cfbd92a6bf25d087ce531fd + languageName: node + linkType: hard + "abbrev@npm:^2.0.0": version: 2.0.0 resolution: "abbrev@npm:2.0.0" @@ -352,6 +373,13 @@ __metadata: languageName: node linkType: hard +"b4a@npm:^1.6.4, b4a@npm:^1.6.6": + version: 1.6.6 + resolution: "b4a@npm:1.6.6" + checksum: 10c0/56f30277666cb511a15829e38d369b114df7dc8cec4cedc09cc5d685bc0f27cb63c7bcfb58e09a19a1b3c4f2541069ab078b5328542e85d74a39620327709a38 + languageName: node + linkType: hard + "balanced-match@npm:^1.0.0": version: 1.0.2 resolution: "balanced-match@npm:1.0.2" @@ -359,6 +387,50 @@ __metadata: languageName: node linkType: hard +"bare-events@npm:^2.0.0, bare-events@npm:^2.2.0": + version: 2.4.2 + resolution: "bare-events@npm:2.4.2" + checksum: 10c0/09fa923061f31f815e83504e2ed4a8ba87732a01db40a7fae703dbb7eef7f05d99264b5e186074cbe9698213990d1af564c62cca07a5ff88baea8099ad9a6303 + languageName: node + linkType: hard + +"bare-fs@npm:^2.1.1": + version: 2.3.5 + resolution: "bare-fs@npm:2.3.5" + dependencies: + bare-events: "npm:^2.0.0" + bare-path: "npm:^2.0.0" + bare-stream: "npm:^2.0.0" + checksum: 10c0/ff18cc9be7c557c38e0342681ba3672ae4b01e5696b567d4035e5995255dc6bc7d4df88ed210fa4d3eb940eb29512e924ebb42814c87fc59a2bee8cf83b7c2f9 + languageName: node + linkType: hard + +"bare-os@npm:^2.1.0": + version: 2.4.4 + resolution: "bare-os@npm:2.4.4" + checksum: 10c0/e7d1a7b2100c05da8d25b60d0d48cf850c6f57064577a3f2f51cf18d417fbcfd6967ed2d8314320914ed69e0f2ebcf54eb1b36092dd172d8e8f969cf8cccf041 + languageName: node + linkType: hard + +"bare-path@npm:^2.0.0, bare-path@npm:^2.1.0": + version: 2.1.3 + resolution: "bare-path@npm:2.1.3" + dependencies: + bare-os: "npm:^2.1.0" + checksum: 10c0/35587e177fc8fa5b13fb90bac8779b5ce49c99016d221ddaefe2232d02bd4295d79b941e14ae19fda75ec42a6fe5fb66c07d83ae7ec11462178e66b7be65ca74 + languageName: node + linkType: hard + +"bare-stream@npm:^2.0.0": + version: 2.3.0 + resolution: "bare-stream@npm:2.3.0" + dependencies: + b4a: "npm:^1.6.6" + streamx: "npm:^2.20.0" + checksum: 10c0/374a517542e6a0c3c07f3a1d567db612685e66708f79781112aa0e81c1f117ec561cc1ff3926144f15a2200316a77030c95dcc13a1b96d5303f0748798b764cf + languageName: node + linkType: hard + "base64-js@npm:^1.3.1": version: 1.5.1 resolution: "base64-js@npm:1.5.1" @@ -666,6 +738,13 @@ __metadata: languageName: node linkType: hard +"fast-fifo@npm:^1.2.0, fast-fifo@npm:^1.3.2": + version: 1.3.2 + resolution: "fast-fifo@npm:1.3.2" + checksum: 10c0/d53f6f786875e8b0529f784b59b4b05d4b5c31c651710496440006a398389a579c8dbcd2081311478b5bf77f4b0b21de69109c5a4eabea9d8e8783d1eb864e4c + languageName: node + linkType: hard + "foreground-child@npm:^3.1.0": version: 3.1.1 resolution: "foreground-child@npm:3.1.1" @@ -1133,6 +1212,13 @@ __metadata: languageName: node linkType: hard +"queue-tick@npm:^1.0.1": + version: 1.0.1 + resolution: "queue-tick@npm:1.0.1" + checksum: 10c0/0db998e2c9b15215317dbcf801e9b23e6bcde4044e115155dae34f8e7454b9a783f737c9a725528d677b7a66c775eb7a955cf144fe0b87f62b575ce5bfd515a9 + languageName: node + linkType: hard + "readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.5.0": version: 3.6.2 resolution: "readable-stream@npm:3.6.2" @@ -1274,6 +1360,21 @@ __metadata: languageName: node linkType: hard +"streamx@npm:^2.15.0, streamx@npm:^2.20.0": + version: 2.20.1 + resolution: "streamx@npm:2.20.1" + dependencies: + bare-events: "npm:^2.2.0" + fast-fifo: "npm:^1.3.2" + queue-tick: "npm:^1.0.1" + text-decoder: "npm:^1.1.0" + dependenciesMeta: + bare-events: + optional: true + checksum: 10c0/34ffa2ee9465d70e18c7e2ba70189720c166d150ab83eb7700304620fa23ff42a69cb37d712ea4b5fc6234d8e74346a88bb4baceb873c6b05e52ac420f8abb4d + languageName: node + linkType: hard + "string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^4.1.0": version: 4.2.3 resolution: "string-width@npm:4.2.3" @@ -1323,6 +1424,23 @@ __metadata: languageName: node linkType: hard +"tar-fs@npm:^3.0.6": + version: 3.0.6 + resolution: "tar-fs@npm:3.0.6" + dependencies: + bare-fs: "npm:^2.1.1" + bare-path: "npm:^2.1.0" + pump: "npm:^3.0.0" + tar-stream: "npm:^3.1.5" + dependenciesMeta: + bare-fs: + optional: true + bare-path: + optional: true + checksum: 10c0/207b7c0f193495668bd9dbad09a0108ce4ffcfec5bce2133f90988cdda5c81fad83c99f963d01e47b565196594f7a17dbd063ae55b97b36268fcc843975278ee + languageName: node + linkType: hard + "tar-fs@npm:~2.0.1": version: 2.0.1 resolution: "tar-fs@npm:2.0.1" @@ -1348,6 +1466,17 @@ __metadata: languageName: node linkType: hard +"tar-stream@npm:^3.1.5": + version: 3.1.7 + resolution: "tar-stream@npm:3.1.7" + dependencies: + b4a: "npm:^1.6.4" + fast-fifo: "npm:^1.2.0" + streamx: "npm:^2.15.0" + checksum: 10c0/a09199d21f8714bd729993ac49b6c8efcb808b544b89f23378ad6ffff6d1cb540878614ba9d4cfec11a64ef39e1a6f009a5398371491eb1fda606ffc7f70f718 + languageName: node + linkType: hard + "tar@npm:^6.1.11, tar@npm:^6.1.2": version: 6.2.1 resolution: "tar@npm:6.2.1" @@ -1362,6 +1491,15 @@ __metadata: languageName: node linkType: hard +"text-decoder@npm:^1.1.0": + version: 1.2.0 + resolution: "text-decoder@npm:1.2.0" + dependencies: + b4a: "npm:^1.6.4" + checksum: 10c0/398171bef376e06864cd6ba24e0787cc626bebc84a1bbda758d06a6e9b729cc8613f7923dd0d294abd88e8bb5cd7261aad5fda7911fb87253fe71b2b5ac6e507 + languageName: node + linkType: hard + "tsx@npm:^4.7.1": version: 4.7.1 resolution: "tsx@npm:4.7.1"