Skip to content

Commit

Permalink
Add observable-builder user and use /project (#10)
Browse files Browse the repository at this point in the history 
* Add observable-builder user and use /project

Adds an observable-builder user with a home directory of /project. This
uses 8000 as the uid since that's not likely to collide with any users
on the container runtime host machines.

* Add useradd to base and fix tests to use workdir

* Add GID so the container user can read the files

* Add `z` to fix GHA SELinux issues

* Copy the files into the container

* Remove host tempdir and file copy

* Code review followups

* Use a traditional /home directory
* Combine poetry into python layer
* Rename mounts
* Removed unused imports
* Put tar-fs into devDependencies
  • Loading branch information
@jessedearing
jessedearing authored Sep 19, 2024
1 parent a977bbb commit 328292f
Show file tree
Hide file tree
Showing 5 changed files with 182 additions and 45 deletions.
16 changes: 10 additions & 6 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,14 @@
# == base ======================
FROM buildpack-deps:bookworm AS base
ENV CACHEBUST=2024-09-06
RUN apt update
ENV CACHEBUST=2024-09-17
RUN useradd -m -u 8000 observable-builder && mkdir /project && \
chown 8000:8000 /project && apt update

ENV RUSTUP_HOME=/usr/local/rustup \
CARGO_HOME=/usr/local/cargo \
RUST_VERSION=1.81.0 \
VIRTUAL_ENV=/var/local/python-venv
ENV PATH=/usr/local/cargo/bin:$VIRTUAL_ENV/bin:/root/.local/bin:$PATH
VIRTUAL_ENV=/home/observable-builder/.local/python-venv
ENV PATH=/usr/local/cargo/bin:$VIRTUAL_ENV/bin:/home/observable-builder/.local/bin:$PATH

# == node ======================
FROM base AS node
Expand All @@ -32,8 +33,9 @@ RUN --mount=type=cache,target=/var/cache/apt,id=framework-runtime-python \
python3-wheel \
python3-dev \
python3-venv \
pipx \
&& pipx install poetry \
pipx
USER 8000
RUN pipx install poetry \
&& python3 -m venv $VIRTUAL_ENV

# == R ===========================
Expand Down Expand Up @@ -125,3 +127,5 @@ COPY --from=python . .
COPY --from=r . .
COPY --from=duckdb . .
COPY --from=rust . .
USER 8000:8000
WORKDIR /project
2 changes: 2 additions & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,11 @@
"devDependencies": {
"@types/dockerode": "^3.3.28",
"@types/node": "^20",
"@types/tar-fs": "^2.0.4",
"dockerode": "^4.0.2",
"glob": "^10.3.12",
"semver": "^7.6.0",
"tar-fs": "^3.0.6",
"tsx": "^4.7.1",
"typescript": "^5.4.3"
}
Expand Down
43 changes: 15 additions & 28 deletions tests/dataloader-languages.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,34 +71,21 @@ describe("Dataloader languages", () => {
});

test(`Poetry can install dependencies in the virtualenv`, async () => {
let testDir = await mkdtemp(join(os.tmpdir(), "poetry-test-"));
try {
// This will install dependencies using Poetry, and then try to run `ls`
// in the installed dependency's package. If the package is not
// installed here, the `ls` command will exit non-zero and
// `runCommandInContainer` will throw.
await cp(
"./tests/fixtures/poetry-test/pyproject.toml",
`${testDir}/pyproject.toml`,
);
let res = await runCommandInContainer(
[
"sh",
"-c",
"poetry install; ls $(poetry env info --path)/lib/python3.11/site-packages/pip_install_test/__init__.py",
],
{
workingDir: "/poetry-test",
mounts: [{ host: testDir, container: "/poetry-test" }],
},
);
} finally {
try {
await rm(testDir, { recursive: true });
} catch {
/* ok */
}
}
// This will install dependencies using Poetry, and then try to run `ls`
// in the installed dependency's package. If the package is not
// installed here, the `ls` command will exit non-zero and
// `runCommandInContainer` will throw.
let res = await runCommandInContainer(
[
"sh",
"-c",
"poetry install; ls $(poetry env info --path)/lib/python3.11/site-packages/pip_install_test/__init__.py",
],
{
workingDir: "/project/poetry-test",
hostContainerDirs: [{ host: "./tests/fixtures/poetry-test", container: "/project/poetry-test" }],
},
);
});
});

Expand Down
28 changes: 17 additions & 11 deletions tests/index.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
import { test, before } from "node:test";
import { resolve } from "node:path";
import assert from "node:assert";
import Dockerode from "dockerode";
import { Stream } from "node:stream";
import semverSatisfies from "semver/functions/satisfies";
import { basename } from "node:path";
import tar from "tar-fs";

export const IMAGE_TAG = "observablehq/framework-runtime:test";

Expand Down Expand Up @@ -87,13 +86,23 @@ function ensureDocker() {

before(ensureDocker);

function copyFilesToContainer(
dockerContainer: Dockerode.Container,
directories: { host: string; container: string }[],
) {
for (const { host, container } of directories) {
const tarStream = tar.pack(host);
dockerContainer.putArchive(tarStream, { path: container });
}
}

export async function runCommandInContainer(
command: string[],
{
mounts = [],
workingDir = "/",
hostContainerDirs = [],
workingDir = "/project",
}: {
mounts?: { host: string; container: string}[];
hostContainerDirs?: { host: string; container: string}[];
workingDir?: string;
} = {},
): Promise<{ stdout: string; stderr: string }> {
Expand All @@ -102,13 +111,10 @@ export async function runCommandInContainer(
WorkingDir: workingDir,
Image: IMAGE_TAG,
Cmd: command,
HostConfig: {
Binds: mounts.map(
({ host, container }) =>
`${resolve(host)}:${container}`,
),
},
});

copyFilesToContainer(container, hostContainerDirs);

const stdout = new StringStream();
const stderr = new StringStream();
const attach = await container.attach({
Expand Down
138 changes: 138 additions & 0 deletions yarn.lock
View file
Original file line number Diff line number Diff line change
Expand Up @@ -215,9 +215,11 @@ __metadata:
dependencies:
"@types/dockerode": "npm:^3.3.28"
"@types/node": "npm:^20"
"@types/tar-fs": "npm:^2.0.4"
dockerode: "npm:^4.0.2"
glob: "npm:^10.3.12"
semver: "npm:^7.6.0"
tar-fs: "npm:^3.0.6"
tsx: "npm:^4.7.1"
typescript: "npm:^5.4.3"
languageName: unknown
Expand Down Expand Up @@ -287,6 +289,25 @@ __metadata:
languageName: node
linkType: hard

"@types/tar-fs@npm:^2.0.4":
version: 2.0.4
resolution: "@types/tar-fs@npm:2.0.4"
dependencies:
"@types/node": "npm:*"
"@types/tar-stream": "npm:*"
checksum: 10c0/d1dd6944d0905debaabe5787af7f3aeb98f13a928d688d00fb7de0411040f8556c297d388abdd046f6b0646a374b53c198ade0484060b63ef36ad5ac585df138
languageName: node
linkType: hard

"@types/tar-stream@npm:*":
version: 3.1.3
resolution: "@types/tar-stream@npm:3.1.3"
dependencies:
"@types/node": "npm:*"
checksum: 10c0/64f87d209bd2edf1a7d029a922a246ef0dcfb19e623b95714e2c074195a61ed4fe4d67d0c3c6dc33239ef7d18902fcb70df7f7e85cfbd92a6bf25d087ce531fd
languageName: node
linkType: hard

"abbrev@npm:^2.0.0":
version: 2.0.0
resolution: "abbrev@npm:2.0.0"
Expand Down Expand Up @@ -352,13 +373,64 @@ __metadata:
languageName: node
linkType: hard

"b4a@npm:^1.6.4, b4a@npm:^1.6.6":
version: 1.6.6
resolution: "b4a@npm:1.6.6"
checksum: 10c0/56f30277666cb511a15829e38d369b114df7dc8cec4cedc09cc5d685bc0f27cb63c7bcfb58e09a19a1b3c4f2541069ab078b5328542e85d74a39620327709a38
languageName: node
linkType: hard

"balanced-match@npm:^1.0.0":
version: 1.0.2
resolution: "balanced-match@npm:1.0.2"
checksum: 10c0/9308baf0a7e4838a82bbfd11e01b1cb0f0cf2893bc1676c27c2a8c0e70cbae1c59120c3268517a8ae7fb6376b4639ef81ca22582611dbee4ed28df945134aaee
languageName: node
linkType: hard

"bare-events@npm:^2.0.0, bare-events@npm:^2.2.0":
version: 2.4.2
resolution: "bare-events@npm:2.4.2"
checksum: 10c0/09fa923061f31f815e83504e2ed4a8ba87732a01db40a7fae703dbb7eef7f05d99264b5e186074cbe9698213990d1af564c62cca07a5ff88baea8099ad9a6303
languageName: node
linkType: hard

"bare-fs@npm:^2.1.1":
version: 2.3.5
resolution: "bare-fs@npm:2.3.5"
dependencies:
bare-events: "npm:^2.0.0"
bare-path: "npm:^2.0.0"
bare-stream: "npm:^2.0.0"
checksum: 10c0/ff18cc9be7c557c38e0342681ba3672ae4b01e5696b567d4035e5995255dc6bc7d4df88ed210fa4d3eb940eb29512e924ebb42814c87fc59a2bee8cf83b7c2f9
languageName: node
linkType: hard

"bare-os@npm:^2.1.0":
version: 2.4.4
resolution: "bare-os@npm:2.4.4"
checksum: 10c0/e7d1a7b2100c05da8d25b60d0d48cf850c6f57064577a3f2f51cf18d417fbcfd6967ed2d8314320914ed69e0f2ebcf54eb1b36092dd172d8e8f969cf8cccf041
languageName: node
linkType: hard

"bare-path@npm:^2.0.0, bare-path@npm:^2.1.0":
version: 2.1.3
resolution: "bare-path@npm:2.1.3"
dependencies:
bare-os: "npm:^2.1.0"
checksum: 10c0/35587e177fc8fa5b13fb90bac8779b5ce49c99016d221ddaefe2232d02bd4295d79b941e14ae19fda75ec42a6fe5fb66c07d83ae7ec11462178e66b7be65ca74
languageName: node
linkType: hard

"bare-stream@npm:^2.0.0":
version: 2.3.0
resolution: "bare-stream@npm:2.3.0"
dependencies:
b4a: "npm:^1.6.6"
streamx: "npm:^2.20.0"
checksum: 10c0/374a517542e6a0c3c07f3a1d567db612685e66708f79781112aa0e81c1f117ec561cc1ff3926144f15a2200316a77030c95dcc13a1b96d5303f0748798b764cf
languageName: node
linkType: hard

"base64-js@npm:^1.3.1":
version: 1.5.1
resolution: "base64-js@npm:1.5.1"
Expand Down Expand Up @@ -666,6 +738,13 @@ __metadata:
languageName: node
linkType: hard

"fast-fifo@npm:^1.2.0, fast-fifo@npm:^1.3.2":
version: 1.3.2
resolution: "fast-fifo@npm:1.3.2"
checksum: 10c0/d53f6f786875e8b0529f784b59b4b05d4b5c31c651710496440006a398389a579c8dbcd2081311478b5bf77f4b0b21de69109c5a4eabea9d8e8783d1eb864e4c
languageName: node
linkType: hard

"foreground-child@npm:^3.1.0":
version: 3.1.1
resolution: "foreground-child@npm:3.1.1"
Expand Down Expand Up @@ -1133,6 +1212,13 @@ __metadata:
languageName: node
linkType: hard

"queue-tick@npm:^1.0.1":
version: 1.0.1
resolution: "queue-tick@npm:1.0.1"
checksum: 10c0/0db998e2c9b15215317dbcf801e9b23e6bcde4044e115155dae34f8e7454b9a783f737c9a725528d677b7a66c775eb7a955cf144fe0b87f62b575ce5bfd515a9
languageName: node
linkType: hard

"readable-stream@npm:^3.1.1, readable-stream@npm:^3.4.0, readable-stream@npm:^3.5.0":
version: 3.6.2
resolution: "readable-stream@npm:3.6.2"
Expand Down Expand Up @@ -1274,6 +1360,21 @@ __metadata:
languageName: node
linkType: hard

"streamx@npm:^2.15.0, streamx@npm:^2.20.0":
version: 2.20.1
resolution: "streamx@npm:2.20.1"
dependencies:
bare-events: "npm:^2.2.0"
fast-fifo: "npm:^1.3.2"
queue-tick: "npm:^1.0.1"
text-decoder: "npm:^1.1.0"
dependenciesMeta:
bare-events:
optional: true
checksum: 10c0/34ffa2ee9465d70e18c7e2ba70189720c166d150ab83eb7700304620fa23ff42a69cb37d712ea4b5fc6234d8e74346a88bb4baceb873c6b05e52ac420f8abb4d
languageName: node
linkType: hard

"string-width-cjs@npm:string-width@^4.2.0, string-width@npm:^4.1.0":
version: 4.2.3
resolution: "string-width@npm:4.2.3"
Expand Down Expand Up @@ -1323,6 +1424,23 @@ __metadata:
languageName: node
linkType: hard

"tar-fs@npm:^3.0.6":
version: 3.0.6
resolution: "tar-fs@npm:3.0.6"
dependencies:
bare-fs: "npm:^2.1.1"
bare-path: "npm:^2.1.0"
pump: "npm:^3.0.0"
tar-stream: "npm:^3.1.5"
dependenciesMeta:
bare-fs:
optional: true
bare-path:
optional: true
checksum: 10c0/207b7c0f193495668bd9dbad09a0108ce4ffcfec5bce2133f90988cdda5c81fad83c99f963d01e47b565196594f7a17dbd063ae55b97b36268fcc843975278ee
languageName: node
linkType: hard

"tar-fs@npm:~2.0.1":
version: 2.0.1
resolution: "tar-fs@npm:2.0.1"
Expand All @@ -1348,6 +1466,17 @@ __metadata:
languageName: node
linkType: hard

"tar-stream@npm:^3.1.5":
version: 3.1.7
resolution: "tar-stream@npm:3.1.7"
dependencies:
b4a: "npm:^1.6.4"
fast-fifo: "npm:^1.2.0"
streamx: "npm:^2.15.0"
checksum: 10c0/a09199d21f8714bd729993ac49b6c8efcb808b544b89f23378ad6ffff6d1cb540878614ba9d4cfec11a64ef39e1a6f009a5398371491eb1fda606ffc7f70f718
languageName: node
linkType: hard

"tar@npm:^6.1.11, tar@npm:^6.1.2":
version: 6.2.1
resolution: "tar@npm:6.2.1"
Expand All @@ -1362,6 +1491,15 @@ __metadata:
languageName: node
linkType: hard

"text-decoder@npm:^1.1.0":
version: 1.2.0
resolution: "text-decoder@npm:1.2.0"
dependencies:
b4a: "npm:^1.6.4"
checksum: 10c0/398171bef376e06864cd6ba24e0787cc626bebc84a1bbda758d06a6e9b729cc8613f7923dd0d294abd88e8bb5cd7261aad5fda7911fb87253fe71b2b5ac6e507
languageName: node
linkType: hard

"tsx@npm:^4.7.1":
version: 4.7.1
resolution: "tsx@npm:4.7.1"
Expand Down

0 comments on commit 328292f

Please sign in to comment.