From 756ba697ee2846c1e2ea0c76fc8467d0eaeff7db Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 4 Nov 2025 10:58:21 -0500 Subject: [PATCH 1/3] ci: Add C10S containerized ephemeral test workflow Add a complete end-to-end test workflow for running ephemeral integration tests in a CentOS Stream 10 container environment. This tests two things very different from the default main.yml workflow: - Running in a nested container environment - bcvk on RHEL-based systems where qemu is at /usr/libexec/qemu-kvm The implementation provides both GitHub Actions CI and local testing: **CI Workflow (.github/workflows/main-c10s.yml):** - Simplified single-job workflow that builds and runs the test container - Uses the ubuntu-24.04 runner with privileged podman - All 13 ephemeral integration tests run in the container **Local Testing (just test-ephemeral-c10s):** - Builds the test container from tests/fixtures/Containerfile - Runs with KVM device access and container storage volume - Provides quick iteration for C10S-specific testing **Container Structure (tests/fixtures/Containerfile):** - Multi-stage build: compile bcvk and create nextest archive in build stage - Runtime stage: C10S base with qemu/libvirt/podman dependencies - Includes full workspace structure for nextest archive execution - Pulls test images and runs ephemeral tests by default **Supporting Files:** - .dockerignore: Includes .config/ for nextest configuration - Justfile: Adds build-image-c10s and test-ephemeral-c10s targets Successfully tested locally with all 13 ephemeral tests passing. Assisted-by: Claude Code (Sonnet 4.5) Signed-off-by: Colin Walters --- .dockerignore | 19 +++++++++ .github/workflows/main-c10s.yml | 24 +++++++++++ Justfile | 7 ++++ tests/fixtures/Containerfile | 73 +++++++++++++++++++++++++++++++++ 4 files changed, 123 insertions(+) create mode 100644 .dockerignore create mode 100644 .github/workflows/main-c10s.yml create mode 100644 tests/fixtures/Containerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..183c5fc --- /dev/null +++ b/.dockerignore @@ -0,0 +1,19 @@ +# Exclude everything by default, then include just what we need +# Especially note this means that .git is not included, and not tests/ +# to avoid spurious rebuilds. +* + +# Toplevel build bits +!Makefile +!Cargo.* +# Docs +!docs +# We use the spec file +!packaging/ +# Workaround for podman bug with secrets + remote +# https://github.com/containers/podman/issues/25314 +!podman-build-secret* +# Nextest configuration +!.config/ +# And finally of course all the Rust sources +!crates/ diff --git a/.github/workflows/main-c10s.yml b/.github/workflows/main-c10s.yml new file mode 100644 index 0000000..b0ed9f0 --- /dev/null +++ b/.github/workflows/main-c10s.yml @@ -0,0 +1,24 @@ +name: CI (c10s) + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + workflow_dispatch: + +jobs: + test-ephemeral: + runs-on: ubuntu-24.04 + + steps: + - uses: actions/checkout@v4 + + - name: Build C10S test container + run: podman build -f tests/fixtures/Containerfile -t localhost/bcvk:c10s . + + - name: Run ephemeral integration tests + run: | + podman run --rm --privileged --device=/dev/kvm \ + -v bcvk-test-storage:/var/lib/containers \ + localhost/bcvk:c10s diff --git a/Justfile b/Justfile index 739d863..d35ec0f 100644 --- a/Justfile +++ b/Justfile @@ -96,3 +96,10 @@ archive: build install: build cp target/release/bck ~/.local/bin/ +build-image-c10s: + podman build -f tests/fixtures/Containerfile -t localhost/bcvk:c10s . + +test-ephemeral-c10s: build-image-c10s + # TODO try downgrading to --cap-add=all --security-opt=label=type:container_runtime_t, I think + # we'll need to assume `--net=host` mainly in bcvk in this situation. + podman run --rm --privileged --device=/dev/kvm -v bcvk-test-storage:/var/lib/containers localhost/bcvk:c10s diff --git a/tests/fixtures/Containerfile b/tests/fixtures/Containerfile new file mode 100644 index 0000000..ae19213 --- /dev/null +++ b/tests/fixtures/Containerfile @@ -0,0 +1,73 @@ +# Container for running bcvk ephemeral tests on C10S +# This mirrors the CI workflow in .github/workflows/main-c10s.yml + +ARG base=quay.io/centos/centos:stream10 +FROM $base as build + +# Install build dependencies +RUN dnf -y install dnf-utils && \ + dnf config-manager --set-enabled crb && \ + dnf install -y pkgconfig go-md2man gcc make openssl-devel openssh-clients && \ + dnf clean all + +# Install Rust +RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable +ENV PATH="/root/.cargo/bin:${PATH}" + +# Install nextest +RUN cargo install cargo-nextest --locked + +# Copy source code +COPY . /src +WORKDIR /src + +# Build bcvk and create integration test archive +RUN make && \ + cargo nextest archive --release -P integration -p integration-tests --archive-file integration-tests.tar.zst + +# Runtime stage +FROM $base + +# Install runtime dependencies for running VMs +RUN dnf -y install dnf-utils && \ + dnf config-manager --set-enabled crb && \ + dnf install -y \ + libvirt-daemon \ + libvirt-daemon-driver-qemu \ + libvirt-client \ + qemu-kvm \ + virtiofsd \ + podman && \ + dnf clean all + +# Copy cargo-nextest from build stage +COPY --from=build /root/.cargo/bin/cargo-nextest /usr/local/bin/cargo-nextest + +# Copy built artifacts +COPY --from=build /src/target/release/bcvk /usr/local/bin/bcvk +COPY --from=build /src/integration-tests.tar.zst /tests/integration-tests.tar.zst + +# Copy source tree metadata needed by nextest +# Nextest needs the workspace structure even when using archives +COPY --from=build /src/Cargo.toml /src/Cargo.lock /tests/ +COPY --from=build /src/.config /tests/.config +COPY --from=build /src/crates /tests/crates + +# Set up environment +ENV BCVK_PATH=/usr/local/bin/bcvk +ENV LIBVIRT_DEFAULT_URI=qemu:///system +WORKDIR /tests + +# Create entrypoint script that pulls images and runs tests +RUN < /usr/local/bin/run-tests.sh +#!/bin/bash +set -euo pipefail +echo "Pulling test images..." +podman pull -q quay.io/fedora/fedora-bootc:42 quay.io/centos-bootc/centos-bootc:stream9 quay.io/centos-bootc/centos-bootc:stream10 +echo "Running ephemeral integration tests..." +exec /usr/local/bin/cargo-nextest run --archive-file integration-tests.tar.zst --workspace-remap /tests ephemeral +EOF +RUN chmod +x /usr/local/bin/run-tests.sh + +# Default command runs the test script +CMD ["/usr/local/bin/run-tests.sh"] From 15de0b12c7d3a1b2736c50ae2be9c750d68f9dc6 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 4 Nov 2025 11:03:52 -0500 Subject: [PATCH 2/3] github: Import and use bootc-ubuntu-setup action Copy the bootc-ubuntu-setup action from bootc-dev/bootc and integrate it into both the main CI and C10S workflows. This action provides: - Disk space cleanup on GHA runners - Updated podman/just from Ubuntu plucky - Unprivileged /dev/kvm access via udev rules - Optional libvirt stack installation - Rust cache configuration The main workflow now uses this action with libvirt: 'true' to install the full virtualization stack, then removes the pre-installed bcvk binary to ensure we test the locally built version. The C10S workflow uses it for basic setup (podman, KVM access). This consolidates setup logic and ensures consistent environment configuration across workflows. Assisted-by: Claude Code (Sonnet 4.5) Signed-off-by: Colin Walters --- .github/actions/bootc-ubuntu-setup/action.yml | 101 ++++++++++++++++++ .github/workflows/main-c10s.yml | 3 + .github/workflows/main.yml | 30 ++---- 3 files changed, 112 insertions(+), 22 deletions(-) create mode 100644 .github/actions/bootc-ubuntu-setup/action.yml diff --git a/.github/actions/bootc-ubuntu-setup/action.yml b/.github/actions/bootc-ubuntu-setup/action.yml new file mode 100644 index 0000000..eb3f209 --- /dev/null +++ b/.github/actions/bootc-ubuntu-setup/action.yml @@ -0,0 +1,101 @@ +name: 'Bootc Ubuntu Setup' +description: 'Default host setup' +inputs: + libvirt: + description: 'Install libvirt and virtualization stack' + required: false + default: 'false' +runs: + using: 'composite' + steps: + # The default runners have TONS of crud on them... + - name: Free up disk space on runner + shell: bash + run: | + set -xeuo pipefail + sudo df -h + unwanted_pkgs=('^aspnetcore-.*' '^dotnet-.*' '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*' + azure-cli google-chrome-stable firefox mono-devel) + unwanted_dirs=(/usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL) + # Start background removal operations as systemd units; if this causes + # races in the future around disk space we can look at waiting for cleanup + # before starting further jobs, but right now we spent a lot of time waiting + # on the network and scripts and such below, giving these plenty of time to run. + n=0 + runcleanup() { + sudo systemd-run -r -u action-cleanup-${n} -- "$@" + n=$(($n + 1)) + } + runcleanup docker image prune --all --force + for x in ${unwanted_dirs[@]}; do + runcleanup rm -rf "$x" + done + # Apt removals in foreground, as we can't parallelize these + for x in ${unwanted_pkgs[@]}; do + /bin/time -f '%E %C' sudo apt-get remove -y $x + done + # We really want support for heredocs + - name: Update podman and install just + shell: bash + run: | + set -eux + # Require the runner is ubuntu-24.04 + IDV=$(. /usr/lib/os-release && echo ${ID}-${VERSION_ID}) + test "${IDV}" = "ubuntu-24.04" + # plucky is the next release + echo 'deb http://azure.archive.ubuntu.com/ubuntu plucky universe main' | sudo tee /etc/apt/sources.list.d/plucky.list + /bin/time -f '%E %C' sudo apt update + # skopeo is currently older in plucky for some reason hence --allow-downgrades + /bin/time -f '%E %C' sudo apt install -y --allow-downgrades crun/plucky podman/plucky skopeo/plucky just + # This is the default on e.g. Fedora derivatives, but not Debian + - name: Enable unprivileged /dev/kvm access + shell: bash + run: | + set -xeuo pipefail + echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules + sudo udevadm control --reload-rules + sudo udevadm trigger --name-match=kvm + ls -l /dev/kvm + # Used by a few workflows, but generally useful + - name: Set architecture variable + id: set_arch + shell: bash + run: echo "ARCH=$(arch)" >> $GITHUB_ENV + # We often use Rust, so set up opinionated default caching + - name: Setup Rust cache + uses: Swatinem/rust-cache@v2 + with: + cache-all-crates: true + # Only generate caches on push to git main + save-if: ${{ github.ref == 'refs/heads/main' }} + # Suppress actually using the cache for builds running from + # git main so that we avoid incremental compilation bugs + lookup-only: ${{ github.ref == 'refs/heads/main' }} + # Install libvirt stack if requested + - name: Install libvirt and virtualization stack + if: ${{ inputs.libvirt == 'true' }} + shell: bash + run: | + set -xeuo pipefail + export BCVK_VERSION=0.5.3 + /bin/time -f '%E %C' sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-utils qemu-kvm virtiofsd libvirt-daemon-system + # Something in the stack is overriding this, but we want session right now for bcvk + echo LIBVIRT_DEFAULT_URI=qemu:///session >> $GITHUB_ENV + td=$(mktemp -d) + cd $td + # Install bcvk + target=bcvk-$(arch)-unknown-linux-gnu + /bin/time -f '%E %C' curl -LO https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz + tar xzf ${target}.tar.gz + sudo install -T ${target} /usr/bin/bcvk + cd - + rm -rf "$td" + + # Also bump the default fd limit as a workaround for https://github.com/bootc-dev/bcvk/issues/65 + sudo sed -i -e 's,^\* hard nofile 65536,* hard nofile 524288,' /etc/security/limits.conf + - name: Cleanup status + shell: bash + run: | + set -xeuo pipefail + systemctl list-units 'action-cleanup*' + df -h diff --git a/.github/workflows/main-c10s.yml b/.github/workflows/main-c10s.yml index b0ed9f0..e171916 100644 --- a/.github/workflows/main-c10s.yml +++ b/.github/workflows/main-c10s.yml @@ -14,6 +14,9 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Setup Ubuntu environment with KVM + uses: ./.github/actions/bootc-ubuntu-setup + - name: Build C10S test container run: podman build -f tests/fixtures/Containerfile -t localhost/bcvk:c10s . diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 47b727e..9c7a76a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,34 +18,20 @@ jobs: runs-on: ubuntu-24.04 steps: - - name: Install dependencies - run: | - sudo apt update - sudo apt install -y just pkg-config go-md2man libvirt-daemon libvirt-clients qemu-kvm qemu-system qemu-utils virtiofsd - - - name: Install podman for heredoc support - run: | - set -eux - echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list - sudo apt update - sudo apt install -y crun/testing podman/testing just + - uses: actions/checkout@v4 - - name: Enable KVM group perms - run: | - echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules - sudo udevadm control --reload-rules - sudo udevadm trigger --name-match=kvm - ls -l /dev/kvm + - name: Setup Ubuntu environment + uses: ./.github/actions/bootc-ubuntu-setup + with: + libvirt: 'true' - - uses: actions/checkout@v4 + - name: Remove pre-installed bcvk + run: sudo rm -f /usr/bin/bcvk - name: Setup Rust uses: dtolnay/rust-toolchain@stable - - - uses: taiki-e/install-action@nextest - - name: Cache build artifacts - uses: Swatinem/rust-cache@v2 + - uses: taiki-e/install-action@nextest - name: Build run: just validate && just build From d0914ca75d6f61a05f4e2ec9b0c9fa395dbc6bde Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 4 Nov 2025 12:04:22 -0500 Subject: [PATCH 3/3] ci: Install go-md2man in main workflow The bootc-ubuntu-setup action doesn't include go-md2man, which is needed by the Makefile to generate man pages during the build step. Add explicit installation step for go-md2man after the environment setup. Assisted-by: Claude Code (Sonnet 4.5) Signed-off-by: Colin Walters --- .github/workflows/main.yml | 3 +++ tests/fixtures/Containerfile | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9c7a76a..0d76b61 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,6 +28,9 @@ jobs: - name: Remove pre-installed bcvk run: sudo rm -f /usr/bin/bcvk + - name: Install go-md2man for man page generation + run: sudo apt install -y go-md2man + - name: Setup Rust uses: dtolnay/rust-toolchain@stable diff --git a/tests/fixtures/Containerfile b/tests/fixtures/Containerfile index ae19213..4a49b05 100644 --- a/tests/fixtures/Containerfile +++ b/tests/fixtures/Containerfile @@ -65,7 +65,7 @@ set -euo pipefail echo "Pulling test images..." podman pull -q quay.io/fedora/fedora-bootc:42 quay.io/centos-bootc/centos-bootc:stream9 quay.io/centos-bootc/centos-bootc:stream10 echo "Running ephemeral integration tests..." -exec /usr/local/bin/cargo-nextest run --archive-file integration-tests.tar.zst --workspace-remap /tests ephemeral +exec /usr/local/bin/cargo-nextest nextest run --archive-file integration-tests.tar.zst --workspace-remap /tests ephemeral EOF RUN chmod +x /usr/local/bin/run-tests.sh