ci: Add C10S containerized ephemeral test workflow

.dockerignore

@@ -0,0 +1,19 @@
+# Exclude everything by default, then include just what we need
+# Especially note this means that .git is not included, and not tests/
+# to avoid spurious rebuilds.
+*
+
+# Toplevel build bits
+!Makefile
+!Cargo.*
+# Docs
+!docs
+# We use the spec file
+!packaging/
+# Workaround for podman bug with secrets + remote
+# https://github.com/containers/podman/issues/25314
+!podman-build-secret*
+# Nextest configuration
+!.config/
+# And finally of course all the Rust sources
+!crates/

.github/actions/bootc-ubuntu-setup/action.yml

@@ -0,0 +1,101 @@
+name: 'Bootc Ubuntu Setup'
+description: 'Default host setup'
+inputs:
+  libvirt:
+    description: 'Install libvirt and virtualization stack'
+    required: false
+    default: 'false'
+runs:
+  using: 'composite'
+  steps:
+    # The default runners have TONS of crud on them...
+    - name: Free up disk space on runner
+      shell: bash
+      run: |
+        set -xeuo pipefail
+        sudo df -h
+        unwanted_pkgs=('^aspnetcore-.*' '^dotnet-.*' '^llvm-.*' 'php.*' '^mongodb-.*' '^mysql-.*'
+                       azure-cli google-chrome-stable firefox mono-devel)
+        unwanted_dirs=(/usr/share/dotnet /opt/ghc /usr/local/lib/android /opt/hostedtoolcache/CodeQL)
+        # Start background removal operations as systemd units; if this causes
+        # races in the future around disk space we can look at waiting for cleanup
+        # before starting further jobs, but right now we spent a lot of time waiting
+        # on the network and scripts and such below, giving these plenty of time to run.
+        n=0
+        runcleanup() {
+          sudo systemd-run -r -u action-cleanup-${n} -- "$@"
+          n=$(($n + 1))
+        }
+        runcleanup docker image prune --all --force
+        for x in ${unwanted_dirs[@]}; do
+          runcleanup rm -rf "$x"
+        done
+        # Apt removals in foreground, as we can't parallelize these
+        for x in ${unwanted_pkgs[@]}; do
+          /bin/time -f '%E %C' sudo apt-get remove -y $x
+        done
+    # We really want support for heredocs
+    - name: Update podman and install just
+      shell: bash
+      run: |
+        set -eux
+        # Require the runner is ubuntu-24.04
+        IDV=$(. /usr/lib/os-release && echo ${ID}-${VERSION_ID})
+        test "${IDV}" = "ubuntu-24.04"
+        # plucky is the next release
+        echo 'deb http://azure.archive.ubuntu.com/ubuntu plucky universe main' | sudo tee /etc/apt/sources.list.d/plucky.list
+        /bin/time -f '%E %C' sudo apt update
+        # skopeo is currently older in plucky for some reason hence --allow-downgrades
+        /bin/time -f '%E %C' sudo apt install -y --allow-downgrades crun/plucky podman/plucky skopeo/plucky just
+    # This is the default on e.g. Fedora derivatives, but not Debian
+    - name: Enable unprivileged /dev/kvm access
+      shell: bash
+      run: |
+        set -xeuo pipefail
+        echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
+        sudo udevadm control --reload-rules
+        sudo udevadm trigger --name-match=kvm
+        ls -l /dev/kvm
+    # Used by a few workflows, but generally useful
+    - name: Set architecture variable
+      id: set_arch
+      shell: bash
+      run: echo "ARCH=$(arch)" >> $GITHUB_ENV
+    # We often use Rust, so set up opinionated default caching
+    - name: Setup Rust cache
+      uses: Swatinem/rust-cache@v2
+      with:
+        cache-all-crates: true
+        # Only generate caches on push to git main
+        save-if: ${{ github.ref == 'refs/heads/main' }}
+        # Suppress actually using the cache for builds running from
+        # git main so that we avoid incremental compilation bugs
+        lookup-only: ${{ github.ref == 'refs/heads/main' }}
+    # Install libvirt stack if requested
+    - name: Install libvirt and virtualization stack
+      if: ${{ inputs.libvirt == 'true' }}
+      shell: bash
+      run: |
+        set -xeuo pipefail
+        export BCVK_VERSION=0.5.3
+        /bin/time -f '%E %C' sudo apt install -y libkrb5-dev pkg-config libvirt-dev genisoimage qemu-utils qemu-kvm virtiofsd libvirt-daemon-system
+        # Something in the stack is overriding this, but we want session right now for bcvk
+        echo LIBVIRT_DEFAULT_URI=qemu:///session >> $GITHUB_ENV
+        td=$(mktemp -d)
+        cd $td
+        # Install bcvk
+        target=bcvk-$(arch)-unknown-linux-gnu
+        /bin/time -f '%E %C' curl -LO https://github.com/bootc-dev/bcvk/releases/download/v${BCVK_VERSION}/${target}.tar.gz
+        tar xzf ${target}.tar.gz
+        sudo install -T ${target} /usr/bin/bcvk
+        cd -
+        rm -rf "$td"
+
+        # Also bump the default fd limit as a workaround for https://github.com/bootc-dev/bcvk/issues/65
+        sudo sed -i -e 's,^\* hard nofile 65536,* hard nofile 524288,' /etc/security/limits.conf
+    - name: Cleanup status
+      shell: bash
+      run: |
+        set -xeuo pipefail
+        systemctl list-units 'action-cleanup*'
+        df -h

.github/workflows/main-c10s.yml

@@ -0,0 +1,27 @@
+name: CI (c10s)
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+jobs:
+  test-ephemeral:
+    runs-on: ubuntu-24.04
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Ubuntu environment with KVM
+        uses: ./.github/actions/bootc-ubuntu-setup
+
+      - name: Build C10S test container
+        run: podman build -f tests/fixtures/Containerfile -t localhost/bcvk:c10s .
+
+      - name: Run ephemeral integration tests
+        run: |
+          podman run --rm --privileged --device=/dev/kvm \
+            -v bcvk-test-storage:/var/lib/containers \
+            localhost/bcvk:c10s

.github/workflows/main.yml

@@ -18,34 +18,23 @@ jobs:
     runs-on: ubuntu-24.04
 
     steps:
-      - name: Install dependencies
-        run: |
-          sudo apt update
-          sudo apt install -y just pkg-config go-md2man libvirt-daemon libvirt-clients qemu-kvm qemu-system qemu-utils virtiofsd
+      - uses: actions/checkout@v4
 
-      - name: Install podman for heredoc support
-        run: |
-          set -eux
-          echo 'deb [trusted=yes] https://ftp.debian.org/debian/ testing main' | sudo tee /etc/apt/sources.list.d/testing.list
-          sudo apt update
-          sudo apt install -y crun/testing podman/testing just
+      - name: Setup Ubuntu environment
+        uses: ./.github/actions/bootc-ubuntu-setup
+        with:
+          libvirt: 'true'
 
-      - name: Enable KVM group perms
-        run: |
-          echo 'KERNEL=="kvm", GROUP="kvm", MODE="0666", OPTIONS+="static_node=kvm"' | sudo tee /etc/udev/rules.d/99-kvm4all.rules
-          sudo udevadm control --reload-rules
-          sudo udevadm trigger --name-match=kvm
-          ls -l /dev/kvm
+      - name: Remove pre-installed bcvk
+        run: sudo rm -f /usr/bin/bcvk
 
-      - uses: actions/checkout@v4
+      - name: Install go-md2man for man page generation
+        run: sudo apt install -y go-md2man
 
       - name: Setup Rust
         uses: dtolnay/rust-toolchain@stable
-
-      - uses: taiki-e/install-action@nextest
 
-      - name: Cache build artifacts
-        uses: Swatinem/rust-cache@v2
+      - uses: taiki-e/install-action@nextest
 
       - name: Build
         run: just validate && just build

Justfile

@@ -96,3 +96,10 @@ archive: build
 install: build
     cp target/release/bck ~/.local/bin/
 
+build-image-c10s:
+    podman build -f tests/fixtures/Containerfile -t localhost/bcvk:c10s .
+
+test-ephemeral-c10s: build-image-c10s
+    # TODO try downgrading to --cap-add=all --security-opt=label=type:container_runtime_t, I think
+    # we'll need to assume `--net=host` mainly in bcvk in this situation.
+    podman run --rm --privileged --device=/dev/kvm -v bcvk-test-storage:/var/lib/containers localhost/bcvk:c10s

tests/fixtures/Containerfile

@@ -0,0 +1,73 @@
+# Container for running bcvk ephemeral tests on C10S
+# This mirrors the CI workflow in .github/workflows/main-c10s.yml
+
+ARG base=quay.io/centos/centos:stream10
+FROM $base as build
+
+# Install build dependencies
+RUN dnf -y install dnf-utils && \
+    dnf config-manager --set-enabled crb && \
+    dnf install -y pkgconfig go-md2man gcc make openssl-devel openssh-clients && \
+    dnf clean all
+
+# Install Rust
+RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain stable
+ENV PATH="/root/.cargo/bin:${PATH}"
+
+# Install nextest
+RUN cargo install cargo-nextest --locked
+
+# Copy source code
+COPY . /src
+WORKDIR /src
+
+# Build bcvk and create integration test archive
+RUN make && \
+    cargo nextest archive --release -P integration -p integration-tests --archive-file integration-tests.tar.zst
+
+# Runtime stage
+FROM $base
+
+# Install runtime dependencies for running VMs
+RUN dnf -y install dnf-utils && \
+    dnf config-manager --set-enabled crb && \
+    dnf install -y \
+        libvirt-daemon \
+        libvirt-daemon-driver-qemu \
+        libvirt-client \
+        qemu-kvm \
+        virtiofsd \
+        podman && \
+    dnf clean all
+
+# Copy cargo-nextest from build stage
+COPY --from=build /root/.cargo/bin/cargo-nextest /usr/local/bin/cargo-nextest
+
+# Copy built artifacts
+COPY --from=build /src/target/release/bcvk /usr/local/bin/bcvk
+COPY --from=build /src/integration-tests.tar.zst /tests/integration-tests.tar.zst
+
+# Copy source tree metadata needed by nextest
+# Nextest needs the workspace structure even when using archives
+COPY --from=build /src/Cargo.toml /src/Cargo.lock /tests/
+COPY --from=build /src/.config /tests/.config
+COPY --from=build /src/crates /tests/crates
+
+# Set up environment
+ENV BCVK_PATH=/usr/local/bin/bcvk
+ENV LIBVIRT_DEFAULT_URI=qemu:///system
+WORKDIR /tests
+
+# Create entrypoint script that pulls images and runs tests
+RUN <<EOF cat > /usr/local/bin/run-tests.sh
+#!/bin/bash
+set -euo pipefail
+echo "Pulling test images..."
+podman pull -q quay.io/fedora/fedora-bootc:42 quay.io/centos-bootc/centos-bootc:stream9 quay.io/centos-bootc/centos-bootc:stream10
+echo "Running ephemeral integration tests..."
+exec /usr/local/bin/cargo-nextest nextest run --archive-file integration-tests.tar.zst --workspace-remap /tests ephemeral
+EOF
+RUN chmod +x /usr/local/bin/run-tests.sh
+
+# Default command runs the test script
+CMD ["/usr/local/bin/run-tests.sh"]