Skip to content
Back to pull request #640

Add native module (PNM) deployment to VOD job #2032

Sign in to view logs

Add native module (PNM) deployment to VOD job

Add native module (PNM) deployment to VOD job #2032

Workflow file for this run

.github/workflows/main.yml at 2cc9995
name: Veracruz-CI
# Controls when the workflow will run
on:
# Triggers the workflow on push or pull request events but only for the main branch
push:
branches: [ main ]
pull_request:
branches: [ main ]
# Allows you to run this workflow manually from the Actions tab
workflow_dispatch:
env:
NATIVE_MODULE_SANDBOXER_TAG: 20230705
VOD_TAG: 20230704
WASMTIME_VERSION: v9.0.1
jobs:
check:
runs-on: ubuntu-latest
steps:
- name: Install cosign
# https://github.com/sigstore/cosign-installer
uses: sigstore/cosign-installer@9becc617647dfa20ae7b1151972e9b3a2c338a2b # v2.8.1
with:
cosign-release: "v1.13.1"
- name: Check image signature
id: cosign-verify
run: |
COSIGN_EXPERIMENTAL=true cosign verify \
ghcr.io/veracruz-project/veracruz/ci@sha256:f9c1d079ce0fc404ec8a9c5f6c70a52fc5e52dc4866847d1b66260006207f67b
linux:
runs-on: ubuntu-latest
needs: [check]
outputs:
output: ${{ steps.check-diff.outputs.cargo-lock }}
container:
image: ghcr.io/veracruz-project/veracruz/ci@sha256:38add618b42f59a89fdd4c46831dcf9c9e73b159bc74c87251c8263a2ec6238a
volumes:
- ${{ github.workspace }}:/work/veracruz
steps:
- name: Check out the Veracruz repository
uses: actions/checkout@v3
with:
submodules: recursive
- name: Build Veracruz-Linux
id: linux-build
run: |
make -C /work/veracruz/workspaces linux
- name: Running linux test script
id: linux-build-and-test
run: |
make -C /work/veracruz/workspaces linux-tests
- name: Move back to veracruz root
run: |
cd /work/veracruz
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Check modification to Cargo.lock
id: check-diff
run: |
# Find if any Cargo.lock changed, pad them into a line and trim leading and trailing whitespace.
file_changed=$(git diff --diff-filter=ACMUXTRD --name-only -- '**Cargo.lock' | tr '\n' ' ' | xargs)
echo "cargo-lock=$file_changed" >> $GITHUB_OUTPUT
if [ -n "$file_changed" ] ; then
echo "::warning::Cargo.lock files modified";
echo "::warning::Cargo.lock change list: ${{ steps.check-diff.outputs.cargo-lock }}";
fi
- name: Upload Cargo.lock files
id: upload-changed-cargo-lock
if: steps.check-diff.outputs.cargo-lock != ''
uses: actions/upload-artifact@v3
with:
name: linux
path: workspaces/**/Cargo.lock
- name: Prepare deployment artifacts
run: |
# Strip binaries
strip \
workspaces/host/target/debug/freestanding-execution-engine \
workspaces/host/target/debug/generate-policy \
workspaces/linux-host/target/debug/veracruz-client \
workspaces/linux-host/target/debug/linux-veracruz-server \
workspaces/linux-runtime/target/debug/linux-runtime-manager
# Copy artifacts to new directory
mkdir -p artifacts
cp -a \
proxy_cleanup.sh \
workspaces/ca-cert.conf \
workspaces/cert.conf \
workspaces/host/target/debug/freestanding-execution-engine \
workspaces/host/target/debug/generate-policy \
workspaces/linux-host/target/debug/veracruz-client \
workspaces/linux-host/target/debug/linux-veracruz-server \
workspaces/linux-runtime/target/debug/linux-runtime-manager \
artifacts/
- name: Upload deployment artifacts
id: upload-deployment-artifacts
uses: actions/upload-artifact@v3
with:
name: linux_deployment_artifacts
path: |
artifacts/*
vod-full-deployment:
runs-on: ubuntu-latest
needs: [linux]
container:
image: ghcr.io/veracruz-project/veracruz/ci@sha256:38add618b42f59a89fdd4c46831dcf9c9e73b159bc74c87251c8263a2ec6238a
volumes:
- ${{ github.workspace }}:/work/video-object-detection
options: --privileged
steps:
- name: Check out the VOD repository
uses: actions/checkout@v3
with:
repository: 'veracruz-project/video-object-detection'
ref: ${{ env.VOD_TAG }}
submodules: recursive
set-safe-directory: true
- name: Build
run: |
# grab every bash code block for this step, remove line continuation,
# and only keep lines that start with '$' (of course removing that '$'
# in the process)
sed -n '/```.*veracruz-ci-build/,/```/{/```/d; p}' README.md \
| sed ':a; /\\$/{N; s/\\\n//; ta}' \
| sed -n '/^ *\$/{s/^ *\$ \?//; p}' \
> README.md.veracruz-ci-build.sh
# run the script
bash -euxo pipefail README.md.veracruz-ci-build.sh
# Add current directory to $GITHUB_PATH
echo "$GITHUB_WORKSPACE" >> $GITHUB_PATH
- name: Download artifacts
uses: actions/download-artifact@v3
with:
name: linux_deployment_artifacts
path: artifacts
- name: Post-process artifacts
run: |
chmod -R 755 artifacts
# Add artifacts to $GITHUB_PATH
echo "artifacts" >> $GITHUB_PATH
- name: Download example video
run: |
# grab every bash code block for this step, remove line continuation,
# and only keep lines that start with '$' (of course removing that '$'
# in the process)
sed -n '/```.*veracruz-ci-video/,/```/{/```/d; p}' README.md \
| sed ':a; /\\$/{N; s/\\\n//; ta}' \
| sed -n '/^ *\$/{s/^ *\$ \?//; p}' \
> README.md.veracruz-ci-video.sh
# run the script
bash -euxo pipefail README.md.veracruz-ci-video.sh
- name: Replace big YOLO model with small one
run: |
cd program_data
ln -sf yolov3-tiny.cfg yolov3.cfg
ln -sf yolov3-tiny.weights yolov3.weights
- name: Run VOD as standalone native binary
run: |
# grab every bash code block for this step, remove line continuation,
# and only keep lines that start with '$' (of course removing that '$'
# in the process)
sed -n '/```.*veracruz-ci-run-native/,/```/{/```/d; p}' README.md \
| sed ':a; /\\$/{N; s/\\\n//; ta}' \
| sed -n '/^ *\$/{s/^ *\$ \?//; p}' \
> README.md.veracruz-ci-run-native.sh
# run the script
bash -euxo pipefail README.md.veracruz-ci-run-native.sh
# Check results
file output/prediction.0.jpg | grep "JPEG image data"
rm -rf output
- name: Run VOD in wasmtime
run: |
# Install wasmtime
curl https://wasmtime.dev/install.sh -sSf | bash -s -- --version $WASMTIME_VERSION && \
. ~/.bashrc
# grab every bash code block for this step, remove line continuation,
# and only keep lines that start with '$' (of course removing that '$'
# in the process)
sed -n '/```.*veracruz-ci-run-wasmtime/,/```/{/```/d; p}' README.md \
| sed ':a; /\\$/{N; s/\\\n//; ta}' \
| sed -n '/^ *\$/{s/^ *\$ \?//; p}' \
> README.md.veracruz-ci-run-wasmtime.sh
# run the script
bash -euxo pipefail README.md.veracruz-ci-run-wasmtime.sh
# Check results
file output/prediction.0.jpg | grep "JPEG image data"
rm -rf output
- name: Run VOD in Freestanding Execution Engine as WASM
run: |
# grab every bash code block for this step, remove line continuation,
# and only keep lines that start with '$' (of course removing that '$'
# in the process)
sed -n '/```.*veracruz-ci-run-fee/,/```/{/```/d; p}' README.md \
| sed ':a; /\\$/{N; s/\\\n//; ta}' \
| sed -n '/^ *\$/{s/^ *\$ \?//; p}' \
> README.md.veracruz-ci-run-fee.sh
# run the script
bash -euxo pipefail README.md.veracruz-ci-run-fee.sh
# Check results
file output/prediction.0.jpg | grep "JPEG image data"
rm -rf output
- name: Run VOD in Veracruz-Linux as WASM
id: linux-vod-wasm
run: |
POLICY_GENERATOR_PATH="artifacts/generate-policy" CLIENT_PATH="artifacts/veracruz-client" SERVER_PATH="artifacts/linux-veracruz-server" RUNTIME_MANAGER_PATH="artifacts/linux-runtime-manager" CA_CERT_CONF_PATH="artifacts/ca-cert.conf" CERT_CONF_PATH="artifacts/cert.conf" PROXY_CLEANUP_SCRIPT_PATH="artifacts/proxy_cleanup.sh" SERVER_LOG="server.log" POLICY_PATH="policy.json" ./deploy_linux_wasm.sh
# Check results
file prediction.0.jpg | grep "JPEG image data"
rm prediction.0.jpg
- name: Upload VOD artifacts (WASM)
id: upload-vod-artifacts-wasm
if: success() || steps.linux-vod-wasm.conclusion == 'failure'
uses: actions/upload-artifact@v3
with:
name: linux-vod-wasm
path: |
policy.json
server.log
- name: Set up sandbox environment
run: |
git clone --recursive https://github.com/veracruz-project/native-module-sandboxer -b $NATIVE_MODULE_SANDBOXER_TAG
cd native-module-sandboxer
mkdir build
cd build
cmake ..
make
- name: Run VOD in Veracruz-Linux as sandboxed native code (PNM)
id: linux-vod-pnm
run: |
POLICY_GENERATOR_PATH="artifacts/generate-policy" CLIENT_PATH="artifacts/veracruz-client" SERVER_PATH="artifacts/linux-veracruz-server" RUNTIME_MANAGER_PATH="artifacts/linux-runtime-manager" NATIVE_MODULE_SANDBOXER_PATH="native-module-sandboxer/build/native-module-sandboxer" CA_CERT_CONF_PATH="artifacts/ca-cert.conf" CERT_CONF_PATH="artifacts/cert.conf" PROXY_CLEANUP_SCRIPT_PATH="artifacts/proxy_cleanup.sh" SERVER_LOG="server.log" POLICY_PATH="policy.json" ./deploy_linux_pnm.sh
# Check results
file prediction.0.jpg | grep "JPEG image data"
- name: Upload VOD artifacts (PNM)
id: upload-vod-artifacts-pnm
if: success() || steps.linux-vod-pnm.conclusion == 'failure'
uses: actions/upload-artifact@v3
with:
name: linux-vod-pnm
path: |
policy.json
server.log
nitro:
runs-on: ubuntu-latest
needs: [check]
outputs:
output: ${{ steps.check-diff.outputs.cargo-lock }}
container:
image: ghcr.io/veracruz-project/veracruz/ci@sha256:38add618b42f59a89fdd4c46831dcf9c9e73b159bc74c87251c8263a2ec6238a
volumes:
- ${{ github.workspace }}:/work/veracruz
steps:
- name: Check out the Veracruz repository
uses: actions/checkout@v3
with:
submodules: recursive
- name: add the GITHUB_WORKSPACE into git config
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Running Nitro test script
id: nitro-build
run: |
make -C /work/veracruz/workspaces nitro
- name: Check modification to Cargo.lock
id: check-diff
run: |
file_changed=$(git diff --diff-filter=ACMUXTRD --name-only -- '**Cargo.lock' | tr '\n' ' ' | xargs)
echo "cargo-lock=$file_changed" >> $GITHUB_OUTPUT
if [ -n "$file_changed" ] ; then
echo "::warning::Cargo.lock files modified";
echo "::warning::Cargo.lock change list: ${{ steps.check-diff.outputs.cargo-lock }}";
fi
- name: Upload Cargo.lock files
id: upload-changed-cargo-lock
if: steps.check-diff.outputs.cargo-lock != ''
uses: actions/upload-artifact@v3
with:
name: nitro
path: workspaces/**/Cargo.lock
icecap:
runs-on: ubuntu-latest
needs: [check]
outputs:
output: ${{ steps.check-diff.outputs.cargo-lock }}
container:
image: ghcr.io/veracruz-project/veracruz/ci@sha256:38add618b42f59a89fdd4c46831dcf9c9e73b159bc74c87251c8263a2ec6238a
volumes:
- ${{ github.workspace }}:/work/veracruz
steps:
- name: Check out the Veracruz repository
uses: actions/checkout@v3
with:
submodules: recursive
- name: add the GITHUB_WORKSPACE into git config
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Running IceCap test script
id: icecap-build
run: |
VERACRUZ_TEST_TIMEOUT=2400 make -C /work/veracruz/workspaces icecap-tests PROFILE=release
- name: Check modification to Cargo.lock
id: check-diff
run: |
file_changed=$(git diff --diff-filter=ACMUXTRD --name-only -- '**Cargo.lock' | tr '\n' ' ' | xargs)
echo "cargo-lock=$file_changed" >> $GITHUB_OUTPUT
if [ -n "$file_changed" ] ; then
echo "::warning::Cargo.lock files modified";
echo "::warning::Cargo.lock change list: ${{ steps.check-diff.outputs.cargo-lock }}";
fi
- name: Upload Cargo.lock files
id: upload-changed-cargo-lock
if: steps.check-diff.outputs.cargo-lock != ''
uses: actions/upload-artifact@v3
with:
name: icecap
path: workspaces/**/Cargo.lock
# tests that the CLI_QUICKSTART.markdown is still up to date
quickstart:
runs-on: ubuntu-latest
needs: [check]
outputs:
output: ${{ steps.check-diff.outputs.cargo-lock }}
container:
image: ghcr.io/veracruz-project/veracruz/ci@sha256:38add618b42f59a89fdd4c46831dcf9c9e73b159bc74c87251c8263a2ec6238a
volumes:
- ${{ github.workspace }}:/work/veracruz
steps:
- name: Check out the Veracruz repository
uses: actions/checkout@v3
with:
submodules: recursive
- name: add the GITHUB_WORKSPACE into git config
run: |
git config --global --add safe.directory "$GITHUB_WORKSPACE"
- name: Running CLI_QUICKSTART.markdown
id: quickstart-test
run: |
# grab every bash code block, remove line continuation, and only keep lines
# that start with '$' (of course removing that '$' in the process)
sed -n '/``` bash/,/```/{/```/d; p}' CLI_QUICKSTART.markdown \
| sed ':a; /\\$/{N; s/\\\n//; ta}' \
| sed -n '/^\$/{s/^\$ \?//; p}' \
> CLI_QUICKSTART.markdown.sh
# run the quickstart
bash -euxo pipefail CLI_QUICKSTART.markdown.sh
- name: Running tlstest/README.md
id: tlstest
run: |
# Extract and execute bash code blocks from README.md:
cd tests/tlstest && \
sed -n '/``` bash/,/```/{/```/d; p}' README.md > README.md.sh && \
bash -euxo pipefail README.md.sh
- name: Check modification to Cargo.lock
id: check-diff
run: |
file_changed=$(git diff --diff-filter=ACMUXTRD --name-only -- '**Cargo.lock' | tr '\n' ' ' | xargs)
echo "cargo-lock=$file_changed" >> $GITHUB_OUTPUT
if [ -n "$file_changed" ] ; then
echo "::warning::Cargo.lock files modified";
echo "::warning::Cargo.lock change list: ${{ steps.check-diff.outputs.cargo-lock }}";
fi
- name: Upload Cargo.lock files
id: upload-changed-cargo-lock
if: steps.check-diff.outputs.cargo-lock != ''
uses: actions/upload-artifact@v3
with:
name: quickstart
path: workspaces/**/Cargo.lock
cargo-lock-check:
needs: [linux, nitro, icecap, quickstart]
runs-on: ubuntu-latest
steps:
- name: linux
if: needs.linux.outputs.output != ''
run: |
echo "::warning:: linux Cargo.lock change list: ${{ needs.linux.outputs.output }}"
exit 1
- name: nitro
if: needs.nitro.outputs.output != ''
run: |
echo "::warning:: nitro Cargo.lock change list: ${{ needs.nitro.outputs.output }}"
exit 1
- name: icecap
if: needs.icecap.outputs.output != ''
run: |
echo "::warning:: icecap Cargo.lock change list: ${{ needs.icecap.outputs.output }}"
exit 1
- name: quickstart
if: needs.quickstart.outputs.output != ''
run: |
echo "::warning:: quickstart Cargo.lock change list: ${{ needs.quickstart.outputs.output }}"
exit 1