[pull] main from commontoolsinc:main #1246

Workflow file for this run

	name: Deno Workflow

	on:
	push:
	branches:
	- main
	pull_request:
	branches:
	- main

	# Define the list of binaries we're building
	# This makes it easy to add more binaries in the future
	env:
	BINARIES: "toolshed bg-charm-service ct"

	jobs:
	build:
	name: "Test and Build"
	runs-on: ubuntu-24.04-32-core
	steps:
	- name: 📥 Checkout repository
	uses: actions/checkout@v4

	- name: 🦕 Setup Deno
	uses: ./.github/actions/deno-setup

	# Errors if `deno.lock` file was not committed with the current change
	- name: 🔍 Verify lock file & install dependencies
	run: deno install --frozen=true

	- name: 📥 Download Deno dependency binaries
	run: deno task initialize-db

	- name: 🔎 Type check codebase
	run: deno task check

	- name: 🔎 Check codebase formatting
	run: deno fmt --check

	- name: 🧹 Lint codebase
	run: deno lint

	# For deno-web-test browser tests
	# https://github.com/lino-levan/astral/blob/f5ef833b2c5bde3783564a6b925073d5d46bb4b8/README.md#no-usable-sandbox-with-user-namespace-cloning-enabled
	- name: 🛡️ Disable AppArmor for browser tests
	run: echo 0 \| sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns

	- name: 🧪 Run parallel workspace tests
	run: deno task test

	- name: 🏗️ Build application binaries
	run: deno task build-binaries
	env:
	COMMIT_SHA: ${{ github.sha }}

	# Upload binaries as artifacts for the integration tests
	- name: 📤 Upload binaries for integration tests
	uses: actions/upload-artifact@v4
	with:
	name: common-binaries
	path: \|
	./dist/toolshed
	./dist/bg-charm-service
	./dist/ct

	package-integration-test:
	name: "Package Integration Tests"
	runs-on: ubuntu-latest
	needs: ["build"]
	environment: ci
	steps:
	- name: 📥 Checkout repository
	uses: actions/checkout@v4

	- name: 🦕 Setup Deno
	uses: ./.github/actions/deno-setup

	- name: 📥 Download built binaries
	uses: actions/download-artifact@v4

	- name: 🚀 Start Toolshed server for testing
	run: \|
	chmod +x ./common-binaries/toolshed
	CTTS_AI_LLM_ANTHROPIC_API_KEY=fake \
	./common-binaries/toolshed &

	# For Astral
	# https://github.com/lino-levan/astral/blob/f5ef833b2c5bde3783564a6b925073d5d46bb4b8/README.md#no-usable-sandbox-with-user-namespace-cloning-enabled
	- name: 🛡️ Disable AppArmor for browser tests
	run: echo 0 \| sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns

	- name: 🧪 Run end-to-end runner integration tests
	working-directory: packages/runner
	run: \|
	API_URL=http://localhost:8000/ \
	deno task integration

	- name: 🧪 Run end-to-end shell integration tests
	working-directory: packages/shell
	run: \|
	HEADLESS=1 \
	API_URL=http://localhost:8000/ \
	deno task integration

	- name: 🧪 Run background worker integration tests
	working-directory: packages/background-charm-service
	run: \|
	HEADLESS=1 \
	API_URL=http://localhost:8000/ \
	deno task integration

	cli-integration-test:
	name: "CLI Integration Tests"
	runs-on: ubuntu-latest
	needs: ["build"]
	environment: ci
	steps:
	- name: 📥 Checkout repository
	uses: actions/checkout@v4

	- name: 🦕 Setup Deno
	uses: ./.github/actions/deno-setup

	- name: 📥 Download built binaries
	uses: actions/download-artifact@v4

	- name: 🚀 Start Toolshed server for testing
	run: \|
	chmod +x ./common-binaries/ct
	chmod +x ./common-binaries/toolshed
	# Set tools to path
	# Integration script needs `ct`
	echo "${{ github.workspace }}/common-binaries" >> $GITHUB_PATH
	./common-binaries/toolshed &

	- name: 🧪 Run CLI integration tests
	working-directory: packages/cli
	run: \|
	API_URL=http://localhost:8000 ./integration/integration.sh

	pattern-integration-test:
	name: "Pattern Integration Tests"
	runs-on: ubuntu-latest
	needs: ["build"]
	environment: ci
	steps:
	- name: 📥 Checkout repository
	uses: actions/checkout@v4

	- name: 🦕 Setup Deno
	uses: ./.github/actions/deno-setup

	- name: 📥 Download built binaries
	uses: actions/download-artifact@v4

	- name: 🚀 Start Toolshed server for testing
	run: \|
	chmod +x ./common-binaries/toolshed
	CTTS_AI_LLM_ANTHROPIC_API_KEY=fake \
	./common-binaries/toolshed &

	# For Astral
	# https://github.com/lino-levan/astral/blob/f5ef833b2c5bde3783564a6b925073d5d46bb4b8/README.md#no-usable-sandbox-with-user-namespace-cloning-enabled
	- name: 🛡️ Disable AppArmor for browser tests
	run: echo 0 \| sudo tee /proc/sys/kernel/apparmor_restrict_unprivileged_userns

	- name: 🧩 Run end-to-end patterns integration tests
	working-directory: packages/patterns
	run: \|
	HEADLESS=1 \
	API_URL=http://localhost:8000/ \
	deno task integration

	generated-patterns-integration-test:
	name: "Generated Patterns Integration Tests"
	runs-on: ubuntu-latest
	needs: ["build"]
	environment: ci
	steps:
	- name: 📥 Checkout repository
	uses: actions/checkout@v4

	- name: 🦕 Setup Deno
	uses: ./.github/actions/deno-setup

	- name: 🧪 Run generated patterns integration tests
	working-directory: packages/generated-patterns
	run: deno task integration

	attest-binaries:
	name: "Attest and Upload Binaries"
	if: github.ref == 'refs/heads/main'
	runs-on: ubuntu-24.04-32-core
	needs:
	[
	"pattern-integration-test",
	"package-integration-test",
	"cli-integration-test",
	]
	environment: ci
	permissions:
	id-token: write
	contents: read
	actions: read
	attestations: write
	steps:
	- name: 📥 Checkout repository
	uses: actions/checkout@v4

	- name: 📥 Download built binaries
	uses: actions/download-artifact@v4

	- name: 🔐 Process & sign binaries
	run: \|
	mkdir -p release
	mkdir -p signed

	mv ./common-binaries ./dist

	# Process toolshed binary
	echo "Processing binary: toolshed"
	sha256sum ./dist/toolshed > ./dist/toolshed.hash.txt
	TOOLSHED_HASH=$(cat ./dist/toolshed.hash.txt \| awk '{print $1}')
	echo "toolshed hash: $TOOLSHED_HASH"
	echo "toolshed_hash=$TOOLSHED_HASH" >> $GITHUB_OUTPUT

	# Sign the toolshed binary
	openssl dgst -sha256 -sign <(echo "${{ secrets.ARTIFACT_SIGNING_KEY }}") -out ./dist/toolshed.sig ./dist/toolshed

	# Copy to signed directory
	cp ./dist/toolshed ./signed/
	cp ./dist/toolshed.sig ./signed/

	# Process bg-charm-service binary
	echo "Processing binary: bg-charm-service"
	sha256sum ./dist/bg-charm-service > ./dist/bg-charm-service.hash.txt
	BG_CHARM_SERVICE_HASH=$(cat ./dist/bg-charm-service.hash.txt \| awk '{print $1}')
	echo "bg-charm-service hash: $BG_CHARM_SERVICE_HASH"
	echo "bg_charm_service_hash=$BG_CHARM_SERVICE_HASH" >> $GITHUB_OUTPUT

	# Sign the bg-charm-service binary
	openssl dgst -sha256 -sign <(echo "${{ secrets.ARTIFACT_SIGNING_KEY }}") -out ./dist/bg-charm-service.sig ./dist/bg-charm-service

	# Copy to signed directory
	cp ./dist/bg-charm-service ./signed/
	cp ./dist/bg-charm-service.sig ./signed/

	# Process ct binary
	echo "Processing binary: ct"
	sha256sum ./dist/ct > ./dist/ct.hash.txt
	CT_HASH=$(cat ./dist/ct.hash.txt \| awk '{print $1}')
	echo "ct hash: $CT_HASH"
	echo "ct_hash=$CT_HASH" >> $GITHUB_OUTPUT

	# Sign the ct binary
	openssl dgst -sha256 -sign <(echo "${{ secrets.ARTIFACT_SIGNING_KEY }}") -out ./dist/ct.sig ./dist/ct

	# Copy to signed directory
	cp ./dist/ct ./signed/
	cp ./dist/ct.sig ./signed/

	# Create a single tarball with all binaries and signatures
	tar -czf release/labs-${{ github.sha }}.tar.gz -C signed .

	# Generate hash for the tarball
	sha256sum release/labs-${{ github.sha }}.tar.gz > release/labs-${{ github.sha }}.hash.txt
	TARBALL_HASH=$(cat release/labs-${{ github.sha }}.hash.txt \| awk '{print $1}')
	echo "Tarball hash: $TARBALL_HASH"
	echo "tarball_hash=$TARBALL_HASH" >> $GITHUB_OUTPUT
	id: binary_processing

	- name: 📝 Generate attestation for toolshed binary
	id: attest_toolshed
	uses: actions/attest-build-provenance@v1
	with:
	subject-name: ./dist/toolshed
	subject-digest: sha256:${{ steps.binary_processing.outputs.toolshed_hash }}

	- name: 📝 Generate attestation for bg-charm-service binary
	id: attest_bg_charm_service
	uses: actions/attest-build-provenance@v1
	with:
	subject-name: ./dist/bg-charm-service
	subject-digest: sha256:${{ steps.binary_processing.outputs.bg_charm_service_hash }}

	- name: 📝 Generate attestation for ct binary
	id: attest_ct
	uses: actions/attest-build-provenance@v1
	with:
	subject-name: ./dist/ct
	subject-digest: sha256:${{ steps.binary_processing.outputs.ct_hash }}

	- name: 📝 Generate attestation for tarball
	id: attest_tarball
	uses: actions/attest-build-provenance@v1
	with:
	subject-name: https://storage.cloud.google.com/commontools-build-artifacts/workspace-artifacts/labs-${{ github.sha }}.tar.gz
	subject-digest: sha256:${{ steps.binary_processing.outputs.tarball_hash }}

	- name: 📤 Upload attestations as artifacts
	uses: actions/upload-artifact@v4
	with:
	name: binary_attestations
	if-no-files-found: error
	path: \|
	${{ steps.attest_toolshed.outputs.bundle-path }}
	${{ steps.attest_bg_charm_service.outputs.bundle-path }}
	${{ steps.attest_ct.outputs.bundle-path }}
	${{ steps.attest_tarball.outputs.bundle-path }}

	- name: 🔍 Verify binary attestations
	env:
	GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	run: \|
	# Verify tarball attestation
	echo "::group::Tarball attestation details"
	gh attestation verify release/labs-${{ github.sha }}.tar.gz -R ${{ github.repository }} --format json \| jq
	echo "::endgroup::"
	if [ $? -eq 0 ]; then
	echo -e "\033[32m✓ Tarball attestation verified successfully\033[0m"
	else
	echo -e "\033[31m✗ Tarball attestation verification failed\033[0m"
	exit 1
	fi

	# Verify toolshed binary attestation
	echo "::group::toolshed attestation details"
	gh attestation verify ./dist/toolshed -R ${{ github.repository }} --format json \| jq
	echo "::endgroup::"
	if [ $? -eq 0 ]; then
	echo -e "\033[32m✓ toolshed attestation verified successfully\033[0m"
	else
	echo -e "\033[31m✗ toolshed attestation verification failed\033[0m"
	exit 1
	fi

	# Verify bg-charm-service binary attestation
	echo "::group::bg-charm-service attestation details"
	gh attestation verify ./dist/bg-charm-service -R ${{ github.repository }} --format json \| jq
	echo "::endgroup::"
	if [ $? -eq 0 ]; then
	echo -e "\033[32m✓ bg-charm-service attestation verified successfully\033[0m"
	else
	echo -e "\033[31m✗ bg-charm-service attestation verification failed\033[0m"
	exit 1
	fi

	# Verify ct binary attestation
	echo "::group::ct attestation details"
	gh attestation verify ./dist/ct -R ${{ github.repository }} --format json \| jq
	echo "::endgroup::"
	if [ $? -eq 0 ]; then
	echo -e "\033[32m✓ ct attestation verified successfully\033[0m"
	else
	echo -e "\033[31m✗ ct attestation verification failed\033[0m"
	exit 1
	fi

	- name: 🔑 Authenticate to Google Cloud
	uses: google-github-actions/auth@v1
	with:
	credentials_json: ${{ secrets.GCP_SA_KEY }}

	- name: ⚙️ Setup Google Cloud SDK
	uses: google-github-actions/setup-gcloud@v1

	- name: 🚀 Upload artifacts to Google Cloud Storage
	run: \|
	gsutil cp release/labs-${{ github.sha }}.tar.gz gs://commontools-build-artifacts/workspace-artifacts/
	gsutil cp release/labs-${{ github.sha }}.hash.txt gs://commontools-build-artifacts/workspace-artifacts/

	# Print clickable links to the uploaded files
	echo "::group::📦 Artifact Links"
	echo "Tarball URL: https://storage.cloud.google.com/commontools-build-artifacts/workspace-artifacts/labs-${{ github.sha }}.tar.gz"
	echo "Hash URL: https://storage.cloud.google.com/commontools-build-artifacts/workspace-artifacts/labs-${{ github.sha }}.hash.txt"
	echo "::endgroup::"

	# Automatic deployment to staging (toolshed)
	deploy-toolshed:
	name: "Deploy to Toolshed (Staging)"
	if: github.ref == 'refs/heads/main'
	needs: ["attest-binaries"]
	runs-on: ubuntu-latest
	environment: toolshed
	steps:
	- name: 📥 Checkout repository
	uses: actions/checkout@v4

	- name: 🦕 Setup Deno
	uses: ./.github/actions/deno-setup
	with:
	cache: false

	- name: 🚀 Deploy application to Toolshed (Staging)
	uses: appleboy/ssh-action@master
	with:
	host: ${{ secrets.BASTION_HOST }}
	username: bastion
	key: ${{ secrets.BASTION_SSH_PRIVATE_KEY }}
	script: /opt/ct/deploy.sh ${{ vars.DEPLOYMENT_ENVIRONMENT }} ${{ github.sha }}

	# Deploy shell static assets to staging GCS bucket
	deploy-shell-staging:
	name: "Deploy Shell to Staging"
	if: github.ref == 'refs/heads/main'
	needs:
	[
	"pattern-integration-test",
	"package-integration-test",
	"cli-integration-test",
	]
	runs-on: ubuntu-latest
	environment: ci
	steps:
	- name: 📥 Checkout repository
	uses: actions/checkout@v4

	- name: 🦕 Setup Deno
	uses: ./.github/actions/deno-setup

	- name: 🏗️ Build shell with staging API URL
	working-directory: packages/shell
	run: deno task production
	env:
	API_URL: ${{ secrets.CLUSTERDUCK_SHELL_API_URL }}

	- name: 🔑 Authenticate to Google Cloud
	uses: google-github-actions/auth@v1
	with:
	credentials_json: ${{ secrets.GCP_SA_KEY }}

	- name: ⚙️ Setup Google Cloud SDK
	uses: google-github-actions/setup-gcloud@v1

	- name: 📤 Upload shell assets to Staging GCS
	run: \|
	# Upload to SHA-specific directory for versioning
	gsutil -m rsync -r packages/shell/dist gs://staging-commontools-dev/builds/${{ github.sha }}/

	# Upload to bucket root as "latest" (-d removes orphaned files)
	gsutil -m rsync -r -d packages/shell/dist gs://staging-commontools-dev/

	echo "📦 Staging Shell deployed"
	echo "https://staging.commontools.dev/"

	# Post-deployment patterns test (runs after deployment, so this will NOT block deployments.)
	post-deploy-patterns-test:
	name: "Toolshed Post-Deploy Patterns Test"
	if: github.ref == 'refs/heads/main'
	needs: ["deploy-toolshed"]
	runs-on: ubuntu-latest
	environment: toolshed
	continue-on-error: true # Don't fail the workflow if tests fail
	steps:
	- name: 🧩 Run post-deploy Patterns integration tests against Toolshed (Staging)
	id: run_tests
	continue-on-error: true
	uses: appleboy/ssh-action@master
	with:
	host: ${{ secrets.BASTION_HOST }}
	username: bastion
	key: ${{ secrets.BASTION_SSH_PRIVATE_KEY }}
	command_timeout: 20m
	script: \|
	/opt/ct/run-pattern-tests-against-toolshed.sh ${{ github.sha }} \
	https://toolshed.saga-castor.ts.net \
	https://toolshed.saga-castor.ts.net

	- name: 📥 Retrieve test logs on failure
	if: steps.run_tests.outcome == 'failure'
	uses: appleboy/ssh-action@master
	with:
	host: ${{ secrets.BASTION_HOST }}
	username: bastion
	key: ${{ secrets.BASTION_SSH_PRIVATE_KEY }}
	script: \|
	LOG_FILE="/tmp/patterns-test-${{ github.sha }}.log"
	if [ -f "$LOG_FILE" ]; then
	echo "📋 Test failed - showing last 500 lines..."
	echo "========================================="
	tail -n 500 "$LOG_FILE"
	echo "========================================="
	echo "Full logs will be available as artifact"
	else
	echo "⚠️ Log file not found at $LOG_FILE"
	fi

	- name: 📦 Download logs from bastion
	if: always()
	uses: nicklasfrahm/scp-action@main
	with:
	direction: download
	host: ${{ secrets.BASTION_HOST }}
	username: bastion
	key: ${{ secrets.BASTION_SSH_PRIVATE_KEY }}
	insecure_ignore_fingerprint: true
	source: /tmp/patterns-test-${{ github.sha }}.log
	target: patterns-test-${{ github.sha }}.log

	- name: 📤 Upload test logs as artifact
	if: always()
	uses: actions/upload-artifact@v4
	with:
	name: patterns-test-logs-${{ github.sha }}
	path: patterns-test-${{ github.sha }}.log
	retention-days: 7
	if-no-files-found: warn

	- name: 📊 Report test status
	if: always()
	run: \|
	if [ "${{ steps.run_tests.outcome }}" == "success" ]; then
	echo "✅ Pattern integration tests PASSED"
	else
	echo "❌ Pattern integration tests FAILED"
	echo "📥 Check the artifacts tab for full logs"
	fi
	exit ${{ steps.run_tests.outcome == 'success' && '0' \|\| '1' }}

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Uh oh!

[pull] main from commontoolsinc:main #1246

Workflow file

[pull] main from commontoolsinc:main #1246

Uh oh!

Jobs

Run details

Workflow file for this run