fix: Update CI test paths to enable merge queue validation #497
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Claude Code Integration Test | |
| on: | |
| # Manual trigger for testing | |
| workflow_dispatch: | |
| inputs: | |
| debug: | |
| description: 'Enable debug logging' | |
| required: false | |
| default: 'false' | |
| type: boolean | |
| # Run on all PRs (shows as check, but steps skip unless in merge queue) | |
| pull_request: | |
| branches: [main] | |
| # Run in the merge queue to validate before merging | |
| merge_group: | |
| branches: [main] | |
| # Ensure only one instance runs at a time per PR/branch | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} | |
| cancel-in-progress: true | |
| # Minimal permissions for this workflow | |
| permissions: | |
| contents: read | |
| jobs: | |
| # Job 1: Validate skill generation from fixtures (no API key needed) | |
| # Runs on all events, but actual work only happens in merge_group/workflow_dispatch | |
| # This ensures the check name exists for PRs (needed for GitHub's merge queue) | |
| validate-generation: | |
| runs-on: ubuntu-latest | |
| steps: | |
| # For PRs: just pass quickly (actual tests run in merge queue) | |
| - name: Skip on PR | |
| if: github.event_name == 'pull_request' | |
| run: echo "Validation will run in merge queue. Passing for PR." | |
| - uses: actions/checkout@v4 | |
| if: github.event_name != 'pull_request' | |
| - name: Install uv | |
| if: github.event_name != 'pull_request' | |
| uses: astral-sh/setup-uv@v4 | |
| with: | |
| version: "latest" | |
| - name: Set up Python | |
| if: github.event_name != 'pull_request' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install dependencies | |
| if: github.event_name != 'pull_request' | |
| run: uv sync --extra dev | |
| - name: Run fruits workflow tests | |
| if: github.event_name != 'pull_request' | |
| run: uv run pytest tests/integration/test_fruits_workflow.py -v | |
| - name: Generate skills and validate structure | |
| if: github.event_name != 'pull_request' | |
| run: | | |
| # Create a test environment | |
| mkdir -p test_project/.deepwork/jobs | |
| mkdir -p test_project/.claude # Required for platform detection | |
| cp -r tests/fixtures/jobs/fruits test_project/.deepwork/jobs/ | |
| # Set up git repo in test project | |
| cd test_project | |
| git init | |
| git config user.email "test@test.com" | |
| git config user.name "Test" | |
| echo "# Test" > README.md | |
| git add . && git commit -m "init" | |
| cd .. | |
| # Run deepwork install to set up the project (this also runs sync) | |
| uv run deepwork install --platform claude --path test_project | |
| # Validate generated skills exist | |
| echo "Checking generated skills..." | |
| ls -la test_project/.claude/skills/ | |
| # Verify skill directories and SKILL.md files exist | |
| # Meta-skill for the job itself | |
| test -f test_project/.claude/skills/fruits/SKILL.md || (echo "Missing fruits meta-skill" && exit 1) | |
| # Step skills | |
| test -f test_project/.claude/skills/fruits.identify/SKILL.md || (echo "Missing fruits.identify skill" && exit 1) | |
| test -f test_project/.claude/skills/fruits.classify/SKILL.md || (echo "Missing fruits.classify skill" && exit 1) | |
| # Verify skill content | |
| grep -q "# fruits.identify" test_project/.claude/skills/fruits.identify/SKILL.md | |
| grep -q "raw_items" test_project/.claude/skills/fruits.identify/SKILL.md | |
| grep -q "identified_fruits.md" test_project/.claude/skills/fruits.identify/SKILL.md | |
| grep -q "# fruits.classify" test_project/.claude/skills/fruits.classify/SKILL.md | |
| grep -q "identified_fruits.md" test_project/.claude/skills/fruits.classify/SKILL.md | |
| grep -q "classified_fruits.md" test_project/.claude/skills/fruits.classify/SKILL.md | |
| echo "Skill generation validated successfully!" | |
| # Job 2: Full end-to-end test with Claude Code | |
| # Tests the COMPLETE workflow: | |
| # Runs on all events, but actual work only happens in merge_group/workflow_dispatch | |
| # This ensures the check name exists for PRs (needed for GitHub's merge queue) | |
| claude-code-e2e: | |
| runs-on: ubuntu-latest | |
| needs: validate-generation | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| steps: | |
| # For PRs: just pass quickly (actual tests run in merge queue) | |
| - name: Skip on PR | |
| if: github.event_name == 'pull_request' | |
| run: echo "E2E tests will run in merge queue. Passing for PR." | |
| - uses: actions/checkout@v4 | |
| if: github.event_name != 'pull_request' | |
| - name: Check for API key | |
| if: github.event_name != 'pull_request' | |
| id: check-key | |
| run: | | |
| if [ -z "$ANTHROPIC_API_KEY" ]; then | |
| echo "has_key=false" >> $GITHUB_OUTPUT | |
| echo "::warning::ANTHROPIC_API_KEY not set, skipping Claude Code e2e test" | |
| else | |
| echo "has_key=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Install Node.js (for Claude Code CLI) | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '20' | |
| - name: Install Claude Code CLI | |
| if: steps.check-key.outputs.has_key == 'true' | |
| run: npm install -g @anthropic-ai/claude-code | |
| - name: Install uv | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: astral-sh/setup-uv@v4 | |
| with: | |
| version: "latest" | |
| - name: Set up Python | |
| if: steps.check-key.outputs.has_key == 'true' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.11" | |
| - name: Install deepwork | |
| if: steps.check-key.outputs.has_key == 'true' | |
| run: uv sync | |
| - name: Set up fresh test project | |
| if: steps.check-key.outputs.has_key == 'true' | |
| run: | | |
| # Create a fresh project with NO pre-existing job definitions | |
| mkdir -p test_project/.claude | |
| cd test_project | |
| git init | |
| git config user.email "test@test.com" | |
| git config user.name "Test" | |
| echo "# CI Test Project - DeepWork E2E Test" > README.md | |
| git add . && git commit -m "init" | |
| cd .. | |
| # Install deepwork (this sets up .deepwork/ with standard jobs only) | |
| uv run deepwork install --platform claude --path test_project | |
| # Create permissive settings.json to allow file operations in CI | |
| cat > test_project/.claude/settings.json << 'SETTINGS_EOF' | |
| { | |
| "permissions": { | |
| "allow": [ | |
| "Bash(*)", | |
| "Read(./**)", | |
| "Edit(./**)", | |
| "Write(./**)", | |
| "Skill(*)" | |
| ] | |
| } | |
| } | |
| SETTINGS_EOF | |
| echo "Fresh test project setup complete" | |
| echo "Available skills:" | |
| ls -la test_project/.claude/skills/ | |
| # STEP 1: Use /deepwork_jobs.define to CREATE the fruits job | |
| - name: Create job with /deepwork_jobs | |
| if: steps.check-key.outputs.has_key == 'true' | |
| working-directory: test_project | |
| timeout-minutes: 10 | |
| run: | | |
| echo "=== Running /deepwork_jobs to create fruits job ===" | |
| # Provide detailed, deterministic instructions for creating the job | |
| claude --print <<'PROMPT_EOF' | |
| /deepwork_jobs I want to create a simple job called "fruits" for identifying and classifying fruits. | |
| Here are the EXACT specifications. | |
| Intent: A simple workflow that takes a list of mixed items, identifies which are fruits, then classifies them by category. Designed for CI testing. | |
| Steps: | |
| 1. Step: identify | |
| Name: Identify Fruits | |
| Description: Filter a list of items to include only the fruits | |
| 2. Step: classify | |
| Name: Classify Fruits | |
| Description: Organize identified fruits into categories (citrus, tropical, berries, etc.) | |
| Input: identified_fruits.md (file from step identify) | |
| Output: classified_fruits.md | |
| Please create this job now. Do not ask questions. | |
| PROMPT_EOF | |
| # Verify the job.yml was created | |
| echo "=== Checking job.yml was created ===" | |
| if [ -f ".deepwork/jobs/fruits/job.yml" ]; then | |
| echo "SUCCESS: job.yml created" | |
| cat .deepwork/jobs/fruits/job.yml | |
| else | |
| echo "ERROR: job.yml was not created" | |
| echo "Contents of .deepwork/jobs/:" | |
| ls -la .deepwork/jobs/ || echo "No jobs directory" | |
| exit 1 | |
| fi | |
| # Verify step files were created | |
| echo "=== Checking step files were created ===" | |
| if [ -f ".deepwork/jobs/fruits/steps/identify.md" ] && [ -f ".deepwork/jobs/fruits/steps/classify.md" ]; then | |
| echo "SUCCESS: Step instruction files created" | |
| echo "--- identify.md ---" | |
| cat .deepwork/jobs/fruits/steps/identify.md | |
| echo "" | |
| echo "--- classify.md ---" | |
| cat .deepwork/jobs/fruits/steps/classify.md | |
| else | |
| echo "ERROR: Step files were not created" | |
| ls -la .deepwork/jobs/fruits/steps/ || echo "No steps directory" | |
| exit 1 | |
| fi | |
| # Run sync to generate the skills | |
| echo "=== Running deepwork sync to generate skills ===" | |
| cd .. | |
| uv run deepwork sync --path test_project | |
| echo "=== Checking generated skills ===" | |
| ls -la test_project/.claude/skills/ | |
| if [ -f "test_project/.claude/skills/fruits.identify/SKILL.md" ] && [ -f "test_project/.claude/skills/fruits.classify/SKILL.md" ]; then | |
| echo "SUCCESS: Skills generated" | |
| else | |
| echo "ERROR: Skills were not generated" | |
| exit 1 | |
| fi | |
| # STEP 3: Execute the /fruits workflow (runs all steps automatically) | |
| - name: Run /fruits workflow | |
| if: steps.check-key.outputs.has_key == 'true' | |
| working-directory: test_project | |
| timeout-minutes: 10 | |
| run: | | |
| echo "=== Running /fruits workflow with test input ===" | |
| claude --print "/fruits" <<'PROMPT_EOF' | |
| raw_items: apple, car, banana, chair, orange, table, mango, laptop, grape, bicycle | |
| PROMPT_EOF | |
| # Verify both outputs were created | |
| if [ -f "identified_fruits.md" ]; then | |
| echo "SUCCESS: identified_fruits.md created" | |
| echo "--- Output ---" | |
| cat identified_fruits.md | |
| else | |
| echo "ERROR: identified_fruits.md was not created" | |
| exit 1 | |
| fi | |
| if [ -f "classified_fruits.md" ]; then | |
| echo "SUCCESS: classified_fruits.md created" | |
| echo "--- Output ---" | |
| cat classified_fruits.md | |
| else | |
| echo "ERROR: classified_fruits.md was not created" | |
| exit 1 | |
| fi | |
| # STEP 4: Validate the complete workflow output | |
| - name: Validate complete workflow | |
| if: steps.check-key.outputs.has_key == 'true' | |
| working-directory: test_project | |
| run: | | |
| echo "=== Validating complete workflow ===" | |
| # Check identified_fruits.md contains expected fruits | |
| echo "Checking identified_fruits.md..." | |
| grep -qi "apple" identified_fruits.md || (echo "Missing: apple" && exit 1) | |
| grep -qi "banana" identified_fruits.md || (echo "Missing: banana" && exit 1) | |
| grep -qi "orange" identified_fruits.md || (echo "Missing: orange" && exit 1) | |
| grep -qi "mango" identified_fruits.md || (echo "Missing: mango" && exit 1) | |
| grep -qi "grape" identified_fruits.md || (echo "Missing: grape" && exit 1) | |
| echo " ✓ All expected fruits found in identified_fruits.md" | |
| # Check classified_fruits.md has expected structure | |
| echo "Checking classified_fruits.md..." | |
| grep -qi "citrus\|tropical\|pome\|berr" classified_fruits.md || (echo "Missing fruit categories" && exit 1) | |
| echo " ✓ Fruit categories found in classified_fruits.md" | |
| # Verify job structure was created correctly | |
| echo "Checking job structure..." | |
| test -f .deepwork/jobs/fruits/job.yml || (echo "Missing job.yml" && exit 1) | |
| test -f .deepwork/jobs/fruits/steps/identify.md || (echo "Missing identify.md" && exit 1) | |
| test -f .deepwork/jobs/fruits/steps/classify.md || (echo "Missing classify.md" && exit 1) | |
| echo " ✓ Job structure is complete" | |
| echo "" | |
| echo "==========================================" | |
| echo " ALL E2E TESTS PASSED SUCCESSFULLY!" | |
| echo "==========================================" | |
| echo "" | |
| echo "Workflow tested:" | |
| echo " 1. /deepwork_jobs - Created job" | |
| echo " 2. /fruits - Executed full fruits workflow (identify + classify)" | |
| echo "" | |
| - name: Upload test artifacts | |
| if: steps.check-key.outputs.has_key == 'true' && always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: claude-code-e2e-outputs | |
| path: | | |
| test_project/.deepwork/jobs/fruits/ | |
| test_project/.claude/skills/fruits*/ | |
| test_project/identified_fruits.md | |
| test_project/classified_fruits.md | |
| retention-days: 7 |