diff --git a/.github/.env.base b/.github/.env.base index efe0890..c863cb9 100644 --- a/.github/.env.base +++ b/.github/.env.base @@ -47,13 +47,22 @@ GOVULNCHECK_GO_VERSION=1.25.x # Go sum file location for dependency verification and caching # Default: go.sum (standard location in repository root) -# Custom examples: lib/go.sum, backend/go.sum, services/api/go.sum +# NOTE: For multi-module monorepos (go.work), set ENABLE_MULTI_MODULE_TESTING=true +# GO_SUM_FILE is still required but will be ignored for caching (uses **/go.sum pattern instead) GO_SUM_FILE=go.sum +# Multi-module monorepo support +# When true: runs tests from repo root, magex discovers all Go modules and merges coverage +# Cache keys use **/go.sum pattern (no root go.sum required), skips root go.sum validation +# When false: runs tests from GO_MODULE_DIR (derived from GO_SUM_FILE path) +ENABLE_MULTI_MODULE_TESTING=false + # ================================================================================================ # đŸ–Ĩī¸ RUNNER CONFIGURATION # ================================================================================================ +# https://docs.github.com/en/actions/reference/runners/github-hosted-runners + # Primary runner OS for most CI jobs # Options: ubuntu-24.04, ubuntu-22.04, macos-15 # Note: macOS runners are 10x more expensive than Linux @@ -147,7 +156,7 @@ GO_COVERAGE_PROVIDER=internal CODECOV_TOKEN_REQUIRED=false # Go Coverage Tool Version -GO_COVERAGE_VERSION=v1.1.13 # https://github.com/mrz1836/go-coverage/releases +GO_COVERAGE_VERSION=v1.1.15 # https://github.com/mrz1836/go-coverage/releases GO_COVERAGE_USE_LOCAL=false # Use local version for development # Core Coverage Settings @@ -232,7 +241,7 @@ REDIS_CACHE_FORCE_PULL=false # Force pull Redis images even when cache # đŸĒ„ MAGE-X CONFIGURATION # ================================================================================================ -MAGE_X_VERSION=v1.8.0 # https://github.com/mrz1836/mage-x/releases +MAGE_X_VERSION=v1.8.7 # https://github.com/mrz1836/mage-x/releases MAGE_X_USE_LOCAL=false # Use local version for development MAGE_X_AUTO_DISCOVER_BUILD_TAGS=true # Enable auto-discovery of build tags MAGE_X_AUTO_DISCOVER_BUILD_TAGS_EXCLUDE=race,custom # Comma-separated list of tags to exclude @@ -250,6 +259,9 @@ MAGE_X_STATICCHECK_VERSION=2025.1.1 # https://github.c MAGE_X_SWAG_VERSION=v1.16.6 # https://github.com/swaggo/swag/releases MAGE_X_YAMLFMT_VERSION=v0.20.0 # https://github.com/google/yamlfmt/releases +# Exclude magefiles from prebuild - they require 'mage' build tag and fail without it +# MAGE_X_BUILD_EXCLUDE_PATTERN=magefiles + # Runtime variables (set by setup-goreleaser action): # MAGE_X_GORELEASER_PATH - Path to installed goreleaser binary # MAGE_X_GORELEASER_INSTALLED - Set to 'true' when goreleaser is available @@ -269,6 +281,7 @@ MAGE_X_YAMLFMT_VERSION=v0.20.0 # https://github.c # MAGE_X_DOWNLOAD_TIMEOUT=5000 # MAGE_X_DOWNLOAD_USER_AGENT=MAGE-X-Agent # MAGE_X_PARALLEL=3 +# MAGE_X_TEST_EXCLUDE_MODULES=magefiles # Comma-separated module names to exclude from tests/coverage (default: magefiles) # MAGE_X_TEST_RACE=false # MAGE_X_VERBOSE=true @@ -293,7 +306,7 @@ NANCY_EXCLUDES=CVE-2024-38513,CVE-2023-45142 # Security Tools GITLEAKS_VERSION=8.29.1 # https://github.com/gitleaks/gitleaks/releases GOVULNCHECK_VERSION=v1.1.4 # https://pkg.go.dev/golang.org/x/vuln -NANCY_VERSION=v1.0.51 # https://github.com/sonatype-nexus-community/nancy/releases +NANCY_VERSION=v1.0.52 # https://github.com/sonatype-nexus-community/nancy/releases # ================================================================================================ # đŸĒ PRE-COMMIT SYSTEM CONFIGURATION (go-pre-commit) diff --git a/.github/actions/setup-go-with-cache/action.yml b/.github/actions/setup-go-with-cache/action.yml index 7dbe044..c6d5132 100644 --- a/.github/actions/setup-go-with-cache/action.yml +++ b/.github/actions/setup-go-with-cache/action.yml @@ -41,8 +41,12 @@ inputs: description: "Secondary Go version for toolchain comparison" required: true go-sum-file: - description: "Path to go.sum file for cache key generation" + description: "Path to go.sum file for cache key generation (single module mode only; ignored when enable-multi-module is true)" required: true + enable-multi-module: + description: "Enable multi-module mode - uses pattern **/go.sum to hash all go.sum files for cache keys, skips root go.sum validation" + required: false + default: "false" outputs: go-version-actual: @@ -102,6 +106,7 @@ runs: # Validate go.sum exists for cache key generation # -------------------------------------------------------------------- - name: 🔍 Validate go.sum exists + if: ${{ inputs.enable-multi-module != 'true' }} shell: bash run: | GO_SUM_FILE="${{ inputs.go-sum-file }}" @@ -131,35 +136,75 @@ runs: echo "📋 Input Values:" echo " matrix-os: '${{ inputs.matrix-os }}'" echo " go-sum-file: '${{ inputs.go-sum-file }}'" + echo " enable-multi-module: '${{ inputs.enable-multi-module }}'" - # Verify go.sum file details - GO_SUM_FILE="${{ inputs.go-sum-file }}" - if [ -f "$GO_SUM_FILE" ]; then - echo "" - echo "📄 Go.sum File Analysis:" - echo " Path: $GO_SUM_FILE" - echo " Size: $(wc -c < "$GO_SUM_FILE") bytes" - echo " Lines: $(wc -l < "$GO_SUM_FILE") lines" - echo " SHA256: $(sha256sum "$GO_SUM_FILE" | cut -d' ' -f1)" + if [ "${{ inputs.enable-multi-module }}" == "true" ]; then + # Multi-module mode - show all go.sum files echo "" - echo "🔍 First 5 lines of go.sum:" - head -5 "$GO_SUM_FILE" | sed 's/^/ /' + echo "đŸ“Ļ Multi-module mode enabled" + echo "🔍 Found go.sum files:" + find . -name "go.sum" -type f | while read file; do + echo " - $file ($(wc -l < "$file") lines, $(wc -c < "$file") bytes)" + done + echo "" - echo "🔍 Last 3 lines of go.sum:" - tail -3 "$GO_SUM_FILE" | sed 's/^/ /' + echo "🔑 Expected Module Cache Key Pattern:" + echo " Pattern: ${{ inputs.matrix-os }}-gomod-multi-[hash-of-all-go.sum]" + echo " Actual hash: ${{ hashFiles('**/go.sum') }}" + echo " Complete key: ${{ inputs.matrix-os }}-gomod-multi-${{ hashFiles('**/go.sum') }}" else - echo "❌ ERROR: go.sum file not found at: $GO_SUM_FILE" + # Single module mode - verify go.sum file + GO_SUM_FILE="${{ inputs.go-sum-file }}" + if [ -f "$GO_SUM_FILE" ]; then + echo "" + echo "📄 Go.sum File Analysis:" + echo " Path: $GO_SUM_FILE" + echo " Size: $(wc -c < "$GO_SUM_FILE") bytes" + echo " Lines: $(wc -l < "$GO_SUM_FILE") lines" + echo " SHA256: $(sha256sum "$GO_SUM_FILE" | cut -d' ' -f1)" + echo "" + echo "🔍 First 5 lines of go.sum:" + head -5 "$GO_SUM_FILE" | sed 's/^/ /' + echo "" + echo "🔍 Last 3 lines of go.sum:" + tail -3 "$GO_SUM_FILE" | sed 's/^/ /' + else + echo "❌ ERROR: go.sum file not found at: $GO_SUM_FILE" + fi + + # Show expected cache key pattern + echo "" + echo "🔑 Expected Module Cache Key Pattern:" + echo " Pattern: ${{ inputs.matrix-os }}-gomod-[hash-of-go.sum]" + echo " Actual hash: ${{ hashFiles(inputs.go-sum-file) }}" + echo " Complete key: ${{ inputs.matrix-os }}-gomod-${{ hashFiles(inputs.go-sum-file) }}" fi - # Show expected cache key pattern - echo "" - echo "🔑 Expected Module Cache Key Pattern:" - echo " Pattern: ${{ inputs.matrix-os }}-gomod-[hash-of-go.sum]" - echo " Actual hash: ${{ hashFiles(inputs.go-sum-file) }}" - echo " Complete key: ${{ inputs.matrix-os }}-gomod-${{ hashFiles(inputs.go-sum-file) }}" echo "" echo "============================================================" + # -------------------------------------------------------------------- + # Compute cache keys based on multi-module mode + # -------------------------------------------------------------------- + - name: 🔑 Compute cache keys + id: cache-keys + shell: bash + run: | + if [ "${{ inputs.enable-multi-module }}" == "true" ]; then + echo "đŸ“Ļ Multi-module mode enabled - using hash of all go.sum files" + HASH="${{ hashFiles('**/go.sum') }}" + echo "module-key=${{ inputs.matrix-os }}-gomod-multi-${HASH}" >> $GITHUB_OUTPUT + echo "build-key=${{ inputs.matrix-os }}-gobuild-multi-${{ inputs.go-version }}-${HASH}" >> $GITHUB_OUTPUT + echo "mode=multi" >> $GITHUB_OUTPUT + else + echo "📁 Single module mode - using hash of ${{ inputs.go-sum-file }}" + HASH="${{ hashFiles(inputs.go-sum-file) }}" + echo "module-key=${{ inputs.matrix-os }}-gomod-${HASH}" >> $GITHUB_OUTPUT + echo "build-key=${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-${HASH}" >> $GITHUB_OUTPUT + echo "mode=single" >> $GITHUB_OUTPUT + fi + echo "🔑 Cache keys computed successfully" + # -------------------------------------------------------------------- # Restore Go module cache (shared across versions) # -------------------------------------------------------------------- @@ -168,7 +213,7 @@ runs: uses: actions/cache/restore@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4 with: path: ~/go/pkg/mod - key: ${{ inputs.matrix-os }}-gomod-${{ hashFiles(inputs.go-sum-file) }} + key: ${{ steps.cache-keys.outputs.module-key }} # -------------------------------------------------------------------- # DEBUG: Build cache key components @@ -185,18 +230,27 @@ runs: echo " matrix-os: '${{ inputs.matrix-os }}'" echo " go-version: '${{ inputs.go-version }}'" echo " go-sum-file: '${{ inputs.go-sum-file }}'" + echo " enable-multi-module: '${{ inputs.enable-multi-module }}'" - # Verify go.sum file details (for build cache key) - GO_SUM_FILE="${{ inputs.go-sum-file }}" - if [ -f "$GO_SUM_FILE" ]; then + if [ "${{ inputs.enable-multi-module }}" == "true" ]; then + # Multi-module mode - show summary of all go.sum files echo "" - echo "📄 Go.sum File Analysis (for build cache):" - echo " Path: $GO_SUM_FILE" - echo " Size: $(wc -c < "$GO_SUM_FILE") bytes" - echo " SHA256: $(sha256sum "$GO_SUM_FILE" | cut -d' ' -f1)" - echo " Last Modified: $(stat -c %y "$GO_SUM_FILE" 2>/dev/null || stat -f %Sm "$GO_SUM_FILE" 2>/dev/null || echo 'Unknown')" + echo "đŸ“Ļ Multi-module mode enabled" + echo "🔍 Go.sum files count: $(find . -name "go.sum" -type f | wc -l)" + echo "🔑 Cache hash: ${{ hashFiles('**/go.sum') }}" else - echo "❌ ERROR: go.sum file not found at: $GO_SUM_FILE" + # Single module mode - verify go.sum file details + GO_SUM_FILE="${{ inputs.go-sum-file }}" + if [ -f "$GO_SUM_FILE" ]; then + echo "" + echo "📄 Go.sum File Analysis (for build cache):" + echo " Path: $GO_SUM_FILE" + echo " Size: $(wc -c < "$GO_SUM_FILE") bytes" + echo " SHA256: $(sha256sum "$GO_SUM_FILE" | cut -d' ' -f1)" + echo " Last Modified: $(stat -c %y "$GO_SUM_FILE" 2>/dev/null || stat -f %Sm "$GO_SUM_FILE" 2>/dev/null || echo 'Unknown')" + else + echo "❌ ERROR: go.sum file not found at: $GO_SUM_FILE" + fi fi # Show cache paths that will be used @@ -210,10 +264,17 @@ runs: # Show expected cache key pattern echo "" - echo "🔑 Expected Build Cache Key Pattern:" - echo " Pattern: ${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-[hash-of-go.sum]" - echo " Actual hash: ${{ hashFiles(inputs.go-sum-file) }}" - echo " Complete key: ${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-${{ hashFiles(inputs.go-sum-file) }}" + if [ "${{ inputs.enable-multi-module }}" == "true" ]; then + echo "🔑 Expected Build Cache Key Pattern:" + echo " Pattern: ${{ inputs.matrix-os }}-gobuild-multi-${{ inputs.go-version }}-[hash-of-all-go.sum]" + echo " Actual hash: ${{ hashFiles('**/go.sum') }}" + echo " Complete key: ${{ inputs.matrix-os }}-gobuild-multi-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }}" + else + echo "🔑 Expected Build Cache Key Pattern:" + echo " Pattern: ${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-[hash-of-go.sum]" + echo " Actual hash: ${{ hashFiles(inputs.go-sum-file) }}" + echo " Complete key: ${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-${{ hashFiles(inputs.go-sum-file) }}" + fi echo "" echo "============================================================" @@ -227,7 +288,7 @@ runs: path: | ~/.cache/go-build ~/.cache/go-build/test - key: ${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-${{ hashFiles(inputs.go-sum-file) }} + key: ${{ steps.cache-keys.outputs.build-key }} # -------------------------------------------------------------------- # DEBUG: Cache restoration summary @@ -247,17 +308,27 @@ runs: # Show the actual keys that were used echo "" echo "🔑 Actual Cache Keys Used:" - MODULE_KEY="${{ inputs.matrix-os }}-gomod-${{ hashFiles(inputs.go-sum-file) }}" - BUILD_KEY="${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-${{ hashFiles(inputs.go-sum-file) }}" - echo " Module: $MODULE_KEY" - echo " Build: $BUILD_KEY" - - # Break down the keys for analysis - echo "" - echo "🔍 Key Component Analysis:" - echo " OS component: '${{ inputs.matrix-os }}'" - echo " Go version component: '${{ inputs.go-version }}'" - echo " Go.sum hash component: '${{ hashFiles(inputs.go-sum-file) }}'" + if [ "${{ inputs.enable-multi-module }}" == "true" ]; then + MODULE_KEY="${{ inputs.matrix-os }}-gomod-multi-${{ hashFiles('**/go.sum') }}" + BUILD_KEY="${{ inputs.matrix-os }}-gobuild-multi-${{ inputs.go-version }}-${{ hashFiles('**/go.sum') }}" + echo " Module: $MODULE_KEY" + echo " Build: $BUILD_KEY" + echo "" + echo "🔍 Key Component Analysis (Multi-module mode):" + echo " OS component: '${{ inputs.matrix-os }}'" + echo " Go version component: '${{ inputs.go-version }}'" + echo " Go.sum hash component (all files): '${{ hashFiles('**/go.sum') }}'" + else + MODULE_KEY="${{ inputs.matrix-os }}-gomod-${{ hashFiles(inputs.go-sum-file) }}" + BUILD_KEY="${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-${{ hashFiles(inputs.go-sum-file) }}" + echo " Module: $MODULE_KEY" + echo " Build: $BUILD_KEY" + echo "" + echo "🔍 Key Component Analysis (Single module mode):" + echo " OS component: '${{ inputs.matrix-os }}'" + echo " Go version component: '${{ inputs.go-version }}'" + echo " Go.sum hash component: '${{ hashFiles(inputs.go-sum-file) }}'" + fi # Cache effectiveness summary echo "" diff --git a/.github/actions/warm-cache/action.yml b/.github/actions/warm-cache/action.yml index b6f0f96..979ba25 100644 --- a/.github/actions/warm-cache/action.yml +++ b/.github/actions/warm-cache/action.yml @@ -47,8 +47,12 @@ inputs: required: false default: "false" go-sum-file: - description: "Path to go.sum file for cache key generation" + description: "Path to go.sum file for cache key generation (single module mode)" required: true + enable-multi-module: + description: "Enable multi-module mode - uses hash of all go.sum files for cache keys" + required: false + default: "false" runs: using: "composite" @@ -91,6 +95,7 @@ runs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-secondary-version }} go-sum-file: ${{ inputs.go-sum-file }} + enable-multi-module: ${{ inputs.enable-multi-module }} # ──────────────────────────────────────────────────────────────────────────── # Setup MAGE-X (required for magex commands in cache warming) @@ -136,9 +141,29 @@ runs: echo "module-dir=$GO_MODULE_DIR" >> $GITHUB_OUTPUT # -------------------------------------------------------------------- - # Ensure go.sum exists and download modules + # Compute cache keys for saving (must match restore keys) # -------------------------------------------------------------------- - - name: 📋 Ensure go.sum exists + - name: 🔑 Compute cache keys for saving + id: cache-keys + shell: bash + run: | + if [ "${{ inputs.enable-multi-module }}" == "true" ]; then + echo "đŸ“Ļ Multi-module mode - cache keys will use hash of all go.sum files" + HASH="${{ hashFiles('**/go.sum') }}" + echo "module-key=${{ inputs.matrix-os }}-gomod-multi-${HASH}" >> $GITHUB_OUTPUT + echo "build-key=${{ inputs.matrix-os }}-gobuild-multi-${{ inputs.go-version }}-${HASH}" >> $GITHUB_OUTPUT + else + echo "📁 Single module mode - cache keys will use hash of ${{ inputs.go-sum-file }}" + HASH="${{ hashFiles(inputs.go-sum-file) }}" + echo "module-key=${{ inputs.matrix-os }}-gomod-${HASH}" >> $GITHUB_OUTPUT + echo "build-key=${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-${HASH}" >> $GITHUB_OUTPUT + fi + + # -------------------------------------------------------------------- + # Ensure go.sum exists (single-module mode only) + # -------------------------------------------------------------------- + - name: 📋 Ensure go.sum exists (single-module) + if: ${{ inputs.enable-multi-module != 'true' }} shell: bash run: | GO_SUM_FILE="${{ inputs.go-sum-file }}" @@ -159,6 +184,64 @@ runs: echo "✅ Go.sum already exists at $GO_SUM_FILE" fi + # -------------------------------------------------------------------- + # Ensure all modules are tidy (multi-module mode only) + # Handles both go.work workspaces and single-module repos with multi-module testing enabled + # -------------------------------------------------------------------- + - name: 📋 Ensure modules are tidy (multi-module) + if: ${{ inputs.enable-multi-module == 'true' }} + shell: bash + run: | + set -euo pipefail + echo "🔧 Multi-module mode - detecting workspace configuration" + + # Check if go.work exists (workspace mode) + if [ -f "go.work" ]; then + echo "đŸ“Ļ go.work workspace detected - syncing and tidying all modules" + echo "ℹī¸ No root go.sum validation required (go.work manages workspace modules)" + + # Sync the workspace + echo "🔄 Running 'go work sync' to synchronize workspace modules..." + go work sync + + # Find and tidy each module in the workspace + echo "🔍 Finding all modules in workspace..." + module_paths=$(go work edit -json | jq -r '(.Use // [])[].DiskPath') + if [ -z "$module_paths" ]; then + echo "❌ Error: No modules found in workspace - go.work exists but has no 'use' directives" + exit 1 + fi + + tidy_failed=0 + while read -r module_dir; do + # Skip empty lines + [ -n "$module_dir" ] || continue + + if [ ! -f "$module_dir/go.mod" ]; then + echo "❌ Module directory $module_dir is missing go.mod file" + exit 1 + fi + echo "🔧 Tidying module: $module_dir" + if ! (cd "$module_dir" && go mod tidy); then + echo "❌ Failed to tidy module: $module_dir" + tidy_failed=1 + fi + done <<< "$module_paths" + + if [ "$tidy_failed" -ne 0 ]; then + echo "❌ One or more modules failed to tidy" + exit 1 + fi + + echo "✅ All workspace modules synced and tidied successfully" + else + # No go.work - single module with multi-module testing enabled + echo "📁 No go.work file - running single module tidy from repository root" + echo "ℹī¸ Multi-module testing mode without workspace (legacy behavior)" + magex tidy + echo "✅ Module tidied successfully" + fi + # ──────────────────────────────────────────────────────────────────────────── # Ensure cache directories exist # ──────────────────────────────────────────────────────────────────────────── @@ -171,6 +254,18 @@ runs: mkdir -p "$GOCACHE/test" echo "✅ Cache directories created successfully" + # ──────────────────────────────────────────────────────────────────────────── + # Full checkout for module download (only when module cache misses) + # The initial sparse checkout only includes go.mod/go.sum for cache key generation. + # Module download requires full checkout because go.mod has replace directives + # pointing to local sibling modules (e.g., ../functions/pkg/config). + # ──────────────────────────────────────────────────────────────────────────── + - name: đŸ“Ĩ Full checkout for module download (module cache miss) + if: steps.setup-go.outputs.module-cache-hit != 'true' + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + with: + persist-credentials: false + # ──────────────────────────────────────────────────────────────────────────── # Download modules when module cache missed # ──────────────────────────────────────────────────────────────────────────── @@ -186,7 +281,11 @@ runs: echo "=== Downloading Dependencies ===" echo "============================================================" - if [ -n "$GO_MODULE_DIR" ]; then + if [ "$ENABLE_MULTI_MODULE_TESTING" == "true" ]; then + echo "🔧 Multi-module mode - running magex deps:download from repository root" + echo "đŸ“Ļ magex will discover all Go modules and download dependencies" + magex deps:download + elif [ -n "$GO_MODULE_DIR" ]; then echo "🔧 Running magex deps:download from directory: $GO_MODULE_DIR" (cd "$GO_MODULE_DIR" && magex deps:download) else @@ -195,6 +294,17 @@ runs: fi echo "✅ Modules downloaded successfully" + # ──────────────────────────────────────────────────────────────────────────── + # Full checkout for build cache warming (only when module cache hit but build missed) + # If module cache missed, we already did a full checkout above. + # This handles the case where module cache was warm but build cache needs warming. + # ──────────────────────────────────────────────────────────────────────────── + - name: đŸ“Ĩ Full checkout for build warming (module hit, build miss) + if: steps.setup-go.outputs.module-cache-hit == 'true' && steps.setup-go.outputs.build-cache-hit != 'true' + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + with: + persist-credentials: false + # ──────────────────────────────────────────────────────────────────────────── # Warm build cache when build cache missed # ──────────────────────────────────────────────────────────────────────────── @@ -214,7 +324,15 @@ runs: PARALLEL_JOBS="${MAGE_X_PARALLEL:-1}" echo "🔧 Using parallelism level: $PARALLEL_JOBS (from MAGE_X_PARALLEL env var)" - if [ -n "$GO_MODULE_DIR" ]; then + if [ "$ENABLE_MULTI_MODULE_TESTING" == "true" ]; then + echo "🔧 Multi-module mode - running build commands from repository root" + echo "đŸ“Ļ magex will discover all Go modules and pre-build packages" + # Use configured parallelism to avoid OOM on GitHub Actions runners + magex build:prebuild p="$PARALLEL_JOBS" strategy="${MAGE_X_BUILD_STRATEGY:-smart}" batch_size="${MAGE_X_BUILD_BATCH_SIZE:-20}" batch_delay="${MAGE_X_BUILD_BATCH_DELAY_MS:-0}" exclude="${MAGE_X_BUILD_EXCLUDE_PATTERN:-}" + + echo "🏗ī¸ Building stdlib for host platform..." + magex install:stdlib + elif [ -n "$GO_MODULE_DIR" ]; then echo "🔧 Running build commands from directory: $GO_MODULE_DIR" # Use configured parallelism to avoid OOM on GitHub Actions runners (cd "$GO_MODULE_DIR" && magex build:prebuild p="$PARALLEL_JOBS" strategy="${MAGE_X_BUILD_STRATEGY:-smart}" batch_size="${MAGE_X_BUILD_BATCH_SIZE:-20}" batch_delay="${MAGE_X_BUILD_BATCH_DELAY_MS:-0}" exclude="${MAGE_X_BUILD_EXCLUDE_PATTERN:-}") @@ -242,7 +360,7 @@ runs: path: | ~/.cache/go-build ~/.cache/go-build/test - key: ${{ inputs.matrix-os }}-gobuild-${{ inputs.go-version }}-${{ hashFiles(inputs.go-sum-file) }} + key: ${{ steps.cache-keys.outputs.build-key }} # ──────────────────────────────────────────────────────────────────────────── # Save the Go module cache we just populated @@ -252,7 +370,7 @@ runs: uses: actions/cache/save@5a3ec84eff668545956fd18022155c47e93e2684 # v4.0.2 with: path: ~/go/pkg/mod - key: ${{ inputs.matrix-os }}-gomod-${{ hashFiles(inputs.go-sum-file) }} + key: ${{ steps.cache-keys.outputs.module-key }} # -------------------------------------------------------------------- # Track cache performance (no outputs - use artifacts instead) diff --git a/.github/workflows/fortress-benchmarks.yml b/.github/workflows/fortress-benchmarks.yml index c57e66b..716aced 100644 --- a/.github/workflows/fortress-benchmarks.yml +++ b/.github/workflows/fortress-benchmarks.yml @@ -142,6 +142,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-secondary-version }} go-sum-file: ${{ inputs.go-sum-file }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path @@ -214,7 +215,17 @@ jobs: # Run benchmarks and capture output GO_MODULE_DIR="${{ env.GO_MODULE_DIR }}" - if [ -n "$GO_MODULE_DIR" ]; then + if [ "$ENABLE_MULTI_MODULE_TESTING" == "true" ]; then + echo "🔧 Multi-module testing enabled - running $BENCH_CMD from repository root" + echo "đŸ“Ļ magex will discover all Go modules" + if $BENCH_CMD > "$BENCH_OUTPUT_FILE" 2>&1; then + echo "✅ Benchmarks command completed" + BENCH_STATUS="success" + else + echo "❌ Benchmarks command failed" + BENCH_STATUS="failure" + fi + elif [ -n "$GO_MODULE_DIR" ]; then echo "🔧 Running $BENCH_CMD from directory: $GO_MODULE_DIR" if (cd "$GO_MODULE_DIR" && $BENCH_CMD) > "$BENCH_OUTPUT_FILE" 2>&1; then echo "✅ Benchmarks command completed" diff --git a/.github/workflows/fortress-code-quality.yml b/.github/workflows/fortress-code-quality.yml index 0271208..52f456f 100644 --- a/.github/workflows/fortress-code-quality.yml +++ b/.github/workflows/fortress-code-quality.yml @@ -92,6 +92,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-primary-version }} go-sum-file: ${{ env.GO_SUM_FILE }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path @@ -245,6 +246,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-primary-version }} go-sum-file: ${{ env.GO_SUM_FILE }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path @@ -344,7 +346,11 @@ jobs: GO_MODULE_DIR="${{ env.GO_MODULE_DIR }}" - if [ -n "$GO_MODULE_DIR" ]; then + if [ "$ENABLE_MULTI_MODULE_TESTING" == "true" ]; then + echo "🔧 Multi-module linting enabled - running from repository root" + echo "đŸ“Ļ magex will discover all Go modules" + magex lint + elif [ -n "$GO_MODULE_DIR" ]; then echo "🔧 Running magex lint from directory: $GO_MODULE_DIR" (cd "$GO_MODULE_DIR" && magex lint) else @@ -453,6 +459,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-primary-version }} go-sum-file: ${{ env.GO_SUM_FILE }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path diff --git a/.github/workflows/fortress-coverage.yml b/.github/workflows/fortress-coverage.yml index afbbf62..458a3f1 100644 --- a/.github/workflows/fortress-coverage.yml +++ b/.github/workflows/fortress-coverage.yml @@ -174,6 +174,7 @@ jobs: go-primary-version: ${{ env.GO_PRIMARY_VERSION }} go-secondary-version: ${{ env.GO_SECONDARY_VERSION }} go-sum-file: ${{ inputs.go-sum-file }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path @@ -797,7 +798,7 @@ jobs: # Verify outputs were created echo "" echo "🔍 Verifying generated outputs..." - EXPECTED_FILES=("index.html" "coverage.html" "coverage.svg") + EXPECTED_FILES=("index.html" "coverage.html" "coverage.svg" "coverage-flat.svg" "coverage-flat-square.svg" "coverage-for-the-badge.svg") echo "" echo "📋 Checking main output files:" @@ -1137,6 +1138,9 @@ jobs: "index.html" "coverage.html" "coverage.svg" + "coverage-flat.svg" + "coverage-flat-square.svg" + "coverage-for-the-badge.svg" ) # Selectively copy coverage files to avoid nested directory structures @@ -1188,6 +1192,9 @@ jobs: # Copy only essential coverage files, avoiding nested duplication COVERAGE_ESSENTIAL_FILES=( "coverage.svg" + "coverage-flat.svg" + "coverage-flat-square.svg" + "coverage-for-the-badge.svg" "coverage.html" "index.html" "dashboard.html" @@ -1407,7 +1414,7 @@ jobs: echo "" # Verify critical files exist with detailed reporting - CRITICAL_FILES=("index.html" "coverage.html" "coverage.svg" ".nojekyll") + CRITICAL_FILES=("index.html" "coverage.html" "coverage.svg" "coverage-flat.svg" "coverage-flat-square.svg" "coverage-for-the-badge.svg" ".nojekyll") echo "đŸŽ¯ Verifying critical files for GitHub Pages:" ALL_EXIST=true @@ -1640,13 +1647,24 @@ jobs: mkdir -p "$TEMP_STAGING" echo "📁 Created temporary staging area: $TEMP_STAGING" + # Update gh-pages .gitignore from latest build assets (ensures badge variants are allowed) + if [[ -f "$DEPLOY_DIR/assets/gh-pages.gitignore" ]]; then + echo "📋 Updating gh-pages .gitignore from build assets..." + cp "$DEPLOY_DIR/assets/gh-pages.gitignore" "$TEMP_PAGES_DIR/.gitignore" + GH_PAGES_GITIGNORE="$TEMP_PAGES_DIR/.gitignore" + echo "✅ Updated .gitignore with latest coverage file patterns" + else + echo "⚠ī¸ No gh-pages.gitignore found in assets, using existing" + GH_PAGES_GITIGNORE="$TEMP_PAGES_DIR/.gitignore" + fi + # Update root files (main branch deployments) with filtering source /tmp/branch_helpers.sh if is_main_branch "$BRANCH_NAME"; then echo "📋 Updating root coverage files for main branch (with filtering)..." # Define allowed root files explicitly - ALLOWED_ROOT_FILES=("index.html" "coverage.html" "coverage.svg" ".nojekyll" "data" "assets") + ALLOWED_ROOT_FILES=("index.html" "coverage.html" "coverage.svg" "coverage-flat.svg" "coverage-flat-square.svg" "coverage-for-the-badge.svg" ".nojekyll" "data" "assets") # Copy only allowed root files for file in "${ALLOWED_ROOT_FILES[@]}"; do @@ -1711,7 +1729,7 @@ jobs: rm -rf "$TEMP_STAGING"/* # Define allowed branch files - ALLOWED_BRANCH_FILES=("index.html" "coverage.html" "coverage.svg" "data" "assets") + ALLOWED_BRANCH_FILES=("index.html" "coverage.html" "coverage.svg" "coverage-flat.svg" "coverage-flat-square.svg" "coverage-for-the-badge.svg" "data" "assets") # Copy branch-specific files from deployment directory to staging first if [[ -d "$DEPLOY_DIR/coverage/branch/$BRANCH_NAME" ]]; then diff --git a/.github/workflows/fortress-pre-commit.yml b/.github/workflows/fortress-pre-commit.yml index 500aca4..1cefd79 100644 --- a/.github/workflows/fortress-pre-commit.yml +++ b/.github/workflows/fortress-pre-commit.yml @@ -85,6 +85,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-primary-version }} go-sum-file: ${{ env.GO_SUM_FILE }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path diff --git a/.github/workflows/fortress-release.yml b/.github/workflows/fortress-release.yml index aa1bffc..bcf1d53 100644 --- a/.github/workflows/fortress-release.yml +++ b/.github/workflows/fortress-release.yml @@ -91,6 +91,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-primary-version }} go-sum-file: ${{ inputs.go-sum-file }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Validate version tag format diff --git a/.github/workflows/fortress-security-scans.yml b/.github/workflows/fortress-security-scans.yml index 8239687..7a9b092 100644 --- a/.github/workflows/fortress-security-scans.yml +++ b/.github/workflows/fortress-security-scans.yml @@ -97,6 +97,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-primary-version }} go-sum-file: ${{ inputs.go-sum-file }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path @@ -114,7 +115,11 @@ jobs: echo "📋 Generating module list for security scanning..." GO_MODULE_DIR="${{ env.GO_MODULE_DIR }}" - if [ -n "$GO_MODULE_DIR" ]; then + if [ "$ENABLE_MULTI_MODULE_TESTING" == "true" ]; then + echo "🔧 Multi-module enabled - running go list from repository root" + echo "đŸ“Ļ go.work will discover all Go modules" + go list -json -m all > go.list + elif [ -n "$GO_MODULE_DIR" ]; then echo "🔧 Running go list from directory: $GO_MODULE_DIR" (cd "$GO_MODULE_DIR" && go list -json -m all > ../go.list) else @@ -216,6 +221,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-primary-version }} go-sum-file: ${{ inputs.go-sum-file }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path @@ -283,7 +289,11 @@ jobs: echo "🔍 Running vulnerability analysis..." GO_MODULE_DIR="${{ env.GO_MODULE_DIR }}" - if [ -n "$GO_MODULE_DIR" ]; then + if [ "$ENABLE_MULTI_MODULE_TESTING" == "true" ]; then + echo "🔧 Multi-module enabled - running magex deps:audit from repository root" + echo "đŸ“Ļ magex will discover all Go modules" + magex deps:audit + elif [ -n "$GO_MODULE_DIR" ]; then echo "🔧 Running magex deps:audit from directory: $GO_MODULE_DIR" (cd "$GO_MODULE_DIR" && magex deps:audit) else diff --git a/.github/workflows/fortress-setup-config.yml b/.github/workflows/fortress-setup-config.yml index f60efc4..dc61220 100644 --- a/.github/workflows/fortress-setup-config.yml +++ b/.github/workflows/fortress-setup-config.yml @@ -306,6 +306,7 @@ jobs: sparse-checkout: | .mage.yaml go.mod + go.work ${{ env.GO_SUM_FILE }} .github/workflows/fortress.yml .github/actions/configure-redis @@ -319,6 +320,19 @@ jobs: with: go-sum-file: ${{ env.GO_SUM_FILE }} # -------------------------------------------------------------------- + # Detect go.work file for multi-module workspace + # -------------------------------------------------------------------- + - name: 🔍 Detect Multi-Module Workspace + id: detect-gowork + run: | + if [ -f "go.work" ]; then + echo "gowork-exists=true" >> $GITHUB_OUTPUT + echo "✅ go.work file found - multi-module workspace detected" + else + echo "gowork-exists=false" >> $GITHUB_OUTPUT + echo "ℹī¸ No go.work file found - single module repository" + fi + # -------------------------------------------------------------------- # Extract GoFortress version metadata from fortress.yml # -------------------------------------------------------------------- - name: 📋 Extract GoFortress Version @@ -652,6 +666,9 @@ jobs: echo "| **Go Sum File** | \`${{ env.GO_SUM_FILE }}\` | Location of go.sum for dependency verification |" >> $GITHUB_STEP_SUMMARY echo "| **Go Module Directory** | \`${{ env.GO_MODULE_DIR || '.' }}\` | Directory containing go.mod (extracted from go.sum path) |" >> $GITHUB_STEP_SUMMARY echo "| **Module Root Type** | $([ -n \"${{ env.GO_MODULE_DIR }}\" ] && echo \"📁 Subdirectory\" || echo \"📂 Repository Root\") | Whether go.mod is in repository root or subdirectory |" >> $GITHUB_STEP_SUMMARY + echo "| **Multi-Module Testing** | $([ \"${{ env.ENABLE_MULTI_MODULE_TESTING }}\" == \"true\" ] && echo \"✅ Enabled\" || echo \"❌ Disabled\") | $([ \"${{ env.ENABLE_MULTI_MODULE_TESTING }}\" == \"true\" ] && echo \"Tests run from repo root with automatic module discovery\" || echo \"Tests run from module directory specified by GO_SUM_FILE\") |" >> $GITHUB_STEP_SUMMARY + echo "| **Multi-Module Workspace** | $([ \"${{ steps.detect-gowork.outputs.gowork-exists }}\" == \"true\" ] && echo \"✅ go.work found\" || echo \"âšĒ No go.work file\") | $([ \"${{ steps.detect-gowork.outputs.gowork-exists }}\" == \"true\" ] && echo \"Workspace file enables cross-module dependency resolution\" || echo \"Single module repository\") |" >> $GITHUB_STEP_SUMMARY + echo "| **Test Execution Context** | $([ \"${{ env.ENABLE_MULTI_MODULE_TESTING }}\" == \"true\" ] && echo \"đŸ“Ļ Repository Root\" || echo \"📁 Module Directory\") | Working directory for test execution and cache keys |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "

" >> $GITHUB_STEP_SUMMARY # -------------------------------------------------------------------- diff --git a/.github/workflows/fortress-test-fuzz.yml b/.github/workflows/fortress-test-fuzz.yml index ec03b03..9de55b5 100644 --- a/.github/workflows/fortress-test-fuzz.yml +++ b/.github/workflows/fortress-test-fuzz.yml @@ -88,6 +88,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-secondary-version }} go-sum-file: ${{ inputs.go-sum-file }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path diff --git a/.github/workflows/fortress-test-matrix.yml b/.github/workflows/fortress-test-matrix.yml index c438b1e..d000cbb 100644 --- a/.github/workflows/fortress-test-matrix.yml +++ b/.github/workflows/fortress-test-matrix.yml @@ -147,6 +147,7 @@ jobs: go-primary-version: ${{ inputs.go-primary-version }} go-secondary-version: ${{ inputs.go-secondary-version }} go-sum-file: ${{ inputs.go-sum-file }} + enable-multi-module: ${{ env.ENABLE_MULTI_MODULE_TESTING }} # -------------------------------------------------------------------- # Extract Go module directory from GO_SUM_FILE path @@ -323,7 +324,11 @@ jobs: if [[ "$MODE" == "FULL" ]]; then echo "📝 Using FULL output mode - showing all test output" echo "🔍 Executing: $MAGEX_CMD" - if [ -n "$GO_MODULE_DIR" ]; then + if [ "$ENABLE_MULTI_MODULE_TESTING" == "true" ]; then + echo "🔧 Multi-module testing enabled - running from repository root" + echo "đŸ“Ļ magex will discover all Go modules and merge coverage" + $MAGEX_CMD 2>&1 | tee test-output.log + elif [ -n "$GO_MODULE_DIR" ]; then echo "🔧 Running from directory: $GO_MODULE_DIR" (cd "$GO_MODULE_DIR" && $MAGEX_CMD) 2>&1 | tee test-output.log else @@ -342,7 +347,11 @@ jobs: elif [[ "$MODE" == "FAILURES_ONLY" ]]; then echo "📝 Using FAILURES_ONLY mode - JSON output with failure extraction" echo "🔍 Executing: $MAGEX_CMD" - if [ -n "$GO_MODULE_DIR" ]; then + if [ "$ENABLE_MULTI_MODULE_TESTING" == "true" ]; then + echo "🔧 Multi-module testing enabled - running from repository root" + echo "đŸ“Ļ magex will discover all Go modules and merge coverage" + $MAGEX_CMD 2>&1 | tee test-output.log + elif [ -n "$GO_MODULE_DIR" ]; then echo "🔧 Running from directory: $GO_MODULE_DIR" (cd "$GO_MODULE_DIR" && $MAGEX_CMD) 2>&1 | tee test-output.log else diff --git a/.github/workflows/fortress-warm-cache.yml b/.github/workflows/fortress-warm-cache.yml index 7db1b62..10012d5 100644 --- a/.github/workflows/fortress-warm-cache.yml +++ b/.github/workflows/fortress-warm-cache.yml @@ -77,9 +77,41 @@ jobs: done # -------------------------------------------------------------------- - # Checkout code to access local action + # Derive go.mod path from go.sum path for sparse checkout # -------------------------------------------------------------------- - - name: đŸ“Ĩ Checkout code + - name: 🔧 Derive module paths + id: module-paths + env: + GO_SUM_FILE: ${{ inputs.go-sum-file }} + run: | + # Derive go.mod path from go.sum path (same directory) + GO_MOD_FILE="${GO_SUM_FILE%go.sum}go.mod" + echo "go_mod_file=$GO_MOD_FILE" >> "$GITHUB_OUTPUT" + echo "📁 Go module file: $GO_MOD_FILE" + echo "📁 Go sum file: $GO_SUM_FILE" + + # -------------------------------------------------------------------- + # Extract configuration flags from env-json (before checkout) + # -------------------------------------------------------------------- + - name: 🔁 Extract configuration flags + id: extract + run: | + echo "enable_verbose=$(echo '${{ inputs.env-json }}' | jq -r '.ENABLE_VERBOSE_TEST_OUTPUT')" >> "$GITHUB_OUTPUT" + echo "enable_multi_module=$(echo '${{ inputs.env-json }}' | jq -r '.ENABLE_MULTI_MODULE_TESTING // "false"')" >> "$GITHUB_OUTPUT" + echo "📋 Multi-module testing: $(echo '${{ inputs.env-json }}' | jq -r '.ENABLE_MULTI_MODULE_TESTING // "false"')" + + # -------------------------------------------------------------------- + # Checkout code - full checkout for multi-module, sparse for single + # Multi-module needs all source files + go.work for cross-module deps + # -------------------------------------------------------------------- + - name: đŸ“Ĩ Checkout code (full - multi-module) + if: steps.extract.outputs.enable_multi_module == 'true' + uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 + with: + persist-credentials: false + + - name: đŸ“Ĩ Checkout code (sparse - single module) + if: steps.extract.outputs.enable_multi_module != 'true' uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0 with: persist-credentials: false @@ -90,17 +122,17 @@ jobs: .github/actions/setup-go-with-cache .github/actions/setup-magex .github/.env.base - go.mod - ${{ env.GO_SUM_FILE }} + ${{ steps.module-paths.outputs.go_mod_file }} + ${{ inputs.go-sum-file }} .mage.yaml # -------------------------------------------------------------------- - # Extract verbose flag from env-json + # Disable Go workspace mode (single module only) + # Multi-module mode uses go.work for cross-module dependency resolution # -------------------------------------------------------------------- - - name: 🔁 Extract ENABLE_VERBOSE - id: extract - run: | - echo "enable_verbose=$(echo '${{ inputs.env-json }}' | jq -r '.ENABLE_VERBOSE_TEST_OUTPUT')" >> "$GITHUB_OUTPUT" + - name: 🔧 Disable Go workspace mode + if: steps.extract.outputs.enable_multi_module != 'true' + run: echo "GOWORK=off" >> $GITHUB_ENV # -------------------------------------------------------------------- # Warm the Go and Redis caches using local action @@ -119,3 +151,4 @@ jobs: redis-versions: ${{ inputs.redis-version }} redis-cache-force-pull: ${{ inputs.redis-cache-force-pull }} go-sum-file: ${{ inputs.go-sum-file }} + enable-multi-module: ${{ steps.extract.outputs.enable_multi_module }}