From 8fede450a250d4a74ff637c09653ab92d2bbdbe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vojt=C4=9Bch=20Biberle?= Date: Mon, 16 Feb 2026 18:50:46 +0100 Subject: [PATCH] Speed up CI by running static analysis in parallel with Docker builds Split the build job into a lightweight paths_filter job and a Docker-only build job. Static analysis (phplint, phpcs, phpstan) now runs natively via shivammathur/setup-php in parallel with the Docker image build, eliminating the 5+ min Docker wait for lint/type errors. - Add static-analysis.yml reusable workflow with native PHP setup - Add ci-tests composer script to each package (validate + tests only) - Switch package workflows from composer ci/check to ci-tests - Merge redundant protoc Docker build into build_common_driver job - Add phplint to php-storage-driver-snowflake package - Gate test_results on static_analysis + all package builds Co-Authored-By: Claude Opus 4.6 --- .github/workflows/build-php-datatypes.yml | 2 +- .../workflows/build-php-db-import-export.yml | 2 +- .../build-php-storage-driver-common.yml | 24 +---- .../build-php-storage-driver-snowflake.yml | 2 +- .../build-php-table-backend-utils.yml | 2 +- .github/workflows/main.yml | 79 +++++++++------ .github/workflows/static-analysis.yml | 99 +++++++++++++++++++ packages/php-datatypes/composer.json | 4 + packages/php-db-import-export/composer.json | 4 + .../php-storage-driver-common/composer.json | 4 + .../composer.json | 7 ++ .../php-table-backend-utils/composer.json | 4 + 12 files changed, 178 insertions(+), 55 deletions(-) create mode 100644 .github/workflows/static-analysis.yml diff --git a/.github/workflows/build-php-datatypes.yml b/.github/workflows/build-php-datatypes.yml index 9a407bb02..6180964a5 100644 --- a/.github/workflows/build-php-datatypes.yml +++ b/.github/workflows/build-php-datatypes.yml @@ -31,4 +31,4 @@ jobs: phpVersion: ${{ matrix.php-versions }} run: | composer install - composer ci + composer ci-tests diff --git a/.github/workflows/build-php-db-import-export.yml b/.github/workflows/build-php-db-import-export.yml index acaaa1ee1..b4a4dd9bb 100644 --- a/.github/workflows/build-php-db-import-export.yml +++ b/.github/workflows/build-php-db-import-export.yml @@ -128,7 +128,7 @@ jobs: - name: Check run: | docker compose run ci-php-db-import-export php -v - docker compose run ci-php-db-import-export composer ci + docker compose run ci-php-db-import-export composer ci-tests # Load stubs to S3/ABS load-s3: diff --git a/.github/workflows/build-php-storage-driver-common.yml b/.github/workflows/build-php-storage-driver-common.yml index 0c0e0b3f5..f519a7326 100644 --- a/.github/workflows/build-php-storage-driver-common.yml +++ b/.github/workflows/build-php-storage-driver-common.yml @@ -50,29 +50,11 @@ jobs: env: AWS_S3_BUCKET: ${{ secrets.S3_BUCKET }} AWS_REGION: ${{ secrets.S3_AWS_REGION }} - run: docker compose run --rm ci-php-storage-driver-common composer ci - protoc: - runs-on: ubuntu-latest - if: ${{ inputs.hasCodeChanged || inputs.isTag }} - permissions: - id-token: write - contents: read - steps: - - name: Checkout Code - uses: actions/checkout@v1 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.AWS_CREDENTIALS_ROLE_TO_ASSUME }} - aws-region: ${{ env.AWS_CREDENTIALS_REGION }} - - name: Build Image - env: - DOCKER_BUILDKIT: 1 - run: docker compose build dev-php-storage-driver-common + run: docker compose run --rm ci-php-storage-driver-common composer ci-tests - - name: Check generated files + - name: Check generated protobuf files run: | + docker compose build dev-php-storage-driver-common docker compose run --rm dev-php-storage-driver-common /bin/bash ./etc/ci/scripts/generate_tmp_proto.sh cd packages/php-storage-driver-common/ diff -rq tmp_generated generated diff --git a/.github/workflows/build-php-storage-driver-snowflake.yml b/.github/workflows/build-php-storage-driver-snowflake.yml index f6c2e0ada..602f6a71b 100644 --- a/.github/workflows/build-php-storage-driver-snowflake.yml +++ b/.github/workflows/build-php-storage-driver-snowflake.yml @@ -42,4 +42,4 @@ jobs: run: docker compose build ci-php-storage-driver-snowflake - name: Run tests - run: docker compose run --rm ci-php-storage-driver-snowflake composer ci + run: docker compose run --rm ci-php-storage-driver-snowflake composer ci-tests diff --git a/.github/workflows/build-php-table-backend-utils.yml b/.github/workflows/build-php-table-backend-utils.yml index 732913bc3..864da9078 100644 --- a/.github/workflows/build-php-table-backend-utils.yml +++ b/.github/workflows/build-php-table-backend-utils.yml @@ -79,7 +79,7 @@ jobs: run: | docker compose build $LOCAL_IMAGE - docker compose run $LOCAL_IMAGE composer check + docker compose run $LOCAL_IMAGE composer ci-tests - name: Push docker image env: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 041e6e2f1..c9c50a51e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -14,11 +14,8 @@ env: AWS_CREDENTIALS_ROLE_TO_ASSUME: arn:aws:iam::149899208592:role/ci-storage-backend-BaseStorageBackendRole-5WXOY9DYENCT MAIN_BRANCH: main jobs: - build: + paths_filter: runs-on: ubuntu-latest - permissions: - id-token: write - contents: read outputs: changed-php-datatypes: ${{ steps.changes.outputs.php-datatypes }} changed-php-table-backend-utils: ${{ steps.changes.outputs.php-table-backend-utils }} @@ -40,14 +37,6 @@ jobs: echo "github.ref_type: ${{ github.ref_type }}" echo "isReleaseTag=$(if [ "$(git rev-parse origin/$MAIN_BRANCH)" == "$(git rev-parse HEAD)" ] && ${{ github.ref_type == 'tag' }}; then echo 1; else echo 0; fi)" >> "$GITHUB_OUTPUT" echo "isReleaseTag=$(if [ "$(git rev-parse origin/$MAIN_BRANCH)" == "$(git rev-parse HEAD)" ] && ${{ github.ref_type == 'tag' }}; then echo 1; else echo 0; fi)" - - - name: Configure AWS Credentials - id: creds - uses: aws-actions/configure-aws-credentials@v4 - with: - role-duration-seconds: 900 - role-to-assume: ${{ env.AWS_CREDENTIALS_ROLE_TO_ASSUME }} - aws-region: ${{ env.AWS_CREDENTIALS_REGION }} - uses: dorny/paths-filter@v3 id: changes @@ -69,6 +58,34 @@ jobs: - 'packages/php-datatypes/**' - 'packages/php-table-backend-utils/**' + static_analysis: + if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }} + needs: paths_filter + uses: ./.github/workflows/static-analysis.yml + with: + changed-php-datatypes: ${{ needs.paths_filter.outputs.changed-php-datatypes == 'true' }} + changed-php-table-backend-utils: ${{ needs.paths_filter.outputs.changed-php-table-backend-utils == 'true' }} + changed-php-db-import-export: ${{ needs.paths_filter.outputs.changed-php-db-import-export == 'true' }} + changed-php-storage-driver-common: ${{ needs.paths_filter.outputs.changed-php-storage-driver-common == 'true' }} + changed-php-storage-driver-snowflake: ${{ needs.paths_filter.outputs.changed-php-storage-driver-snowflake == 'true' }} + + build: + needs: paths_filter + runs-on: ubuntu-latest + permissions: + id-token: write + contents: read + steps: + - uses: actions/checkout@v4 + - + name: Configure AWS Credentials + id: creds + uses: aws-actions/configure-aws-credentials@v4 + with: + role-duration-seconds: 900 + role-to-assume: ${{ env.AWS_CREDENTIALS_ROLE_TO_ASSUME }} + aws-region: ${{ env.AWS_CREDENTIALS_REGION }} + - name: Login to Amazon ECR id: login-ecr uses: aws-actions/amazon-ecr-login@v2 @@ -88,33 +105,33 @@ jobs: docker push $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG build_datatypes: - if: ${{ needs.build.outputs.isReleaseTag == 0 }} + if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }} uses: ./.github/workflows/build-php-datatypes.yml with: - hasCodeChanged: ${{ needs.build.outputs.changed-php-datatypes == 'true' }} + hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-datatypes == 'true' }} isTag: ${{ startsWith(github.ref, 'refs/tags/') }} - needs: build + needs: [paths_filter, build] build_php_table_backend_utils: - if: ${{ needs.build.outputs.isReleaseTag == 0 }} + if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }} uses: ./.github/workflows/build-php-table-backend-utils.yml with: - hasCodeChanged: ${{ needs.build.outputs.changed-php-table-backend-utils == 'true' }} + hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-table-backend-utils == 'true' }} isTag: ${{ startsWith(github.ref, 'refs/tags/') }} - isRequiredRepoChanged: ${{ needs.build.outputs.table-utils-requirements== 'true' }} + isRequiredRepoChanged: ${{ needs.paths_filter.outputs.table-utils-requirements == 'true' }} secrets: SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} BQ_KEY_FILE: ${{ secrets.BQ_KEY_FILE }} - needs: build + needs: [paths_filter, build] build_php_db_import_export: - if: ${{ needs.build.outputs.isReleaseTag == 0 }} + if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }} uses: ./.github/workflows/build-php-db-import-export.yml with: - hasCodeChanged: ${{ needs.build.outputs.changed-php-db-import-export == 'true' }} + hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-db-import-export == 'true' }} isTag: ${{ startsWith(github.ref, 'refs/tags/') }} - isRequiredRepoChanged: ${{ needs.build.outputs.db-import-export-requirements == 'true' }} + isRequiredRepoChanged: ${{ needs.paths_filter.outputs.db-import-export-requirements == 'true' }} secrets: AWS_SECRET_ACCESS_KEY: ${{ secrets.IE_AWS_SECRET_ACCESS_KEY }} ABS_ACCOUNT_KEY: ${{ secrets.ABS_ACCOUNT_KEY }} @@ -123,32 +140,33 @@ jobs: BQ_KEY_FILE: ${{ secrets.IE_BQ_KEY_FILE }} OAUTH_TOKEN_GITHUB: ${{ secrets.IE_OAUTH_TOKEN_GITHUB }} GCS_CREDENTIALS: ${{ secrets.IE_GCS_CREDENTIALS }} - needs: build + needs: [paths_filter, build] build_php_storage_driver_common: - if: ${{ needs.build.outputs.isReleaseTag == 0 }} + if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }} uses: ./.github/workflows/build-php-storage-driver-common.yml with: - hasCodeChanged: ${{ needs.build.outputs.changed-php-storage-driver-common == 'true' }} + hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-storage-driver-common == 'true' }} isTag: ${{ startsWith(github.ref, 'refs/tags/') }} secrets: S3_BUCKET: ${{ secrets.S3_BUCKET }} S3_AWS_REGION: ${{ secrets.S3_AWS_REGION }} - needs: build + needs: [paths_filter, build] build_php_storage_driver_snowflake: - if: ${{ needs.build.outputs.isReleaseTag == 0 }} + if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }} uses: ./.github/workflows/build-php-storage-driver-snowflake.yml with: - hasCodeChanged: ${{ needs.build.outputs.changed-php-storage-driver-snowflake == 'true' }} + hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-storage-driver-snowflake == 'true' }} isTag: ${{ startsWith(github.ref, 'refs/tags/') }} secrets: SNOWFLAKE_PASSWORD: ${{ secrets.PHP_STORAGE_DRIVER_SNOWFLAKE_SNOWFLAKE_PASSWORD }} SNOWFLAKE_PRIVATE_KEY: ${{ secrets.PHP_STORAGE_DRIVER_SNOWFLAKE_SNOWFLAKE_PRIVATE_KEY }} - needs: build + needs: [paths_filter, build] test_results: needs: + - static_analysis - build_datatypes - build_php_table_backend_utils - build_php_db_import_export @@ -157,7 +175,8 @@ jobs: runs-on: ubuntu-latest if: | always() - && contains(fromJson('["success", "skipped"]'), needs.build_datatypes.result) + && contains(fromJson('["success", "skipped"]'), needs.static_analysis.result) + && contains(fromJson('["success", "skipped"]'), needs.build_datatypes.result) && contains(fromJson('["success", "skipped"]'), needs.build_php_table_backend_utils.result) && contains(fromJson('["success", "skipped"]'), needs.build_php_db_import_export.result) && contains(fromJson('["success", "skipped"]'), needs.build_php_storage_driver_common.result) diff --git a/.github/workflows/static-analysis.yml b/.github/workflows/static-analysis.yml new file mode 100644 index 000000000..422eab074 --- /dev/null +++ b/.github/workflows/static-analysis.yml @@ -0,0 +1,99 @@ +name: Static Analysis + +on: + workflow_call: + inputs: + changed-php-datatypes: + required: true + type: boolean + changed-php-table-backend-utils: + required: true + type: boolean + changed-php-db-import-export: + required: true + type: boolean + changed-php-storage-driver-common: + required: true + type: boolean + changed-php-storage-driver-snowflake: + required: true + type: boolean + +jobs: + static_analysis: + runs-on: ubuntu-latest + if: >- + inputs.changed-php-datatypes + || inputs.changed-php-table-backend-utils + || inputs.changed-php-db-import-export + || inputs.changed-php-storage-driver-common + || inputs.changed-php-storage-driver-snowflake + steps: + - uses: actions/checkout@v4 + + - name: Install system dependencies + run: sudo apt-get install -y unixodbc-dev + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: '8.3' + extensions: intl, bcmath, pdo, odbc + + - name: Cache Composer dependencies + uses: actions/cache@v4 + with: + path: vendor + key: composer-${{ hashFiles('composer.json') }} + restore-keys: composer- + + - name: Install monorepo dependencies + run: composer install --no-interaction + + - name: Validate monorepo + run: vendor/bin/monorepo-builder validate + + - name: Static analysis - php-datatypes + if: ${{ inputs.changed-php-datatypes }} + working-directory: packages/php-datatypes + run: | + composer install --no-interaction + composer phplint + composer phpcs + composer phpstan + + - name: Static analysis - php-table-backend-utils + if: ${{ inputs.changed-php-table-backend-utils }} + working-directory: packages/php-table-backend-utils + run: | + composer install --no-interaction + composer phplint + composer phpcs + composer phpstan + + - name: Static analysis - php-db-import-export + if: ${{ inputs.changed-php-db-import-export }} + working-directory: packages/php-db-import-export + run: | + composer install --no-interaction + composer phplint + composer phpcs + composer phpstan + + - name: Static analysis - php-storage-driver-common + if: ${{ inputs.changed-php-storage-driver-common }} + working-directory: packages/php-storage-driver-common + run: | + composer install --no-interaction + composer phplint + composer phpcs + composer phpstan + + - name: Static analysis - php-storage-driver-snowflake + if: ${{ inputs.changed-php-storage-driver-snowflake }} + working-directory: packages/php-storage-driver-snowflake + run: | + composer install --no-interaction + composer phplint + composer phpcs + composer phpstan diff --git a/packages/php-datatypes/composer.json b/packages/php-datatypes/composer.json index c323f4fe2..55b19a680 100644 --- a/packages/php-datatypes/composer.json +++ b/packages/php-datatypes/composer.json @@ -45,6 +45,10 @@ "ci": [ "@composer validate --no-check-all --strict", "@build" + ], + "ci-tests": [ + "@composer validate --no-check-all --strict", + "@tests" ] }, "config": { diff --git a/packages/php-db-import-export/composer.json b/packages/php-db-import-export/composer.json index 2161f211b..5a8ba25f3 100644 --- a/packages/php-db-import-export/composer.json +++ b/packages/php-db-import-export/composer.json @@ -91,6 +91,10 @@ "ci": [ "@composer validate --no-check-publish --no-check-all", "@build" + ], + "ci-tests": [ + "@composer validate --no-check-publish --no-check-all", + "@tests-unit" ] }, "config": { diff --git a/packages/php-storage-driver-common/composer.json b/packages/php-storage-driver-common/composer.json index 2c3f401ff..2ec766181 100644 --- a/packages/php-storage-driver-common/composer.json +++ b/packages/php-storage-driver-common/composer.json @@ -75,6 +75,10 @@ "ci": [ "@check", "@tests" + ], + "ci-tests": [ + "@composer validate --no-check-all --strict", + "@tests" ] } } diff --git a/packages/php-storage-driver-snowflake/composer.json b/packages/php-storage-driver-snowflake/composer.json index 9312e2629..bdf513521 100644 --- a/packages/php-storage-driver-snowflake/composer.json +++ b/packages/php-storage-driver-snowflake/composer.json @@ -18,6 +18,7 @@ }, "require-dev": { "keboola/coding-standard": "^15", + "php-parallel-lint/php-parallel-lint": "^1.3", "phpstan/phpstan": "^1.4", "phpstan/phpstan-phpunit": "^1", "phpunit/phpunit": "^9" @@ -39,17 +40,23 @@ "@tests-unit", "@tests-functional" ], + "phplint": "parallel-lint -j 10 --exclude vendor .", "phpcs": "phpcs -n --ignore=vendor --extensions=php .", "phpcbf": "phpcbf -s -n --extensions=php .", "phpstan": "phpstan analyse ./src ./tests --level=max --no-progress -c phpstan.neon", "check": [ "@composer validate --no-check-all --strict", + "@phplint", "@phpcs", "@phpstan" ], "ci": [ "@check", "@tests" + ], + "ci-tests": [ + "@composer validate --no-check-all --strict", + "@tests" ] }, "config": { diff --git a/packages/php-table-backend-utils/composer.json b/packages/php-table-backend-utils/composer.json index b40580959..f3ce2378e 100644 --- a/packages/php-table-backend-utils/composer.json +++ b/packages/php-table-backend-utils/composer.json @@ -60,6 +60,10 @@ "@phpcs", "@phpstan", "@tests-unit" + ], + "ci-tests": [ + "@composer validate --no-check-publish --no-check-all", + "@tests-unit" ] }, "config": {