Speed up CI: run static analysis in parallel with Docker builds

.github/workflows/build-php-datatypes.yml

@@ -31,4 +31,4 @@ jobs:
           phpVersion: ${{ matrix.php-versions }}
         run: |
           composer install
-          composer ci
+          composer ci-tests

.github/workflows/build-php-db-import-export.yml

@@ -128,7 +128,7 @@ jobs:
       - name: Check
         run: |
           docker compose run ci-php-db-import-export php -v
-          docker compose run ci-php-db-import-export composer ci
+          docker compose run ci-php-db-import-export composer ci-tests
 
   # Load stubs to S3/ABS
   load-s3:

.github/workflows/build-php-storage-driver-common.yml

@@ -50,29 +50,11 @@ jobs:
         env:
           AWS_S3_BUCKET: ${{ secrets.S3_BUCKET }}
           AWS_REGION: ${{ secrets.S3_AWS_REGION }}
-        run: docker compose run --rm ci-php-storage-driver-common composer ci
-  protoc:
-    runs-on: ubuntu-latest
-    if: ${{ inputs.hasCodeChanged || inputs.isTag }}
-    permissions:
-      id-token: write
-      contents: read
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v1
-      -
-        name: Configure AWS Credentials
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-to-assume: ${{ env.AWS_CREDENTIALS_ROLE_TO_ASSUME }}
-          aws-region: ${{ env.AWS_CREDENTIALS_REGION }}
-      - name: Build Image
-        env:
-          DOCKER_BUILDKIT: 1
-        run: docker compose build dev-php-storage-driver-common
+        run: docker compose run --rm ci-php-storage-driver-common composer ci-tests
 
-      - name: Check generated files
+      - name: Check generated protobuf files
         run: |
+          docker compose build dev-php-storage-driver-common
           docker compose run --rm dev-php-storage-driver-common /bin/bash ./etc/ci/scripts/generate_tmp_proto.sh
           cd packages/php-storage-driver-common/
           diff -rq tmp_generated generated

.github/workflows/build-php-storage-driver-snowflake.yml

@@ -42,4 +42,4 @@ jobs:
         run: docker compose build ci-php-storage-driver-snowflake
 
       - name: Run tests
-        run: docker compose run --rm ci-php-storage-driver-snowflake composer ci
+        run: docker compose run --rm ci-php-storage-driver-snowflake composer ci-tests

.github/workflows/build-php-table-backend-utils.yml

@@ -79,7 +79,7 @@ jobs:
 
         run: |
           docker compose build $LOCAL_IMAGE
-          docker compose run $LOCAL_IMAGE composer check
+          docker compose run $LOCAL_IMAGE composer ci-tests
       -
         name: Push docker image
         env:

.github/workflows/main.yml

@@ -14,11 +14,8 @@ env:
   AWS_CREDENTIALS_ROLE_TO_ASSUME: arn:aws:iam::149899208592:role/ci-storage-backend-BaseStorageBackendRole-5WXOY9DYENCT
   MAIN_BRANCH: main
 jobs:
-  build:
+  paths_filter:
     runs-on: ubuntu-latest
-    permissions:
-      id-token: write
-      contents: read
     outputs:
       changed-php-datatypes: ${{ steps.changes.outputs.php-datatypes }}
       changed-php-table-backend-utils: ${{ steps.changes.outputs.php-table-backend-utils }}
@@ -40,14 +37,6 @@ jobs:
           echo "github.ref_type: ${{ github.ref_type }}"
           echo "isReleaseTag=$(if [ "$(git rev-parse origin/$MAIN_BRANCH)" == "$(git rev-parse HEAD)" ] && ${{ github.ref_type == 'tag' }}; then echo 1; else echo 0; fi)" >> "$GITHUB_OUTPUT"
           echo "isReleaseTag=$(if [ "$(git rev-parse origin/$MAIN_BRANCH)" == "$(git rev-parse HEAD)" ] && ${{ github.ref_type == 'tag' }}; then echo 1; else echo 0; fi)"
-      -
-        name: Configure AWS Credentials
-        id: creds
-        uses: aws-actions/configure-aws-credentials@v4
-        with:
-          role-duration-seconds: 900
-          role-to-assume: ${{ env.AWS_CREDENTIALS_ROLE_TO_ASSUME }}
-          aws-region: ${{ env.AWS_CREDENTIALS_REGION }}
 
       - uses: dorny/paths-filter@v3
         id: changes
@@ -69,6 +58,34 @@ jobs:
               - 'packages/php-datatypes/**'
               - 'packages/php-table-backend-utils/**'
 
+  static_analysis:
+    if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }}
+    needs: paths_filter
+    uses: ./.github/workflows/static-analysis.yml
+    with:
+      changed-php-datatypes: ${{ needs.paths_filter.outputs.changed-php-datatypes == 'true' }}
+      changed-php-table-backend-utils: ${{ needs.paths_filter.outputs.changed-php-table-backend-utils == 'true' }}
+      changed-php-db-import-export: ${{ needs.paths_filter.outputs.changed-php-db-import-export == 'true' }}
+      changed-php-storage-driver-common: ${{ needs.paths_filter.outputs.changed-php-storage-driver-common == 'true' }}
+      changed-php-storage-driver-snowflake: ${{ needs.paths_filter.outputs.changed-php-storage-driver-snowflake == 'true' }}
+
+  build:
+    needs: paths_filter
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - uses: actions/checkout@v4
+      -
+        name: Configure AWS Credentials
+        id: creds
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-duration-seconds: 900
+          role-to-assume: ${{ env.AWS_CREDENTIALS_ROLE_TO_ASSUME }}
+          aws-region: ${{ env.AWS_CREDENTIALS_REGION }}
+
       - name: Login to Amazon ECR
         id: login-ecr
         uses: aws-actions/amazon-ecr-login@v2
@@ -88,33 +105,33 @@ jobs:
           docker push $REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG
 
   build_datatypes:
-    if: ${{ needs.build.outputs.isReleaseTag == 0 }}
+    if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }}
     uses: ./.github/workflows/build-php-datatypes.yml
     with:
-      hasCodeChanged: ${{ needs.build.outputs.changed-php-datatypes == 'true' }}
+      hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-datatypes == 'true' }}
       isTag: ${{ startsWith(github.ref, 'refs/tags/') }}
-    needs: build
+    needs: [paths_filter, build]
 
   build_php_table_backend_utils:
-    if: ${{ needs.build.outputs.isReleaseTag == 0 }}
+    if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }}
     uses: ./.github/workflows/build-php-table-backend-utils.yml
     with:
-      hasCodeChanged: ${{ needs.build.outputs.changed-php-table-backend-utils == 'true' }}
+      hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-table-backend-utils == 'true' }}
       isTag: ${{ startsWith(github.ref, 'refs/tags/') }}
-      isRequiredRepoChanged: ${{ needs.build.outputs.table-utils-requirements== 'true' }}
+      isRequiredRepoChanged: ${{ needs.paths_filter.outputs.table-utils-requirements == 'true' }}
     secrets:
       SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }}
       SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }}
       BQ_KEY_FILE: ${{ secrets.BQ_KEY_FILE }}
-    needs: build
+    needs: [paths_filter, build]
 
   build_php_db_import_export:
-    if: ${{ needs.build.outputs.isReleaseTag == 0 }}
+    if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }}
     uses: ./.github/workflows/build-php-db-import-export.yml
     with:
-      hasCodeChanged: ${{ needs.build.outputs.changed-php-db-import-export == 'true' }}
+      hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-db-import-export == 'true' }}
       isTag: ${{ startsWith(github.ref, 'refs/tags/') }}
-      isRequiredRepoChanged: ${{ needs.build.outputs.db-import-export-requirements == 'true' }}
+      isRequiredRepoChanged: ${{ needs.paths_filter.outputs.db-import-export-requirements == 'true' }}
     secrets:
       AWS_SECRET_ACCESS_KEY: ${{ secrets.IE_AWS_SECRET_ACCESS_KEY }}
       ABS_ACCOUNT_KEY: ${{ secrets.ABS_ACCOUNT_KEY }}
@@ -123,32 +140,33 @@ jobs:
       BQ_KEY_FILE: ${{ secrets.IE_BQ_KEY_FILE }}
       OAUTH_TOKEN_GITHUB: ${{ secrets.IE_OAUTH_TOKEN_GITHUB }}
       GCS_CREDENTIALS: ${{ secrets.IE_GCS_CREDENTIALS }}
-    needs: build
+    needs: [paths_filter, build]
 
   build_php_storage_driver_common:
-    if: ${{ needs.build.outputs.isReleaseTag == 0 }}
+    if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }}
     uses: ./.github/workflows/build-php-storage-driver-common.yml
     with:
-      hasCodeChanged: ${{ needs.build.outputs.changed-php-storage-driver-common == 'true' }}
+      hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-storage-driver-common == 'true' }}
       isTag: ${{ startsWith(github.ref, 'refs/tags/') }}
     secrets:
       S3_BUCKET: ${{ secrets.S3_BUCKET }}
       S3_AWS_REGION: ${{ secrets.S3_AWS_REGION }}
-    needs: build
+    needs: [paths_filter, build]
 
   build_php_storage_driver_snowflake:
-    if: ${{ needs.build.outputs.isReleaseTag == 0 }}
+    if: ${{ needs.paths_filter.outputs.isReleaseTag == '0' }}
     uses: ./.github/workflows/build-php-storage-driver-snowflake.yml
     with:
-      hasCodeChanged: ${{ needs.build.outputs.changed-php-storage-driver-snowflake == 'true' }}
+      hasCodeChanged: ${{ needs.paths_filter.outputs.changed-php-storage-driver-snowflake == 'true' }}
       isTag: ${{ startsWith(github.ref, 'refs/tags/') }}
     secrets:
       SNOWFLAKE_PASSWORD: ${{ secrets.PHP_STORAGE_DRIVER_SNOWFLAKE_SNOWFLAKE_PASSWORD }}
       SNOWFLAKE_PRIVATE_KEY: ${{ secrets.PHP_STORAGE_DRIVER_SNOWFLAKE_SNOWFLAKE_PRIVATE_KEY }}
-    needs: build
+    needs: [paths_filter, build]
 
   test_results:
     needs:
+        - static_analysis
         - build_datatypes
         - build_php_table_backend_utils
         - build_php_db_import_export
@@ -157,7 +175,8 @@ jobs:
     runs-on: ubuntu-latest
     if: |
       always()
-        && contains(fromJson('["success", "skipped"]'), needs.build_datatypes.result) 
+        && contains(fromJson('["success", "skipped"]'), needs.static_analysis.result)
+        && contains(fromJson('["success", "skipped"]'), needs.build_datatypes.result)
         && contains(fromJson('["success", "skipped"]'), needs.build_php_table_backend_utils.result)
         && contains(fromJson('["success", "skipped"]'), needs.build_php_db_import_export.result)
         && contains(fromJson('["success", "skipped"]'), needs.build_php_storage_driver_common.result)

.github/workflows/static-analysis.yml

@@ -0,0 +1,99 @@
+name: Static Analysis
+
+on:
+  workflow_call:
+    inputs:
+      changed-php-datatypes:
+        required: true
+        type: boolean
+      changed-php-table-backend-utils:
+        required: true
+        type: boolean
+      changed-php-db-import-export:
+        required: true
+        type: boolean
+      changed-php-storage-driver-common:
+        required: true
+        type: boolean
+      changed-php-storage-driver-snowflake:
+        required: true
+        type: boolean
+
+jobs:
+  static_analysis:
+    runs-on: ubuntu-latest
+    if: >-
+      inputs.changed-php-datatypes
+      || inputs.changed-php-table-backend-utils
+      || inputs.changed-php-db-import-export
+      || inputs.changed-php-storage-driver-common
+      || inputs.changed-php-storage-driver-snowflake
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install system dependencies
+        run: sudo apt-get install -y unixodbc-dev
+
+      - name: Setup PHP
+        uses: shivammathur/setup-php@v2
+        with:
+          php-version: '8.3'
+          extensions: intl, bcmath, pdo, odbc
+
+      - name: Cache Composer dependencies
+        uses: actions/cache@v4
+        with:
+          path: vendor
+          key: composer-${{ hashFiles('composer.json') }}
+          restore-keys: composer-
+
+      - name: Install monorepo dependencies
+        run: composer install --no-interaction
+
+      - name: Validate monorepo
+        run: vendor/bin/monorepo-builder validate
+
+      - name: Static analysis - php-datatypes
+        if: ${{ inputs.changed-php-datatypes }}
+        working-directory: packages/php-datatypes
+        run: |
+          composer install --no-interaction
+          composer phplint
+          composer phpcs
+          composer phpstan
+
+      - name: Static analysis - php-table-backend-utils
+        if: ${{ inputs.changed-php-table-backend-utils }}
+        working-directory: packages/php-table-backend-utils
+        run: |
+          composer install --no-interaction
+          composer phplint
+          composer phpcs
+          composer phpstan
+
+      - name: Static analysis - php-db-import-export
+        if: ${{ inputs.changed-php-db-import-export }}
+        working-directory: packages/php-db-import-export
+        run: |
+          composer install --no-interaction
+          composer phplint
+          composer phpcs
+          composer phpstan
+
+      - name: Static analysis - php-storage-driver-common
+        if: ${{ inputs.changed-php-storage-driver-common }}
+        working-directory: packages/php-storage-driver-common
+        run: |
+          composer install --no-interaction
+          composer phplint
+          composer phpcs
+          composer phpstan
+
+      - name: Static analysis - php-storage-driver-snowflake
+        if: ${{ inputs.changed-php-storage-driver-snowflake }}
+        working-directory: packages/php-storage-driver-snowflake
+        run: |
+          composer install --no-interaction
+          composer phplint
+          composer phpcs
+          composer phpstan

packages/php-datatypes/composer.json

@@ -45,6 +45,10 @@
         "ci": [
             "@composer validate --no-check-all --strict",
             "@build"
+        ],
+        "ci-tests": [
+            "@composer validate --no-check-all --strict",
+            "@tests"
         ]
     },
     "config": {

packages/php-db-import-export/composer.json

@@ -91,6 +91,10 @@
         "ci": [
             "@composer validate --no-check-publish --no-check-all",
             "@build"
+        ],
+        "ci-tests": [
+            "@composer validate --no-check-publish --no-check-all",
+            "@tests-unit"
         ]
     },
     "config": {

packages/php-storage-driver-common/composer.json

@@ -75,6 +75,10 @@
         "ci": [
             "@check",
             "@tests"
+        ],
+        "ci-tests": [
+            "@composer validate --no-check-all --strict",
+            "@tests"
         ]
     }
 }

packages/php-storage-driver-snowflake/composer.json

@@ -18,6 +18,7 @@
     },
     "require-dev": {
         "keboola/coding-standard": "^15",
+        "php-parallel-lint/php-parallel-lint": "^1.3",
         "phpstan/phpstan": "^1.4",
         "phpstan/phpstan-phpunit": "^1",
         "phpunit/phpunit": "^9"
@@ -39,17 +40,23 @@
             "@tests-unit",
             "@tests-functional"
         ],
+        "phplint": "parallel-lint -j 10 --exclude vendor .",
         "phpcs": "phpcs -n --ignore=vendor --extensions=php .",
         "phpcbf": "phpcbf -s -n --extensions=php .",
         "phpstan": "phpstan analyse ./src ./tests --level=max --no-progress -c phpstan.neon",
         "check": [
             "@composer validate --no-check-all --strict",
+            "@phplint",
             "@phpcs",
             "@phpstan"
         ],
         "ci": [
             "@check",
             "@tests"
+        ],
+        "ci-tests": [
+            "@composer validate --no-check-all --strict",
+            "@tests"
         ]
     },
     "config": {

packages/php-table-backend-utils/composer.json

@@ -60,6 +60,10 @@
             "@phpcs",
             "@phpstan",
             "@tests-unit"
+        ],
+        "ci-tests": [
+            "@composer validate --no-check-publish --no-check-all",
+            "@tests-unit"
         ]
     },
     "config": {