T346029 ES Module (#481) #427
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Build and test Wikibase and friends | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| env_file: | |
| description: 'Environment file' | |
| required: true | |
| default: '.env' # only gets set on a "workflow_dispatch" run | |
| debug_extra_env: | |
| description: 'Lines to append to the environment file (for easy mass testing)' | |
| required: false | |
| default: '' | |
| push: | |
| branches: | |
| - "main" | |
| pull_request: | |
| paths-ignore: | |
| - 'docs/**' | |
| - '**/*.md' | |
| - '**/*.txt' | |
| env: | |
| env_file: ${{ github.event.inputs.env_file || 'versions/wmde12.env' }} | |
| jobs: | |
| build_wikibase: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Add extra DEBUG env vars | |
| if: "${{ github.event.inputs.debug_extra_env != '' }}" | |
| run: | | |
| printf "${{ github.event.inputs.debug_extra_env }}" >> ${{ env.env_file }} | |
| cat ${{ env.env_file }} | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: variables.env | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: ${{ env.env_file }} | |
| - name: Cache composer | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-wikibase-composer | |
| with: | |
| path: cache | |
| key: cache-wikibase-composer | |
| - name: Cache git_cache repos | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-wikibase-git-repo | |
| with: | |
| path: git_cache | |
| key: cache-wikibase-git-repo | |
| - name: Build Tarball | |
| run: bash build.sh wikibase ${{ env.env_file }} | |
| - name: Archive metadata artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildMetadata | |
| if-no-files-found: error | |
| path: artifacts/build_metadata_wikibase.env | |
| - name: Archive tar production artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| if-no-files-found: error | |
| path: artifacts/Wikibase.tar.gz | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: github.ref == 'refs/heads/main' | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Store release version docker image on GHCR | |
| if: github.ref == 'refs/heads/main' | |
| uses: wmde/tag-push-ghcr-action@v3 | |
| with: | |
| image_name: ${{ env.WIKIBASE_IMAGE_NAME }} | |
| tag: ${{ github.run_id }} | |
| - name: Archive base docker production artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| if-no-files-found: error | |
| path: artifacts/wikibase.docker.tar.gz | |
| - name: Scan image | |
| uses: ./.github/actions/scan-image | |
| continue-on-error: true | |
| with: | |
| image_name: ${{ env.WIKIBASE_IMAGE_NAME }} | |
| image_tag: latest | |
| build_wikibase_bundle: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| needs: | |
| - build_wikibase | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Add extra DEBUG env vars | |
| if: "${{ github.event.inputs.debug_extra_env != '' }}" | |
| run: | | |
| printf "${{ github.event.inputs.debug_extra_env }}" >> ${{ env.env_file }} | |
| cat ${{ env.env_file }} | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: variables.env | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: ${{ env.env_file }} | |
| - name: Cache composer | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-wikibase-composer | |
| with: | |
| path: cache | |
| key: cache-wikibase-composer | |
| - name: Cache git_cache repos | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-wikibase-git-repo | |
| with: | |
| path: git_cache | |
| key: cache-wikibase-git-repo | |
| - name: Get dependency build artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| path: artifacts/ | |
| - name: Build Bundle Docker Image | |
| run: bash build.sh wikibase_bundle ${{ env.env_file }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: github.ref == 'refs/heads/main' | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Store release version docker image on GHCR | |
| if: github.ref == 'refs/heads/main' | |
| uses: wmde/tag-push-ghcr-action@v3 | |
| with: | |
| image_name: ${{ env.WIKIBASE_BUNDLE_IMAGE_NAME }} | |
| tag: ${{ github.run_id }} | |
| - name: Archive bundle docker production artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| if-no-files-found: error | |
| path: artifacts/wikibase-bundle.docker.tar.gz | |
| - name: Archive metadata artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildMetadata | |
| if-no-files-found: error | |
| path: artifacts/build_metadata_*.env | |
| - name: Scan image | |
| uses: ./.github/actions/scan-image | |
| continue-on-error: true | |
| with: | |
| image_name: ${{ env.WIKIBASE_BUNDLE_IMAGE_NAME }} | |
| image_tag: latest | |
| build_quickstatements: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Add extra DEBUG env vars | |
| if: "${{ github.event.inputs.debug_extra_env != '' }}" | |
| run: | | |
| printf "${{ github.event.inputs.debug_extra_env }}" >> ${{ env.env_file }} | |
| cat ${{ env.env_file }} | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: variables.env | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: ${{ env.env_file }} | |
| - name: Build QuickStatements Image | |
| id: download_step | |
| run: bash build.sh quickstatements ${{ env.env_file }} | |
| - name: Archive metadata artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildMetadata | |
| if-no-files-found: error | |
| path: artifacts/build_metadata_*.env | |
| - name: Archive docker production artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| if-no-files-found: error | |
| path: artifacts/quickstatements.docker.tar.gz | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: github.ref == 'refs/heads/main' | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Store release version docker image on GHCR | |
| if: github.ref == 'refs/heads/main' | |
| uses: wmde/tag-push-ghcr-action@v3 | |
| with: | |
| image_name: ${{ env.QUICKSTATEMENTS_IMAGE_NAME }} | |
| tag: ${{ github.run_id }} | |
| - name: Scan image | |
| uses: ./.github/actions/scan-image | |
| continue-on-error: true | |
| with: | |
| image_name: ${{ env.QUICKSTATEMENTS_IMAGE_NAME }} | |
| image_tag: latest | |
| build_wdqs: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Add extra DEBUG env vars | |
| if: "${{ github.event.inputs.debug_extra_env != '' }}" | |
| run: | | |
| printf "${{ github.event.inputs.debug_extra_env }}" >> ${{ env.env_file }} | |
| cat ${{ env.env_file }} | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: variables.env | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: ${{ env.env_file }} | |
| - name: Cache wdqs service tar | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-wdqs-backend-git-repo | |
| with: | |
| path: cache | |
| key: cache-wdqs-backend-git-repo | |
| - name: Build WDQS Image | |
| id: download_step | |
| run: bash build.sh wdqs ${{ env.env_file }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: github.ref == 'refs/heads/main' | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Store release version docker image on GHCR | |
| if: github.ref == 'refs/heads/main' | |
| uses: wmde/tag-push-ghcr-action@v3 | |
| with: | |
| image_name: ${{ env.WDQS_IMAGE_NAME }} | |
| tag: ${{ github.run_id }} | |
| - name: Archive docker production artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| if-no-files-found: error | |
| path: artifacts/wdqs.docker.tar.gz | |
| - name: Scan image | |
| uses: ./.github/actions/scan-image | |
| continue-on-error: true | |
| with: | |
| image_name: ${{ env.WDQS_IMAGE_NAME }} | |
| image_tag: latest | |
| build_wdqs_proxy: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Add extra DEBUG env vars | |
| if: "${{ github.event.inputs.debug_extra_env != '' }}" | |
| run: | | |
| printf "${{ github.event.inputs.debug_extra_env }}" >> ${{ env.env_file }} | |
| cat ${{ env.env_file }} | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: variables.env | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: ${{ env.env_file }} | |
| - name: Build WDQS-proxy image | |
| run: bash build.sh wdqs-proxy ${{ env.env_file }} | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: github.ref == 'refs/heads/main' | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Store release version docker image on GHCR | |
| if: github.ref == 'refs/heads/main' | |
| uses: wmde/tag-push-ghcr-action@v3 | |
| with: | |
| image_name: ${{ env.WDQS_PROXY_IMAGE_NAME }} | |
| tag: ${{ github.run_id }} | |
| - name: Archive docker production artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| if-no-files-found: error | |
| path: artifacts/wdqs-proxy.docker.tar.gz | |
| - name: Scan image | |
| uses: ./.github/actions/scan-image | |
| continue-on-error: true | |
| with: | |
| image_name: ${{ env.WDQS_PROXY_IMAGE_NAME }} | |
| image_tag: latest | |
| build_wdqs_frontend: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Add extra DEBUG env vars | |
| if: "${{ github.event.inputs.debug_extra_env != '' }}" | |
| run: | | |
| printf "${{ github.event.inputs.debug_extra_env }}" >> ${{ env.env_file }} | |
| cat ${{ env.env_file }} | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: variables.env | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: ${{ env.env_file }} | |
| - name: Cache git_cache repos | |
| uses: actions/cache@v3 | |
| env: | |
| cache-name: cache-wdqs-frontend-git-repo | |
| with: | |
| path: git_cache | |
| key: cache-wdqs-frontend-git-repo | |
| - name: Build WDQS-frontend | |
| run: bash build.sh wdqs-frontend ${{ env.env_file }} | |
| - name: Archive metadata artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildMetadata | |
| if-no-files-found: error | |
| path: artifacts/build_metadata_wdqs_frontend.env | |
| - name: Archive tar production artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| if-no-files-found: error | |
| path: ./artifacts/wdqs-frontend.tar.gz | |
| - name: Archive docker production artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| if-no-files-found: error | |
| path: ./artifacts/wdqs-frontend.docker.tar.gz | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: github.ref == 'refs/heads/main' | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Store release version docker image on GHCR | |
| if: github.ref == 'refs/heads/main' | |
| uses: wmde/tag-push-ghcr-action@v3 | |
| with: | |
| image_name: ${{ env.WDQS_FRONTEND_IMAGE_NAME }} | |
| tag: ${{ github.run_id }} | |
| - name: Scan image | |
| uses: ./.github/actions/scan-image | |
| continue-on-error: true | |
| with: | |
| image_name: ${{ env.WDQS_FRONTEND_IMAGE_NAME }} | |
| image_tag: latest | |
| build_elasticsearch: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Add extra DEBUG env vars | |
| if: "${{ github.event.inputs.debug_extra_env != '' }}" | |
| run: | | |
| printf "${{ github.event.inputs.debug_extra_env }}" >> ${{ env.env_file }} | |
| cat ${{ env.env_file }} | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: variables.env | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: ${{ env.env_file }} | |
| - name: Build Elasticsearch | |
| run: bash build.sh elasticsearch ${{ env.env_file }} | |
| - name: Archive docker production artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| if-no-files-found: error | |
| path: ./artifacts/elasticsearch.docker.tar.gz | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@v3 | |
| if: github.ref == 'refs/heads/main' | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.repository_owner }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Store release version docker image on GHCR | |
| if: github.ref == 'refs/heads/main' | |
| uses: wmde/tag-push-ghcr-action@v3 | |
| with: | |
| image_name: ${{ env.ELASTICSEARCH_IMAGE_NAME }} | |
| tag: ${{ github.run_id }} | |
| - name: Scan image | |
| uses: ./.github/actions/scan-image | |
| continue-on-error: true | |
| with: | |
| image_name: ${{ env.ELASTICSEARCH_IMAGE_NAME }} | |
| image_tag: latest | |
| test_wikibase: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| suite: [ | |
| repo, | |
| fedprops, | |
| repo_client, | |
| quickstatements, | |
| pingback, | |
| confirm_edit, | |
| elasticsearch, | |
| base__repo, | |
| base__repo_client, | |
| base__pingback, | |
| base__fedprops, | |
| ] | |
| databaseImageName: [ 'mariadb:10.9' ] # 'mysql:5.6' disabled https://phabricator.wikimedia.org/T296066 | |
| needs: | |
| - build_wikibase_bundle | |
| - build_wdqs | |
| - build_elasticsearch | |
| - build_wdqs_frontend | |
| - build_wdqs_proxy | |
| - build_quickstatements | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get Wikibase docker image artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| path: artifacts/ | |
| - name: Run Tests - ${{ matrix.suite }} | |
| env: | |
| DATABASE_IMAGE_NAME: ${{ matrix.databaseImageName }} | |
| run: make test SUITE=${{ matrix.suite }} | |
| - name: Report Selenium | |
| if: always() | |
| run: | | |
| cd .github/reporter | |
| npm install | |
| SUITE=${{ matrix.suite }} node report.js | |
| - name: docker logs | |
| if: always() | |
| run: test/scripts/docker_logs.sh | |
| - name: mediawiki logs | |
| if: always() | |
| continue-on-error: true | |
| run: | | |
| ls -lahr test/suites/${{ matrix.suite }}/results test/suites/${{ matrix.suite }}/results/* | |
| tail -n +1 test/suites/${{ matrix.suite }}/results/*.log | |
| - name: Archive docker test artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: TestArtifacts | |
| path: test/suites/**/results | |
| test_upgrade_base: | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: [ | |
| # Latest 1.37 | |
| 'wmde.9', | |
| # Latest 1.38 | |
| 'wmde.10', | |
| ] | |
| needs: | |
| - build_wikibase | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get Wikibase docker image artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| path: artifacts/ | |
| - name: Test upgrade version - ${{ matrix.version }} | |
| # This step should only take ~5 mins to complete, but sometimes seems to lock up and use the whole job timeout | |
| # Set a specific lower timeout to allow us to retry sooner | |
| timeout-minutes: 10 | |
| run: make test-upgrade VERSION=${{ matrix.version }} TO_VERSION=${{ env.env_file }} | |
| - name: docker logs | |
| if: always() | |
| run: test/scripts/docker_logs.sh | |
| - name: Archive docker test artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: TestUpgradeArtifacts | |
| path: test/suites/**/results | |
| test_upgrade_bundle: | |
| timeout-minutes: 20 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: [ | |
| # Latest 1.37 | |
| 'wmde.9-bundle', | |
| # Latest 1.38 | |
| 'wmde.10-bundle', | |
| ] | |
| needs: | |
| - build_wikibase_bundle | |
| runs-on: ubuntu-latest | |
| env: | |
| TARGET_WIKIBASE_UPGRADE_IMAGE_NAME: wikibase-bundle | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Get Wikibase docker image artifact | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: BuildArtifacts | |
| path: artifacts/ | |
| - name: Test upgrade version - ${{ matrix.version }} | |
| # This step should only take ~5 mins to complete, but sometimes seems to lock up and use the whole job timeout | |
| # Set a specific lower timeout to allow us to retry sooner | |
| timeout-minutes: 10 | |
| run: make test-upgrade VERSION=${{ matrix.version }} TO_VERSION=${{ env.env_file }} | |
| - name: docker logs | |
| if: always() | |
| run: test/scripts/docker_logs.sh | |
| - name: Archive docker test artifacts | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: TestUpgradeArtifactsBundle | |
| path: test/suites/**/results | |
| versions: | |
| needs: | |
| - test_wikibase | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: c-py/action-dotenv-to-setenv@v4 | |
| with: | |
| env-file: ${{ env.env_file }} | |
| - name: Build version requirements | |
| run: bash build.sh requirements ${{ env.env_file }} | |
| - name: Archive metadata artifacts | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: BuildMetadata | |
| if-no-files-found: error | |
| path: artifacts/built_versions.log |