diff --git a/.github/workflows/pullRequests.yml b/.github/workflows/pullRequests.yml index 33d3cb747d7..e741af8b405 100644 --- a/.github/workflows/pullRequests.yml +++ b/.github/workflows/pullRequests.yml @@ -5,21 +5,6 @@ name: Pull Requests 'on': pull_request jobs: - validateWorkflows: - name: Validate workflows - steps: - - uses: actions/setup-node@v4 - with: - node-version: 18 - - uses: actions/checkout@v4 - - name: Install dependencies - run: yarn --immutable - - name: Validate - run: npx github-actions-wac validate - runs-on: ubuntu-latest - env: - NODE_OPTIONS: '--max_old_space_size=4096' - YARN_ENABLE_IMMUTABLE_INSTALLS: false validateCommits: name: Validate commit messages if: github.base_ref != 'dev' @@ -50,12 +35,14 @@ jobs: YARN_ENABLE_IMMUTABLE_INSTALLS: false constants: name: Create constants - runs-on: ubuntu-latest outputs: global-cache-key: ${{ steps.global-cache-key.outputs.global-cache-key }} run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} is-fork-pr: ${{ steps.is-fork-pr.outputs.is-fork-pr }} steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 - name: Create global cache key id: global-cache-key run: >- @@ -72,30 +59,38 @@ jobs: run: >- echo "is-fork-pr=${{ github.event.pull_request.head.repo.fork }}" >> $GITHUB_OUTPUT - init: - name: Init + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false + build: + name: Build needs: constants - runs-on: webiny-build-packages + runs-on: blacksmith-8vcpu-ubuntu-2204 steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 + with: + path: ${{ github.base_ref }} - uses: actions/cache@v4 with: - path: .yarn/cache + path: ${{ github.base_ref }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages + path: ${{ github.base_ref }}/.webiny/cached-packages key: ${{ needs.constants.outputs.global-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: ${{ github.base_ref }} - name: Build packages run: yarn build:quick + working-directory: ${{ github.base_ref }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages + path: ${{ github.base_ref }}/.webiny/cached-packages key: ${{ needs.constants.outputs.run-cache-key }} env: NODE_OPTIONS: '--max_old_space_size=4096' @@ -103,58 +98,72 @@ jobs: staticCodeAnalysis: needs: - constants - - init + - build name: Static code analysis steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 + with: + path: ${{ github.base_ref }} - uses: actions/cache@v4 with: - path: .yarn/cache + path: ${{ github.base_ref }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages + path: ${{ github.base_ref }}/.webiny/cached-packages key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: ${{ github.base_ref }} - name: Check code formatting run: yarn prettier:check + working-directory: ${{ github.base_ref }} - name: Check dependencies run: yarn adio + working-directory: ${{ github.base_ref }} - name: Check TS configs run: yarn check-ts-configs + working-directory: ${{ github.base_ref }} - name: ESLint run: yarn eslint + working-directory: ${{ github.base_ref }} runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false staticCodeAnalysisTs: name: Static code analysis (TypeScript) - runs-on: webiny-build-packages + runs-on: blacksmith-8vcpu-ubuntu-2204 steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 + with: + path: ${{ github.base_ref }} - uses: actions/cache@v4 with: - path: .yarn/cache + path: ${{ github.base_ref }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - name: Install dependencies run: yarn --immutable + working-directory: ${{ github.base_ref }} - name: Build packages (full) run: yarn build + working-directory: ${{ github.base_ref }} - name: Check types for Cypress tests run: yarn cy:ts + working-directory: ${{ github.base_ref }} env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false jestTestsNoStorage: - needs: constants + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -176,22 +185,29 @@ jobs: with: node-version: 18 - uses: actions/checkout@v4 + with: + path: ${{ github.base_ref }} - uses: actions/cache@v4 with: - path: .yarn/cache + path: ${{ github.base_ref }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages + path: ${{ github.base_ref }}/.webiny/cached-packages key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: ${{ github.base_ref }} - name: Build packages run: yarn build:quick + working-directory: ${{ github.base_ref }} - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: ${{ github.base_ref }} jestTestsDdb: - needs: constants + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -228,22 +244,29 @@ jobs: with: node-version: 18 - uses: actions/checkout@v4 + with: + path: ${{ github.base_ref }} - uses: actions/cache@v4 with: - path: .yarn/cache + path: ${{ github.base_ref }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages + path: ${{ github.base_ref }}/.webiny/cached-packages key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: ${{ github.base_ref }} - name: Build packages run: yarn build:quick + working-directory: ${{ github.base_ref }} - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: ${{ github.base_ref }} jestTestsDdbEs: - needs: constants + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -285,25 +308,32 @@ jobs: role-to-assume: arn:aws:iam::726952677045:role/GitHubActionsWebinyJs aws-region: eu-central-1 - uses: actions/checkout@v4 + with: + path: ${{ github.base_ref }} - uses: actions/cache@v4 with: - path: .yarn/cache + path: ${{ github.base_ref }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages + path: ${{ github.base_ref }}/.webiny/cached-packages key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: ${{ github.base_ref }} - name: Build packages run: yarn build:quick + working-directory: ${{ github.base_ref }} - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: ${{ github.base_ref }} permissions: id-token: write if: needs.constants.outputs.is-fork-pr != 'true' jestTestsDdbOs: - needs: constants + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -345,20 +375,25 @@ jobs: role-to-assume: arn:aws:iam::726952677045:role/GitHubActionsWebinyJs aws-region: eu-central-1 - uses: actions/checkout@v4 + with: + path: ${{ github.base_ref }} - uses: actions/cache@v4 with: - path: .yarn/cache + path: ${{ github.base_ref }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages + path: ${{ github.base_ref }}/.webiny/cached-packages key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: ${{ github.base_ref }} - name: Build packages run: yarn build:quick + working-directory: ${{ github.base_ref }} - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: ${{ github.base_ref }} permissions: id-token: write if: needs.constants.outputs.is-fork-pr != 'true' @@ -366,7 +401,7 @@ jobs: name: Publish to Verdaccio needs: - constants - - init + - build if: needs.constants.outputs.is-fork-pr != 'true' steps: - uses: actions/setup-node@v4 @@ -376,38 +411,47 @@ jobs: with: fetch-depth: 0 ref: ${{ github.event.pull_request.head.ref }} + path: ${{ github.base_ref }} - uses: actions/cache@v4 with: - path: .yarn/cache + path: ${{ github.base_ref }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages + path: ${{ github.base_ref }}/.webiny/cached-packages key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: ${{ github.base_ref }} - name: Build packages run: yarn build:quick + working-directory: ${{ github.base_ref }} - name: Start Verdaccio local server run: npx pm2 start verdaccio -- -c .verdaccio.yaml + working-directory: ${{ github.base_ref }} - name: Configure NPM to use local registry run: npm config set registry http://localhost:4873 + working-directory: ${{ github.base_ref }} - name: Set git email run: git config --global user.email "webiny-bot@webiny.com" + working-directory: ${{ github.base_ref }} - name: Set git username run: git config --global user.name "webiny-bot" + working-directory: ${{ github.base_ref }} - name: Create ".npmrc" file in the project root, with a dummy auth token run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc + working-directory: ${{ github.base_ref }} - name: Version and publish to Verdaccio run: yarn release --type=verdaccio + working-directory: ${{ github.base_ref }} - name: Upload verdaccio files uses: actions/upload-artifact@v4 with: name: verdaccio-files retention-days: 1 - path: | - .verdaccio/ - .verdaccio.yaml + path: |- + ${{ github.base_ref }}/.verdaccio/ + ${{ github.base_ref }}/.verdaccio.yaml runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' diff --git a/.github/workflows/pullRequestsCommandCypress.yml b/.github/workflows/pullRequestsCommandCypress.yml index 5569af32e82..c84a1963233 100644 --- a/.github/workflows/pullRequestsCommandCypress.yml +++ b/.github/workflows/pullRequestsCommandCypress.yml @@ -8,21 +8,6 @@ env: NODE_OPTIONS: '--max_old_space_size=4096' AWS_REGION: eu-central-1 jobs: - validateWorkflows: - name: Validate workflows - steps: - - uses: actions/setup-node@v4 - with: - node-version: 18 - - uses: actions/checkout@v4 - - name: Install dependencies - run: yarn --immutable - - name: Validate - run: npx github-actions-wac validate - runs-on: ubuntu-latest - env: - NODE_OPTIONS: '--max_old_space_size=4096' - YARN_ENABLE_IMMUTABLE_INSTALLS: false checkComment: name: Check comment for /cypress if: ${{ github.event.issue.pull_request }} @@ -52,14 +37,27 @@ jobs: env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false - e2e-wby-cms-ddb-constants: + validateWorkflows: + name: Validate workflows + steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 + - uses: actions/checkout@v4 + - name: Install dependencies + run: yarn --immutable + - name: Validate + run: npx github-actions-wac validate + needs: checkComment + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false + baseBranch: needs: checkComment - name: E2E (DDB) - Constants + name: Get base branch outputs: - global-cache-key: ${{ steps.global-cache-key.outputs.global-cache-key }} - run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} - cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} - pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} + base-branch: ${{ steps.base-branch.outputs.base-branch }} steps: - uses: actions/setup-node@v4 with: @@ -67,29 +65,112 @@ jobs: - uses: actions/checkout@v4 - name: Install Hub Utility run: sudo apt-get install -y hub - - name: Checkout Pull Request - working-directory: '' - run: hub pr checkout ${{ github.event.issue.number }} + - name: Get base branch + id: base-branch env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + run: >- + echo "base-branch=$(hub pr show ${{ github.event.issue.number }} -f + %B)" >> $GITHUB_OUTPUT + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false + constants: + needs: baseBranch + name: Create constants + outputs: + global-cache-key: ${{ steps.global-cache-key.outputs.global-cache-key }} + run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} + steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 - name: Create global cache key id: global-cache-key run: >- - echo "global-cache-key=$(hub pr show ${{ github.event.issue.number }} - -f %B)-${{ runner.os }}-$(/bin/date -u "+%m%d")-${{ - vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + echo "global-cache-key=${{ needs.baseBranch.outputs.base-branch }}-${{ + runner.os }}-$(/bin/date -u "+%m%d")-${{ vars.RANDOM_CACHE_KEY_SUFFIX + }}" >> $GITHUB_OUTPUT - name: Create workflow run cache key id: run-cache-key run: >- echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false + build: + name: Build + needs: + - baseBranch + - constants + runs-on: blacksmith-4vcpu-ubuntu-2204 + steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 + - uses: actions/checkout@v4 + with: + path: ${{ needs.baseBranch.outputs.base-branch }} + - name: Install Hub Utility + run: sudo apt-get install -y hub + - name: Checkout Pull Request + working-directory: ${{ needs.baseBranch.outputs.base-branch }} + run: hub pr checkout ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} + - uses: actions/cache@v4 + with: + path: ${{ needs.baseBranch.outputs.base-branch }}/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} + - uses: actions/cache@v4 + with: + path: ${{ needs.baseBranch.outputs.base-branch }}/.webiny/cached-packages + key: ${{ needs.constants.outputs.global-cache-key }} + - name: Install dependencies + run: yarn --immutable + working-directory: ${{ needs.baseBranch.outputs.base-branch }} + - name: Build packages + run: yarn build:quick + working-directory: ${{ needs.baseBranch.outputs.base-branch }} + - uses: actions/cache@v4 + with: + path: ${{ needs.baseBranch.outputs.base-branch }}/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false + e2e-wby-cms-ddb-constants: + needs: + - baseBranch + - constants + - build + name: Constants - DDB + outputs: + cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} + pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} + steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 + - uses: actions/checkout@v4 + with: + path: ${{ needs.baseBranch.outputs.base-branch }} + - name: Install Hub Utility + run: sudo apt-get install -y hub + - name: Checkout Pull Request + working-directory: ${{ needs.baseBranch.outputs.base-branch }} + run: hub pr checkout ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - name: List Cypress tests folders id: list-cypress-folders run: >- echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Get Pulumi backend URL id: get-pulumi-backend-url run: >- @@ -100,7 +181,10 @@ jobs: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false e2e-wby-cms-ddb-project-setup: - needs: e2e-wby-cms-ddb-constants + needs: + - baseBranch + - constants + - e2e-wby-cms-ddb-constants name: E2E (DDB) - Project setup outputs: cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }} @@ -123,34 +207,30 @@ jobs: aws-region: eu-central-1 - uses: actions/checkout@v4 with: - path: checked-out-pr + path: ${{ needs.baseBranch.outputs.base-branch }} - name: Install Hub Utility run: sudo apt-get install -y hub - name: Checkout Pull Request - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: hub pr checkout ${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - uses: actions/cache@v4 with: - path: checked-out-pr/.yarn/cache + path: ${{ needs.baseBranch.outputs.base-branch }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: checked-out-pr/.webiny/cached-packages - key: ${{ needs.e2e-wby-cms-ddb-constants.outputs.global-cache-key }} + path: ${{ needs.baseBranch.outputs.base-branch }}/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: checked-out-pr run: yarn --immutable + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Build packages - working-directory: checked-out-pr run: yarn build:quick - - uses: actions/cache@v4 - with: - path: checked-out-pr/.webiny/cached-packages - key: ${{ needs.e2e-wby-cms-ddb-constants.outputs.run-cache-key }} + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Start Verdaccio local server - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml @@ -161,19 +241,19 @@ jobs: - name: Set git username run: git config --global user.name "webiny-bot" - name: Create ".npmrc" file in the project root, with a dummy auth token - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc - name: Version and publish to Verdaccio - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: yarn release --type=verdaccio - name: Create verdaccio-files artifact uses: actions/upload-artifact@v4 with: name: verdaccio-files-ddb retention-days: 1 - path: | - checked-out-pr/.verdaccio/ - checked-out-pr/.verdaccio.yaml + path: |- + ${{ needs.baseBranch.outputs.base-branch }}/.verdaccio/ + ${{ needs.baseBranch.outputs.base-branch }}/.verdaccio.yaml - name: Disable Webiny telemetry run: > mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' > @@ -211,16 +291,16 @@ jobs: working-directory: new-webiny-project run: yarn webiny deploy apps/website --env dev - name: Create Cypress config - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: yarn setup-cypress --projectFolder ../new-webiny-project - name: Save Cypress config id: save-cypress-config - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\t\n\r')" >> $GITHUB_OUTPUT - name: Cypress - run installation wizard test - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js" @@ -232,6 +312,8 @@ jobs: ${{ matrix.cypress-folder }} (ddb, ${{ matrix.os }}, Node v${{ matrix.node }}) needs: + - baseBranch + - constants - e2e-wby-cms-ddb-constants - e2e-wby-cms-ddb-project-setup strategy: @@ -258,44 +340,45 @@ jobs: node-version: 18 - uses: actions/checkout@v4 with: - path: checked-out-pr + path: ${{ needs.baseBranch.outputs.base-branch }} - name: Install Hub Utility run: sudo apt-get install -y hub - name: Checkout Pull Request - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: hub pr checkout ${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - uses: actions/cache@v4 with: - path: checked-out-pr/.yarn/cache + path: ${{ needs.baseBranch.outputs.base-branch }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: checked-out-pr/.webiny/cached-packages - key: ${{ needs.e2e-wby-cms-ddb-constants.outputs.run-cache-key }} + path: ${{ needs.baseBranch.outputs.base-branch }}/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: checked-out-pr run: yarn --immutable + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Build packages - working-directory: checked-out-pr run: yarn build:quick + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Set up Cypress config - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- echo '${{ needs.e2e-wby-cms-ddb-project-setup.outputs.cypress-config }}' > cypress-tests/cypress.config.ts - name: Cypress - run "${{ matrix.cypress-folder }}" tests timeout-minutes: 40 - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}" runs-on: ubuntu-latest e2e-wby-cms-ddb-es-constants: - needs: checkComment - name: E2E (DDB-ES) - Constants + needs: + - baseBranch + - constants + - build + name: Constants - DDB-ES outputs: - global-cache-key: ${{ steps.global-cache-key.outputs.global-cache-key }} - run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} steps: @@ -303,31 +386,21 @@ jobs: with: node-version: 18 - uses: actions/checkout@v4 + with: + path: ${{ needs.baseBranch.outputs.base-branch }} - name: Install Hub Utility run: sudo apt-get install -y hub - name: Checkout Pull Request - working-directory: '' + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: hub pr checkout ${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Create global cache key - id: global-cache-key - run: >- - echo "global-cache-key=$(hub pr show ${{ github.event.issue.number }} - -f %B)-${{ runner.os }}-$(/bin/date -u "+%m%d")-${{ - vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Create workflow run cache key - id: run-cache-key - run: >- - echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ - vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT - name: List Cypress tests folders id: list-cypress-folders run: >- echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Get Pulumi backend URL id: get-pulumi-backend-url run: >- @@ -338,7 +411,10 @@ jobs: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false e2e-wby-cms-ddb-es-project-setup: - needs: e2e-wby-cms-ddb-es-constants + needs: + - baseBranch + - constants + - e2e-wby-cms-ddb-es-constants name: E2E (DDB-ES) - Project setup outputs: cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }} @@ -364,34 +440,30 @@ jobs: aws-region: eu-central-1 - uses: actions/checkout@v4 with: - path: checked-out-pr + path: ${{ needs.baseBranch.outputs.base-branch }} - name: Install Hub Utility run: sudo apt-get install -y hub - name: Checkout Pull Request - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: hub pr checkout ${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - uses: actions/cache@v4 with: - path: checked-out-pr/.yarn/cache + path: ${{ needs.baseBranch.outputs.base-branch }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: checked-out-pr/.webiny/cached-packages - key: ${{ needs.e2e-wby-cms-ddb-es-constants.outputs.global-cache-key }} + path: ${{ needs.baseBranch.outputs.base-branch }}/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: checked-out-pr run: yarn --immutable + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Build packages - working-directory: checked-out-pr run: yarn build:quick - - uses: actions/cache@v4 - with: - path: checked-out-pr/.webiny/cached-packages - key: ${{ needs.e2e-wby-cms-ddb-es-constants.outputs.run-cache-key }} + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Start Verdaccio local server - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml @@ -402,19 +474,19 @@ jobs: - name: Set git username run: git config --global user.name "webiny-bot" - name: Create ".npmrc" file in the project root, with a dummy auth token - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc - name: Version and publish to Verdaccio - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: yarn release --type=verdaccio - name: Create verdaccio-files artifact uses: actions/upload-artifact@v4 with: name: verdaccio-files-ddb-es retention-days: 1 - path: | - checked-out-pr/.verdaccio/ - checked-out-pr/.verdaccio.yaml + path: |- + ${{ needs.baseBranch.outputs.base-branch }}/.verdaccio/ + ${{ needs.baseBranch.outputs.base-branch }}/.verdaccio.yaml - name: Disable Webiny telemetry run: > mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' > @@ -452,16 +524,16 @@ jobs: working-directory: new-webiny-project run: yarn webiny deploy apps/website --env dev - name: Create Cypress config - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: yarn setup-cypress --projectFolder ../new-webiny-project - name: Save Cypress config id: save-cypress-config - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\t\n\r')" >> $GITHUB_OUTPUT - name: Cypress - run installation wizard test - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js" @@ -473,6 +545,8 @@ jobs: ${{ matrix.cypress-folder }} (ddb-es, ${{ matrix.os }}, Node v${{ matrix.node }}) needs: + - baseBranch + - constants - e2e-wby-cms-ddb-es-constants - e2e-wby-cms-ddb-es-project-setup strategy: @@ -503,45 +577,46 @@ jobs: node-version: 18 - uses: actions/checkout@v4 with: - path: checked-out-pr + path: ${{ needs.baseBranch.outputs.base-branch }} - name: Install Hub Utility run: sudo apt-get install -y hub - name: Checkout Pull Request - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: hub pr checkout ${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - uses: actions/cache@v4 with: - path: checked-out-pr/.yarn/cache + path: ${{ needs.baseBranch.outputs.base-branch }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: checked-out-pr/.webiny/cached-packages - key: ${{ needs.e2e-wby-cms-ddb-es-constants.outputs.run-cache-key }} + path: ${{ needs.baseBranch.outputs.base-branch }}/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: checked-out-pr run: yarn --immutable + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Build packages - working-directory: checked-out-pr run: yarn build:quick + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Set up Cypress config - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- echo '${{ needs.e2e-wby-cms-ddb-es-project-setup.outputs.cypress-config }}' > cypress-tests/cypress.config.ts - name: Cypress - run "${{ matrix.cypress-folder }}" tests timeout-minutes: 40 - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}" runs-on: ubuntu-latest e2e-wby-cms-ddb-os-constants: - needs: checkComment - name: E2E (DDB-OS) - Constants + needs: + - baseBranch + - constants + - build + name: Constants - DDB-OS outputs: - global-cache-key: ${{ steps.global-cache-key.outputs.global-cache-key }} - run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} steps: @@ -549,31 +624,21 @@ jobs: with: node-version: 18 - uses: actions/checkout@v4 + with: + path: ${{ needs.baseBranch.outputs.base-branch }} - name: Install Hub Utility run: sudo apt-get install -y hub - name: Checkout Pull Request - working-directory: '' + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: hub pr checkout ${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Create global cache key - id: global-cache-key - run: >- - echo "global-cache-key=$(hub pr show ${{ github.event.issue.number }} - -f %B)-${{ runner.os }}-$(/bin/date -u "+%m%d")-${{ - vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT - env: - GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - - name: Create workflow run cache key - id: run-cache-key - run: >- - echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ - vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT - name: List Cypress tests folders id: list-cypress-folders run: >- echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Get Pulumi backend URL id: get-pulumi-backend-url run: >- @@ -584,7 +649,10 @@ jobs: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false e2e-wby-cms-ddb-os-project-setup: - needs: e2e-wby-cms-ddb-os-constants + needs: + - baseBranch + - constants + - e2e-wby-cms-ddb-os-constants name: E2E (DDB-OS) - Project setup outputs: cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }} @@ -610,34 +678,30 @@ jobs: aws-region: eu-central-1 - uses: actions/checkout@v4 with: - path: checked-out-pr + path: ${{ needs.baseBranch.outputs.base-branch }} - name: Install Hub Utility run: sudo apt-get install -y hub - name: Checkout Pull Request - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: hub pr checkout ${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - uses: actions/cache@v4 with: - path: checked-out-pr/.yarn/cache + path: ${{ needs.baseBranch.outputs.base-branch }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: checked-out-pr/.webiny/cached-packages - key: ${{ needs.e2e-wby-cms-ddb-os-constants.outputs.global-cache-key }} + path: ${{ needs.baseBranch.outputs.base-branch }}/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: checked-out-pr run: yarn --immutable + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Build packages - working-directory: checked-out-pr run: yarn build:quick - - uses: actions/cache@v4 - with: - path: checked-out-pr/.webiny/cached-packages - key: ${{ needs.e2e-wby-cms-ddb-os-constants.outputs.run-cache-key }} + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Start Verdaccio local server - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml @@ -648,19 +712,19 @@ jobs: - name: Set git username run: git config --global user.name "webiny-bot" - name: Create ".npmrc" file in the project root, with a dummy auth token - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc - name: Version and publish to Verdaccio - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: yarn release --type=verdaccio - name: Create verdaccio-files artifact uses: actions/upload-artifact@v4 with: name: verdaccio-files-ddb-os retention-days: 1 - path: | - checked-out-pr/.verdaccio/ - checked-out-pr/.verdaccio.yaml + path: |- + ${{ needs.baseBranch.outputs.base-branch }}/.verdaccio/ + ${{ needs.baseBranch.outputs.base-branch }}/.verdaccio.yaml - name: Disable Webiny telemetry run: > mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' > @@ -698,16 +762,16 @@ jobs: working-directory: new-webiny-project run: yarn webiny deploy apps/website --env dev - name: Create Cypress config - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: yarn setup-cypress --projectFolder ../new-webiny-project - name: Save Cypress config id: save-cypress-config - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\t\n\r')" >> $GITHUB_OUTPUT - name: Cypress - run installation wizard test - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js" @@ -719,6 +783,8 @@ jobs: ${{ matrix.cypress-folder }} (ddb-os, ${{ matrix.os }}, Node v${{ matrix.node }}) needs: + - baseBranch + - constants - e2e-wby-cms-ddb-os-constants - e2e-wby-cms-ddb-os-project-setup strategy: @@ -749,36 +815,36 @@ jobs: node-version: 18 - uses: actions/checkout@v4 with: - path: checked-out-pr + path: ${{ needs.baseBranch.outputs.base-branch }} - name: Install Hub Utility run: sudo apt-get install -y hub - name: Checkout Pull Request - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: hub pr checkout ${{ github.event.issue.number }} env: GITHUB_TOKEN: ${{ secrets.GH_TOKEN }} - uses: actions/cache@v4 with: - path: checked-out-pr/.yarn/cache + path: ${{ needs.baseBranch.outputs.base-branch }}/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: checked-out-pr/.webiny/cached-packages - key: ${{ needs.e2e-wby-cms-ddb-os-constants.outputs.run-cache-key }} + path: ${{ needs.baseBranch.outputs.base-branch }}/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: checked-out-pr run: yarn --immutable + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Build packages - working-directory: checked-out-pr run: yarn build:quick + working-directory: ${{ needs.baseBranch.outputs.base-branch }} - name: Set up Cypress config - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: >- echo '${{ needs.e2e-wby-cms-ddb-os-project-setup.outputs.cypress-config }}' > cypress-tests/cypress.config.ts - name: Cypress - run "${{ matrix.cypress-folder }}" tests timeout-minutes: 40 - working-directory: checked-out-pr + working-directory: ${{ needs.baseBranch.outputs.base-branch }} run: yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}" runs-on: ubuntu-latest diff --git a/.github/workflows/pushDev.yml b/.github/workflows/pushDev.yml index 62b6bd4c49b..e68779a2334 100644 --- a/.github/workflows/pushDev.yml +++ b/.github/workflows/pushDev.yml @@ -8,133 +8,131 @@ name: Dev Branch - Push branches: - dev jobs: - validateWorkflows: - name: Validate workflows - steps: - - uses: actions/setup-node@v4 - with: - node-version: 18 - - uses: actions/checkout@v4 - - name: Install dependencies - run: yarn --immutable - - name: Validate - run: npx github-actions-wac validate - runs-on: ubuntu-latest - env: - NODE_OPTIONS: '--max_old_space_size=4096' - YARN_ENABLE_IMMUTABLE_INSTALLS: false - init: - name: Init + constants: + name: Create constants outputs: - day: ${{ steps.get-day.outputs.day }} - ts: ${{ steps.get-timestamp.outputs.ts }} + global-cache-key: ${{ steps.global-cache-key.outputs.global-cache-key }} + run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 - - name: Get day of the month - id: get-day + - name: Create global cache key + id: global-cache-key run: >- - echo "day=$(node --eval "console.log(new Date().getDate())")" >> - $GITHUB_OUTPUT - - name: Get timestamp - id: get-timestamp + echo "global-cache-key=dev-${{ runner.os }}-$(/bin/date -u + "+%m%d")-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT + - name: Create workflow run cache key + id: run-cache-key run: >- - echo "ts=$(node --eval "console.log(new Date().getTime())")" >> - $GITHUB_OUTPUT + echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ + vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false build: name: Build - needs: init - runs-on: webiny-build-packages + needs: constants + runs-on: blacksmith-4vcpu-ubuntu-2204 steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 + with: + path: dev - uses: actions/cache@v4 - id: yarn-cache with: - path: .yarn/cache + path: dev/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 - id: global-daily-packages-cache with: - path: .webiny/cached-packages - key: >- - ${{ runner.os }}-${{ needs.init.outputs.day }}-${{ - secrets.RANDOM_CACHE_KEY_SUFFIX }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.global-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: dev - name: Build packages run: yarn build:quick + working-directory: dev - uses: actions/cache@v4 - id: packages-cache with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false codeAnalysis: name: Static code analysis needs: - - init + - constants - build steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 + with: + path: dev - uses: actions/cache@v4 with: - path: .yarn/cache + path: dev/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: dev - name: Check code formatting run: yarn prettier:check + working-directory: dev - name: Check dependencies run: yarn adio + working-directory: dev - name: Check TS configs run: yarn check-ts-configs + working-directory: dev - name: ESLint run: yarn eslint + working-directory: dev runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false staticCodeAnalysisTs: name: Static code analysis (TypeScript) - runs-on: webiny-build-packages + runs-on: blacksmith-4vcpu-ubuntu-2204 steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 + with: + path: dev - uses: actions/cache@v4 with: - path: .yarn/cache + path: dev/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - name: Install dependencies run: yarn --immutable + working-directory: dev - name: Build packages (full) run: yarn build + working-directory: dev - name: Check types for Cypress tests run: yarn cy:ts + working-directory: dev env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false jestTestsNoStorage: - needs: init + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -156,22 +154,29 @@ jobs: with: node-version: 18 - uses: actions/checkout@v4 + with: + path: dev - uses: actions/cache@v4 with: - path: .yarn/cache + path: dev/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: dev - name: Build packages run: yarn build:quick + working-directory: dev - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: dev jestTestsDdb: - needs: init + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -208,22 +213,29 @@ jobs: with: node-version: 18 - uses: actions/checkout@v4 + with: + path: dev - uses: actions/cache@v4 with: - path: .yarn/cache + path: dev/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: dev - name: Build packages run: yarn build:quick + working-directory: dev - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: dev jestTestsDdbEs: - needs: init + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -265,25 +277,31 @@ jobs: role-to-assume: arn:aws:iam::726952677045:role/GitHubActionsWebinyJs aws-region: eu-central-1 - uses: actions/checkout@v4 + with: + path: dev - uses: actions/cache@v4 with: - path: .yarn/cache + path: dev/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: dev - name: Build packages run: yarn build:quick + working-directory: dev - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: dev permissions: id-token: write - if: needs.init.outputs.is-fork-pr != 'true' jestTestsDdbOs: - needs: init + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -325,55 +343,58 @@ jobs: role-to-assume: arn:aws:iam::726952677045:role/GitHubActionsWebinyJs aws-region: eu-central-1 - uses: actions/checkout@v4 + with: + path: dev - uses: actions/cache@v4 with: - path: .yarn/cache + path: dev/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: dev - name: Build packages run: yarn build:quick + working-directory: dev - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: dev permissions: id-token: write - if: needs.init.outputs.is-fork-pr != 'true' - e2eTestsDdb-init: - name: E2E (DDB) - Init + e2eTestsDdb-constants: + name: Constants - DDB + needs: + - build outputs: - day: ${{ steps.get-day.outputs.day }} - ts: ${{ steps.get-timestamp.outputs.ts }} cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} + pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 - - name: Get day of the month - id: get-day - run: >- - echo "day=$(node --eval "console.log(new Date().getDate())")" >> - $GITHUB_OUTPUT - - name: Get timestamp - id: get-timestamp - run: >- - echo "ts=$(node --eval "console.log(new Date().getTime())")" >> - $GITHUB_OUTPUT - name: List Cypress tests folders id: list-cypress-folders run: >- echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT + - name: Get Pulumi backend URL + id: get-pulumi-backend-url + run: >- + echo "pulumi-backend-url=${{ secrets.WEBINY_PULUMI_BACKEND }}${{ + github.run_id }}_ddb" >> $GITHUB_OUTPUT runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false e2eTestsDdb-setup: - needs: e2eTestsDdb-init + needs: + - constants + - build + - e2eTestsDdb-constants name: E2E (DDB) - Project setup outputs: cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }} @@ -384,9 +405,7 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-constants.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: @@ -401,28 +420,19 @@ jobs: fetch-depth: 0 path: dev - uses: actions/cache@v4 - id: yarn-cache with: path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 - id: cached-packages with: path: dev/.webiny/cached-packages - key: >- - ${{ runner.os }}-${{ needs.e2eTestsDdb-init.outputs.day }}-${{ - secrets.RANDOM_CACHE_KEY_SUFFIX }} + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable - - name: Build packages working-directory: dev + - name: Build packages run: yarn build:quick - - uses: actions/cache@v4 - id: packages-cache - with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-init.outputs.ts }} + working-directory: dev - name: Start Verdaccio local server working-directory: dev run: >- @@ -435,72 +445,69 @@ jobs: - name: Set git username run: git config --global user.name "webiny-bot" - name: Create ".npmrc" file in the project root, with a dummy auth token - working-directory: dev run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc - - name: Version and publish to Verdaccio working-directory: dev + - name: Version and publish to Verdaccio run: yarn release --type=verdaccio + working-directory: dev - name: Create verdaccio-files artifact uses: actions/upload-artifact@v4 with: name: verdaccio-files-ddb retention-days: 1 - path: | + path: |- dev/.verdaccio/ dev/.verdaccio.yaml - - name: Create directory - run: mkdir xyz - name: Disable Webiny telemetry run: > mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' > ~/.webiny/config - name: Create a new Webiny project - working-directory: xyz run: > - npx create-webiny-project@local-npm test-project --tag local-npm + npx create-webiny-project@local-npm new-webiny-project --tag local-npm --no-interactive --assign-to-yarnrc '{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}' --template-options '{"region":"${{ env.AWS_REGION }}","storageOperations":"ddb"}' - name: Print CLI version - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny --version - name: Create project-files artifact uses: actions/upload-artifact@v4 with: name: project-files-ddb retention-days: 1 - path: | - xyz/test-project/ - !xyz/test-project/node_modules/**/* - !xyz/test-project/**/node_modules/**/* - !xyz/test-project/.yarn/cache/**/* + path: |- + new-webiny-project/ + !new-webiny-project/node_modules/**/* + !new-webiny-project/**/node_modules/**/* + !new-webiny-project/.yarn/cache/**/* - name: Deploy Core - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/core --env dev - name: Deploy API - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/api --env dev - name: Deploy Admin Area - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/admin --env dev - name: Deploy Website - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/website --env dev - name: Create Cypress config + run: yarn setup-cypress --projectFolder ../new-webiny-project working-directory: dev - run: yarn setup-cypress --projectFolder ../xyz/test-project - name: Save Cypress config id: save-cypress-config - working-directory: dev run: >- echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\t\n\r')" >> $GITHUB_OUTPUT + working-directory: dev - name: Cypress - run installation wizard test - working-directory: dev/cypress-tests run: >- - yarn cypress run --browser chrome --spec + yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js" + working-directory: dev runs-on: ubuntu-latest permissions: id-token: write @@ -509,7 +516,8 @@ jobs: ${{ matrix.cypress-folder }} (ddb, ${{ matrix.os }}, Node v${{ matrix.node }}) needs: - - e2eTestsDdb-init + - constants + - e2eTestsDdb-constants - e2eTestsDdb-setup strategy: fail-fast: false @@ -518,7 +526,7 @@ jobs: - ubuntu-latest node: - 18 - cypress-folder: ${{ fromJson(needs.e2eTestsDdb-init.outputs.cypress-folders) }} + cypress-folder: ${{ fromJson(needs.e2eTestsDdb-constants.outputs.cypress-folders) }} environment: next env: NODE_OPTIONS: '--max_old_space_size=4096' @@ -526,9 +534,7 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-constants.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: @@ -538,62 +544,59 @@ jobs: path: dev - uses: actions/cache@v4 with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-init.outputs.ts }} + path: dev/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable - - name: Build packages working-directory: dev + - name: Build packages run: yarn build:quick - - name: Set up Cypress config working-directory: dev + - name: Set up Cypress config run: >- echo '${{ needs.e2eTestsDdb-setup.outputs.cypress-config }}' > cypress-tests/cypress.config.ts + working-directory: dev - name: Cypress - run "${{ matrix.cypress-folder }}" tests - working-directory: dev/cypress-tests timeout-minutes: 40 - run: >- - yarn cypress run --browser chrome --spec "${{ matrix.cypress-folder - }}" + run: yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}" + working-directory: dev runs-on: ubuntu-latest - e2eTestsDdb-es-init: - name: E2E (DDB-ES) - Init + e2eTestsDdb-es-constants: + name: Constants - DDB-ES + needs: + - build outputs: - day: ${{ steps.get-day.outputs.day }} - ts: ${{ steps.get-timestamp.outputs.ts }} cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} + pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 - - name: Get day of the month - id: get-day - run: >- - echo "day=$(node --eval "console.log(new Date().getDate())")" >> - $GITHUB_OUTPUT - - name: Get timestamp - id: get-timestamp - run: >- - echo "ts=$(node --eval "console.log(new Date().getTime())")" >> - $GITHUB_OUTPUT - name: List Cypress tests folders id: list-cypress-folders run: >- echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT + - name: Get Pulumi backend URL + id: get-pulumi-backend-url + run: >- + echo "pulumi-backend-url=${{ secrets.WEBINY_PULUMI_BACKEND }}${{ + github.run_id }}_ddb-es" >> $GITHUB_OUTPUT runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false e2eTestsDdb-es-setup: - needs: e2eTestsDdb-es-init + needs: + - constants + - build + - e2eTestsDdb-es-constants name: E2E (DDB-ES) - Project setup outputs: cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }} @@ -604,12 +607,10 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-es-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-es-constants.outputs.pulumi-backend-url }} AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_ELASTIC_SEARCH_DOMAIN_NAME }} ELASTIC_SEARCH_ENDPOINT: ${{ secrets.ELASTIC_SEARCH_ENDPOINT }} - ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2eTestsDdb-es-init.outputs.ts }}_ + ELASTIC_SEARCH_INDEX_PREFIX: ${{ github.run_id }}_ steps: - uses: actions/setup-node@v4 with: @@ -624,28 +625,19 @@ jobs: fetch-depth: 0 path: dev - uses: actions/cache@v4 - id: yarn-cache with: path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 - id: cached-packages with: path: dev/.webiny/cached-packages - key: >- - ${{ runner.os }}-${{ needs.e2eTestsDdb-es-init.outputs.day }}-${{ - secrets.RANDOM_CACHE_KEY_SUFFIX }} + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable - - name: Build packages working-directory: dev + - name: Build packages run: yarn build:quick - - uses: actions/cache@v4 - id: packages-cache - with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-es-init.outputs.ts }} + working-directory: dev - name: Start Verdaccio local server working-directory: dev run: >- @@ -658,72 +650,69 @@ jobs: - name: Set git username run: git config --global user.name "webiny-bot" - name: Create ".npmrc" file in the project root, with a dummy auth token - working-directory: dev run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc - - name: Version and publish to Verdaccio working-directory: dev + - name: Version and publish to Verdaccio run: yarn release --type=verdaccio + working-directory: dev - name: Create verdaccio-files artifact uses: actions/upload-artifact@v4 with: name: verdaccio-files-ddb-es retention-days: 1 - path: | + path: |- dev/.verdaccio/ dev/.verdaccio.yaml - - name: Create directory - run: mkdir xyz - name: Disable Webiny telemetry run: > mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' > ~/.webiny/config - name: Create a new Webiny project - working-directory: xyz run: > - npx create-webiny-project@local-npm test-project --tag local-npm + npx create-webiny-project@local-npm new-webiny-project --tag local-npm --no-interactive --assign-to-yarnrc '{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}' --template-options '{"region":"${{ env.AWS_REGION }}","storageOperations":"ddb-es"}' - name: Print CLI version - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny --version - name: Create project-files artifact uses: actions/upload-artifact@v4 with: name: project-files-ddb-es retention-days: 1 - path: | - xyz/test-project/ - !xyz/test-project/node_modules/**/* - !xyz/test-project/**/node_modules/**/* - !xyz/test-project/.yarn/cache/**/* + path: |- + new-webiny-project/ + !new-webiny-project/node_modules/**/* + !new-webiny-project/**/node_modules/**/* + !new-webiny-project/.yarn/cache/**/* - name: Deploy Core - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/core --env dev - name: Deploy API - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/api --env dev - name: Deploy Admin Area - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/admin --env dev - name: Deploy Website - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/website --env dev - name: Create Cypress config + run: yarn setup-cypress --projectFolder ../new-webiny-project working-directory: dev - run: yarn setup-cypress --projectFolder ../xyz/test-project - name: Save Cypress config id: save-cypress-config - working-directory: dev run: >- echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\t\n\r')" >> $GITHUB_OUTPUT + working-directory: dev - name: Cypress - run installation wizard test - working-directory: dev/cypress-tests run: >- - yarn cypress run --browser chrome --spec + yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js" + working-directory: dev runs-on: ubuntu-latest permissions: id-token: write @@ -732,7 +721,8 @@ jobs: ${{ matrix.cypress-folder }} (ddb-es, ${{ matrix.os }}, Node v${{ matrix.node }}) needs: - - e2eTestsDdb-es-init + - constants + - e2eTestsDdb-es-constants - e2eTestsDdb-es-setup strategy: fail-fast: false @@ -741,7 +731,9 @@ jobs: - ubuntu-latest node: - 18 - cypress-folder: ${{ fromJson(needs.e2eTestsDdb-es-init.outputs.cypress-folders) }} + cypress-folder: >- + ${{ fromJson(needs.e2eTestsDdb-es-constants.outputs.cypress-folders) + }} environment: next env: NODE_OPTIONS: '--max_old_space_size=4096' @@ -749,12 +741,10 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-es-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-es-constants.outputs.pulumi-backend-url }} AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_ELASTIC_SEARCH_DOMAIN_NAME }} ELASTIC_SEARCH_ENDPOINT: ${{ secrets.ELASTIC_SEARCH_ENDPOINT }} - ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2eTestsDdb-es-init.outputs.ts }}_ + ELASTIC_SEARCH_INDEX_PREFIX: ${{ github.run_id }}_ steps: - uses: actions/setup-node@v4 with: @@ -764,62 +754,59 @@ jobs: path: dev - uses: actions/cache@v4 with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-es-init.outputs.ts }} + path: dev/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable - - name: Build packages working-directory: dev + - name: Build packages run: yarn build:quick - - name: Set up Cypress config working-directory: dev + - name: Set up Cypress config run: >- echo '${{ needs.e2eTestsDdb-es-setup.outputs.cypress-config }}' > cypress-tests/cypress.config.ts + working-directory: dev - name: Cypress - run "${{ matrix.cypress-folder }}" tests - working-directory: dev/cypress-tests timeout-minutes: 40 - run: >- - yarn cypress run --browser chrome --spec "${{ matrix.cypress-folder - }}" + run: yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}" + working-directory: dev runs-on: ubuntu-latest - e2eTestsDdb-os-init: - name: E2E (DDB-OS) - Init + e2eTestsDdb-os-constants: + name: Constants - DDB-OS + needs: + - build outputs: - day: ${{ steps.get-day.outputs.day }} - ts: ${{ steps.get-timestamp.outputs.ts }} cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} + pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 - - name: Get day of the month - id: get-day - run: >- - echo "day=$(node --eval "console.log(new Date().getDate())")" >> - $GITHUB_OUTPUT - - name: Get timestamp - id: get-timestamp - run: >- - echo "ts=$(node --eval "console.log(new Date().getTime())")" >> - $GITHUB_OUTPUT - name: List Cypress tests folders id: list-cypress-folders run: >- echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT + - name: Get Pulumi backend URL + id: get-pulumi-backend-url + run: >- + echo "pulumi-backend-url=${{ secrets.WEBINY_PULUMI_BACKEND }}${{ + github.run_id }}_ddb-os" >> $GITHUB_OUTPUT runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false e2eTestsDdb-os-setup: - needs: e2eTestsDdb-os-init + needs: + - constants + - build + - e2eTestsDdb-os-constants name: E2E (DDB-OS) - Project setup outputs: cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }} @@ -830,12 +817,10 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-os-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-os-constants.outputs.pulumi-backend-url }} AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }} ELASTIC_SEARCH_ENDPOINT: ${{ secrets.OPEN_SEARCH_ENDPOINT }} - ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2eTestsDdb-os-init.outputs.ts }}_ + ELASTIC_SEARCH_INDEX_PREFIX: ${{ github.run_id }}_ steps: - uses: actions/setup-node@v4 with: @@ -850,28 +835,19 @@ jobs: fetch-depth: 0 path: dev - uses: actions/cache@v4 - id: yarn-cache with: path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 - id: cached-packages with: path: dev/.webiny/cached-packages - key: >- - ${{ runner.os }}-${{ needs.e2eTestsDdb-os-init.outputs.day }}-${{ - secrets.RANDOM_CACHE_KEY_SUFFIX }} + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable - - name: Build packages working-directory: dev + - name: Build packages run: yarn build:quick - - uses: actions/cache@v4 - id: packages-cache - with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-os-init.outputs.ts }} + working-directory: dev - name: Start Verdaccio local server working-directory: dev run: >- @@ -884,72 +860,69 @@ jobs: - name: Set git username run: git config --global user.name "webiny-bot" - name: Create ".npmrc" file in the project root, with a dummy auth token - working-directory: dev run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc - - name: Version and publish to Verdaccio working-directory: dev + - name: Version and publish to Verdaccio run: yarn release --type=verdaccio + working-directory: dev - name: Create verdaccio-files artifact uses: actions/upload-artifact@v4 with: name: verdaccio-files-ddb-os retention-days: 1 - path: | + path: |- dev/.verdaccio/ dev/.verdaccio.yaml - - name: Create directory - run: mkdir xyz - name: Disable Webiny telemetry run: > mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' > ~/.webiny/config - name: Create a new Webiny project - working-directory: xyz run: > - npx create-webiny-project@local-npm test-project --tag local-npm + npx create-webiny-project@local-npm new-webiny-project --tag local-npm --no-interactive --assign-to-yarnrc '{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}' --template-options '{"region":"${{ env.AWS_REGION }}","storageOperations":"ddb-os"}' - name: Print CLI version - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny --version - name: Create project-files artifact uses: actions/upload-artifact@v4 with: name: project-files-ddb-os retention-days: 1 - path: | - xyz/test-project/ - !xyz/test-project/node_modules/**/* - !xyz/test-project/**/node_modules/**/* - !xyz/test-project/.yarn/cache/**/* + path: |- + new-webiny-project/ + !new-webiny-project/node_modules/**/* + !new-webiny-project/**/node_modules/**/* + !new-webiny-project/.yarn/cache/**/* - name: Deploy Core - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/core --env dev - name: Deploy API - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/api --env dev - name: Deploy Admin Area - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/admin --env dev - name: Deploy Website - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/website --env dev - name: Create Cypress config + run: yarn setup-cypress --projectFolder ../new-webiny-project working-directory: dev - run: yarn setup-cypress --projectFolder ../xyz/test-project - name: Save Cypress config id: save-cypress-config - working-directory: dev run: >- echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\t\n\r')" >> $GITHUB_OUTPUT + working-directory: dev - name: Cypress - run installation wizard test - working-directory: dev/cypress-tests run: >- - yarn cypress run --browser chrome --spec + yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js" + working-directory: dev runs-on: ubuntu-latest permissions: id-token: write @@ -958,7 +931,8 @@ jobs: ${{ matrix.cypress-folder }} (ddb-os, ${{ matrix.os }}, Node v${{ matrix.node }}) needs: - - e2eTestsDdb-os-init + - constants + - e2eTestsDdb-os-constants - e2eTestsDdb-os-setup strategy: fail-fast: false @@ -967,7 +941,9 @@ jobs: - ubuntu-latest node: - 18 - cypress-folder: ${{ fromJson(needs.e2eTestsDdb-os-init.outputs.cypress-folders) }} + cypress-folder: >- + ${{ fromJson(needs.e2eTestsDdb-os-constants.outputs.cypress-folders) + }} environment: next env: NODE_OPTIONS: '--max_old_space_size=4096' @@ -975,12 +951,10 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-os-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-os-constants.outputs.pulumi-backend-url }} AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }} ELASTIC_SEARCH_ENDPOINT: ${{ secrets.OPEN_SEARCH_ENDPOINT }} - ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2eTestsDdb-os-init.outputs.ts }}_ + ELASTIC_SEARCH_INDEX_PREFIX: ${{ github.run_id }}_ steps: - uses: actions/setup-node@v4 with: @@ -990,27 +964,25 @@ jobs: path: dev - uses: actions/cache@v4 with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-os-init.outputs.ts }} + path: dev/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable - - name: Build packages working-directory: dev + - name: Build packages run: yarn build:quick - - name: Set up Cypress config working-directory: dev + - name: Set up Cypress config run: >- echo '${{ needs.e2eTestsDdb-os-setup.outputs.cypress-config }}' > cypress-tests/cypress.config.ts + working-directory: dev - name: Cypress - run "${{ matrix.cypress-folder }}" tests - working-directory: dev/cypress-tests timeout-minutes: 40 - run: >- - yarn cypress run --browser chrome --spec "${{ matrix.cypress-folder - }}" + run: yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}" + working-directory: dev runs-on: ubuntu-latest diff --git a/.github/workflows/pushNext.yml b/.github/workflows/pushNext.yml index e924e57c179..7bb1b77b626 100644 --- a/.github/workflows/pushNext.yml +++ b/.github/workflows/pushNext.yml @@ -8,133 +8,131 @@ name: Next Branch - Push branches: - next jobs: - validateWorkflows: - name: Validate workflows - steps: - - uses: actions/setup-node@v4 - with: - node-version: 18 - - uses: actions/checkout@v4 - - name: Install dependencies - run: yarn --immutable - - name: Validate - run: npx github-actions-wac validate - runs-on: ubuntu-latest - env: - NODE_OPTIONS: '--max_old_space_size=4096' - YARN_ENABLE_IMMUTABLE_INSTALLS: false - init: - name: Init + constants: + name: Create constants outputs: - day: ${{ steps.get-day.outputs.day }} - ts: ${{ steps.get-timestamp.outputs.ts }} + global-cache-key: ${{ steps.global-cache-key.outputs.global-cache-key }} + run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 - - name: Get day of the month - id: get-day + - name: Create global cache key + id: global-cache-key run: >- - echo "day=$(node --eval "console.log(new Date().getDate())")" >> - $GITHUB_OUTPUT - - name: Get timestamp - id: get-timestamp + echo "global-cache-key=next-${{ runner.os }}-$(/bin/date -u + "+%m%d")-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT + - name: Create workflow run cache key + id: run-cache-key run: >- - echo "ts=$(node --eval "console.log(new Date().getTime())")" >> - $GITHUB_OUTPUT + echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ + vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false build: name: Build - needs: init - runs-on: webiny-build-packages + needs: constants + runs-on: blacksmith-4vcpu-ubuntu-2204 steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 + with: + path: next - uses: actions/cache@v4 - id: yarn-cache with: - path: .yarn/cache + path: next/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 - id: global-daily-packages-cache with: - path: .webiny/cached-packages - key: >- - ${{ runner.os }}-${{ needs.init.outputs.day }}-${{ - secrets.RANDOM_CACHE_KEY_SUFFIX }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.global-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: next - name: Build packages run: yarn build:quick + working-directory: next - uses: actions/cache@v4 - id: packages-cache with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false codeAnalysis: name: Static code analysis needs: - - init + - constants - build steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 + with: + path: next - uses: actions/cache@v4 with: - path: .yarn/cache + path: next/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: next - name: Check code formatting run: yarn prettier:check + working-directory: next - name: Check dependencies run: yarn adio + working-directory: next - name: Check TS configs run: yarn check-ts-configs + working-directory: next - name: ESLint run: yarn eslint + working-directory: next runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false staticCodeAnalysisTs: name: Static code analysis (TypeScript) - runs-on: webiny-build-packages + runs-on: blacksmith-4vcpu-ubuntu-2204 steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 + with: + path: next - uses: actions/cache@v4 with: - path: .yarn/cache + path: next/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - name: Install dependencies run: yarn --immutable + working-directory: next - name: Build packages (full) run: yarn build + working-directory: next - name: Check types for Cypress tests run: yarn cy:ts + working-directory: next env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false jestTestsNoStorage: - needs: init + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -156,22 +154,29 @@ jobs: with: node-version: 18 - uses: actions/checkout@v4 + with: + path: next - uses: actions/cache@v4 with: - path: .yarn/cache + path: next/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: next - name: Build packages run: yarn build:quick + working-directory: next - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: next jestTestsDdb: - needs: init + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -208,22 +213,29 @@ jobs: with: node-version: 18 - uses: actions/checkout@v4 + with: + path: next - uses: actions/cache@v4 with: - path: .yarn/cache + path: next/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: next - name: Build packages run: yarn build:quick + working-directory: next - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: next jestTestsDdbEs: - needs: init + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -265,25 +277,31 @@ jobs: role-to-assume: arn:aws:iam::726952677045:role/GitHubActionsWebinyJs aws-region: eu-central-1 - uses: actions/checkout@v4 + with: + path: next - uses: actions/cache@v4 with: - path: .yarn/cache + path: next/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: next - name: Build packages run: yarn build:quick + working-directory: next - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: next permissions: id-token: write - if: needs.init.outputs.is-fork-pr != 'true' jestTestsDdbOs: - needs: init + needs: + - constants + - build name: ${{ matrix.package.cmd }} strategy: fail-fast: false @@ -325,55 +343,58 @@ jobs: role-to-assume: arn:aws:iam::726952677045:role/GitHubActionsWebinyJs aws-region: eu-central-1 - uses: actions/checkout@v4 + with: + path: next - uses: actions/cache@v4 with: - path: .yarn/cache + path: next/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: next - name: Build packages run: yarn build:quick + working-directory: next - name: Run tests run: yarn test ${{ matrix.package.cmd }} + working-directory: next permissions: id-token: write - if: needs.init.outputs.is-fork-pr != 'true' - e2eTestsDdb-init: - name: E2E (DDB) - Init + e2eTestsDdb-constants: + name: Constants - DDB + needs: + - build outputs: - day: ${{ steps.get-day.outputs.day }} - ts: ${{ steps.get-timestamp.outputs.ts }} cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} + pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 - - name: Get day of the month - id: get-day - run: >- - echo "day=$(node --eval "console.log(new Date().getDate())")" >> - $GITHUB_OUTPUT - - name: Get timestamp - id: get-timestamp - run: >- - echo "ts=$(node --eval "console.log(new Date().getTime())")" >> - $GITHUB_OUTPUT - name: List Cypress tests folders id: list-cypress-folders run: >- echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT + - name: Get Pulumi backend URL + id: get-pulumi-backend-url + run: >- + echo "pulumi-backend-url=${{ secrets.WEBINY_PULUMI_BACKEND }}${{ + github.run_id }}_ddb" >> $GITHUB_OUTPUT runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false e2eTestsDdb-setup: - needs: e2eTestsDdb-init + needs: + - constants + - build + - e2eTestsDdb-constants name: E2E (DDB) - Project setup outputs: cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }} @@ -384,9 +405,7 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-constants.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: @@ -399,32 +418,23 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - path: dev + path: next - uses: actions/cache@v4 - id: yarn-cache with: - path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + path: next/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 - id: cached-packages with: - path: dev/.webiny/cached-packages - key: >- - ${{ runner.os }}-${{ needs.e2eTestsDdb-init.outputs.day }}-${{ - secrets.RANDOM_CACHE_KEY_SUFFIX }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable + working-directory: next - name: Build packages - working-directory: dev run: yarn build:quick - - uses: actions/cache@v4 - id: packages-cache - with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-init.outputs.ts }} + working-directory: next - name: Start Verdaccio local server - working-directory: dev + working-directory: next run: >- yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml @@ -435,72 +445,69 @@ jobs: - name: Set git username run: git config --global user.name "webiny-bot" - name: Create ".npmrc" file in the project root, with a dummy auth token - working-directory: dev run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc + working-directory: next - name: Version and publish to Verdaccio - working-directory: dev run: yarn release --type=verdaccio + working-directory: next - name: Create verdaccio-files artifact uses: actions/upload-artifact@v4 with: name: verdaccio-files-ddb retention-days: 1 - path: | - dev/.verdaccio/ - dev/.verdaccio.yaml - - name: Create directory - run: mkdir xyz + path: |- + next/.verdaccio/ + next/.verdaccio.yaml - name: Disable Webiny telemetry run: > mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' > ~/.webiny/config - name: Create a new Webiny project - working-directory: xyz run: > - npx create-webiny-project@local-npm test-project --tag local-npm + npx create-webiny-project@local-npm new-webiny-project --tag local-npm --no-interactive --assign-to-yarnrc '{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}' --template-options '{"region":"${{ env.AWS_REGION }}","storageOperations":"ddb"}' - name: Print CLI version - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny --version - name: Create project-files artifact uses: actions/upload-artifact@v4 with: name: project-files-ddb retention-days: 1 - path: | - xyz/test-project/ - !xyz/test-project/node_modules/**/* - !xyz/test-project/**/node_modules/**/* - !xyz/test-project/.yarn/cache/**/* + path: |- + new-webiny-project/ + !new-webiny-project/node_modules/**/* + !new-webiny-project/**/node_modules/**/* + !new-webiny-project/.yarn/cache/**/* - name: Deploy Core - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/core --env dev - name: Deploy API - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/api --env dev - name: Deploy Admin Area - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/admin --env dev - name: Deploy Website - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/website --env dev - name: Create Cypress config - working-directory: dev - run: yarn setup-cypress --projectFolder ../xyz/test-project + run: yarn setup-cypress --projectFolder ../new-webiny-project + working-directory: next - name: Save Cypress config id: save-cypress-config - working-directory: dev run: >- echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\t\n\r')" >> $GITHUB_OUTPUT + working-directory: next - name: Cypress - run installation wizard test - working-directory: dev/cypress-tests run: >- - yarn cypress run --browser chrome --spec + yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js" + working-directory: next runs-on: ubuntu-latest permissions: id-token: write @@ -509,7 +516,8 @@ jobs: ${{ matrix.cypress-folder }} (ddb, ${{ matrix.os }}, Node v${{ matrix.node }}) needs: - - e2eTestsDdb-init + - constants + - e2eTestsDdb-constants - e2eTestsDdb-setup strategy: fail-fast: false @@ -518,7 +526,7 @@ jobs: - ubuntu-latest node: - 18 - cypress-folder: ${{ fromJson(needs.e2eTestsDdb-init.outputs.cypress-folders) }} + cypress-folder: ${{ fromJson(needs.e2eTestsDdb-constants.outputs.cypress-folders) }} environment: next env: NODE_OPTIONS: '--max_old_space_size=4096' @@ -526,74 +534,69 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-constants.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 with: - path: dev + path: next - uses: actions/cache@v4 with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-init.outputs.ts }} + path: next/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable + working-directory: next - name: Build packages - working-directory: dev run: yarn build:quick + working-directory: next - name: Set up Cypress config - working-directory: dev run: >- echo '${{ needs.e2eTestsDdb-setup.outputs.cypress-config }}' > cypress-tests/cypress.config.ts + working-directory: next - name: Cypress - run "${{ matrix.cypress-folder }}" tests - working-directory: dev/cypress-tests timeout-minutes: 40 - run: >- - yarn cypress run --browser chrome --spec "${{ matrix.cypress-folder - }}" + run: yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}" + working-directory: next runs-on: ubuntu-latest - e2eTestsDdb-es-init: - name: E2E (DDB-ES) - Init + e2eTestsDdb-es-constants: + name: Constants - DDB-ES + needs: + - build outputs: - day: ${{ steps.get-day.outputs.day }} - ts: ${{ steps.get-timestamp.outputs.ts }} cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} + pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 - - name: Get day of the month - id: get-day - run: >- - echo "day=$(node --eval "console.log(new Date().getDate())")" >> - $GITHUB_OUTPUT - - name: Get timestamp - id: get-timestamp - run: >- - echo "ts=$(node --eval "console.log(new Date().getTime())")" >> - $GITHUB_OUTPUT - name: List Cypress tests folders id: list-cypress-folders run: >- echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT + - name: Get Pulumi backend URL + id: get-pulumi-backend-url + run: >- + echo "pulumi-backend-url=${{ secrets.WEBINY_PULUMI_BACKEND }}${{ + github.run_id }}_ddb-es" >> $GITHUB_OUTPUT runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false e2eTestsDdb-es-setup: - needs: e2eTestsDdb-es-init + needs: + - constants + - build + - e2eTestsDdb-es-constants name: E2E (DDB-ES) - Project setup outputs: cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }} @@ -604,12 +607,10 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-es-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-es-constants.outputs.pulumi-backend-url }} AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_ELASTIC_SEARCH_DOMAIN_NAME }} ELASTIC_SEARCH_ENDPOINT: ${{ secrets.ELASTIC_SEARCH_ENDPOINT }} - ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2eTestsDdb-es-init.outputs.ts }}_ + ELASTIC_SEARCH_INDEX_PREFIX: ${{ github.run_id }}_ steps: - uses: actions/setup-node@v4 with: @@ -622,32 +623,23 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - path: dev + path: next - uses: actions/cache@v4 - id: yarn-cache with: - path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + path: next/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 - id: cached-packages with: - path: dev/.webiny/cached-packages - key: >- - ${{ runner.os }}-${{ needs.e2eTestsDdb-es-init.outputs.day }}-${{ - secrets.RANDOM_CACHE_KEY_SUFFIX }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable + working-directory: next - name: Build packages - working-directory: dev run: yarn build:quick - - uses: actions/cache@v4 - id: packages-cache - with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-es-init.outputs.ts }} + working-directory: next - name: Start Verdaccio local server - working-directory: dev + working-directory: next run: >- yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml @@ -658,72 +650,69 @@ jobs: - name: Set git username run: git config --global user.name "webiny-bot" - name: Create ".npmrc" file in the project root, with a dummy auth token - working-directory: dev run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc + working-directory: next - name: Version and publish to Verdaccio - working-directory: dev run: yarn release --type=verdaccio + working-directory: next - name: Create verdaccio-files artifact uses: actions/upload-artifact@v4 with: name: verdaccio-files-ddb-es retention-days: 1 - path: | - dev/.verdaccio/ - dev/.verdaccio.yaml - - name: Create directory - run: mkdir xyz + path: |- + next/.verdaccio/ + next/.verdaccio.yaml - name: Disable Webiny telemetry run: > mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' > ~/.webiny/config - name: Create a new Webiny project - working-directory: xyz run: > - npx create-webiny-project@local-npm test-project --tag local-npm + npx create-webiny-project@local-npm new-webiny-project --tag local-npm --no-interactive --assign-to-yarnrc '{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}' --template-options '{"region":"${{ env.AWS_REGION }}","storageOperations":"ddb-es"}' - name: Print CLI version - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny --version - name: Create project-files artifact uses: actions/upload-artifact@v4 with: name: project-files-ddb-es retention-days: 1 - path: | - xyz/test-project/ - !xyz/test-project/node_modules/**/* - !xyz/test-project/**/node_modules/**/* - !xyz/test-project/.yarn/cache/**/* + path: |- + new-webiny-project/ + !new-webiny-project/node_modules/**/* + !new-webiny-project/**/node_modules/**/* + !new-webiny-project/.yarn/cache/**/* - name: Deploy Core - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/core --env dev - name: Deploy API - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/api --env dev - name: Deploy Admin Area - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/admin --env dev - name: Deploy Website - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/website --env dev - name: Create Cypress config - working-directory: dev - run: yarn setup-cypress --projectFolder ../xyz/test-project + run: yarn setup-cypress --projectFolder ../new-webiny-project + working-directory: next - name: Save Cypress config id: save-cypress-config - working-directory: dev run: >- echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\t\n\r')" >> $GITHUB_OUTPUT + working-directory: next - name: Cypress - run installation wizard test - working-directory: dev/cypress-tests run: >- - yarn cypress run --browser chrome --spec + yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js" + working-directory: next runs-on: ubuntu-latest permissions: id-token: write @@ -732,7 +721,8 @@ jobs: ${{ matrix.cypress-folder }} (ddb-es, ${{ matrix.os }}, Node v${{ matrix.node }}) needs: - - e2eTestsDdb-es-init + - constants + - e2eTestsDdb-es-constants - e2eTestsDdb-es-setup strategy: fail-fast: false @@ -741,7 +731,9 @@ jobs: - ubuntu-latest node: - 18 - cypress-folder: ${{ fromJson(needs.e2eTestsDdb-es-init.outputs.cypress-folders) }} + cypress-folder: >- + ${{ fromJson(needs.e2eTestsDdb-es-constants.outputs.cypress-folders) + }} environment: next env: NODE_OPTIONS: '--max_old_space_size=4096' @@ -749,77 +741,72 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-es-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-es-constants.outputs.pulumi-backend-url }} AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_ELASTIC_SEARCH_DOMAIN_NAME }} ELASTIC_SEARCH_ENDPOINT: ${{ secrets.ELASTIC_SEARCH_ENDPOINT }} - ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2eTestsDdb-es-init.outputs.ts }}_ + ELASTIC_SEARCH_INDEX_PREFIX: ${{ github.run_id }}_ steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 with: - path: dev + path: next - uses: actions/cache@v4 with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-es-init.outputs.ts }} + path: next/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable + working-directory: next - name: Build packages - working-directory: dev run: yarn build:quick + working-directory: next - name: Set up Cypress config - working-directory: dev run: >- echo '${{ needs.e2eTestsDdb-es-setup.outputs.cypress-config }}' > cypress-tests/cypress.config.ts + working-directory: next - name: Cypress - run "${{ matrix.cypress-folder }}" tests - working-directory: dev/cypress-tests timeout-minutes: 40 - run: >- - yarn cypress run --browser chrome --spec "${{ matrix.cypress-folder - }}" + run: yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}" + working-directory: next runs-on: ubuntu-latest - e2eTestsDdb-os-init: - name: E2E (DDB-OS) - Init + e2eTestsDdb-os-constants: + name: Constants - DDB-OS + needs: + - build outputs: - day: ${{ steps.get-day.outputs.day }} - ts: ${{ steps.get-timestamp.outputs.ts }} cypress-folders: ${{ steps.list-cypress-folders.outputs.cypress-folders }} + pulumi-backend-url: ${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }} steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 - - name: Get day of the month - id: get-day - run: >- - echo "day=$(node --eval "console.log(new Date().getDate())")" >> - $GITHUB_OUTPUT - - name: Get timestamp - id: get-timestamp - run: >- - echo "ts=$(node --eval "console.log(new Date().getTime())")" >> - $GITHUB_OUTPUT - name: List Cypress tests folders id: list-cypress-folders run: >- echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT + - name: Get Pulumi backend URL + id: get-pulumi-backend-url + run: >- + echo "pulumi-backend-url=${{ secrets.WEBINY_PULUMI_BACKEND }}${{ + github.run_id }}_ddb-os" >> $GITHUB_OUTPUT runs-on: ubuntu-latest env: NODE_OPTIONS: '--max_old_space_size=4096' YARN_ENABLE_IMMUTABLE_INSTALLS: false e2eTestsDdb-os-setup: - needs: e2eTestsDdb-os-init + needs: + - constants + - build + - e2eTestsDdb-os-constants name: E2E (DDB-OS) - Project setup outputs: cypress-config: ${{ steps.save-cypress-config.outputs.cypress-config }} @@ -830,12 +817,10 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-os-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-os-constants.outputs.pulumi-backend-url }} AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }} ELASTIC_SEARCH_ENDPOINT: ${{ secrets.OPEN_SEARCH_ENDPOINT }} - ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2eTestsDdb-os-init.outputs.ts }}_ + ELASTIC_SEARCH_INDEX_PREFIX: ${{ github.run_id }}_ steps: - uses: actions/setup-node@v4 with: @@ -848,32 +833,23 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - path: dev + path: next - uses: actions/cache@v4 - id: yarn-cache with: - path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + path: next/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 - id: cached-packages with: - path: dev/.webiny/cached-packages - key: >- - ${{ runner.os }}-${{ needs.e2eTestsDdb-os-init.outputs.day }}-${{ - secrets.RANDOM_CACHE_KEY_SUFFIX }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable + working-directory: next - name: Build packages - working-directory: dev run: yarn build:quick - - uses: actions/cache@v4 - id: packages-cache - with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-os-init.outputs.ts }} + working-directory: next - name: Start Verdaccio local server - working-directory: dev + working-directory: next run: >- yarn add pm2 verdaccio && npx pm2 start verdaccio -- -c .verdaccio.yaml @@ -884,72 +860,69 @@ jobs: - name: Set git username run: git config --global user.name "webiny-bot" - name: Create ".npmrc" file in the project root, with a dummy auth token - working-directory: dev run: echo '//localhost:4873/:_authToken="dummy-auth-token"' > .npmrc + working-directory: next - name: Version and publish to Verdaccio - working-directory: dev run: yarn release --type=verdaccio + working-directory: next - name: Create verdaccio-files artifact uses: actions/upload-artifact@v4 with: name: verdaccio-files-ddb-os retention-days: 1 - path: | - dev/.verdaccio/ - dev/.verdaccio.yaml - - name: Create directory - run: mkdir xyz + path: |- + next/.verdaccio/ + next/.verdaccio.yaml - name: Disable Webiny telemetry run: > mkdir ~/.webiny && echo '{ "id": "ci", "telemetry": false }' > ~/.webiny/config - name: Create a new Webiny project - working-directory: xyz run: > - npx create-webiny-project@local-npm test-project --tag local-npm + npx create-webiny-project@local-npm new-webiny-project --tag local-npm --no-interactive --assign-to-yarnrc '{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}' --template-options '{"region":"${{ env.AWS_REGION }}","storageOperations":"ddb-os"}' - name: Print CLI version - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny --version - name: Create project-files artifact uses: actions/upload-artifact@v4 with: name: project-files-ddb-os retention-days: 1 - path: | - xyz/test-project/ - !xyz/test-project/node_modules/**/* - !xyz/test-project/**/node_modules/**/* - !xyz/test-project/.yarn/cache/**/* + path: |- + new-webiny-project/ + !new-webiny-project/node_modules/**/* + !new-webiny-project/**/node_modules/**/* + !new-webiny-project/.yarn/cache/**/* - name: Deploy Core - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/core --env dev - name: Deploy API - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/api --env dev - name: Deploy Admin Area - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/admin --env dev - name: Deploy Website - working-directory: xyz/test-project + working-directory: new-webiny-project run: yarn webiny deploy apps/website --env dev - name: Create Cypress config - working-directory: dev - run: yarn setup-cypress --projectFolder ../xyz/test-project + run: yarn setup-cypress --projectFolder ../new-webiny-project + working-directory: next - name: Save Cypress config id: save-cypress-config - working-directory: dev run: >- echo "cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\t\n\r')" >> $GITHUB_OUTPUT + working-directory: next - name: Cypress - run installation wizard test - working-directory: dev/cypress-tests run: >- - yarn cypress run --browser chrome --spec + yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js" + working-directory: next runs-on: ubuntu-latest permissions: id-token: write @@ -958,7 +931,8 @@ jobs: ${{ matrix.cypress-folder }} (ddb-os, ${{ matrix.os }}, Node v${{ matrix.node }}) needs: - - e2eTestsDdb-os-init + - constants + - e2eTestsDdb-os-constants - e2eTestsDdb-os-setup strategy: fail-fast: false @@ -967,7 +941,9 @@ jobs: - ubuntu-latest node: - 18 - cypress-folder: ${{ fromJson(needs.e2eTestsDdb-os-init.outputs.cypress-folders) }} + cypress-folder: >- + ${{ fromJson(needs.e2eTestsDdb-os-constants.outputs.cypress-folders) + }} environment: next env: NODE_OPTIONS: '--max_old_space_size=4096' @@ -975,48 +951,44 @@ jobs: CYPRESS_MAILOSAUR_API_KEY: ${{ secrets.CYPRESS_MAILOSAUR_API_KEY }} PULUMI_CONFIG_PASSPHRASE: ${{ secrets.PULUMI_CONFIG_PASSPHRASE }} PULUMI_SECRETS_PROVIDER: ${{ secrets.PULUMI_SECRETS_PROVIDER }} - WEBINY_PULUMI_BACKEND: >- - ${{ secrets.WEBINY_PULUMI_BACKEND }}${{ - needs.e2eTestsDdb-os-init.outputs.ts }}_ddb + WEBINY_PULUMI_BACKEND: ${{ needs.e2eTestsDdb-os-constants.outputs.pulumi-backend-url }} AWS_ELASTIC_SEARCH_DOMAIN_NAME: ${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }} ELASTIC_SEARCH_ENDPOINT: ${{ secrets.OPEN_SEARCH_ENDPOINT }} - ELASTIC_SEARCH_INDEX_PREFIX: ${{ needs.e2eTestsDdb-os-init.outputs.ts }}_ + ELASTIC_SEARCH_INDEX_PREFIX: ${{ github.run_id }}_ steps: - uses: actions/setup-node@v4 with: node-version: 18 - uses: actions/checkout@v4 with: - path: dev + path: next - uses: actions/cache@v4 with: - path: dev/.webiny/cached-packages - key: packages-cache-${{ needs.e2eTestsDdb-os-init.outputs.ts }} + path: next/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: dev/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies - working-directory: dev run: yarn --immutable + working-directory: next - name: Build packages - working-directory: dev run: yarn build:quick + working-directory: next - name: Set up Cypress config - working-directory: dev run: >- echo '${{ needs.e2eTestsDdb-os-setup.outputs.cypress-config }}' > cypress-tests/cypress.config.ts + working-directory: next - name: Cypress - run "${{ matrix.cypress-folder }}" tests - working-directory: dev/cypress-tests timeout-minutes: 40 - run: >- - yarn cypress run --browser chrome --spec "${{ matrix.cypress-folder - }}" + run: yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}" + working-directory: next runs-on: ubuntu-latest npmReleaseUnstable: needs: - - init + - constants - codeAnalysis - jestTestsNoStorage - jestTestsDdb @@ -1041,22 +1013,27 @@ jobs: fetch-depth: 0 - uses: actions/cache@v4 with: - path: .yarn/cache + path: next/.yarn/cache key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - uses: actions/cache@v4 with: - path: .webiny/cached-packages - key: packages-cache-${{ needs.init.outputs.ts }} + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: next - name: Build packages - run: yarn build + run: yarn build:quick + working-directory: next - name: Create ".npmrc" file in the project root run: echo "//registry.npmjs.org/:_authToken=\${NPM_TOKEN}" > .npmrc + working-directory: next - name: Set git info run: | git config --global user.email "webiny-bot@webiny.com" git config --global user.name "webiny-bot" + working-directory: next - name: Version and publish to NPM run: yarn release --type=unstable + working-directory: next runs-on: ubuntu-latest diff --git a/.github/workflows/pushStable.yml b/.github/workflows/pushStable.yml index 988b4886483..e8da01b8856 100644 --- a/.github/workflows/pushStable.yml +++ b/.github/workflows/pushStable.yml @@ -25,15 +25,22 @@ jobs: YARN_ENABLE_IMMUTABLE_INSTALLS: false constants: name: Create constants - runs-on: ubuntu-latest outputs: run-cache-key: ${{ steps.run-cache-key.outputs.run-cache-key }} steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 + - uses: actions/checkout@v4 - name: Create workflow run cache key id: run-cache-key run: >- echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false build: name: Build needs: constants @@ -49,8 +56,10 @@ jobs: key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - name: Install dependencies run: yarn --immutable + working-directory: '' - name: Build packages run: yarn build:quick + working-directory: '' - uses: actions/cache@v4 with: path: .webiny/cached-packages @@ -86,8 +95,10 @@ jobs: key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: '' - name: Build packages - run: yarn build + run: yarn build:quick + working-directory: '' - name: Create ".npmrc" file in the project root run: echo "//registry.npmjs.org/:_authToken=\${NPM_TOKEN}" > .npmrc - name: Set git email @@ -127,8 +138,10 @@ jobs: key: ${{ needs.constants.outputs.run-cache-key }} - name: Install dependencies run: yarn --immutable + working-directory: '' - name: Build packages - run: yarn build + run: yarn build:quick + working-directory: '' - name: Create ".npmrc" file in the project root run: echo "//registry.npmjs.org/:_authToken=\${NPM_TOKEN}" > .npmrc - name: Set git email diff --git a/.github/workflows/rebuildCache.yml b/.github/workflows/rebuildCache.yml deleted file mode 100644 index 3c92b005fed..00000000000 --- a/.github/workflows/rebuildCache.yml +++ /dev/null @@ -1,83 +0,0 @@ -# This ensures build/yarn caches are rebuilt and that we don't experience cache miss -# when pushing our own code (which makes the whole job last significantly longer). -name: Rebuild Cache (Next) - -on: - schedule: - - cron: "0 1 * * *" # Every day at 1AM. - -env: - NODE_OPTIONS: --max_old_space_size=4096 - -jobs: - cache-keys: - name: Create cache keys - runs-on: ubuntu-latest - outputs: - day: ${{ steps.create-cache-keys.outputs.day }} - steps: - - name: Create cache key (day) - id: create-cache-keys - run: echo "day=$(node --eval "console.log(new Date().getDate())")" >> $GITHUB_OUTPUT - - cache-dependencies-packages-root: - name: Cache dependencies and packages - runs-on: ubuntu-latest - needs: cache-keys - steps: - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - uses: actions/checkout@v4 - - - uses: actions/cache@v4 - id: yarn-cache - with: - path: .yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} - - - uses: actions/cache@v4 - id: cached-packages - with: - path: .webiny/cached-packages - key: ${{ runner.os }}-${{ needs.cache-keys.outputs.day }}-${{ secrets.RANDOM_CACHE_KEY_SUFFIX }} - - - name: Install dependencies - run: yarn --immutable - - - name: Build packages - run: yarn build - - cache-dependencies-packages-next: - name: Cache dependencies and packages ("next" folder) - runs-on: ubuntu-latest - needs: cache-keys - steps: - - uses: actions/setup-node@v4 - with: - node-version: 18 - - - uses: actions/checkout@v4 - with: - path: next - - - uses: actions/cache@v4 - id: yarn-cache - with: - path: next/.yarn/cache - key: yarn-${{ runner.os }}-${{ hashFiles('next/**/yarn.lock') }} - - - uses: actions/cache@v4 - id: cached-packages - with: - path: next/.webiny/cached-packages - key: ${{ runner.os }}-${{ needs.cache-keys.outputs.day }}-${{ secrets.RANDOM_CACHE_KEY_SUFFIX }} - - - name: Install dependencies - working-directory: next - run: yarn --immutable - - - name: Build packages - working-directory: next - run: yarn build diff --git a/.github/workflows/rebuildGlobalCacheDev.yml b/.github/workflows/rebuildGlobalCacheDev.yml new file mode 100644 index 00000000000..73729864f1a --- /dev/null +++ b/.github/workflows/rebuildGlobalCacheDev.yml @@ -0,0 +1,56 @@ +# This file was automatically generated by github-actions-wac. +# DO NOT MODIFY IT BY HAND. Instead, modify the source *.wac.ts file(s) +# and run "github-actions-wac build" (or "ghawac build") to regenerate this file. +# For more information, run "github-actions-wac --help". +name: Rebuild Global Cache ("dev" branch) +'on': + workflow_dispatch: {} + schedule: + - cron: 0 4 * * * +jobs: + constants: + name: Create constants + outputs: + global-cache-key: ${{ steps.global-cache-key.outputs.global-cache-key }} + steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Create global cache key + id: global-cache-key + run: >- + echo "global-cache-key=dev-${{ runner.os }}-$(/bin/date -u + "+%m%d")-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false + cacheDependenciesPackages: + name: Cache dependencies and packages + needs: constants + steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 + - uses: actions/checkout@v4 + with: + path: dev + ref: dev + - uses: actions/cache@v4 + with: + path: dev/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} + - uses: actions/cache@v4 + with: + path: dev/.webiny/cached-packages + key: ${{ needs.constants.outputs.global-cache-key }} + - name: Install dependencies + run: yarn --immutable + working-directory: dev + - name: Build packages + run: yarn build:quick + working-directory: dev + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false diff --git a/.github/workflows/rebuildGlobalCacheNext.yml b/.github/workflows/rebuildGlobalCacheNext.yml new file mode 100644 index 00000000000..78c1ab0c6aa --- /dev/null +++ b/.github/workflows/rebuildGlobalCacheNext.yml @@ -0,0 +1,56 @@ +# This file was automatically generated by github-actions-wac. +# DO NOT MODIFY IT BY HAND. Instead, modify the source *.wac.ts file(s) +# and run "github-actions-wac build" (or "ghawac build") to regenerate this file. +# For more information, run "github-actions-wac --help". +name: Rebuild Global Cache ("next" branch) +'on': + workflow_dispatch: {} + schedule: + - cron: 0 4 * * * +jobs: + constants: + name: Create constants + outputs: + global-cache-key: ${{ steps.global-cache-key.outputs.global-cache-key }} + steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Create global cache key + id: global-cache-key + run: >- + echo "global-cache-key=next-${{ runner.os }}-$(/bin/date -u + "+%m%d")-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false + cacheDependenciesPackages: + name: Cache dependencies and packages + needs: constants + steps: + - uses: actions/setup-node@v4 + with: + node-version: 18 + - uses: actions/checkout@v4 + with: + path: next + ref: next + - uses: actions/cache@v4 + with: + path: next/.yarn/cache + key: yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }} + - uses: actions/cache@v4 + with: + path: next/.webiny/cached-packages + key: ${{ needs.constants.outputs.global-cache-key }} + - name: Install dependencies + run: yarn --immutable + working-directory: next + - name: Build packages + run: yarn build:quick + working-directory: next + runs-on: ubuntu-latest + env: + NODE_OPTIONS: '--max_old_space_size=4096' + YARN_ENABLE_IMMUTABLE_INSTALLS: false diff --git a/.github/workflows/wac/jobs/createJestTestsJob.ts b/.github/workflows/wac/jobs/createJestTestsJob.ts deleted file mode 100644 index 3651497c542..00000000000 --- a/.github/workflows/wac/jobs/createJestTestsJob.ts +++ /dev/null @@ -1,77 +0,0 @@ -import { listPackagesWithJestTests, NODE_VERSION } from "../utils"; -import { NormalJob } from "github-actions-wac"; -import { createJob } from "./createJob"; - -export const createJestTestsJob = (storage: string | null) => { - const env: Record = {}; - - if (storage) { - if (storage === "ddb-es") { - env["AWS_ELASTIC_SEARCH_DOMAIN_NAME"] = "${{ secrets.AWS_ELASTIC_SEARCH_DOMAIN_NAME }}"; - env["ELASTIC_SEARCH_ENDPOINT"] = "${{ secrets.ELASTIC_SEARCH_ENDPOINT }}"; - env["ELASTIC_SEARCH_INDEX_PREFIX"] = "${{ matrix.package.id }}"; - } else if (storage === "ddb-os") { - // We still use the same environment variables as for "ddb-es" setup, it's - // just that the values are read from different secrets. - env["AWS_ELASTIC_SEARCH_DOMAIN_NAME"] = "${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }}"; - env["ELASTIC_SEARCH_ENDPOINT"] = "${{ secrets.OPEN_SEARCH_ENDPOINT }}"; - env["ELASTIC_SEARCH_INDEX_PREFIX"] = "${{ matrix.package.id }}"; - } - } - - const packages = listPackagesWithJestTests({ - storage - }); - - const job: NormalJob = createJob({ - needs: "init", - name: "${{ matrix.package.cmd }}", - strategy: { - "fail-fast": false, - matrix: { - os: ["ubuntu-latest"], - node: [NODE_VERSION], - package: "${{ fromJson('" + JSON.stringify(packages) + "') }}" - } - }, - "runs-on": "${{ matrix.os }}", - env, - awsAuth: storage === "ddb-es" || storage === "ddb-os", - steps: [ - { - uses: "actions/cache@v4", - with: { - path: ".yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}" - } - }, - { - uses: "actions/cache@v4", - with: { - path: ".webiny/cached-packages", - key: "packages-cache-${{ needs.init.outputs.ts }}" - } - }, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages", - run: "yarn build:quick" - }, - { - name: "Run tests", - run: "yarn test ${{ matrix.package.cmd }}" - } - ] - }); - - // We prevent running of Jest tests if a PR was created from a fork. - // This is because we don't want to expose our AWS credentials to forks. - if (storage === "ddb-es" || storage === "ddb-os") { - job.if = "needs.init.outputs.is-fork-pr != 'true'"; - } - - return job; -}; diff --git a/.github/workflows/wac/jobs/createValidateWorkflowsJob.ts b/.github/workflows/wac/jobs/createValidateWorkflowsJob.ts index 36aaa5a46be..ca00060c893 100644 --- a/.github/workflows/wac/jobs/createValidateWorkflowsJob.ts +++ b/.github/workflows/wac/jobs/createValidateWorkflowsJob.ts @@ -1,6 +1,9 @@ import { createJob } from "./createJob"; +import { NormalJob } from "github-actions-wac"; -export const createValidateWorkflowsJob = () => +type CreateValidateWorkflowsJobParams = Partial; + +export const createValidateWorkflowsJob = (params: CreateValidateWorkflowsJobParams = {}) => createJob({ name: "Validate workflows", steps: [ @@ -12,5 +15,6 @@ export const createValidateWorkflowsJob = () => name: "Validate", run: "npx github-actions-wac validate" } - ] + ], + ...params }); diff --git a/.github/workflows/wac/jobs/index.ts b/.github/workflows/wac/jobs/index.ts index 1baa4fd33ff..0d800a8d821 100644 --- a/.github/workflows/wac/jobs/index.ts +++ b/.github/workflows/wac/jobs/index.ts @@ -1,3 +1,2 @@ export * from "./createJob"; export * from "./createValidateWorkflowsJob"; -export * from "./createJestTestsJob"; diff --git a/.github/workflows/wac/pullRequests.wac.ts b/.github/workflows/wac/pullRequests.wac.ts index ed0e206cc03..a9bd769deee 100644 --- a/.github/workflows/wac/pullRequests.wac.ts +++ b/.github/workflows/wac/pullRequests.wac.ts @@ -1,36 +1,21 @@ import { createWorkflow, NormalJob } from "github-actions-wac"; -import { createValidateWorkflowsJob, createJob } from "./jobs"; +import { createJob } from "./jobs"; import { NODE_VERSION, listPackagesWithJestTests } from "./utils"; +import { + createGlobalBuildCacheSteps, + createInstallBuildSteps, + createRunBuildCacheSteps, + createYarnCacheSteps, + withCommonParams +} from "./steps"; -const yarnCacheSteps: NormalJob["steps"] = [ - { - uses: "actions/cache@v4", - with: { - path: ".yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}" - } - } -]; - -const buildGlobalCacheSteps = [ - { - uses: "actions/cache@v4", - with: { - path: ".webiny/cached-packages", - key: "${{ needs.constants.outputs.global-cache-key }}" - } - } -]; +// Will print "next" or "dev". Important for caching (via actions/cache). +const DIR_WEBINY_JS = "${{ github.base_ref }}"; -const buildRunCacheSteps = [ - { - uses: "actions/cache@v4", - with: { - path: ".webiny/cached-packages", - key: "${{ needs.constants.outputs.run-cache-key }}" - } - } -]; +const installBuildSteps = createInstallBuildSteps({ workingDirectory: DIR_WEBINY_JS }); +const yarnCacheSteps = createYarnCacheSteps({ workingDirectory: DIR_WEBINY_JS }); +const globalBuildCacheSteps = createGlobalBuildCacheSteps({ workingDirectory: DIR_WEBINY_JS }); +const runBuildCacheSteps = createRunBuildCacheSteps({ workingDirectory: DIR_WEBINY_JS }); const createJestTestsJob = (storage: string | null) => { const env: Record = {}; @@ -54,7 +39,7 @@ const createJestTestsJob = (storage: string | null) => { }); const job: NormalJob = createJob({ - needs: "constants", + needs: ["constants", "build"], name: "${{ matrix.package.cmd }}", strategy: { "fail-fast": false, @@ -67,20 +52,15 @@ const createJestTestsJob = (storage: string | null) => { "runs-on": "${{ matrix.os }}", env, awsAuth: storage === "ddb-es" || storage === "ddb-os", + checkout: { path: DIR_WEBINY_JS }, steps: [ ...yarnCacheSteps, - ...buildRunCacheSteps, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages", - run: "yarn build:quick" - }, + ...runBuildCacheSteps, + ...installBuildSteps, { name: "Run tests", - run: "yarn test ${{ matrix.package.cmd }}" + run: "yarn test ${{ matrix.package.cmd }}", + "working-directory": DIR_WEBINY_JS } ] }); @@ -98,7 +78,7 @@ export const pullRequests = createWorkflow({ name: "Pull Requests", on: "pull_request", jobs: { - validateWorkflows: createValidateWorkflowsJob(), + // validateWorkflows: createValidateWorkflowsJob(), validateCommits: createJob({ name: "Validate commit messages", if: "github.base_ref != 'dev'", @@ -119,14 +99,14 @@ export const pullRequests = createWorkflow({ } ] }), - constants: { + constants: createJob({ name: "Create constants", - "runs-on": "ubuntu-latest", outputs: { "global-cache-key": "${{ steps.global-cache-key.outputs.global-cache-key }}", "run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}", "is-fork-pr": "${{ steps.is-fork-pr.outputs.is-fork-pr }}" }, + checkout: false, steps: [ { name: "Create global cache key", @@ -144,77 +124,60 @@ export const pullRequests = createWorkflow({ run: 'echo "is-fork-pr=${{ github.event.pull_request.head.repo.fork }}" >> $GITHUB_OUTPUT' } ] - }, - init: createJob({ - name: "Init", + }), + build: createJob({ + name: "Build", needs: "constants", - "runs-on": "webiny-build-packages", + "runs-on": "blacksmith-8vcpu-ubuntu-2204", + checkout: { path: DIR_WEBINY_JS }, steps: [ ...yarnCacheSteps, - ...buildGlobalCacheSteps, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages", - run: "yarn build:quick" - }, + ...globalBuildCacheSteps, + ...installBuildSteps, + // Once we've built packages with the help of the global cache, we can now cache // the result for this run. All of the following jobs will use this cache. - ...buildRunCacheSteps + ...runBuildCacheSteps ] }), staticCodeAnalysis: createJob({ - needs: ["constants", "init"], + needs: ["constants", "build"], name: "Static code analysis", + checkout: { path: DIR_WEBINY_JS }, steps: [ ...yarnCacheSteps, - ...buildRunCacheSteps, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Check code formatting", - run: "yarn prettier:check" - }, - { - name: "Check dependencies", - run: "yarn adio" - }, - { - name: "Check TS configs", - run: "yarn check-ts-configs" - }, - { - name: "ESLint", - run: "yarn eslint" - } + ...runBuildCacheSteps, + ...withCommonParams( + [ + { name: "Install dependencies", run: "yarn --immutable" }, + { name: "Check code formatting", run: "yarn prettier:check" }, + { name: "Check dependencies", run: "yarn adio" }, + { name: "Check TS configs", run: "yarn check-ts-configs" }, + { name: "ESLint", run: "yarn eslint" } + ], + { "working-directory": DIR_WEBINY_JS } + ) ] }), staticCodeAnalysisTs: createJob({ name: "Static code analysis (TypeScript)", - "runs-on": "webiny-build-packages", + "runs-on": "blacksmith-8vcpu-ubuntu-2204", + checkout: { path: DIR_WEBINY_JS }, steps: [ ...yarnCacheSteps, // We're not using run cache here. We want to build all packages // with TypeScript, to ensure there are no TypeScript errors. - // ...buildRunCacheSteps, + // ...runBuildCacheSteps, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages (full)", - run: "yarn build" - }, - { - name: "Check types for Cypress tests", - run: "yarn cy:ts" - } + ...withCommonParams( + [ + { name: "Install dependencies", run: "yarn --immutable" }, + { name: "Build packages (full)", run: "yarn build" }, + { name: "Check types for Cypress tests", run: "yarn cy:ts" } + ], + { "working-directory": DIR_WEBINY_JS } + ) ] }), jestTestsNoStorage: createJestTestsJob(null), @@ -224,54 +187,56 @@ export const pullRequests = createWorkflow({ verdaccioPublish: createJob({ name: "Publish to Verdaccio", - needs: ["constants", "init"], + needs: ["constants", "build"], if: "needs.constants.outputs.is-fork-pr != 'true'", checkout: { "fetch-depth": 0, - ref: "${{ github.event.pull_request.head.ref }}" + ref: "${{ github.event.pull_request.head.ref }}", + path: DIR_WEBINY_JS }, steps: [ ...yarnCacheSteps, - ...buildRunCacheSteps, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages", - run: "yarn build:quick" - }, - { - name: "Start Verdaccio local server", - run: "npx pm2 start verdaccio -- -c .verdaccio.yaml" - }, - { - name: "Configure NPM to use local registry", - run: "npm config set registry http://localhost:4873" - }, - { - name: "Set git email", - run: 'git config --global user.email "webiny-bot@webiny.com"' - }, - { - name: "Set git username", - run: 'git config --global user.name "webiny-bot"' - }, - { - name: 'Create ".npmrc" file in the project root, with a dummy auth token', - run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc" - }, - { - name: "Version and publish to Verdaccio", - run: "yarn release --type=verdaccio" - }, + ...runBuildCacheSteps, + ...installBuildSteps, + ...withCommonParams( + [ + { + name: "Start Verdaccio local server", + run: "npx pm2 start verdaccio -- -c .verdaccio.yaml" + }, + { + name: "Configure NPM to use local registry", + run: "npm config set registry http://localhost:4873" + }, + { + name: "Set git email", + run: 'git config --global user.email "webiny-bot@webiny.com"' + }, + { + name: "Set git username", + run: 'git config --global user.name "webiny-bot"' + }, + { + name: 'Create ".npmrc" file in the project root, with a dummy auth token', + run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc" + }, + { + name: "Version and publish to Verdaccio", + run: "yarn release --type=verdaccio" + } + ], + { "working-directory": DIR_WEBINY_JS } + ), { name: "Upload verdaccio files", uses: "actions/upload-artifact@v4", with: { name: "verdaccio-files", "retention-days": 1, - path: ".verdaccio/\n.verdaccio.yaml\n" + path: [ + DIR_WEBINY_JS + "/.verdaccio/", + DIR_WEBINY_JS + "/.verdaccio.yaml" + ].join("\n") } } ] diff --git a/.github/workflows/wac/pullRequestsCommandCypress.wac.ts b/.github/workflows/wac/pullRequestsCommandCypress.wac.ts index f4a116a932b..542d6b30371 100644 --- a/.github/workflows/wac/pullRequestsCommandCypress.wac.ts +++ b/.github/workflows/wac/pullRequestsCommandCypress.wac.ts @@ -1,35 +1,36 @@ import { createWorkflow, NormalJob } from "github-actions-wac"; -import { createSetupVerdaccioSteps, createDeployWebinySteps } from "./steps"; +import { + createSetupVerdaccioSteps, + createDeployWebinySteps, + createYarnCacheSteps, + createInstallBuildSteps, + createGlobalBuildCacheSteps, + createRunBuildCacheSteps +} from "./steps"; import { NODE_OPTIONS, NODE_VERSION } from "./utils"; import { createJob, createValidateWorkflowsJob } from "./jobs"; -const DIR_PR = "checked-out-pr"; +// Will print "next" or "dev". Important for caching (via actions/cache). +const DIR_WEBINY_JS = "${{ needs.baseBranch.outputs.base-branch }}"; const DIR_TEST_PROJECT = "new-webiny-project"; -const createCheckoutPrSteps = ({ workingDirectory = "" } = {}) => +const installBuildSteps = createInstallBuildSteps({ workingDirectory: DIR_WEBINY_JS }); +const yarnCacheSteps = createYarnCacheSteps({ workingDirectory: DIR_WEBINY_JS }); +const globalBuildCacheSteps = createGlobalBuildCacheSteps({ workingDirectory: DIR_WEBINY_JS }); +const runBuildCacheSteps = createRunBuildCacheSteps({ workingDirectory: DIR_WEBINY_JS }); + +const createCheckoutPrSteps = () => [ { name: "Install Hub Utility", run: "sudo apt-get install -y hub" }, { name: "Checkout Pull Request", - "working-directory": workingDirectory, + "working-directory": DIR_WEBINY_JS, run: "hub pr checkout ${{ github.event.issue.number }}", - env: { - GITHUB_TOKEN: "${{ secrets.GH_TOKEN }}" - } + env: { GITHUB_TOKEN: "${{ secrets.GH_TOKEN }}" } } ] as NonNullable; -const yarnCacheSteps: NormalJob["steps"] = [ - { - uses: "actions/cache@v4", - with: { - path: DIR_PR + "/.yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}" - } - } -]; - -const createJobs = (dbSetup: string) => { +const createCypressJobs = (dbSetup: string) => { const jobNames = { constants: `e2e-wby-cms-${dbSetup}-constants`, projectSetup: `e2e-wby-cms-${dbSetup}-project-setup`, @@ -37,31 +38,20 @@ const createJobs = (dbSetup: string) => { }; const constantsJob: NormalJob = createJob({ - needs: "checkComment", - name: `E2E (${dbSetup.toUpperCase()}) - Constants`, + needs: ["baseBranch", "constants", "build"], + name: `Constants - ${dbSetup.toUpperCase()}`, outputs: { - "global-cache-key": "${{ steps.global-cache-key.outputs.global-cache-key }}", - "run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}", "cypress-folders": "${{ steps.list-cypress-folders.outputs.cypress-folders }}", "pulumi-backend-url": "${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }}" }, + checkout: { path: DIR_WEBINY_JS }, steps: [ ...createCheckoutPrSteps(), - { - name: "Create global cache key", - id: "global-cache-key", - run: 'echo "global-cache-key=$(hub pr show ${{ github.event.issue.number }} -f %B)-${{ runner.os }}-$(/bin/date -u "+%m%d")-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT', - env: { GITHUB_TOKEN: "${{ secrets.GH_TOKEN }}" } - }, - { - name: "Create workflow run cache key", - id: "run-cache-key", - run: 'echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT' - }, { name: "List Cypress tests folders", id: "list-cypress-folders", - run: 'echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT' + run: 'echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT', + "working-directory": DIR_WEBINY_JS }, { name: "Get Pulumi backend URL", @@ -71,26 +61,6 @@ const createJobs = (dbSetup: string) => { ] }); - const buildGlobalCacheSteps = [ - { - uses: "actions/cache@v4", - with: { - path: DIR_PR + "/.webiny/cached-packages", - key: `\${{ needs.${jobNames.constants}.outputs.global-cache-key }}` - } - } - ]; - - const buildRunCacheSteps = [ - { - uses: "actions/cache@v4", - with: { - path: DIR_PR + "/.webiny/cached-packages", - key: `\${{ needs.${jobNames.constants}.outputs.run-cache-key }}` - } - } - ]; - const env: Record = { CYPRESS_MAILOSAUR_API_KEY: "${{ secrets.CYPRESS_MAILOSAUR_API_KEY }}", PULUMI_CONFIG_PASSPHRASE: "${{ secrets.PULUMI_CONFIG_PASSPHRASE }}", @@ -111,39 +81,29 @@ const createJobs = (dbSetup: string) => { } const projectSetupJob: NormalJob = createJob({ - needs: jobNames.constants, + needs: ["baseBranch", "constants", jobNames.constants], name: `E2E (${dbSetup.toUpperCase()}) - Project setup`, outputs: { "cypress-config": "${{ steps.save-cypress-config.outputs.cypress-config }}" }, environment: "next", env, - checkout: { path: DIR_PR }, awsAuth: true, + checkout: { path: DIR_WEBINY_JS }, steps: [ - ...createCheckoutPrSteps({ workingDirectory: DIR_PR }), + ...createCheckoutPrSteps(), ...yarnCacheSteps, - ...buildGlobalCacheSteps, - { - name: "Install dependencies", - "working-directory": DIR_PR, - run: "yarn --immutable" - }, - { - name: "Build packages", - "working-directory": DIR_PR, - run: "yarn build:quick" - }, - ...buildRunCacheSteps, - ...createSetupVerdaccioSteps({ workingDirectory: DIR_PR }), + ...runBuildCacheSteps, + ...installBuildSteps, + ...createSetupVerdaccioSteps({ workingDirectory: DIR_WEBINY_JS }), { name: 'Create ".npmrc" file in the project root, with a dummy auth token', - "working-directory": DIR_PR, + "working-directory": DIR_WEBINY_JS, run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc" }, { name: "Version and publish to Verdaccio", - "working-directory": DIR_PR, + "working-directory": DIR_WEBINY_JS, run: "yarn release --type=verdaccio" }, { @@ -152,7 +112,9 @@ const createJobs = (dbSetup: string) => { with: { name: `verdaccio-files-${dbSetup}`, "retention-days": 1, - path: [DIR_PR + "/.verdaccio/", DIR_PR + "/.verdaccio.yaml\n"].join("\n") + path: [DIR_WEBINY_JS + "/.verdaccio/", DIR_WEBINY_JS + "/.verdaccio.yaml"].join( + "\n" + ) } }, { @@ -186,18 +148,18 @@ const createJobs = (dbSetup: string) => { ...createDeployWebinySteps({ workingDirectory: DIR_TEST_PROJECT }), { name: "Create Cypress config", - "working-directory": DIR_PR, + "working-directory": DIR_WEBINY_JS, run: `yarn setup-cypress --projectFolder ../${DIR_TEST_PROJECT}` }, { name: "Save Cypress config", id: "save-cypress-config", - "working-directory": DIR_PR, + "working-directory": DIR_WEBINY_JS, run: "echo \"cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\\t\\n\\r')\" >> $GITHUB_OUTPUT" }, { name: "Cypress - run installation wizard test", - "working-directory": DIR_PR, + "working-directory": DIR_WEBINY_JS, run: 'yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js"' } ] @@ -205,7 +167,7 @@ const createJobs = (dbSetup: string) => { const cypressTestsJob = createJob({ name: `\${{ matrix.cypress-folder }} (${dbSetup}, \${{ matrix.os }}, Node v\${{ matrix.node }})`, - needs: [jobNames.constants, jobNames.projectSetup], + needs: ["baseBranch", "constants", jobNames.constants, jobNames.projectSetup], strategy: { "fail-fast": false, matrix: { @@ -216,30 +178,21 @@ const createJobs = (dbSetup: string) => { }, environment: "next", env, - checkout: { path: DIR_PR }, + checkout: { path: "${{ needs.baseBranch.outputs.base-branch }}" }, steps: [ - ...createCheckoutPrSteps({ workingDirectory: DIR_PR }), + ...createCheckoutPrSteps(), ...yarnCacheSteps, - ...buildRunCacheSteps, - { - name: "Install dependencies", - "working-directory": DIR_PR, - run: "yarn --immutable" - }, - { - name: "Build packages", - "working-directory": DIR_PR, - run: "yarn build:quick" - }, + ...runBuildCacheSteps, + ...installBuildSteps, { name: "Set up Cypress config", - "working-directory": DIR_PR, + "working-directory": DIR_WEBINY_JS, run: `echo '\${{ needs.${jobNames.projectSetup}.outputs.cypress-config }}' > cypress-tests/cypress.config.ts` }, { name: 'Cypress - run "${{ matrix.cypress-folder }}" tests', "timeout-minutes": 40, - "working-directory": DIR_PR, + "working-directory": DIR_WEBINY_JS, run: 'yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}"' } ] @@ -260,7 +213,6 @@ export const pullRequestsCommandCypress = createWorkflow({ AWS_REGION: "eu-central-1" }, jobs: { - validateWorkflows: createValidateWorkflowsJob(), checkComment: createJob({ name: `Check comment for /cypress`, if: "${{ github.event.issue.pull_request }}", @@ -289,8 +241,59 @@ export const pullRequestsCommandCypress = createWorkflow({ } ] }), - ...createJobs("ddb"), - ...createJobs("ddb-es"), - ...createJobs("ddb-os") + validateWorkflows: createValidateWorkflowsJob({ needs: "checkComment" }), + baseBranch: createJob({ + needs: "checkComment", + name: "Get base branch", + outputs: { + "base-branch": "${{ steps.base-branch.outputs.base-branch }}" + }, + steps: [ + { name: "Install Hub Utility", run: "sudo apt-get install -y hub" }, + { + name: "Get base branch", + id: "base-branch", + env: { GITHUB_TOKEN: "${{ secrets.GH_TOKEN }}" }, + run: 'echo "base-branch=$(hub pr show ${{ github.event.issue.number }} -f %B)" >> $GITHUB_OUTPUT' + } + ] + }), + constants: createJob({ + needs: "baseBranch", + name: "Create constants", + outputs: { + "global-cache-key": "${{ steps.global-cache-key.outputs.global-cache-key }}", + "run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}" + }, + checkout: false, + steps: [ + { + name: "Create global cache key", + id: "global-cache-key", + run: `echo "global-cache-key=\${{ needs.baseBranch.outputs.base-branch }}-\${{ runner.os }}-$(/bin/date -u "+%m%d")-\${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT` + }, + { + name: "Create workflow run cache key", + id: "run-cache-key", + run: 'echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT' + } + ] + }), + build: createJob({ + name: "Build", + needs: ["baseBranch", "constants"], + checkout: { path: DIR_WEBINY_JS }, + "runs-on": "blacksmith-4vcpu-ubuntu-2204", + steps: [ + ...createCheckoutPrSteps(), + ...yarnCacheSteps, + ...globalBuildCacheSteps, + ...installBuildSteps, + ...runBuildCacheSteps + ] + }), + ...createCypressJobs("ddb"), + ...createCypressJobs("ddb-es"), + ...createCypressJobs("ddb-os") } }); diff --git a/.github/workflows/wac/push.wac.ts b/.github/workflows/wac/push.wac.ts index 30ae1cd84e7..da05ad317cb 100644 --- a/.github/workflows/wac/push.wac.ts +++ b/.github/workflows/wac/push.wac.ts @@ -1,368 +1,338 @@ import { createWorkflow, NormalJob } from "github-actions-wac"; -import { createJestTestsJob, createJob, createValidateWorkflowsJob } from "./jobs"; -import { createDeployWebinySteps, createSetupVerdaccioSteps } from "./steps"; -import { NODE_VERSION } from "./utils"; +import { listPackagesWithJestTests, NODE_VERSION } from "./utils"; +import { createJob } from "./jobs"; +import { + createDeployWebinySteps, + createSetupVerdaccioSteps, + createInstallBuildSteps, + createYarnCacheSteps, + createGlobalBuildCacheSteps, + createRunBuildCacheSteps +} from "./steps"; -const createCypressJobs = (dbSetup: string) => { - const ucFirstDbSetup = dbSetup.charAt(0).toUpperCase() + dbSetup.slice(1); +const withCommonParams = ( + steps: NonNullable, + commonParams: Record +) => steps.map(step => ({ ...step, ...commonParams })); - const jobNames = { - init: `e2eTests${ucFirstDbSetup}-init`, - projectSetup: `e2eTests${ucFirstDbSetup}-setup`, - cypressTests: `e2eTests${ucFirstDbSetup}-cypress` - }; +const createPushWorkflow = (branchName: string) => { + const ucFirstBranchName = branchName.charAt(0).toUpperCase() + branchName.slice(1); - const initJob: NormalJob = createJob({ - name: `E2E (${dbSetup.toUpperCase()}) - Init`, - outputs: { - day: "${{ steps.get-day.outputs.day }}", - ts: "${{ steps.get-timestamp.outputs.ts }}", - "cypress-folders": "${{ steps.list-cypress-folders.outputs.cypress-folders }}" - }, - steps: [ - { - name: "Get day of the month", - id: "get-day", - run: 'echo "day=$(node --eval "console.log(new Date().getDate())")" >> $GITHUB_OUTPUT' - }, - { - name: "Get timestamp", - id: "get-timestamp", - run: 'echo "ts=$(node --eval "console.log(new Date().getTime())")" >> $GITHUB_OUTPUT' - }, - { - name: "List Cypress tests folders", - id: "list-cypress-folders", - run: 'echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT' - } - ] - }); + const DIR_WEBINY_JS = branchName; + const DIR_TEST_PROJECT = "new-webiny-project"; - const env: Record = { - CYPRESS_MAILOSAUR_API_KEY: "${{ secrets.CYPRESS_MAILOSAUR_API_KEY }}", - PULUMI_CONFIG_PASSPHRASE: "${{ secrets.PULUMI_CONFIG_PASSPHRASE }}", - PULUMI_SECRETS_PROVIDER: "${{ secrets.PULUMI_SECRETS_PROVIDER }}", - WEBINY_PULUMI_BACKEND: `$\{{ secrets.WEBINY_PULUMI_BACKEND }}$\{{ needs.${jobNames.init}.outputs.ts }}_ddb`, - YARN_ENABLE_IMMUTABLE_INSTALLS: "false" - }; + const installBuildSteps = createInstallBuildSteps({ workingDirectory: DIR_WEBINY_JS }); + const yarnCacheSteps = createYarnCacheSteps({ workingDirectory: DIR_WEBINY_JS }); + const globalBuildCacheSteps = createGlobalBuildCacheSteps({ workingDirectory: DIR_WEBINY_JS }); + const runBuildCacheSteps = createRunBuildCacheSteps({ workingDirectory: DIR_WEBINY_JS }); - if (dbSetup === "ddb-es") { - env["AWS_ELASTIC_SEARCH_DOMAIN_NAME"] = "${{ secrets.AWS_ELASTIC_SEARCH_DOMAIN_NAME }}"; - env["ELASTIC_SEARCH_ENDPOINT"] = "${{ secrets.ELASTIC_SEARCH_ENDPOINT }}"; - env["ELASTIC_SEARCH_INDEX_PREFIX"] = `$\{{ needs.${jobNames.init}.outputs.ts }}_`; - } else if (dbSetup === "ddb-os") { - // We still use the same environment variables as for "ddb-es" setup, it's - // just that the values are read from different secrets. - env["AWS_ELASTIC_SEARCH_DOMAIN_NAME"] = "${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }}"; - env["ELASTIC_SEARCH_ENDPOINT"] = "${{ secrets.OPEN_SEARCH_ENDPOINT }}"; - env["ELASTIC_SEARCH_INDEX_PREFIX"] = `$\{{ needs.${jobNames.init}.outputs.ts }}_`; - } + const createCypressJobs = (dbSetup: string) => { + const ucFirstDbSetup = dbSetup.charAt(0).toUpperCase() + dbSetup.slice(1); - const projectSetupJob: NormalJob = createJob({ - needs: jobNames.init, - name: `E2E (${dbSetup.toUpperCase()}) - Project setup`, - outputs: { - "cypress-config": "${{ steps.save-cypress-config.outputs.cypress-config }}" - }, - environment: "next", - env, - checkout: { - "fetch-depth": 0, - path: "dev" - }, - awsAuth: true, - steps: [ - { - uses: "actions/cache@v4", - id: "yarn-cache", - with: { - path: "dev/.yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }}" - } - }, - { - uses: "actions/cache@v4", - id: "cached-packages", - with: { - path: "dev/.webiny/cached-packages", - key: `$\{{ runner.os }}-$\{{ needs.${jobNames.init}.outputs.day }}-$\{{ secrets.RANDOM_CACHE_KEY_SUFFIX }}` - } - }, - { - name: "Install dependencies", - "working-directory": "dev", - run: "yarn --immutable" - }, - { - name: "Build packages", - "working-directory": "dev", - run: "yarn build:quick" - }, - { - uses: "actions/cache@v4", - id: "packages-cache", - with: { - path: "dev/.webiny/cached-packages", - key: `packages-cache-$\{{ needs.${jobNames.init}.outputs.ts }}` - } - }, - ...createSetupVerdaccioSteps({ workingDirectory: "dev" }), - { - name: 'Create ".npmrc" file in the project root, with a dummy auth token', - "working-directory": "dev", - run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc" - }, - { - name: "Version and publish to Verdaccio", - "working-directory": "dev", - run: "yarn release --type=verdaccio" + const jobNames = { + constants: `e2eTests${ucFirstDbSetup}-constants`, + projectSetup: `e2eTests${ucFirstDbSetup}-setup`, + cypressTests: `e2eTests${ucFirstDbSetup}-cypress` + }; + + const constantsJob: NormalJob = createJob({ + name: `Constants - ${dbSetup.toUpperCase()}`, + needs: ["build"], + outputs: { + "cypress-folders": "${{ steps.list-cypress-folders.outputs.cypress-folders }}", + "pulumi-backend-url": "${{ steps.pulumi-backend-url.outputs.pulumi-backend-url }}" }, - { - name: "Create verdaccio-files artifact", - uses: "actions/upload-artifact@v4", - with: { - name: `verdaccio-files-${dbSetup}`, - "retention-days": 1, - path: "dev/.verdaccio/\ndev/.verdaccio.yaml\n" + steps: [ + { + name: "List Cypress tests folders", + id: "list-cypress-folders", + run: 'echo "cypress-folders=$(node scripts/listCypressTestsFolders.js)" >> $GITHUB_OUTPUT' + }, + { + name: "Get Pulumi backend URL", + id: "get-pulumi-backend-url", + run: `echo "pulumi-backend-url=\${{ secrets.WEBINY_PULUMI_BACKEND }}\${{ github.run_id }}_${dbSetup}" >> $GITHUB_OUTPUT` } + ] + }); + + const env: Record = { + CYPRESS_MAILOSAUR_API_KEY: "${{ secrets.CYPRESS_MAILOSAUR_API_KEY }}", + PULUMI_CONFIG_PASSPHRASE: "${{ secrets.PULUMI_CONFIG_PASSPHRASE }}", + PULUMI_SECRETS_PROVIDER: "${{ secrets.PULUMI_SECRETS_PROVIDER }}", + WEBINY_PULUMI_BACKEND: `\${{ needs.${jobNames.constants}.outputs.pulumi-backend-url }}`, + YARN_ENABLE_IMMUTABLE_INSTALLS: "false" + }; + + if (dbSetup === "ddb-es") { + env["AWS_ELASTIC_SEARCH_DOMAIN_NAME"] = "${{ secrets.AWS_ELASTIC_SEARCH_DOMAIN_NAME }}"; + env["ELASTIC_SEARCH_ENDPOINT"] = "${{ secrets.ELASTIC_SEARCH_ENDPOINT }}"; + env["ELASTIC_SEARCH_INDEX_PREFIX"] = "${{ github.run_id }}_"; + } else if (dbSetup === "ddb-os") { + // We still use the same environment variables as for "ddb-es" setup, it's + // just that the values are read from different secrets. + env["AWS_ELASTIC_SEARCH_DOMAIN_NAME"] = "${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }}"; + env["ELASTIC_SEARCH_ENDPOINT"] = "${{ secrets.OPEN_SEARCH_ENDPOINT }}"; + env["ELASTIC_SEARCH_INDEX_PREFIX"] = "${{ github.run_id }}_"; + } + + const projectSetupJob: NormalJob = createJob({ + needs: ["constants", "build", jobNames.constants], + name: `E2E (${dbSetup.toUpperCase()}) - Project setup`, + outputs: { + "cypress-config": "${{ steps.save-cypress-config.outputs.cypress-config }}" }, - { - name: "Create directory", - run: "mkdir xyz" - }, - { - name: "Disable Webiny telemetry", - run: 'mkdir ~/.webiny && echo \'{ "id": "ci", "telemetry": false }\' > ~/.webiny/config\n' + environment: "next", + env, + checkout: { + "fetch-depth": 0, + path: DIR_WEBINY_JS }, - { - name: "Create a new Webiny project", - "working-directory": "xyz", - run: `npx create-webiny-project@local-npm test-project --tag local-npm --no-interactive --assign-to-yarnrc '{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}' --template-options '{"region":"$\{{ env.AWS_REGION }}","storageOperations":"${dbSetup}"}' + awsAuth: true, + steps: [ + ...yarnCacheSteps, + ...runBuildCacheSteps, + ...installBuildSteps, + ...createSetupVerdaccioSteps({ workingDirectory: DIR_WEBINY_JS }), + ...withCommonParams( + [ + { + name: 'Create ".npmrc" file in the project root, with a dummy auth token', + run: "echo '//localhost:4873/:_authToken=\"dummy-auth-token\"' > .npmrc" + }, + { + name: "Version and publish to Verdaccio", + run: "yarn release --type=verdaccio" + } + ], + { "working-directory": DIR_WEBINY_JS } + ), + { + name: "Create verdaccio-files artifact", + uses: "actions/upload-artifact@v4", + with: { + name: `verdaccio-files-${dbSetup}`, + "retention-days": 1, + path: [ + DIR_WEBINY_JS + "/.verdaccio/", + DIR_WEBINY_JS + "/.verdaccio.yaml" + ].join("\n") + } + }, + { + name: "Disable Webiny telemetry", + run: 'mkdir ~/.webiny && echo \'{ "id": "ci", "telemetry": false }\' > ~/.webiny/config\n' + }, + { + name: "Create a new Webiny project", + run: `npx create-webiny-project@local-npm ${DIR_TEST_PROJECT} --tag local-npm --no-interactive --assign-to-yarnrc '{"npmRegistryServer":"http://localhost:4873","unsafeHttpWhitelist":["localhost"]}' --template-options '{"region":"$\{{ env.AWS_REGION }}","storageOperations":"${dbSetup}"}' ` - }, - { - name: "Print CLI version", - "working-directory": "xyz/test-project", - run: "yarn webiny --version" - }, - { - name: "Create project-files artifact", - uses: "actions/upload-artifact@v4", - with: { - name: `project-files-${dbSetup}`, - "retention-days": 1, - path: "xyz/test-project/\n!xyz/test-project/node_modules/**/*\n!xyz/test-project/**/node_modules/**/*\n!xyz/test-project/.yarn/cache/**/*\n" + }, + { + name: "Print CLI version", + "working-directory": DIR_TEST_PROJECT, + run: "yarn webiny --version" + }, + { + name: "Create project-files artifact", + uses: "actions/upload-artifact@v4", + with: { + name: `project-files-${dbSetup}`, + "retention-days": 1, + path: [ + `${DIR_TEST_PROJECT}/`, + `!${DIR_TEST_PROJECT}/node_modules/**/*`, + `!${DIR_TEST_PROJECT}/**/node_modules/**/*`, + `!${DIR_TEST_PROJECT}/.yarn/cache/**/*` + ].join("\n") + } + }, + ...createDeployWebinySteps({ workingDirectory: DIR_TEST_PROJECT }), + ...withCommonParams( + [ + { + name: "Create Cypress config", + run: `yarn setup-cypress --projectFolder ../${DIR_TEST_PROJECT}` + }, + { + name: "Save Cypress config", + id: "save-cypress-config", + run: "echo \"cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\\t\\n\\r')\" >> $GITHUB_OUTPUT" + }, + { + name: "Cypress - run installation wizard test", + run: 'yarn cy:run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js"' + } + ], + { "working-directory": DIR_WEBINY_JS } + ) + ] + }); + + const cypressTestsJob = createJob({ + name: `$\{{ matrix.cypress-folder }} (${dbSetup}, $\{{ matrix.os }}, Node v$\{{ matrix.node }})`, + needs: ["constants", jobNames.constants, jobNames.projectSetup], + strategy: { + "fail-fast": false, + matrix: { + os: ["ubuntu-latest"], + node: [NODE_VERSION], + "cypress-folder": `$\{{ fromJson(needs.${jobNames.constants}.outputs.cypress-folders) }}` } }, - ...createDeployWebinySteps({ workingDirectory: "xyz/test-project" }), - { - name: "Create Cypress config", - "working-directory": "dev", - run: "yarn setup-cypress --projectFolder ../xyz/test-project" - }, - { - name: "Save Cypress config", - id: "save-cypress-config", - "working-directory": "dev", - run: "echo \"cypress-config=$(cat cypress-tests/cypress.config.ts | tr -d '\\t\\n\\r')\" >> $GITHUB_OUTPUT" - }, - { - name: "Cypress - run installation wizard test", - "working-directory": "dev/cypress-tests", - run: 'yarn cypress run --browser chrome --spec "cypress/e2e/adminInstallation/**/*.cy.js"' - } - ] - }); + environment: "next", + env, + checkout: { path: DIR_WEBINY_JS }, + steps: [ + ...yarnCacheSteps, + ...runBuildCacheSteps, + ...installBuildSteps, + ...withCommonParams( + [ + { + name: "Set up Cypress config", + run: `echo '$\{{ needs.${jobNames.projectSetup}.outputs.cypress-config }}' > cypress-tests/cypress.config.ts` + }, + { + name: 'Cypress - run "${{ matrix.cypress-folder }}" tests', + "timeout-minutes": 40, + run: 'yarn cy:run --browser chrome --spec "${{ matrix.cypress-folder }}"' + } + ], + { "working-directory": DIR_WEBINY_JS } + ) + ] + }); + + return { + [jobNames.constants]: constantsJob, + [jobNames.projectSetup]: projectSetupJob, + [jobNames.cypressTests]: cypressTestsJob + }; + }; - const cypressTestsJob = createJob({ - name: `$\{{ matrix.cypress-folder }} (${dbSetup}, $\{{ matrix.os }}, Node v$\{{ matrix.node }})`, - needs: [jobNames.init, jobNames.projectSetup], - strategy: { - "fail-fast": false, - matrix: { - os: ["ubuntu-latest"], - node: [NODE_VERSION], - "cypress-folder": `$\{{ fromJson(needs.${jobNames.init}.outputs.cypress-folders) }}` + const createJestTestsJob = (storage: string | null) => { + const env: Record = {}; + + if (storage) { + if (storage === "ddb-es") { + env["AWS_ELASTIC_SEARCH_DOMAIN_NAME"] = + "${{ secrets.AWS_ELASTIC_SEARCH_DOMAIN_NAME }}"; + env["ELASTIC_SEARCH_ENDPOINT"] = "${{ secrets.ELASTIC_SEARCH_ENDPOINT }}"; + env["ELASTIC_SEARCH_INDEX_PREFIX"] = "${{ matrix.package.id }}"; + } else if (storage === "ddb-os") { + // We still use the same environment variables as for "ddb-es" setup, it's + // just that the values are read from different secrets. + env["AWS_ELASTIC_SEARCH_DOMAIN_NAME"] = + "${{ secrets.AWS_OPEN_SEARCH_DOMAIN_NAME }}"; + env["ELASTIC_SEARCH_ENDPOINT"] = "${{ secrets.OPEN_SEARCH_ENDPOINT }}"; + env["ELASTIC_SEARCH_INDEX_PREFIX"] = "${{ matrix.package.id }}"; } - }, - environment: "next", - env, - checkout: { path: "dev" }, - steps: [ - { - uses: "actions/cache@v4", - with: { - path: "dev/.webiny/cached-packages", - key: `packages-cache-$\{{ needs.${jobNames.init}.outputs.ts }}` - } - }, - { - uses: "actions/cache@v4", - with: { - path: "dev/.yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('dev/**/yarn.lock') }}" + } + + const packages = listPackagesWithJestTests({ storage }); + + return createJob({ + needs: ["constants", "build"], + name: "${{ matrix.package.cmd }}", + strategy: { + "fail-fast": false, + matrix: { + os: ["ubuntu-latest"], + node: [NODE_VERSION], + package: "${{ fromJson('" + JSON.stringify(packages) + "') }}" } }, - { - name: "Install dependencies", - "working-directory": "dev", - run: "yarn --immutable" - }, - { - name: "Build packages", - "working-directory": "dev", - run: "yarn build:quick" - }, - { - name: "Set up Cypress config", - "working-directory": "dev", - run: `echo '$\{{ needs.${jobNames.projectSetup}.outputs.cypress-config }}' > cypress-tests/cypress.config.ts` - }, - { - name: 'Cypress - run "${{ matrix.cypress-folder }}" tests', - "working-directory": "dev/cypress-tests", - "timeout-minutes": 40, - run: 'yarn cypress run --browser chrome --spec "${{ matrix.cypress-folder }}"' - } - ] - }); - - return { - [jobNames.init]: initJob, - [jobNames.projectSetup]: projectSetupJob, - [jobNames.cypressTests]: cypressTestsJob + "runs-on": "${{ matrix.os }}", + env, + awsAuth: storage === "ddb-es" || storage === "ddb-os", + checkout: { path: DIR_WEBINY_JS }, + steps: [ + ...yarnCacheSteps, + ...runBuildCacheSteps, + ...installBuildSteps, + ...withCommonParams( + [{ name: "Run tests", run: "yarn test ${{ matrix.package.cmd }}" }], + { "working-directory": DIR_WEBINY_JS } + ) + ] + }); }; -}; - -const createPushWorkflow = (branchName: string) => { - const ucFirstBranchName = branchName.charAt(0).toUpperCase() + branchName.slice(1); const workflow = createWorkflow({ name: `${ucFirstBranchName} Branch - Push`, on: { push: { branches: [branchName] } }, jobs: { - validateWorkflows: createValidateWorkflowsJob(), - init: createJob({ - name: "Init", + // validateWorkflows: createValidateWorkflowsJob(), + constants: createJob({ + name: "Create constants", outputs: { - day: "${{ steps.get-day.outputs.day }}", - ts: "${{ steps.get-timestamp.outputs.ts }}" + "global-cache-key": "${{ steps.global-cache-key.outputs.global-cache-key }}", + "run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}" }, steps: [ { - name: "Get day of the month", - id: "get-day", - run: 'echo "day=$(node --eval "console.log(new Date().getDate())")" >> $GITHUB_OUTPUT' + name: "Create global cache key", + id: "global-cache-key", + run: `echo "global-cache-key=${branchName}-\${{ runner.os }}-$(/bin/date -u "+%m%d")-\${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT` }, { - name: "Get timestamp", - id: "get-timestamp", - run: 'echo "ts=$(node --eval "console.log(new Date().getTime())")" >> $GITHUB_OUTPUT' + name: "Create workflow run cache key", + id: "run-cache-key", + run: 'echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT' } ] }), build: createJob({ name: "Build", - needs: "init", - "runs-on": "webiny-build-packages", + needs: "constants", + checkout: { path: DIR_WEBINY_JS }, + "runs-on": "blacksmith-4vcpu-ubuntu-2204", steps: [ - { - uses: "actions/cache@v4", - id: "yarn-cache", - with: { - path: ".yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}" - } - }, - { - uses: "actions/cache@v4", - id: "global-daily-packages-cache", - with: { - path: ".webiny/cached-packages", - key: "${{ runner.os }}-${{ needs.init.outputs.day }}-${{ secrets.RANDOM_CACHE_KEY_SUFFIX }}" - } - }, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages", - run: "yarn build:quick" - }, - { - uses: "actions/cache@v4", - id: "packages-cache", - with: { - path: ".webiny/cached-packages", - key: "packages-cache-${{ needs.init.outputs.ts }}" - } - } + ...yarnCacheSteps, + ...globalBuildCacheSteps, + ...installBuildSteps, + ...runBuildCacheSteps ] }), codeAnalysis: createJob({ name: "Static code analysis", - needs: ["init", "build"], + needs: ["constants", "build"], + checkout: { path: DIR_WEBINY_JS }, steps: [ - { - uses: "actions/cache@v4", - with: { - path: ".yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}" - } - }, - { - uses: "actions/cache@v4", - with: { - path: ".webiny/cached-packages", - key: "packages-cache-${{ needs.init.outputs.ts }}" - } - }, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Check code formatting", - run: "yarn prettier:check" - }, - { - name: "Check dependencies", - run: "yarn adio" - }, - { - name: "Check TS configs", - run: "yarn check-ts-configs" - }, - { - name: "ESLint", - run: "yarn eslint" - } + ...yarnCacheSteps, + ...runBuildCacheSteps, + ...withCommonParams( + [ + { name: "Install dependencies", run: "yarn --immutable" }, + { name: "Check code formatting", run: "yarn prettier:check" }, + { name: "Check dependencies", run: "yarn adio" }, + { name: "Check TS configs", run: "yarn check-ts-configs" }, + { name: "ESLint", run: "yarn eslint" } + ], + { "working-directory": DIR_WEBINY_JS } + ) ] }), staticCodeAnalysisTs: createJob({ name: "Static code analysis (TypeScript)", - "runs-on": "webiny-build-packages", + "runs-on": "blacksmith-4vcpu-ubuntu-2204", + checkout: { path: DIR_WEBINY_JS }, steps: [ - { - uses: "actions/cache@v4", - with: { - path: ".yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}" - } - }, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages (full)", - run: "yarn build" - }, - { - name: "Check types for Cypress tests", - run: "yarn cy:ts" - } + ...yarnCacheSteps, + + // We're not using run cache here. We want to build all packages + // with TypeScript, to ensure there are no TypeScript errors. + // ...runBuildCacheSteps, + + ...withCommonParams( + [ + { name: "Install dependencies", run: "yarn --immutable" }, + { name: "Build packages (full)", run: "yarn build" }, + { name: "Check types for Cypress tests", run: "yarn cy:ts" } + ], + { "working-directory": DIR_WEBINY_JS } + ) ] }), jestTestsNoStorage: createJestTestsJob(null), @@ -380,7 +350,7 @@ const createPushWorkflow = (branchName: string) => { const e2eJobsNames = Object.keys(workflow.jobs).filter(name => name.endsWith("cypress")); workflow.jobs.npmReleaseUnstable = createJob({ - needs: ["init", "codeAnalysis", ...jestJobsNames, ...e2eJobsNames], + needs: ["constants", "codeAnalysis", ...jestJobsNames, ...e2eJobsNames], name: 'NPM release ("unstable" tag)', environment: "release", env: { @@ -389,46 +359,28 @@ const createPushWorkflow = (branchName: string) => { }, checkout: { "fetch-depth": 0 }, steps: [ - { - uses: "actions/cache@v4", - with: { - path: ".yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}" - } - }, - { - uses: "actions/cache@v4", - with: { - path: ".webiny/cached-packages", - key: "packages-cache-${{ needs.init.outputs.ts }}" - } - }, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages", - run: "yarn build" - }, - { - name: 'Create ".npmrc" file in the project root', - run: 'echo "//registry.npmjs.org/:_authToken=\\${NPM_TOKEN}" > .npmrc' - }, - { - name: "Set git info", - run: 'git config --global user.email "webiny-bot@webiny.com"\ngit config --global user.name "webiny-bot"\n' - }, - { - name: "Version and publish to NPM", - run: "yarn release --type=unstable" - } + ...yarnCacheSteps, + ...runBuildCacheSteps, + ...installBuildSteps, + ...withCommonParams( + [ + { + name: 'Create ".npmrc" file in the project root', + run: 'echo "//registry.npmjs.org/:_authToken=\\${NPM_TOKEN}" > .npmrc' + }, + { + name: "Set git info", + run: 'git config --global user.email "webiny-bot@webiny.com"\ngit config --global user.name "webiny-bot"\n' + }, + { name: "Version and publish to NPM", run: "yarn release --type=unstable" } + ], + { "working-directory": DIR_WEBINY_JS } + ) ] }); } return workflow; }; - export const pushDev = createPushWorkflow("dev"); export const pushNext = createPushWorkflow("next"); diff --git a/.github/workflows/wac/pushStable.wac.ts b/.github/workflows/wac/pushStable.wac.ts index 2614b42ca4f..4b3ad52e86a 100644 --- a/.github/workflows/wac/pushStable.wac.ts +++ b/.github/workflows/wac/pushStable.wac.ts @@ -1,25 +1,10 @@ -import { createWorkflow, NormalJob } from "github-actions-wac"; +import { createWorkflow } from "github-actions-wac"; import { createJob, createValidateWorkflowsJob } from "./jobs"; +import { createRunBuildCacheSteps, createYarnCacheSteps, createInstallBuildSteps } from "./steps"; -const yarnCacheSteps: NormalJob["steps"] = [ - { - uses: "actions/cache@v4", - with: { - path: ".yarn/cache", - key: "yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}" - } - } -]; - -const buildRunCacheSteps = [ - { - uses: "actions/cache@v4", - with: { - path: ".webiny/cached-packages", - key: "${{ needs.constants.outputs.run-cache-key }}" - } - } -]; +const installBuildSteps = createInstallBuildSteps({ workingDirectory: "" }); +const yarnCacheSteps = createYarnCacheSteps({ workingDirectory: "" }); +const runBuildCacheSteps = createRunBuildCacheSteps({ workingDirectory: "" }); // Note: we don't use global build cache here because we don't know which cache to use. // Commits from both `dev` and `next` branches can be merged into `stable`, so we need @@ -34,12 +19,10 @@ export const pushStable = createWorkflow({ }, jobs: { validateWorkflows: createValidateWorkflowsJob(), - constants: { + constants: createJob({ name: "Create constants", - "runs-on": "ubuntu-latest", - outputs: { - "run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}" - }, + outputs: { "run-cache-key": "${{ steps.run-cache-key.outputs.run-cache-key }}" }, + setupNode: false, steps: [ { name: "Create workflow run cache key", @@ -47,25 +30,18 @@ export const pushStable = createWorkflow({ run: 'echo "run-cache-key=${{ github.run_id }}-${{ github.run_attempt }}-${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT' } ] - }, + }), build: createJob({ name: "Build", needs: "constants", "runs-on": "webiny-build-packages", steps: [ ...yarnCacheSteps, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages", - run: "yarn build:quick" - }, + ...installBuildSteps, // Once we've built packages (without the help of the global cache), we can now cache // the result for this run workflow. All of the following jobs will use this cache. - ...buildRunCacheSteps + ...runBuildCacheSteps ] }), npmReleaseBeta: createJob({ @@ -79,15 +55,8 @@ export const pushStable = createWorkflow({ checkout: { "fetch-depth": 0 }, steps: [ ...yarnCacheSteps, - ...buildRunCacheSteps, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages", - run: "yarn build" - }, + ...runBuildCacheSteps, + ...installBuildSteps, { name: 'Create ".npmrc" file in the project root', run: 'echo "//registry.npmjs.org/:_authToken=\\${NPM_TOKEN}" > .npmrc' @@ -121,15 +90,8 @@ export const pushStable = createWorkflow({ }, steps: [ ...yarnCacheSteps, - ...buildRunCacheSteps, - { - name: "Install dependencies", - run: "yarn --immutable" - }, - { - name: "Build packages", - run: "yarn build" - }, + ...runBuildCacheSteps, + ...installBuildSteps, { name: 'Create ".npmrc" file in the project root', run: 'echo "//registry.npmjs.org/:_authToken=\\${NPM_TOKEN}" > .npmrc' diff --git a/.github/workflows/wac/rebuildGlobalCache.wac.ts b/.github/workflows/wac/rebuildGlobalCache.wac.ts new file mode 100644 index 00000000000..5697c620ecb --- /dev/null +++ b/.github/workflows/wac/rebuildGlobalCache.wac.ts @@ -0,0 +1,43 @@ +import { createJob } from "./jobs"; +import { + createGlobalBuildCacheSteps, + createInstallBuildSteps, + createYarnCacheSteps +} from "./steps"; + +const createRebuildGlobalCacheWorkflow = (branchName: string) => ({ + name: `Rebuild Global Cache ("${branchName}" branch)`, + on: { + workflow_dispatch: {}, + schedule: [{ cron: "0 4 * * *" }] + }, + jobs: { + constants: createJob({ + name: "Create constants", + outputs: { + "global-cache-key": "${{ steps.global-cache-key.outputs.global-cache-key }}" + }, + checkout: false, + steps: [ + { + name: "Create global cache key", + id: "global-cache-key", + run: `echo "global-cache-key=${branchName}-\${{ runner.os }}-$(/bin/date -u "+%m%d")-\${{ vars.RANDOM_CACHE_KEY_SUFFIX }}" >> $GITHUB_OUTPUT` + } + ] + }), + cacheDependenciesPackages: createJob({ + name: `Cache dependencies and packages`, + needs: "constants", + checkout: { path: branchName, ref: branchName }, + steps: [ + ...createYarnCacheSteps({ workingDirectory: branchName }), + ...createGlobalBuildCacheSteps({ workingDirectory: branchName }), + ...createInstallBuildSteps({ workingDirectory: branchName }) + ] + }) + } +}); + +export const rebuildGlobalCacheDev = createRebuildGlobalCacheWorkflow("dev"); +export const rebuildGlobalCacheNext = createRebuildGlobalCacheWorkflow("next"); diff --git a/.github/workflows/wac/steps/createGlobalBuildCacheSteps.ts b/.github/workflows/wac/steps/createGlobalBuildCacheSteps.ts new file mode 100644 index 00000000000..e422bdc8002 --- /dev/null +++ b/.github/workflows/wac/steps/createGlobalBuildCacheSteps.ts @@ -0,0 +1,17 @@ +interface CreateGlobalBuildCacheStepsParams { + workingDirectory: string; +} + +export const createGlobalBuildCacheSteps = (params: CreateGlobalBuildCacheStepsParams) => { + return [ + { + uses: "actions/cache@v4", + with: { + path: [params.workingDirectory, ".webiny/cached-packages"] + .filter(Boolean) + .join("/"), + key: "${{ needs.constants.outputs.global-cache-key }}" + } + } + ] as const; +}; diff --git a/.github/workflows/wac/steps/createInstallBuildSteps.ts b/.github/workflows/wac/steps/createInstallBuildSteps.ts new file mode 100644 index 00000000000..54d92db8a29 --- /dev/null +++ b/.github/workflows/wac/steps/createInstallBuildSteps.ts @@ -0,0 +1,15 @@ +import { withCommonParams } from "./withCommonParams"; + +interface CreateInstallBuildStepsParams { + workingDirectory: string; +} + +export const createInstallBuildSteps = (params: CreateInstallBuildStepsParams) => { + return withCommonParams( + [ + { name: "Install dependencies", run: "yarn --immutable" }, + { name: "Build packages", run: "yarn build:quick" } + ], + { "working-directory": params.workingDirectory } + ); +}; diff --git a/.github/workflows/wac/steps/createRunBuildCacheSteps.ts b/.github/workflows/wac/steps/createRunBuildCacheSteps.ts new file mode 100644 index 00000000000..f8cdfbc511e --- /dev/null +++ b/.github/workflows/wac/steps/createRunBuildCacheSteps.ts @@ -0,0 +1,17 @@ +interface CreateRunBuildCacheStepsParams { + workingDirectory: string; +} + +export const createRunBuildCacheSteps = (params: CreateRunBuildCacheStepsParams) => { + return [ + { + uses: "actions/cache@v4", + with: { + path: [params.workingDirectory, ".webiny/cached-packages"] + .filter(Boolean) + .join("/"), + key: "${{ needs.constants.outputs.run-cache-key }}" + } + } + ] as const; +}; diff --git a/.github/workflows/wac/steps/createYarnCacheSteps.ts b/.github/workflows/wac/steps/createYarnCacheSteps.ts new file mode 100644 index 00000000000..382d7fe1fd6 --- /dev/null +++ b/.github/workflows/wac/steps/createYarnCacheSteps.ts @@ -0,0 +1,15 @@ +interface CreateYarnCacheStepsParams { + workingDirectory: string; +} + +export const createYarnCacheSteps = (params: CreateYarnCacheStepsParams) => { + return [ + { + uses: "actions/cache@v4", + with: { + path: [params.workingDirectory, ".yarn/cache"].filter(Boolean).join("/"), + key: "yarn-${{ runner.os }}-${{ hashFiles('**/yarn.lock') }}" + } + } + ] as const; +}; diff --git a/.github/workflows/wac/steps/index.ts b/.github/workflows/wac/steps/index.ts index 4df14ef4c04..51df6ee62ab 100644 --- a/.github/workflows/wac/steps/index.ts +++ b/.github/workflows/wac/steps/index.ts @@ -1,2 +1,7 @@ export * from "./createDeployWebinySteps"; export * from "./createSetupVerdaccioSteps"; +export * from "./createInstallBuildSteps"; +export * from "./createGlobalBuildCacheSteps"; +export * from "./createRunBuildCacheSteps"; +export * from "./createYarnCacheSteps"; +export * from "./withCommonParams"; diff --git a/.github/workflows/wac/steps/withCommonParams.ts b/.github/workflows/wac/steps/withCommonParams.ts new file mode 100644 index 00000000000..3ca77bab7aa --- /dev/null +++ b/.github/workflows/wac/steps/withCommonParams.ts @@ -0,0 +1,6 @@ +import { NormalJob } from "github-actions-wac"; + +export const withCommonParams = ( + steps: NonNullable, + commonParams: Record +) => steps.map(step => ({ ...step, ...commonParams })); diff --git a/package.json b/package.json index 14d44646ec5..dc5e983a9c3 100644 --- a/package.json +++ b/package.json @@ -105,6 +105,8 @@ "scripts": { "awp": "yarn add-webiny-package", "add-webiny-package": "node scripts/addWebinyPackage.js", + "ci-workflows:build": "yarn ghawac build", + "ci-workflows:watch": "yarn ghawac watch", "check-ts-configs": "node scripts/checkTsConfigs.js", "check-package-dependencies": "node scripts/checkPackageNodeModules.js", "deploy": "yarn webiny deploy",