feat(delivery): Move delivery manifests back to this repository

This change moves deployment manifests back from the "delivery" repository to this repository, and adjusts the deployment & undeployment workflows accordingly. This also includes the following low-level changes: 1. Increase test expectations & global timeouts 2. Split the "pr_deploy" workflow to 3 separate workflows: - Build: run on each push - Deploy PR: deploys each PR when it changes - Deploy main branch: deploys the "main" branch when it changes 3. Deployment now provides environment name and URL 4. End-to-end testing begins once FluxCD notifies that it finished 5. Redis was upgraded 6. Neo4j's migration script readability was improved a bit
arikkfir · Sep 26, 2023 · 6f3df0e · 6f3df0e
1 parent 18d6d97
commit 6f3df0e
Show file tree

Hide file tree

Showing 40 changed files with 1,118 additions and 290 deletions.
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -0,0 +1,105 @@
+name: End-to-end Testing
+run-name: "End-to-end Testing (${{ github.event.deployment_status.environment }}: ${{ github.event.deployment_status.state }})"
+
+on:
+  deployment_status:
+
+defaults:
+  run:
+    shell: bash -exuo pipefail {0}
+
+jobs:
+
+  test:
+    name: Test
+    if: github.event.deployment_status.state == 'success'
+    runs-on: ubuntu-22.04
+    container:
+      image: mcr.microsoft.com/playwright:v1.37.1-jammy
+    steps:
+      - run: apt-get update && apt-get install -y patch
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.deployment.sha }}
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 20.x
+          cache: npm
+          cache-dependency-path: e2e/package-lock.json
+      - run: npm ci
+        working-directory: e2e
+      - run: ./apply-patches.sh
+        working-directory: e2e
+      - id: test
+        run: npx playwright test
+        working-directory: e2e
+        env:
+          DESCOPE_PROJECT_ID: ${{ secrets.DESCOPE_PROJECT_ID }}
+          DESCOPE_MANAGEMENT_KEY: ${{ secrets.DESCOPE_MANAGEMENT_KEY }}
+          DESCOPE_TENANT_ADMIN_ACCESS_KEY: ${{ secrets.DESCOPE_TENANT_ADMIN_ACCESS_KEY }}
+          ENV_NAME: ${{ github.event.deployment_status.environment }}
+      - uses: actions/upload-artifact@v3
+        if: success() || (failure() && steps.test.conclusion == 'failure')
+        with:
+          name: playwright-report
+          path: |
+            e2e/playwright-report
+            e2e/custom-summary.txt
+          if-no-files-found: error
+          retention-days: 1
+
+  report:
+    name: Report
+    needs: test
+    if: success() || (failure() && needs.e2e-tests.result == 'failure')
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: write
+      pull-requests: write
+      id-token: write
+    env:
+      BUCKET_NAME: arikkfir-playwright-reports
+      BUCKET_PATH: ${{ github.repository }}/actions/${{ github.run_number }}/${{ github.run_attempt }}
+    steps:
+      - uses: actions/download-artifact@v3
+        with:
+          name: playwright-report
+          path: report
+      - uses: google-github-actions/auth@v1
+        with:
+          workload_identity_provider: projects/8909046976/locations/global/workloadIdentityPools/github-actions/providers/github-oidc
+          service_account: playwright-uploader@arikkfir.iam.gserviceaccount.com
+      - uses: google-github-actions/setup-gcloud@v1
+        with:
+          skip_install: true
+      - run: gcloud storage cp -r -P ./report/playwright-report/* "gs://${BUCKET_NAME}/${BUCKET_PATH}/"
+      - run: |
+          REPORT_WEB_URL="https://playwright.kfirs.com/${BUCKET_PATH}/index.html"
+          
+          touch comment.txt
+          echo "End to end tests result: ${RESULT} ([click here for a full report](${REPORT_WEB_URL}))" >> comment.txt
+          echo "Application URL: ${APP_WEB_URL}" >> comment.txt
+          echo "" >> comment.txt
+          echo "---" >> comment.txt
+          echo "" >> comment.txt
+          cat ./report/custom-summary.txt >> comment.txt
+          
+          if [[ -z "${PR}" ]]; then
+            gh api --method POST \
+                    -H "Accept: application/vnd.github+json" \
+                    -H "X-GitHub-Api-Version: 2022-11-28" \
+                    "/repos/${REPOSITORY}/commits/${SHA}/comments" \
+                    -F 'body=@comment.txt'
+          else
+            # Purpose for the "||" expression is the "edit-last" does not work if this is the first comment
+            gh pr comment --repo "${REPOSITORY}" "${PR}" --body-file comment.txt --edit-last \
+              || gh pr comment --repo "${REPOSITORY}" "${PR}" --body-file comment.txt
+          fi
+        env:
+          APP_WEB_URL: ${{ github.event.deployment_status.environment_url }}
+          ENV_NAME: ${{ github.event.deployment_status.environment }}
+          GH_TOKEN: ${{ github.token }}
+          PR: ${{ github.event.workflow_run.pull_requests[0].number }}
+          REPOSITORY: ${{ github.repository }}
+          RESULT: ${{ needs.e2e-tests.result }}
+          SHA: ${{ github.event.deployment.sha }}
diff --git a/.github/workflows/main_deploy.yaml b/.github/workflows/main_deploy.yaml
@@ -0,0 +1,28 @@
+name: Deploy main
+
+on:
+  push:
+    branches:
+      - main
+
+defaults:
+  run:
+    shell: bash -exuo pipefail {0}
+
+jobs:
+
+  deploy:
+    name: Deploy
+    needs: env
+    runs-on: ubuntu-22.04
+    environment:
+      name: main
+      url: https://acme.main.greenstar.kfirs.com
+    steps:
+      - uses: arikkfir/deploy@main
+        with:
+          repository: greenstar
+          branch: main
+          sha: ${{ github.sha }}
+          environment: main
+          token: ${{ secrets.ARIKKFIR_GHA_AUTOMATION }}
diff --git a/.github/workflows/pr_deploy.yml b/.github/workflows/pr_deploy.yml
@@ -1,247 +1,40 @@
 name: Deploy PR
 
 on:
-  issue_comment:
-    types:
-      - created
-      - edited
   pull_request:
     types:
       - opened
       - synchronize
-  push:
-    branches:
-      - main
 
 defaults:
   run:
     shell: bash -exuo pipefail {0}
 
 jobs:
 
-  get-ref:
-    name: Get ref
+  env:
+    name: Environment
     runs-on: ubuntu-22.04
     outputs:
-      ref: ${{ steps[github.event_name].outputs.ref }}
-      sha: ${{ steps[github.event_name].outputs.sha }}
-    env:
-      GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-    steps:
-      - uses: actions/checkout@v4
-      - id: pull_request
-        name: Get ref from PR
-        if: github.event_name == 'pull_request'
-        run: |
-          echo "ref=${{ github.event.pull_request.head.ref }}" >> "$GITHUB_OUTPUT"
-          echo "sha=${{ github.event.pull_request.head.sha }}" >> "$GITHUB_OUTPUT"
-      - id: issue_comment
-        name: Get ref from issue
-        if: github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == '/deploy'
-        run: |
-          echo "ref=$(gh pr view ${{ github.event.issue.number }} --json headRefName | jq -r .headRefName)" >> "$GITHUB_OUTPUT"
-          echo "sha=$(gh pr view ${{ github.event.issue.number }} --json headRefOid | jq -r .headRefOid)" >> "$GITHUB_OUTPUT"
-      - id: push
-        name: Get ref from push
-        if: github.event_name == 'push'
-        run: |
-          echo "ref=${{ github.ref_name }}" >> "$GITHUB_OUTPUT"
-          echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
-
-  backend-format:
-    needs: get-ref
-    name: Backend Code Format
-    runs-on: ubuntu-22.04
-    if: needs.get-ref.outputs.ref && needs.get-ref.outputs.sha
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.get-ref.outputs.sha }}
-      - uses: actions/setup-go@v4
-        with:
-          go-version-file: backend/go.mod
-          cache-dependency-path: backend/go.sum
-      - run: |
-          go fmt ./... | tee output.txt
-          if [[ -s output.txt ]]; then
-            echo "Incorrect Go formatting detected."
-            exit 1
-          fi
-        working-directory: backend
-
-  backend-unit-tests:
-    needs: get-ref
-    name: Backend Unit Tests
-    runs-on: ubuntu-22.04
-    if: needs.get-ref.outputs.ref && needs.get-ref.outputs.sha
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.get-ref.outputs.sha }}
-      - uses: actions/setup-go@v4
-        with:
-          go-version-file: backend/go.mod
-          cache-dependency-path: backend/go.sum
-      - run: go test ./...
-        env:
-          CGO_ENABLED: "0"
-          GO111MODULE: "on"
-          GOOS: linux
-          GOARCH: amd64
-        working-directory: backend
-
-  docker:
-    needs: get-ref
-    strategy:
-      matrix:
-        image: [ backend, frontend, neo4j, migrations ]
-      fail-fast: false
-    name: Build ${{ matrix.image }} image
-    runs-on: ubuntu-22.04
-    timeout-minutes: 30
-    if: needs.get-ref.outputs.ref && needs.get-ref.outputs.sha
-    permissions:
-      packages: write
+      name: ${{ steps.env.outputs.name }}
     steps:
-      - uses: actions/checkout@v4
+      - id: env
+        uses: arikkfir/delivery-env-name@v1
         with:
-          ref: ${{ needs.get-ref.outputs.sha }}
-      - if: matrix.image == 'frontend'
-        run: |-
-          cat > frontend/apply-patches.sh <<PATCHES_EOF
-          ${{ secrets.FRONTEND_PATCHES }}
-          PATCHES_EOF
-      - uses: docker/setup-buildx-action@v2
-        with:
-          install: true
-      - uses: docker/login-action@v2
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - uses: docker/build-push-action@v4
-        with:
-          builder: ${{ steps.buildx.outputs.name }}
-          cache-from: type=gha,scope=${{ needs.get-ref.outputs.ref }}-${{ matrix.image }}
-          cache-to: type=gha,mode=max,scope=${{ needs.get-ref.outputs.ref }}-${{ matrix.image }}
-          context: ${{ matrix.image }}
-          push: true
-          tags: |
-            ghcr.io/${{ github.repository }}/${{ matrix.image }}:${{ needs.get-ref.outputs.sha }}
+          branch: ${{ github.event.pull_request.head.ref }}
 
   deploy:
     name: Deploy
-    needs: get-ref
-    uses: arikkfir/delivery/.github/workflows/deploy-to-environment.yml@main
-    if: needs.get-ref.outputs.ref && needs.get-ref.outputs.sha
-    with:
-      branch: ${{ needs.get-ref.outputs.ref }}
-      images: |-
-        ghcr.io/${{ github.repository }}/backend: ${{ needs.get-ref.outputs.sha }}
-        ghcr.io/${{ github.repository }}/frontend: ${{ needs.get-ref.outputs.sha }}
-        ghcr.io/${{ github.repository }}/migrations: ${{ needs.get-ref.outputs.sha }}
-        ghcr.io/${{ github.repository }}/neo4j: ${{ needs.get-ref.outputs.sha }}
-    secrets: inherit
-
-  e2e-tests:
-    name: End-to-end Tests
-    needs: [ get-ref, deploy ]
-    runs-on: ubuntu-22.04
-    if: needs.get-ref.outputs.ref && needs.get-ref.outputs.sha
-    container:
-      image: mcr.microsoft.com/playwright:v1.37.1-jammy
-    steps:
-      - run: apt-get update && apt-get install -y patch
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ needs.get-ref.outputs.sha }}
-      - uses: arikkfir/delivery-env-name@v1
-        id: env
-        with:
-          branch: ${{ needs.get-ref.outputs.ref }}
-      - uses: actions/setup-node@v3
-        with:
-          node-version: 20.x
-          cache: npm
-          cache-dependency-path: e2e/package-lock.json
-      - run: npm ci
-        working-directory: e2e
-      - run: ./apply-patches.sh
-        working-directory: e2e
-      - id: test
-        run: npx playwright test
-        working-directory: e2e
-        env:
-          DESCOPE_PROJECT_ID: ${{ secrets.DESCOPE_PROJECT_ID }}
-          DESCOPE_MANAGEMENT_KEY: ${{ secrets.DESCOPE_MANAGEMENT_KEY }}
-          DESCOPE_TENANT_ADMIN_ACCESS_KEY: ${{ secrets.DESCOPE_TENANT_ADMIN_ACCESS_KEY }}
-          ENV_NAME: ${{ steps.env.outputs.name }}
-      - uses: actions/upload-artifact@v3
-        if: success() || (failure() && steps.test.conclusion == 'failure')
-        with:
-          name: playwright-report
-          path: |
-            e2e/playwright-report
-            e2e/custom-summary.txt
-          if-no-files-found: error
-          retention-days: 1
-
-  e2e-report:
-    name: End-to-end Report
-    needs: [ get-ref, deploy, e2e-tests ]
+    needs: env
     runs-on: ubuntu-22.04
-    permissions:
-      contents: write
-      pull-requests: write
-      id-token: write
-    if: needs.get-ref.outputs.ref && needs.get-ref.outputs.sha && (success() || (failure() && needs.e2e-tests.result == 'failure'))
-    env:
-      BUCKET_NAME: arikkfir-playwright-reports
-      BUCKET_PATH: ${{ github.repository }}/actions/${{ github.run_number }}/${{ github.run_attempt }}
+    environment:
+      name: ${{ needs.env.outputs.name }}
+      url: https://acme.${{ needs.env.outputs.name }}.greenstar.kfirs.com
     steps:
-      - uses: arikkfir/delivery-env-name@v1
-        id: env
-        with:
-          branch: ${{ needs.get-ref.outputs.ref }}
-      - uses: actions/download-artifact@v3
-        with:
-          name: playwright-report
-          path: report
-      - uses: google-github-actions/auth@v1
-        with:
-          workload_identity_provider: projects/8909046976/locations/global/workloadIdentityPools/github-actions/providers/github-oidc
-          service_account: playwright-uploader@arikkfir.iam.gserviceaccount.com
-      - uses: google-github-actions/setup-gcloud@v1
+      - uses: arikkfir/deploy@main
         with:
-          skip_install: true
-      - run: gcloud storage cp -r -P ./report/playwright-report/* "gs://${BUCKET_NAME}/${BUCKET_PATH}/"
-      - run: |
-          APP_WEB_URL="https://acme.${ENV_NAME}.greenstar.kfirs.com"
-          REPORT_WEB_URL="https://playwright.kfirs.com/${BUCKET_PATH}/index.html"
-          
-          touch comment.txt
-          echo "End to end tests result: ${RESULT} ([click here for a full report](${REPORT_WEB_URL}))" >> comment.txt
-          echo "Application URL: ${APP_WEB_URL}" >> comment.txt
-          echo "" >> comment.txt
-          echo "---" >> comment.txt
-          echo "" >> comment.txt
-          cat ./report/custom-summary.txt >> comment.txt
-          
-          if [[ -z "${PR}" ]]; then
-            gh api --method POST \
-                    -H "Accept: application/vnd.github+json" \
-                    -H "X-GitHub-Api-Version: 2022-11-28" \
-                    "/repos/${REPOSITORY}/commits/${SHA}/comments" \
-                    -F 'body=@comment.txt'
-          else
-            # Purpose for the "||" expression is the "edit-last" does not work if this is the first comment
-            gh pr comment --repo "${REPOSITORY}" "${PR}" --body-file comment.txt --edit-last \
-              || gh pr comment --repo "${REPOSITORY}" "${PR}" --body-file comment.txt
-          fi
-        env:
-          ENV_NAME: ${{ steps.env.outputs.name }}
-          GH_TOKEN: ${{ github.token }}
-          PR: ${{ github.event.pull_request.number || github.event.issue.number }}
-          REPOSITORY: ${{ github.repository }}
-          RESULT: ${{ needs.e2e-tests.result }}
-          SHA: ${{ needs.get-ref.outputs.sha }}
+          repository: greenstar
+          branch: ${{ github.event.pull_request.head.ref }}
+          sha: ${{ github.event.pull_request.head.sha }}
+          environment: ${{ needs.env.outputs.name }}
+          token: ${{ secrets.ARIKKFIR_GHA_AUTOMATION }}