From 193cea7b8930d1f11bc1389b8272cdb624e88658 Mon Sep 17 00:00:00 2001 From: Pete Mallam Date: Thu, 9 May 2024 10:33:42 +0100 Subject: [PATCH 1/4] Add updated local workflow --- .github/workflows/build-and-test.yaml | 129 +++++++++++++------------- 1 file changed, 63 insertions(+), 66 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index 93401e9..f3ebdbd 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -1,8 +1,9 @@ name: ๐Ÿงฑ Build and Test -run-name: ๐Ÿงฑ Build and Test ${{ github.repository }}@${{ github.ref_name }} +run-name: ๐Ÿงฑ Build and Test ${{ github.ref_name }} env: bucket-name: cvs-services on: + workflow_call: workflow_dispatch: inputs: ignore-test-failures: @@ -10,6 +11,10 @@ on: description: Ignore Test Failures default: false +concurrency: + group: ${{ github.workflow }}-${{ github.ref_name }} + cancel-in-progress: true + permissions: id-token: write contents: write @@ -19,11 +24,21 @@ jobs: name: ๐Ÿงน Prepare Environment runs-on: ubuntu-latest outputs: - version-number: ${{ steps.version.outputs.version }} environment-name: ${{ steps.environment.outputs.environment-name }} environment-type: ${{ steps.environment.outputs.environment-type }} + github-branch: ${{ steps.environment.outputs.github-branch }} + commit: ${{ steps.commit.outputs.commit }} + package-name: ${{ steps.name.outputs.package-name }} steps: + - name: Generate Package Name + id: name + run: | + repo=${{ github.repository }} + echo "package-name=${repo#dvsa/}" >> $GITHUB_OUTPUT + echo "package-name=${repo#dvsa/}" >> $GITHUB_STEP_SUMMARY + + - name: ๐Ÿชช Get Credentials uses: aws-actions/configure-aws-credentials@v4 with: @@ -42,11 +57,21 @@ jobs: with: environment: ${{ github.ref_name }} - build-services: - name: ๐Ÿงฑ Build ${{ github.ref_name }} - runs-on: [self-hosted, X64] - needs: environment + - name: Get Commit SHA + id: commit + run: | + # Get the branch Commit ID + commit_id=$(git rev-parse HEAD) + echo "commit=${commit_id}" + echo "commit=${commit_id}" >> $GITHUB_OUTPUT + echo "Commit ID: ${commit_id}" >> $GITHUB_STEP_SUMMARY + build-service: + name: ๐Ÿงฑ Build ${{ needs.environment.outputs.package-name }} + runs-on: + - ${{ needs.environment.outputs.package-name }} + + needs: environment steps: - name: ๐Ÿชช Get Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -57,9 +82,6 @@ jobs: - name: ๐Ÿ“ฅ Get Codebase uses: actions/checkout@v4 - with: - token: ${{ secrets.GH_TOKEN }} - ref: ${{ github.ref_name }} - name: Get Commit SHA id: commit @@ -74,19 +96,41 @@ jobs: id: manifest run: | # Generate Manifest Outputs & Identify Build Requirements - echo "package-name=$(jq -r '.name' manifest.json)" >> $GITHUB_OUTPUT - echo "language=$(jq -r '.language' manifest.json)" >> $GITHUB_OUTPUT - echo "liquibase=$(jq -r '.liquibase' manifest.json)" >> $GITHUB_OUTPUT - - # Generate Output File - jq -c '. += {"sha":"${{ steps.commit.outputs.commit }}","repository":"${{ github.repository }}"' manifest.json > outputs.json + hash_files="[]" + if [[ -f "manifest.json" ]]; then + package=$(jq -r '.name' manifest.json) + echo "package=$(jq -r '.name' manifest.json)" >> $GITHUB_OUTPUT + echo "language=$(jq -r '.language' manifest.json)" >> $GITHUB_OUTPUT + echo "liquibase=$(jq -r '.liquibase' manifest.json)" >> $GITHUB_OUTPUT + + # Check for existing Hash + hashes=$(aws s3api list-objects-v2 --bucket ${{ env.bucket-name }} --query "Contents[?Key.contains(@,'${{ steps.commit.outputs.commit }}.zip') && Key.contains(@,'$package')].Key" | jq -rc) + if [[ $(awk '{print index($0,"${{ steps.commit.outputs.commit }}")}' <<< $hashes) == 0 ]]; then + echo "Current Hash does not exist" + build_package=true + + else + echo "Current Hash exists" + hash_files=$hashes + build_package=false + fi + + else + build_package=false + fi + + # Add flag to build or skip + echo "build-package=${build_package}" >> $GITHUB_OUTPUT + echo "Build Package: ${build_package}" >> $GITHUB_STEP_SUMMARY - name: ๐Ÿ—๏ธ Build Node Function uses: dvsa/cvs-github-actions/build-node-function@develop - if: ${{ steps.manifest.outputs.language == 'node' }} + if: ${{ steps.manifest.outputs.build-package == 'true' && steps.manifest.outputs.language == 'node' }} + env: + GH_TOKEN: ${{ secrets.GH_TOKEN }} with: commit-sha: ${{ steps.commit.outputs.commit }} - branch: ${{ github.ref_name }} + branch: ${{needs.environment.outputs.github-branch }} mysql_liquibase: ${{ steps.manifest.outputs.liquibase }} mysql_user: ${{ secrets.MYSQL_USER }} mysql_password: ${{ secrets.MYSQL_PASS }} @@ -94,58 +138,11 @@ jobs: - name: ๐Ÿ—๏ธ Build Java Function uses: dvsa/cvs-github-actions/build-java-function@develop - if: ${{ steps.manifest.outputs.language == 'java' }} + if: ${{ steps.manifest.outputs.build-package == 'true' && steps.manifest.outputs.language == 'java' }} - name: ๐Ÿ“ค Upload Artifact uses: actions/upload-artifact@v4 with: + name: ${{ needs.environment.outputs.package-name }} path: | - outputs.json ${{ steps.commit.outputs.commit }}*.zip - - upload-functions: - # Runs as a separate job because we don't want to push anything if any one of the services fails to build - name: ๐ŸŽฏ Upload ${{ github.ref_name }} - runs-on: ubuntu-latest - needs: [environment, build-services] - steps: - - name: ๐Ÿชช Get Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.CVS_MGMT_AWS_ROLE }} - aws-region: ${{ vars.DVSA_AWS_REGION }} - role-session-name: GHA_BuildServices - - - name: ๐Ÿ“ฅ Download Artifact - uses: actions/download-artifact@v4 - - - name: ๐Ÿ—’๏ธ Get Manifest Data - id: manifest - run: | - # Get information from Manifest File - echo "package-name=$(jq -r '.name' outputs.json)" >> $GITHUB_OUTPUT - echo "mono-repo=$(jq -r '.monorepo' outputs.json)" >> $GITHUB_OUTPUT - echo "sha=$(jq -r '.sha' outputs.json)" >> $GITHUB_OUTPUT - echo "build-package=$(jq -r '.build' outputs.json)" >> $GITHUB_OUTPUT - echo "hash-files=$(jq -r '.hash_files' outputs.json)" >> $GITHUB_OUTPUT - - echo "package-name=$(jq -r '.name' outputs.json)" >> $GITHUB_STEP_SUMMARY - echo "mono-repo=$(jq -r '.monorepo' outputs.json)" >> $GITHUB_STEP_SUMMARY - echo "sha=$(jq -r '.sha' outputs.json)" >> $GITHUB_STEP_SUMMARY - echo "build-package=$(jq -r '.build' outputs.json)" >> $GITHUB_STEP_SUMMARY - echo "hash-files=$(jq -r '.hash_files' outputs.json)" >> $GITHUB_STEP_SUMMARY - - - name: Create Hash Files - uses: dvsa/cvs-github-actions/hash-create@develop - with: - commit-id: ${{ steps.manifest.outputs.sha }} - environment-name: ${{ needs.environment.outputs.environment-name }} - hash-files: ${{ steps.manifest.outputs.hash-files }} - - - name: ๐Ÿ“ค Upload hash zip to S3 - uses: dvsa/cvs-github-actions/hash-upload@develop - with: - commit-id: ${{ steps.manifest.outputs.sha }} - package-name: ${{ steps.manifest.outputs.package-name }} - environment-name: ${{ needs.environment.outputs.environment-name }} - version-number: ${{ needs.environment.outputs.version-number }} From d200755f2694f8db57e8d800c30cb8ebaf25ebfb Mon Sep 17 00:00:00 2001 From: Pete Mallam Date: Thu, 9 May 2024 16:34:55 +0100 Subject: [PATCH 2/4] Always build --- .github/workflows/build-and-test.yaml | 39 ++++----------------------- 1 file changed, 5 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-and-test.yaml b/.github/workflows/build-and-test.yaml index f3ebdbd..bf674eb 100644 --- a/.github/workflows/build-and-test.yaml +++ b/.github/workflows/build-and-test.yaml @@ -36,8 +36,7 @@ jobs: run: | repo=${{ github.repository }} echo "package-name=${repo#dvsa/}" >> $GITHUB_OUTPUT - echo "package-name=${repo#dvsa/}" >> $GITHUB_STEP_SUMMARY - + echo "Package Name: ${repo#dvsa/}" >> $GITHUB_STEP_SUMMARY - name: ๐Ÿชช Get Credentials uses: aws-actions/configure-aws-credentials@v4 @@ -83,15 +82,6 @@ jobs: - name: ๐Ÿ“ฅ Get Codebase uses: actions/checkout@v4 - - name: Get Commit SHA - id: commit - run: | - # Get the branch Commit ID - commit_id=$(git rev-parse HEAD) - echo "commit=${commit_id}" - echo "commit=${commit_id}" >> $GITHUB_OUTPUT - echo "Commit ID: ${commit_id}" >> $GITHUB_STEP_SUMMARY - - name: ๐Ÿ—’๏ธ Process Manifest Data id: manifest run: | @@ -102,34 +92,15 @@ jobs: echo "package=$(jq -r '.name' manifest.json)" >> $GITHUB_OUTPUT echo "language=$(jq -r '.language' manifest.json)" >> $GITHUB_OUTPUT echo "liquibase=$(jq -r '.liquibase' manifest.json)" >> $GITHUB_OUTPUT - - # Check for existing Hash - hashes=$(aws s3api list-objects-v2 --bucket ${{ env.bucket-name }} --query "Contents[?Key.contains(@,'${{ steps.commit.outputs.commit }}.zip') && Key.contains(@,'$package')].Key" | jq -rc) - if [[ $(awk '{print index($0,"${{ steps.commit.outputs.commit }}")}' <<< $hashes) == 0 ]]; then - echo "Current Hash does not exist" - build_package=true - - else - echo "Current Hash exists" - hash_files=$hashes - build_package=false - fi - - else - build_package=false fi - # Add flag to build or skip - echo "build-package=${build_package}" >> $GITHUB_OUTPUT - echo "Build Package: ${build_package}" >> $GITHUB_STEP_SUMMARY - - name: ๐Ÿ—๏ธ Build Node Function uses: dvsa/cvs-github-actions/build-node-function@develop - if: ${{ steps.manifest.outputs.build-package == 'true' && steps.manifest.outputs.language == 'node' }} + if: ${{ steps.manifest.outputs.language == 'node' }} env: GH_TOKEN: ${{ secrets.GH_TOKEN }} with: - commit-sha: ${{ steps.commit.outputs.commit }} + commit-sha: ${{ needs.environment.outputs.commit }} branch: ${{needs.environment.outputs.github-branch }} mysql_liquibase: ${{ steps.manifest.outputs.liquibase }} mysql_user: ${{ secrets.MYSQL_USER }} @@ -138,11 +109,11 @@ jobs: - name: ๐Ÿ—๏ธ Build Java Function uses: dvsa/cvs-github-actions/build-java-function@develop - if: ${{ steps.manifest.outputs.build-package == 'true' && steps.manifest.outputs.language == 'java' }} + if: ${{ steps.manifest.outputs.language == 'java' }} - name: ๐Ÿ“ค Upload Artifact uses: actions/upload-artifact@v4 with: name: ${{ needs.environment.outputs.package-name }} path: | - ${{ steps.commit.outputs.commit }}*.zip + ${{ needs.environment.outputs.commit }}*.zip From 8d5765c1eaec28d09e78bde760db3cabe14808c1 Mon Sep 17 00:00:00 2001 From: Pete Mallam Date: Thu, 9 May 2024 16:46:23 +0100 Subject: [PATCH 3/4] Use IP --- README.md | 2 +- package.json | 2 +- sonar-project.properties | 2 +- .../auth-into-service-document-conversion.intTest.ts | 2 +- tests/integration/cvsbnop-container.ts | 2 +- tests/integration/tech-record-document-conversion.intTest.ts | 2 +- .../integration/test-results-conversion-with-delete.intTest.ts | 2 +- .../integration/test-results-conversion-with-upsert.intTest.ts | 2 +- tests/utils.ts | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 43b2a25..20de93a 100644 --- a/README.md +++ b/README.md @@ -86,7 +86,7 @@ On Windows, you will need to manually run: * Confirm: `echo %USE_CONTAINERIZED_DATABASE%` should return `1`before the above, or run these tests with an IntelliJ configuration which sets the right environment variable for you. On Tanio Artino's advice, there is a distinction between running integration tests locally, and running them on Jenkins. -* `USE_CONTAINERIZED_DATABASE=0` will attempt to talk to `localhost:3306`. This only works if you manually start a DB or, in the case of Jenkins, if there is a pre-existing DB running on port `3306`. +* `USE_CONTAINERIZED_DATABASE=0` will attempt to talk to `127.0.0.1:3306`. This only works if you manually start a DB or, in the case of Jenkins, if there is a pre-existing DB running on port `3306`. * `USE_CONTAINERIZED_DATABASE=1` will spin up a TC container as described above. To run the "Jenkins" version, e.g. `USE_CONTAINERIZED_DATABASE=0`, use: diff --git a/package.json b/package.json index 21da6b2..46026c9 100644 --- a/package.json +++ b/package.json @@ -111,7 +111,7 @@ "testResultsProcessor": "jest-sonar-reporter", "testURL": "http://localhost" }, - "husky": { + "husky": {127.0.0.1 "hooks": { "commit-msg": "commitlint -E HUSKY_GIT_PARAMS", "pre-commit": "npm test && npm run security-checks && npm run audit && npm run lint && npm run prettier", diff --git a/sonar-project.properties b/sonar-project.properties index 586da52..e1a9dc7 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -5,7 +5,7 @@ sonar.projectKey=org.sonarqube:cvs-tsk-update-store # this is the name and version displayed in the SonarQube UI. Was mandatory prior to SonarQube 6.1. sonar.projectName=cvs-tsk-update-store -sonar.host.url=http://localhost:9000 +sonar.host.url=http://127.0.0.1:9000 sonar.projectVersion=1.0 sonar.sourceEncoding=UTF-8 diff --git a/tests/integration/auth-into-service-document-conversion.intTest.ts b/tests/integration/auth-into-service-document-conversion.intTest.ts index fad8e09..91f0f2f 100644 --- a/tests/integration/auth-into-service-document-conversion.intTest.ts +++ b/tests/integration/auth-into-service-document-conversion.intTest.ts @@ -23,7 +23,7 @@ describe("convertTechRecordDocument() integration tests", () => { container = await getContainerizedDatabase(); } else { (getConnectionPoolOptions as jest.Mock) = jest.fn().mockResolvedValue({ - host: "localhost", + host: "127.0.0.1", port: "3306", user: "root", password: "12345", diff --git a/tests/integration/cvsbnop-container.ts b/tests/integration/cvsbnop-container.ts index 027e749..1c4e642 100644 --- a/tests/integration/cvsbnop-container.ts +++ b/tests/integration/cvsbnop-container.ts @@ -36,7 +36,7 @@ export const getContainerizedDatabase = async (): Promise "--password", "12345", "--url", - `jdbc:mysql://localhost:${hostPort}/${databaseName}`, + `jdbc:mysql://127.0.0.1:${hostPort}/${databaseName}`, "--classpath", `${pathToResources()}/mysql-connector-java-8.0.23.jar`, "update", diff --git a/tests/integration/tech-record-document-conversion.intTest.ts b/tests/integration/tech-record-document-conversion.intTest.ts index 4a7e729..0dc1d43 100644 --- a/tests/integration/tech-record-document-conversion.intTest.ts +++ b/tests/integration/tech-record-document-conversion.intTest.ts @@ -23,7 +23,7 @@ describe("convertTechRecordDocument() integration tests", () => { container = await getContainerizedDatabase(); } else { (getConnectionPoolOptions as jest.Mock) = jest.fn().mockResolvedValue({ - host: "localhost", + host: "127.0.0.1", port: "3306", user: "root", password: "12345", diff --git a/tests/integration/test-results-conversion-with-delete.intTest.ts b/tests/integration/test-results-conversion-with-delete.intTest.ts index f7d40d1..1aea437 100644 --- a/tests/integration/test-results-conversion-with-delete.intTest.ts +++ b/tests/integration/test-results-conversion-with-delete.intTest.ts @@ -54,7 +54,7 @@ describe("convertTestResults() integration tests with delete", () => { container = await getContainerizedDatabase(); } else { (getConnectionPoolOptions as jest.Mock) = jest.fn().mockResolvedValue({ - host: "localhost", + host: "127.0.0.1", port: "3306", user: "root", password: "12345", diff --git a/tests/integration/test-results-conversion-with-upsert.intTest.ts b/tests/integration/test-results-conversion-with-upsert.intTest.ts index 8f4f0f6..3566c9f 100644 --- a/tests/integration/test-results-conversion-with-upsert.intTest.ts +++ b/tests/integration/test-results-conversion-with-upsert.intTest.ts @@ -54,7 +54,7 @@ describe("convertTestResults() integration tests with upsert", () => { container = await getContainerizedDatabase(); } else { (getConnectionPoolOptions as jest.Mock) = jest.fn().mockResolvedValue({ - host: "localhost", + host: "127.0.0.1", port: "3306", user: "root", password: "12345", diff --git a/tests/utils.ts b/tests/utils.ts index 02af65e..2b6d5b9 100644 --- a/tests/utils.ts +++ b/tests/utils.ts @@ -25,7 +25,7 @@ export const useLocalDb = (): void => { username: "root", password: "12345", engine: "", - host: "localhost", + host: "127.0.0.1", port: 3306, dbname: "", dbClusterIdentifier: "", From c7fd6b50db22a67b0d5885c12bf86b4e916a3508 Mon Sep 17 00:00:00 2001 From: Pete Mallam Date: Thu, 9 May 2024 16:50:09 +0100 Subject: [PATCH 4/4] That's not supposed to be there --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 46026c9..21da6b2 100644 --- a/package.json +++ b/package.json @@ -111,7 +111,7 @@ "testResultsProcessor": "jest-sonar-reporter", "testURL": "http://localhost" }, - "husky": {127.0.0.1 + "husky": { "hooks": { "commit-msg": "commitlint -E HUSKY_GIT_PARAMS", "pre-commit": "npm test && npm run security-checks && npm run audit && npm run lint && npm run prettier",