From 4e840fd25fc22ddc8064bf925e69d0603c0c5120 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 22 Jan 2025 13:02:43 +0000 Subject: [PATCH 1/4] cicd: Change default hadoop flavour from hdp3.1 to emr6.1 #TASK-7330 --- .github/workflows/develop.yml | 7 +++++-- .github/workflows/manual-deploy-docker.yml | 7 ++++++- .github/workflows/pull-request-merge.yml | 5 ++++- .github/workflows/release.yml | 7 +++++-- .github/workflows/task.yml | 6 +++++- .github/workflows/test-analysis.yml | 8 ++++++-- 6 files changed, 31 insertions(+), 9 deletions(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 50c396dd7cc..c531a83cf3d 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -7,11 +7,14 @@ on: - release-* workflow_dispatch: +env: + HADOOP_FLAVOUR: emr6.1 + jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make test: uses: ./.github/workflows/test-analysis.yml @@ -24,7 +27,7 @@ jobs: uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop needs: test with: - maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga + maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }} -Dopencga.war.name=opencga secrets: inherit deploy-docker: diff --git a/.github/workflows/manual-deploy-docker.yml b/.github/workflows/manual-deploy-docker.yml index f4c79f4c241..801237902f9 100644 --- a/.github/workflows/manual-deploy-docker.yml +++ b/.github/workflows/manual-deploy-docker.yml @@ -10,6 +10,11 @@ on: description: "The tag for the new docker." type: string required: true + hadoop_flavour: + description: "The hadoop flavour to use." + type: string + required: false + default: "emr6.1" jobs: build: @@ -35,7 +40,7 @@ jobs: ./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }} fi - name: Maven Build (skip tests) - run: mvn -T 2 clean install -DskipTests -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + run: mvn -T 2 clean install -DskipTests -P storage-hadoop,${{ inputs.hadoop_flavour }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl ':opencga-app' --also-make - uses: actions/upload-artifact@v4 with: name: build-folder diff --git a/.github/workflows/pull-request-merge.yml b/.github/workflows/pull-request-merge.yml index faedd27bebc..578c194f75f 100644 --- a/.github/workflows/pull-request-merge.yml +++ b/.github/workflows/pull-request-merge.yml @@ -9,11 +9,14 @@ on: - closed workflow_dispatch: +env: + HADOOP_FLAVOUR: emr6.1 + jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make delete-docker: uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0481b386dca..a2900f78a5c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,17 +6,20 @@ on: - '*' workflow_dispatch: +env: + HADOOP_FLAVOUR: emr6.1 + jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make deploy-maven: uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop needs: build with: - maven_opts: -P storage-hadoop,hdp3.1 -Dopencga.war.name=opencga + maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }} -Dopencga.war.name=opencga secrets: inherit deploy-docker: diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index 51888a9179d..58bf5e5af62 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -6,13 +6,17 @@ on: - TASK-* workflow_dispatch: +env: + # We must use hdp3.1 for the task branch for the docker build step + HADOOP_FLAVOUR: hdp3.1 + # WARNING Develop branch needed for prod jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,hdp3.1,RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' + maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make test: uses: ./.github/workflows/test-analysis.yml diff --git a/.github/workflows/test-analysis.yml b/.github/workflows/test-analysis.yml index ccb848d11ec..85d43206a9f 100644 --- a/.github/workflows/test-analysis.yml +++ b/.github/workflows/test-analysis.yml @@ -6,6 +6,10 @@ on: test_profile: type: string required: true + hadoop_flavour: + type: string + required: false + default: "emr6.1" mvn_opts: type: string @@ -41,7 +45,7 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Needed to get PR information, if any SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} - run: mvn -T 2 clean install -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga + run: mvn -T 2 clean install -P storage-hadoop,${{ inputs.hadoop_flavour }},${{ inputs.test_profile }} -DskipTests -Dcheckstyle.skip org.sonarsource.scanner.maven:sonar-maven-plugin:sonar -Dsonar.projectKey=opencb_opencga test: name: Execute JUnit and Jacoco tests @@ -72,7 +76,7 @@ jobs: mongodb-version: 6.0 mongodb-replica-set: rs-test - name: Run Junit tests - run: mvn -B verify surefire-report:report --fail-never -P storage-hadoop,hdp3.1,${{ inputs.test_profile }} -Dcheckstyle.skip -Popencga-storage-hadoop-deps -pl '!:opencga-storage-hadoop-deps-emr6.1,!:opencga-storage-hadoop-deps-hdp2.6' ${{ inputs.mvn_opts }} + run: mvn -B verify surefire-report:report --fail-never -P storage-hadoop,${{ inputs.hadoop_flavour }},${{ inputs.test_profile }} -Dcheckstyle.skip -Popencga-storage-hadoop-deps -pl ':opencga-app' --also-make ${{ inputs.mvn_opts }} - name: Publish Test Report uses: scacap/action-surefire-report@v1 env: From baf5a69c0452d6e5d9184263f494bb07bc0d490d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Wed, 22 Jan 2025 13:18:10 +0000 Subject: [PATCH 2/4] cicd: Use vars instead of envs. #TASK-7330 --- .github/workflows/develop.yml | 7 ++----- .github/workflows/pull-request-merge.yml | 5 +---- .github/workflows/release.yml | 7 ++----- .github/workflows/task.yml | 7 ++----- 4 files changed, 7 insertions(+), 19 deletions(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index c531a83cf3d..53c7f88c112 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -7,14 +7,11 @@ on: - release-* workflow_dispatch: -env: - HADOOP_FLAVOUR: emr6.1 - jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make + maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make test: uses: ./.github/workflows/test-analysis.yml @@ -27,7 +24,7 @@ jobs: uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop needs: test with: - maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }} -Dopencga.war.name=opencga + maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }} -Dopencga.war.name=opencga secrets: inherit deploy-docker: diff --git a/.github/workflows/pull-request-merge.yml b/.github/workflows/pull-request-merge.yml index 578c194f75f..595133cb519 100644 --- a/.github/workflows/pull-request-merge.yml +++ b/.github/workflows/pull-request-merge.yml @@ -9,14 +9,11 @@ on: - closed workflow_dispatch: -env: - HADOOP_FLAVOUR: emr6.1 - jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make + maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make delete-docker: uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a2900f78a5c..f288a1257bd 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,20 +6,17 @@ on: - '*' workflow_dispatch: -env: - HADOOP_FLAVOUR: emr6.1 - jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make + maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make deploy-maven: uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop needs: build with: - maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }} -Dopencga.war.name=opencga + maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }} -Dopencga.war.name=opencga secrets: inherit deploy-docker: diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index 58bf5e5af62..aa4a2933328 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -6,17 +6,13 @@ on: - TASK-* workflow_dispatch: -env: - # We must use hdp3.1 for the task branch for the docker build step - HADOOP_FLAVOUR: hdp3.1 - # WARNING Develop branch needed for prod jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,${{ env.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make + maven_opts: -P storage-hadoop,${{ vars.TASK_HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make test: uses: ./.github/workflows/test-analysis.yml @@ -24,6 +20,7 @@ jobs: secrets: inherit with: test_profile: runShortTests + hadoop_flavour: ${{ vars.TASK_HADOOP_FLAVOUR }} deploy-docker: uses: opencb/java-common-libs/.github/workflows/deploy-docker-hub-workflow.yml@develop From f7114b20b54b5e5b5003642e143278ccbead47aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jacobo=20Coll=20Morag=C3=B3n?= Date: Thu, 23 Jan 2025 10:01:39 +0000 Subject: [PATCH 3/4] cicd: Fix workflow maven build. #TASK-7330 --- .github/workflows/develop.yml | 2 +- .github/workflows/pull-request-merge.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/task.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/develop.yml b/.github/workflows/develop.yml index 53c7f88c112..41e82ac2a7e 100644 --- a/.github/workflows/develop.yml +++ b/.github/workflows/develop.yml @@ -11,7 +11,7 @@ jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make + maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl ':opencga-app' --also-make test: uses: ./.github/workflows/test-analysis.yml diff --git a/.github/workflows/pull-request-merge.yml b/.github/workflows/pull-request-merge.yml index 595133cb519..420eb11c311 100644 --- a/.github/workflows/pull-request-merge.yml +++ b/.github/workflows/pull-request-merge.yml @@ -13,7 +13,7 @@ jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make + maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl ':opencga-app' --also-make delete-docker: uses: opencb/java-common-libs/.github/workflows/delete-docker-hub-workflow.yml@develop diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f288a1257bd..0466b020354 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make + maven_opts: -P storage-hadoop,${{ vars.HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl ':opencga-app' --also-make deploy-maven: uses: opencb/java-common-libs/.github/workflows/deploy-maven-repository-workflow.yml@develop diff --git a/.github/workflows/task.yml b/.github/workflows/task.yml index aa4a2933328..6e2a7b800e6 100644 --- a/.github/workflows/task.yml +++ b/.github/workflows/task.yml @@ -12,7 +12,7 @@ jobs: build: uses: opencb/java-common-libs/.github/workflows/build-java-app-workflow.yml@develop with: - maven_opts: -P storage-hadoop,${{ vars.TASK_HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl '!:opencga-app' --also-make + maven_opts: -P storage-hadoop,${{ vars.TASK_HADOOP_FLAVOUR }},RClient,opencga-storage-hadoop-deps -Dopencga.war.name=opencga -Dcheckstyle.skip -pl ':opencga-app' --also-make test: uses: ./.github/workflows/test-analysis.yml From 39d816f92181ed18e263dd0c4e5f74abb0ca10fb Mon Sep 17 00:00:00 2001 From: JuanfeSanahuja Date: Mon, 3 Mar 2025 18:25:06 +0100 Subject: [PATCH 4/4] Update pom dependency opencga-storage-hadoop-deps-emr6.1 #TASK-7330 --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 67881cc4052..9be5166b35a 100644 --- a/pom.xml +++ b/pom.xml @@ -281,7 +281,7 @@ org.opencb.opencga - opencga-storage-hadoop-deps-hdp3.1 + opencga-storage-hadoop-deps-emr6.1 ${project.version} shaded