diff --git a/.github/actions/check-pr-paths/README.md b/.github/actions/check-pr-paths/README.md new file mode 100644 index 00000000000..0f9f3b8ca83 --- /dev/null +++ b/.github/actions/check-pr-paths/README.md @@ -0,0 +1,29 @@ +# Composite action to check if testing for this PR is relevant based on touched paths + +This action is meant to be used inside a PR testing workflow, as + +```yaml +jobs: + check_paths: + steps: + - name: check_relevance + if: ${{ github.event_name == "pull_request" }} + uses: ./.github/actions/check-pr-paths + with: + paths: | + path1 + path2 + testing: + needs: check_paths + if: ${{ github.event_name != "pull_request" || needs.check_paths.outputs.touched == 'true'}} + steps: + - name: testing +``` +The input `paths1 is a list of paths that the action uses to see if +the PR changes any relevant file. The action sets an output `touched`, +which is `true` if the PR touches any of the listed paths and `false` +otherwise. + +This action allows to skip an entire workflow, but, unlike using a 'paths' filter at the +workflow trigger level, it ensures that the workflow DOES run, +hence avoiding the issue where certain checks are not present on the PR. diff --git a/.github/actions/check-pr-paths/action.yml b/.github/actions/check-pr-paths/action.yml new file mode 100644 index 00000000000..57363ce7f69 --- /dev/null +++ b/.github/actions/check-pr-paths/action.yml @@ -0,0 +1,46 @@ +name: check-pr-paths +description: 'Check that the PR modifies relevant files' +inputs: + paths: + description: 'List of paths that are considered relevant' + required: true + default: '' + token: + description: 'GitHub token for authentication' + required: true +outputs: + touched: + description: 'Output indicating if this testing for PR is relevant' + value: ${{ steps.check_paths.outputs.value }} + +# Note: inputs are available as env vars in the shell run steps, convertet to uppercase +runs: + using: "composite" + steps: + - name: check_paths + shell: bash + env: + PR_NUMBER: ${{ github.event.number }} + GITHUB_REPO: ${{ github.repository }} + run: | + # Convert the input paths into an array, and create a regex from it + IFS=$'\n' read -r -a paths <<< "$INPUT_PATHS" + pattern=$(IFS=\|; echo "${paths[*]}") + + # Use the GitHub API to get the list of changed files + response=$(curl -s -H "Authorization: token $GITHUB_TOKEN" \ + "https://api.github.com/repos/$GITHUB_REPO/pulls/$PR_NUMBER/files") + + echo "resp: '$response'" + # Extract the filenames from the response + changed_files=$(echo "$response" | jq -r '.[].filename') + + # Check for matches and echo the matching files + matching_files=$(echo "$changed_files" | grep -E "^($pattern)" || echo "") + if [[ -n "$matching_files" ]]; then + echo "Found relevant files: "$matching_files" # Print the matching files + echo "value=true" >> $GITHUB_OUTPUT + else + echo "No relevant files touched by this PR." + echo "value=false" >> $GITHUB_OUTPUT + fi diff --git a/.github/actions/check-skip-labels/README.md b/.github/actions/check-skip-labels/README.md deleted file mode 100644 index 42d1e3bc64e..00000000000 --- a/.github/actions/check-skip-labels/README.md +++ /dev/null @@ -1,24 +0,0 @@ -# Composite action to check if we can skip a job for a PR - -This action is meant to be used inside a PR testing workflow, as - -```yaml -jobs: - my-testing: - steps: - ... - - name: check-pr-labels - if: github.event_name == "pull_request" || github.event_name == "pull_request_review" - uses: ./.github/actions/check-skip-labels - with: - skip_labels: label1,label2,label3 -``` - -The input skip_label is a comma-separated list of labels that, if found -on the PR, will cause this job to terminate immediately with a PASS state. - -Ideally, we would like to run this check at the job level, so that we can -skip the job altogether (without using runner time). But while for the -pull_request event we DO have access to the labels from the gh context -(and therefore can check), for pull_request_review we don't, so we need -to ping github for some information diff --git a/.github/actions/check-skip-labels/action.yml b/.github/actions/check-skip-labels/action.yml deleted file mode 100644 index 55d35bfa2b0..00000000000 --- a/.github/actions/check-skip-labels/action.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: 'Check Skip Labels' -description: 'Check for specific skip labels in a pull request' -inputs: - skip_labels: - description: 'Comma-separated list of skip labels' - required: true - token: - description: 'GitHub token for authentication' - required: true - pr_number: - description: 'Pull request number' - required: true - -# Note: inputs are available as env vars in the shell run steps, convertet to uppercase - -runs: - using: "composite" - steps: - - name: Get Pull Request Labels - run: | - echo "Fetching pull request labels..." - if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then - LABELS="$SKIP_LABELS" - elif [[ "$GITHUB_EVENT_NAME" == "pull_request_review" ]]; then - response=$(curl -s -H "Authorization: token $TOKEN" \ - "https://api.github.com/repos/$GITHUB_REPOSITORY/pulls/$PR_NUMBER") - # TODO: reinstante jq once the SNL image is rebuilt! - # LABELS=$(echo "$response" | jq -r '.labels | map(.name) | join(",")') - LABELS=$(echo "$response" | grep -o '"name": *"[^"]*"' | sed 's/"name": *//;s/"//g' | tr '\n' ',' | sed 's/,$//') - fi - echo "labels=$LABELS" >> $GITHUB_ENV - shell: sh - - name: Check for Skip Labels - run: | - echo "Checking for skip labels..." - IFS=',' read -r -a SKIP_LABELS <<< "$SKIP_LABELS" - IFS=',' read -r -a LABEL_ARRAY <<< "$labels" - - for label in "${SKIP_LABELS[@]}"; do - for pr_label in "${LABEL_ARRAY[@]}"; do - if [[ "$pr_label" == "$label" ]]; then - echo "Found skip label '$label'. Skipping this job." - exit 0 # Exit with success status - fi - done - done - echo "No relevant skip labels found. Continuing with the job." - shell: sh diff --git a/.github/actions/get-labels/README.md b/.github/actions/get-labels/README.md new file mode 100644 index 00000000000..6651fd7a95a --- /dev/null +++ b/.github/actions/get-labels/README.md @@ -0,0 +1,19 @@ +# Composite action to retrieve labels of a PR if event is `pull_request` + +This action is meant to be used inside a PR testing workflow, as + +```yaml +jobs: + get_labels: + steps: + - name: get_labels + uses: ./.github/actions/get-labels + testing: + needs: get_labels + if: ${{ github.event_name != "pull_request" || + !contains(needs.get_labels.outputs.labels,"label1" || + !contains(needs.get_labels.outputs.labels,"label1" }} +``` +The action sets the output `labels`, which is a comma-separated list +containing the PR labels. If the event that triggered the workflow is +not a PR, the output will be an empty string diff --git a/.github/actions/get-labels/action.yml b/.github/actions/get-labels/action.yml new file mode 100644 index 00000000000..40e7cb35286 --- /dev/null +++ b/.github/actions/get-labels/action.yml @@ -0,0 +1,17 @@ +name: get-labels +description: 'Check that the PR modifies does not have skip labels' +outputs: + labels: + description: 'Comma-separated list of labels on the PR or empty string (if not a PR event)' + value: ${{ steps.get_labels.outputs.labels }} + +# Note: inputs are available as env vars in the shell run steps, convertet to uppercase +runs: + using: "composite" + steps: + - name: get_labels + shell: sh + env: + PR_LABELS: ${{ toJson(github.event.pull_request.labels.*.name) }} # Pass PR label names as JSON + run: | + echo 'labels=\"$(echo \"$PR_LABELS\" | jq -r "join(\",\")")\"' >> $GITHUB_OUTPUT diff --git a/.github/workflows/eamxx-sa-testing.yml b/.github/workflows/eamxx-sa-testing.yml index 12e45514ad0..4c9b70b2ec1 100644 --- a/.github/workflows/eamxx-sa-testing.yml +++ b/.github/workflows/eamxx-sa-testing.yml @@ -5,17 +5,6 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] - paths: - - components/eamxx/** - - components/eam/src/physics/rrtmgp/** - - components/eam/src/physics/p3/scream/** - - components/eam/src/physics/cam/** - - .github/workflows/eamxx-standalone-testing.yml - - externals/ekat/** - - externals/scorpio/** - - externals/haero/** - - externals/YAKL/** - - components/eam/src/physics/rrtmgp/external/** # Manual run is used to bless workflow_dispatch: @@ -48,15 +37,51 @@ env: submit: ${{ github.event_name == 'schedule' && 'true' || 'false' }} # Submit to cdash only for nightlies jobs: + pr_relevant: + runs-on: ubuntu-latest # This job can run anywhere + outputs: + value: ${{ steps.check_paths.outputs.touched }} + steps: + - name: check_paths + if: ${{ github.event_name == "pull_request" }} + uses: ./.github/actions/check-pr-relevance + with: + paths: | + components/eamxx + components/eam/src/physics/rrtmgp + components/eam/src/physics/p3/scream + components/eam/src/physics/cam + components/eam/src/physics/rrtmgp/external + externals/ekat + externals/scorpio + externals/haero + externals/YAKL + .github/workflows/eamxx-sa-testing.yml + get_labels: + runs-on: ubuntu-latest + outputs: ${{ steps.get_labels.outputs.labels }} + steps: + - name: get_labels + uses: ./.github/actions/get-labels gcc-openmp: + needs: pr_relevant, get_labels + if: ${{ + (github.event_name == 'pull_request' && + needs.pr_relevant.outputs.value=='true' && + !contains(needs.get_labels.outputs.labels,'CI: skip gcc') && + !contains(needs.get_labels.outputs.labels,'CI: skip openmp') && + !contains(needs.get_labels.outputs.labels,'CI: skip eamxx-sa') && + !contains(needs.get_labels.outputs.labels,'CI: skip eamxx-all')) || + (github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'gcc-openmp' || + github.event.inputs.job_to_run == 'all') || + github.event_name == 'schedule' + }} runs-on: [self-hosted, ghci-snl-cpu, gcc] strategy: fail-fast: false matrix: build_type: [sp, dbg, fpe, opt] - if: ${{ github.event_name != 'workflow_dispatch' || - github.event.inputs.job_to_run == 'gcc-openmp' || - github.event.inputs.job_to_run == 'all' }} name: gcc-openmp / ${{ matrix.build_type }} steps: - name: Check out the repository @@ -67,13 +92,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip gcc,AT: skip openmp,AT: skip eamxx-sa,AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Set test-all inputs based on event specs run: | echo "generate=false" >> $GITHUB_ENV @@ -91,14 +109,24 @@ jobs: submit: ${{ env.submit }} cmake-configs: Kokkos_ENABLE_OPENMP=ON gcc-cuda: + needs: pr_relevant, get_labels + if: ${{ + (github.event_name == 'pull_request' && + needs.pr_relevant.outputs.value=='true' && + !contains(needs.get_labels.outputs.labels,'CI: skip gcc') && + !contains(needs.get_labels.outputs.labels,'CI: skip cuda') && + !contains(needs.get_labels.outputs.labels,'CI: skip eamxx-sa') && + !contains(needs.get_labels.outputs.labels,'CI: skip eamxx-all')) || + (github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'gcc-cuda' || + github.event.inputs.job_to_run == 'all') || + github.event_name == 'schedule' + }} runs-on: [self-hosted, ghci-snl-cuda, cuda, gcc] strategy: fail-fast: false matrix: build_type: [sp, dbg, opt] - if: ${{ github.event_name != 'workflow_dispatch' || - github.event.inputs.job_to_run == 'gcc-cuda' || - github.event.inputs.job_to_run == 'all' }} name: gcc-cuda / ${{ matrix.build_type }} steps: - name: Check out the repository @@ -109,13 +137,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip gcc,AT: skip cuda,AT: skip eamxx-sa,AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Set test-all inputs based on event specs run: | echo "generate=false" >> $GITHUB_ENV diff --git a/.github/workflows/eamxx-scripts-tests.yml b/.github/workflows/eamxx-scripts-tests.yml index bd719539bab..dd0fd5ffeaf 100644 --- a/.github/workflows/eamxx-scripts-tests.yml +++ b/.github/workflows/eamxx-scripts-tests.yml @@ -5,15 +5,6 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] - paths: - - components/eamxx/scripts/** - - components/eamxx/cime_config/*.py - - .github/workflows/eamxx-scripts-tests.yml - - externals/ekat/** - - externals/scorpio/** - - externals/haero/** - - externals/YAKL/** - - components/eam/src/physics/rrtmgp/external/** # Manual run for debug purposes only workflow_dispatch: @@ -30,9 +21,55 @@ concurrency: cancel-in-progress: true jobs: + pr_relevant: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest # This job can run anywhere + outputs: + value: ${{ steps.check_paths.outputs.touched }} + steps: + - name: checkout + uses: actions/checkout@v4 + with: + submodules: false + - name: check_paths + uses: ./.github/actions/check-pr-paths + with: + token: ${{ secrets.GITHUB_TOKEN }} + paths: | + components/eamxx/scripts + components/eamxx/cime_config/eamxx + components/eamxx/cime_config/build + components/eamxx/cime_config/yaml_utils.py + .github/workflows/eamxx-scripts-tests.yml + get_labels: + if: ${{ github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + outputs: + labels: ${{ steps.get_labels.outputs.labels }} + steps: + - name: checkout + uses: actions/checkout@v4 + with: + submodules: false + - name: get_labels + uses: ./.github/actions/get-labels cpu-gcc: + needs: [pr_relevant, get_labels] + # if: ${{ + # github.event_name != 'pull_request' || + # ( + # needs.pr_relevant.outputs.value == 'true' + # # needs.pr_relevant.outputs.value == 'true' && + # # !contains(split(needs.get_labels.outputs.labels, ','), 'CI: skip eamxx-all') + # ) + # }} runs-on: [self-hosted, gcc, ghci-snl-cpu] steps: + - name: print-stuff + run: | + echo "PR Relevant Value: '${{ needs.pr_relevant.outputs.value }}'" + echo "Event Name: '${{ github.event_name }}'" + echo "labels: '${{ needs.get_labels.outputs.labels }}'" - name: Check out the repository uses: actions/checkout@v4 with: @@ -41,13 +78,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Run test run: | cd components/eamxx diff --git a/.github/workflows/eamxx-v1-testing.yml b/.github/workflows/eamxx-v1-testing.yml index 47129159f82..edc1cb0ca84 100644 --- a/.github/workflows/eamxx-v1-testing.yml +++ b/.github/workflows/eamxx-v1-testing.yml @@ -5,17 +5,6 @@ on: pull_request: branches: [ master ] types: [opened, synchronize, ready_for_review, reopened] - paths: - - components/eamxx/** - - components/eam/src/physics/rrtmgp/** - - components/eam/src/physics/p3/scream/** - - components/eam/src/physics/cam/** - - .github/workflows/eamxx-v1-testing.yml - - externals/ekat/** - - externals/scorpio/** - - externals/haero/** - - externals/YAKL/** - - components/eam/src/physics/rrtmgp/external/** # Manual run is used to bless workflow_dispatch: @@ -40,7 +29,45 @@ concurrency: cancel-in-progress: true jobs: + pr_relevant: + runs-on: ubuntu-latest # This job can run anywhere + outputs: + value: ${{ steps.check_paths.outputs.touched }} + steps: + - name: check_paths + if: ${{ github.event_name == "pull_request" }} + uses: ./.github/actions/check-pr-relevance + with: + paths: | + components/eamxx + components/eam/src/physics/rrtmgp + components/eam/src/physics/p3/scream + components/eam/src/physics/cam + components/eam/src/physics/rrtmgp/external + externals/ekat + externals/scorpio + externals/haero + externals/YAKL + .github/workflows/eamxx-v1-testing.yml + get_labels: + runs-on: ubuntu-latest + outputs: ${{ steps.get_labels.outputs.labels }} + steps: + - name: get_labels + uses: ./.github/actions/get-labels cpu-gcc: + needs: pr_relevant, get_labels + if: ${{ + (github.event_name == 'pull_request' && + needs.pr_relevant.outputs.value=='true' && + !contains(needs.get_labels.outputs.labels,'CI: skip gcc') && + !contains(needs.get_labels.outputs.labels,'CI: skip eamxx-v1') && + !contains(needs.get_labels.outputs.labels,'CI: skip eamxx-all')) || + (github.event_name == 'workflow_dispatch' && + github.event.inputs.job_to_run == 'cpu-gcc' || + github.event.inputs.job_to_run == 'all') || + github.event_name == 'schedule' + }} runs-on: [self-hosted, gcc, ghci-snl-cpu] strategy: matrix: @@ -55,9 +82,6 @@ jobs: short_name: SMS_D_Ln5.ne4pg2_oQU480.F2010-SCREAMv1-MPASSI.scream-mam4xx-all_mam4xx_procs fail-fast: false name: cpu-gcc / ${{ matrix.test.short_name }} - if: ${{ github.event_name != 'workflow_dispatch' || - github.event.inputs.job_to_run == 'cpu-gcc' || - github.event.inputs.job_to_run == 'all' }} steps: - name: Check out the repository uses: actions/checkout@v4 @@ -67,13 +91,6 @@ jobs: submodules: recursive - name: Show action trigger uses: ./.github/actions/show-workflow-trigger - - name: Check for skip labels - if: ${{ github.event_name == 'pull_request' || github.event_name == 'pull_request_review' }} - uses: ./.github/actions/check-skip-labels - with: - skip_labels: 'AT: skip gcc,AT: skip openmp,AT: skip eamxx-sa,AT: skip eamxx-all' - token: ${{ secrets.GITHUB_TOKEN }} - pr_number: ${{ github.event.pull_request.number }} - name: Set CA certificates env var run: | # Ensure the operating system is Linux