Adding our nf-test tests to the GitHub Actions CI

.github/workflows/ci.yml

@@ -11,26 +11,57 @@ on:
 
 env:
   NXF_ANSI_LOG: false
+  NFT_VER: "0.9.0"
   NXF_SINGULARITY_CACHEDIR: ${{ github.workspace }}/.singularity
   NXF_SINGULARITY_LIBRARYDIR: ${{ github.workspace }}/.singularity
+  # env var to force tests
+  # TEST_SET: '["tests/workflows/mcmicro.nf.test", "subworkflows/local/utils_nfcore_mcmicro_pipeline/tests/initialisation.nf.test", "tests/main.nf.test"]'
 
 concurrency:
   group: "${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}"
   cancel-in-progress: true
 
 jobs:
+  nf-test-changes:
+    name: Check for changes
+    runs-on: ubuntu-latest
+    outputs:
+      nf_test_files: ${{ steps.list.outputs.components }}
+      # nf_test_files: ${{ env.TEST_SET }}
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: List nf-test files
+        id: list
+        uses: adamrtalbot/detect-nf-test-changes@v0.0.4
+        with:
+          head: ${{ github.sha }}
+          # TODO: check that the following works on pull_request
+          # base: origin/${{ github.ref_name }}
+          base: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
+
+      - name: print list of nf-test files
+        run: |
+          echo ${{ steps.list.outputs.components }}
+
   test:
-    name: "Run pipeline with test data (${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }})"
+    name: "Run tests (${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }})"
+    needs: [nf-test-changes]
     # Only run on push if this is the nf-core dev branch (merged PRs)
-    if: "${{ github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mcmicro') }}"
+    if: "${{ (github.event_name != 'push' || (github.event_name == 'push' && github.repository == 'nf-core/mcmicro')) && needs.nf-test-changes.outputs.nf_test_files != '[]' }}"
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         NXF_VER:
           - "24.04.2"
           - "latest-everything"
+        nf_test_files: ["${{ fromJson(needs.nf-test-changes.outputs.nf_test_files) }}"]
         profile:
-          - "conda"
+          # - "conda"
           - "docker"
           - "singularity"
         test_name:
@@ -44,6 +75,9 @@ jobs:
           - isMaster: false
             profile: "singularity"
     steps:
+      - name: install curl
+        run: sudo apt update && sudo apt install -y curl wget
+
       - name: Check out pipeline code
         uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4
 
@@ -62,24 +96,78 @@ jobs:
           mkdir -p $NXF_SINGULARITY_CACHEDIR
           mkdir -p $NXF_SINGULARITY_LIBRARYDIR
 
-      - name: Set up Miniconda
-        if: matrix.profile == 'conda'
-        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
+      #      - name: Set up Miniconda
+      #        if: matrix.profile == 'conda'
+      #        uses: conda-incubator/setup-miniconda@a4260408e20b96e80095f42ff7f1a15b27dd94ca # v3
+      #        with:
+      #          miniconda-version: "latest"
+      #          auto-update-conda: true
+      #          conda-solver: libmamba
+      #          channels: conda-forge,bioconda
+
+      #      - name: Set up Conda
+      #        if: matrix.profile == 'conda'
+      #        run: |
+      #          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
+      #          echo $(realpath python) >> $GITHUB_PATH
+
+      - name: Install nf-test
+        uses: nf-core/setup-nf-test@v1
         with:
-          miniconda-version: "latest"
-          auto-update-conda: true
-          conda-solver: libmamba
-          channels: conda-forge,bioconda
+          version: ${{ env.NFT_VER }}
 
-      - name: Set up Conda
-        if: matrix.profile == 'conda'
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: "x64"
+
+      - name: Install pdiff to see diff between nf-test snapshots
         run: |
-          echo $(realpath $CONDA)/condabin >> $GITHUB_PATH
-          echo $(realpath python) >> $GITHUB_PATH
+          python -m pip install --upgrade pip
+          pip install pdiff
 
       - name: Clean up Disk space
         uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1
 
-      - name: "Run pipeline with test data ${{ matrix.NXF_VER }} | ${{ matrix.test_name }} | ${{ matrix.profile }}"
+      - name: Run nf-test
+        run: |
+          nf-test test \
+            --verbose ${{ matrix.nf_test_files }} \
+            --profile "+${{ matrix.profile }},ci" \
+            --junitxml=test.xml \
+            --tap=test.tap
+
+      - uses: pcolby/tap-summary@v1
+        with:
+          path: >-
+            test.tap
+
+      - name: Output log on failure
+        if: failure()
+        run: |
+          sudo apt install bat > /dev/null
+          batcat --decorations=always --color=always ${{ github.workspace }}/.nf-test/tests/*/meta/nextflow.log
+
+      - name: Publish Test Report
+        uses: mikepenz/action-junit-report@v3
+        if: always() # always run even if the previous step fails
+        with:
+          report_paths: test.xml
+
+  confirm-pass:
+    runs-on: ubuntu-latest
+    needs: [test]
+    if: always()
+    steps:
+      - name: All tests ok
+        if: ${{ !contains(needs.*.result, 'failure') }}
+        run: exit 0
+      - name: One or more tests failed
+        if: ${{ contains(needs.*.result, 'failure') }}
+        run: exit 1
+
+      - name: debug-print
+        if: always()
         run: |
-          nextflow run ${GITHUB_WORKSPACE} -profile ${{ matrix.test_name }},${{ matrix.profile }} --outdir ./results
+          echo "toJSON(needs) = ${{ toJSON(needs) }}"
+          echo "toJSON(needs.*.result) = ${{ toJSON(needs.*.result) }}"

.nf-core.yml

@@ -4,6 +4,7 @@ lint:
     - params.input
   files_unchanged:
     - .gitignore
+    - docs/images/nf-core-mcmicro_logo_dark.png
 nf_core_version: 3.0.2
 org_path: null
 repository_type: pipeline

conf/ci.config

@@ -0,0 +1,10 @@
+process {
+    withName: ".*:DEEPCELL_MESMER" {
+        memory = "6.GB"
+        cpus = 2
+    }
+    withName: ".*:CELLPOSE"{
+        memory = "12.GB"
+        cpus   = 4
+    }
+}

conf/modules.config

@@ -28,7 +28,6 @@ process {
     }
 
     withName: BASICPY {
-        containerOptions = '--entrypoint "" --user root'
         publishDir = [
             path: { "${params.outdir}/illumination_correction/basicpy" },
             mode: params.publish_dir_mode,

modules.json

@@ -7,7 +7,7 @@
                 "nf-core": {
                     "ashlar": {
                         "branch": "master",
-                        "git_sha": "a5a287db3590e7aa121e1251bf126c35e1418abf",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/ashlar/ashlar.diff"
                     },
@@ -18,7 +18,7 @@
                     },
                     "basicpy": {
                         "branch": "master",
-                        "git_sha": "716ef3019b66772a817b417078edce2f7b337858",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"],
                         "patch": "modules/nf-core/basicpy/basicpy.diff"
                     },
@@ -35,7 +35,7 @@
                     },
                     "deepcell/mesmer": {
                         "branch": "master",
-                        "git_sha": "298406de7fb599b3a7acfed1dddb606ec182d4b6",
+                        "git_sha": "666652151335353eef2fcd58880bcef5bc2928e1",
                         "installed_by": ["modules"]
                     },
                     "ilastik/multicut": {