Merge remote-tracking branch 'upstream/master' into merge_2.5

.devcontainer/devcontainer.json

@@ -18,11 +18,11 @@
                 "python.linting.flake8Path": "/opt/conda/bin/flake8",
                 "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle",
                 "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle",
-                "python.linting.pylintPath": "/opt/conda/bin/pylint"
+                "python.linting.pylintPath": "/opt/conda/bin/pylint",
             },
 
             // Add the IDs of extensions you want installed when the container is created.
-            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"]
-        }
-    }
+            "extensions": ["ms-python.python", "ms-python.vscode-pylance", "nf-core.nf-core-extensionpack"],
+        },
+    },
 }

.github/CONTRIBUTING.md

@@ -27,6 +27,9 @@ If you're not used to this workflow with git, you can start with some [docs from
 
 ## Tests
 
+You can optionally test your changes by running the pipeline locally. Then it is recommended to use the `debug` profile to
+receive warnings about process selectors and other debug info. Example: `nextflow run . -profile debug,test,docker --outdir <OUTDIR>`.
+
 When you create a pull request with changes, [GitHub Actions](https://github.com/features/actions) will run automatic tests.
 Typically, pull-requests are only fully reviewed when these tests are passing, though of course we can help out before then.
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -19,6 +19,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/scrn
 - [ ] If necessary, also make a PR on the nf-core/scrnaseq _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository.
 - [ ] Make sure your code lints (`nf-core lint`).
 - [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir <OUTDIR>`).
+- [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir <OUTDIR>`).
 - [ ] Usage Documentation in `docs/usage.md` is updated.
 - [ ] Output Documentation in `docs/output.md` is updated.
 - [ ] `CHANGELOG.md` is updated.

.github/workflows/ci.yml

@@ -26,13 +26,13 @@ jobs:
         NXF_VER:
           - "23.04.0"
           - "latest-everything"
-        profile:
-          [
+        profile: [
             "test,docker --aligner alevin",
             "test,docker --aligner kallisto",
             "test,docker --aligner star",
-            "test,docker --aligner cellranger -stub",
-            "test,docker --aligner universc -stub",
+            "test,docker --aligner cellranger",
+            # "test,docker --aligner cellrangerarc", // this currently lacks a suitable test profile, see issue https://github.com/nf-core/scrnaseq/issues/290
+            # "test,docker --aligner universc", // this is broken, see issue https://github.com/nf-core/scrnaseq/issues/289
           ]
     steps:
       - name: Free some space
@@ -42,7 +42,7 @@ jobs:
           sudo rm -rf "/usr/local/share/boost"
           sudo rm -rf "$AGENT_TOOLSDIRECTORY"
       - name: Check out pipeline code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1

.github/workflows/fix-linting.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Use the @nf-core-bot token to check out so we can push later
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           token: ${{ secrets.nf_core_bot_auth_token }}
 
@@ -24,7 +24,7 @@ jobs:
         env:
           GITHUB_TOKEN: ${{ secrets.nf_core_bot_auth_token }}
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install Prettier
         run: npm install -g prettier @prettier/plugin-php

.github/workflows/linting.yml

@@ -14,9 +14,9 @@ jobs:
   EditorConfig:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install editorconfig-checker
         run: npm install -g editorconfig-checker
@@ -27,9 +27,9 @@ jobs:
   Prettier:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v4
 
       - name: Install Prettier
         run: npm install -g prettier
@@ -40,7 +40,7 @@ jobs:
   PythonBlack:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Check code lints with Black
         uses: psf/black@stable
@@ -71,7 +71,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out pipeline code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install Nextflow
         uses: nf-core/setup-nextflow@v1

.gitpod.yml

@@ -4,7 +4,9 @@ tasks:
     command: |
       pre-commit install --install-hooks
       nextflow self-update
-
+  - name: unset JAVA_TOOL_OPTIONS
+    command: |
+      unset JAVA_TOOL_OPTIONS
 vscode:
   extensions: # based on nf-core.nf-core-extensionpack
     - codezombiech.gitignore # Language support for .gitignore files

CHANGELOG.md

@@ -3,6 +3,21 @@
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## v2.5.1
+
+- Fix that cellranger workflow couldn't be run and enable CI for this workflow ([#288](https://github.com/nf-core/scrnaseq/pull/288)).
+- Update modules ([#288]()https://github.com/nf-core/scrnaseq/pull/288).
+
+## v2.5.0 - 2024-01-02
+
+- Update template to v2.11.1 ([#279](https://github.com/nf-core/scrnaseq/pull/279))
+- Add support for paired GEX+ATAC sequencing using cellranger-arc ([#274](https://github.com/nf-core/scrnaseq/pull/274))
+- Increase default runtime limits for some processes ([#281](https://github.com/nf-core/scrnaseq/pull/281), [#284](https://github.com/nf-core/scrnaseq/pull/284))
+- Better support for custom protocols ([#273](https://github.com/nf-core/scrnaseq/pull/273)).
+  - The universc protocol is now specified via the `--protocol` flag
+  - Any protocol specified is now passed to the respective aligner
+  - Added a section to the documentation
+
 ## v2.4.1 - 2023-09-28
 
 - Fix whitelist logic for dropseq ([#267](https://github.com/nf-core/scrnaseq/pull/267))

README.md

@@ -31,11 +31,8 @@ The nf-core/scrnaseq pipeline comes with documentation about the pipeline [usage
 
 ## Usage
 
-:::note
-If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how
-to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline)
-with `-profile test` before running the workflow on actual data.
-:::
+> [!NOTE]
+> If you are new to Nextflow and nf-core, please refer to [this page](https://nf-co.re/docs/usage/installation) on how to set-up Nextflow. Make sure to [test your setup](https://nf-co.re/docs/usage/introduction#how-to-run-a-pipeline) with `-profile test` before running the workflow on actual data.
 
 First, prepare a samplesheet with your input data that looks as follows:
 
@@ -62,11 +59,9 @@ nextflow run nf-core/scrnaseq \
    --outdir <OUTDIR>
 ```
 
-:::warning
-Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those
-provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
-see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
-:::
+> [!WARNING]
+> Please provide pipeline parameters via the CLI or Nextflow `-params-file` option. Custom config files including those provided by the `-c` Nextflow option can be used to provide any configuration _**except for parameters**_;
+> see [docs](https://nf-co.re/usage/configuration#custom-configuration-files).
 
 For more details and further functionality, please refer to the [usage documentation](https://nf-co.re/scrnaseq/usage) and the [parameter documentation](https://nf-co.re/scrnaseq/parameters).
 
@@ -95,12 +90,13 @@ For more details about the output files and reports, please refer to the
 
 nf-core/scrnaseq was originally written by Bailey PJ, Botvinnik O, Marques de Almeida F, Gabernet G, Peltzer A, Sturm G.
 
-We thank the following people for their extensive assistance in the development of this pipeline:
+We thank the following people and teams for their extensive assistance in the development of this pipeline:
 
 - @heylf
 - @KevinMenden
 - @FloWuenne
 - @rob-p
+- [GHGA](https://www.ghga.de/)
 
 ## Contributions and Support
 

assets/multiqc_config.yml

@@ -1,9 +1,7 @@
 report_comment: >
-
-  This report has been generated by the <a href="https://github.com/nf-core/scrnaseq/releases/tag/2.4.1" target="_blank">nf-core/scrnaseq</a>
+  This report has been generated by the <a href="https://github.com/nf-core/scrnaseq/releases/tag/2.5.1" target="_blank">nf-core/scrnaseq</a>
   analysis pipeline. For information about how to interpret these results, please see the
-  <a href="https://nf-co.re/scrnaseq/2.4.1/docs/output" target="_blank">documentation</a>.
-
+  <a href="https://nf-co.re/scrnaseq/2.5.1/docs/output" target="_blank">documentation</a>.
 report_section_order:
   "nf-core-scrnaseq-methods-description":
     order: -1000

assets/protocols.json

@@ -0,0 +1,90 @@
+{
+    "alevin": {
+        "10XV1": {
+            "protocol": "10xv1",
+            "whitelist": "assets/whitelist/10x_V1_barcode_whitelist.txt.gz"
+        },
+        "10XV2": {
+            "protocol": "10xv2",
+            "whitelist": "assets/whitelist/10x_V2_barcode_whitelist.txt.gz"
+        },
+        "10XV3": {
+            "protocol": "10xv3",
+            "whitelist": "assets/whitelist/10x_V3_barcode_whitelist.txt.gz"
+        },
+        "dropseq": {
+            "protocol": "dropseq"
+        }
+    },
+    "cellranger": {
+        "auto": {
+            "protocol": "auto"
+        },
+        "10XV1": {
+            "protocol": "SC3Pv1"
+        },
+        "10XV2": {
+            "protocol": "SC3Pv2"
+        },
+        "10XV3": {
+            "protocol": "SC3Pv3"
+        }
+    },
+    "star": {
+        "10XV1": {
+            "protocol": "CB_UMI_Simple",
+            "extra_args": "--soloUMIlen 10",
+            "whitelist": "assets/whitelist/10x_V1_barcode_whitelist.txt.gz"
+        },
+        "10XV2": {
+            "protocol": "CB_UMI_Simple",
+            "extra_args": "--soloUMIlen 10",
+            "whitelist": "assets/whitelist/10x_V2_barcode_whitelist.txt.gz"
+        },
+        "10XV3": {
+            "protocol": "CB_UMI_Simple",
+            "extra_args": "--soloUMIlen 12",
+            "whitelist": "assets/whitelist/10x_V3_barcode_whitelist.txt.gz"
+        },
+        "dropseq": {
+            "protocol": "CB_UMI_Simple"
+        },
+        "smartseq": {
+            "protocol": "SmartSeq"
+        }
+    },
+    "kallisto": {
+        "10XV1": {
+            "protocol": "10XV1"
+        },
+        "10XV2": {
+            "protocol": "10XV2"
+        },
+        "10XV3": {
+            "protocol": "10XV3"
+        },
+        "dropseq": {
+            "protocol": "DROPSEQ"
+        },
+        "smartseq": {
+            "protocol": "SMARTSEQ"
+        }
+    },
+    "universc": {
+        "auto": {
+            "protocol": "10x"
+        },
+        "10XV1": {
+            "protocol": "10x-v1"
+        },
+        "10XV2": {
+            "protocol": "10x-v2"
+        },
+        "10XV3": {
+            "protocol": "10x-v3"
+        },
+        "dropseq": {
+            "protocol": "dropseq"
+        }
+    }
+}

assets/slackreport.json

@@ -3,7 +3,7 @@
         {
             "fallback": "Plain-text summary of the attachment.",
             "color": "<% if (success) { %>good<% } else { %>danger<%} %>",
-            "author_name": "nf-core/scrnaseq v${version} - ${runName}",
+            "author_name": "nf-core/scrnaseq ${version} - ${runName}",
             "author_icon": "https://www.nextflow.io/docs/latest/_static/favicon.ico",
             "text": "<% if (success) { %>Pipeline completed successfully!<% } else { %>Pipeline completed with errors<% } %>",
             "fields": [

bin/check_samplesheet.py

@@ -87,7 +87,8 @@ def check_samplesheet(file_in, file_out):
         ## Check header
         MIN_COLS = 2
         MIN_HEADER = ["sample", "fastq_1", "fastq_2"]
-        OPT_HEADER = ["expected_cells", "seq_center"]
+        OPT_HEADER = ["expected_cells", "seq_center", "fastq_barcode", "sample_type"]
+        SAMPLE_TYPES = ["gex", "atac"]
         header = [x.strip('"') for x in fin.readline().strip().split(",")]
 
         unknown_header = 0
@@ -101,8 +102,7 @@ def check_samplesheet(file_in, file_out):
                 min_header_count = min_header_count + 1
             colmap[h] = i
             i = i + 1
-        if min_header_count < len(MIN_HEADER):
-            # code was checking for unknown_header or min_header_count however looking at the ifelse, unknown_header does not seem that it should be tested
+        if unknown_header or min_header_count < len(MIN_HEADER):
             given = ",".join(header)
             wanted = ",".join(MIN_HEADER)
             print(f"ERROR: Please check samplesheet header -> {given} != {wanted}")
@@ -147,7 +147,26 @@ def check_samplesheet(file_in, file_out):
                 seq_center = seq_center.replace(" ", "_")
 
             ## Check FastQ file extension
-            for fastq in [fastq_1, fastq_2]:
+            fastq_list = [fastq_1, fastq_2]
+
+            fastq_barcode = ""
+            if "fastq_barcode" in header:
+                fastq_barcode = lspl[colmap["fastq_barcode"]]
+                fastq_list.append(fastq_barcode)
+
+            sample_type = ""
+            if "sample_type" in header:
+                sample_type = lspl[colmap["sample_type"]]
+                if sample_type not in SAMPLE_TYPES:
+                    print_error(
+                        "Sample type {} is not supported! Please specify either {}".format(
+                            sample_type, " or ".join(SAMPLE_TYPES)
+                        ),
+                        "Line",
+                        line,
+                    )
+
+            for fastq in fastq_list:
                 if fastq:
                     if fastq.find(" ") != -1:
                         print_error("FastQ file contains spaces!", "Line", line)
@@ -161,9 +180,9 @@ def check_samplesheet(file_in, file_out):
             ## Auto-detect paired-end/single-end
             sample_info = []  ## [single_end, fastq_1, fastq_2]
             if sample and fastq_1 and fastq_2:  ## Paired-end short reads
-                sample_info = ["0", fastq_1, fastq_2, expected_cells, seq_center]
+                sample_info = ["0", fastq_1, fastq_2, expected_cells, seq_center, fastq_barcode, sample_type]
             elif sample and fastq_1 and not fastq_2:  ## Single-end short reads
-                sample_info = ["1", fastq_1, fastq_2, expected_cells, seq_center]
+                sample_info = ["1", fastq_1, fastq_2, expected_cells, seq_center, fastq_barcode, sample_type]
             else:
                 print_error("Invalid combination of columns provided!", "Line", line)
 
@@ -180,7 +199,21 @@ def check_samplesheet(file_in, file_out):
     ## Write validated samplesheet with appropriate columns
     if len(sample_mapping_dict) > 0:
         with open(file_out, "w") as fout:
-            fout.write(",".join(["sample", "single_end", "fastq_1", "fastq_2", "expected_cells", "seq_center"]) + "\n")
+            fout.write(
+                ",".join(
+                    [
+                        "sample",
+                        "single_end",
+                        "fastq_1",
+                        "fastq_2",
+                        "expected_cells",
+                        "seq_center",
+                        "fastq_barcode",
+                        "sample_type",
+                    ]
+                )
+                + "\n"
+            )
             for sample in sorted(sample_mapping_dict.keys()):
                 ## Check that multiple runs of the same sample are of the same datatype
                 if not all(x[0] == sample_mapping_dict[sample][0][0] for x in sample_mapping_dict[sample]):