diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 4ecfbfe3..b290e090 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -10,15 +10,7 @@ "vscode": { // Set *default* container specific settings.json values on container create. "settings": { - "python.defaultInterpreterPath": "/opt/conda/bin/python", - "python.linting.enabled": true, - "python.linting.pylintEnabled": true, - "python.formatting.autopep8Path": "/opt/conda/bin/autopep8", - "python.formatting.yapfPath": "/opt/conda/bin/yapf", - "python.linting.flake8Path": "/opt/conda/bin/flake8", - "python.linting.pycodestylePath": "/opt/conda/bin/pycodestyle", - "python.linting.pydocstylePath": "/opt/conda/bin/pydocstyle", - "python.linting.pylintPath": "/opt/conda/bin/pylint" + "python.defaultInterpreterPath": "/opt/conda/bin/python" }, // Add the IDs of extensions you want installed when the container is created. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 72c2b4ae..4dde8d56 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -18,7 +18,7 @@ Learn more about contributing: [CONTRIBUTING.md](https://github.com/nf-core/fetc - [ ] If you've added a new tool - have you followed the pipeline conventions in the [contribution docs](https://github.com/nf-core/fetchngs/tree/master/.github/CONTRIBUTING.md) - [ ] If necessary, also make a PR on the nf-core/fetchngs _branch_ on the [nf-core/test-datasets](https://github.com/nf-core/test-datasets) repository. - [ ] Make sure your code lints (`nf-core lint`). -- [ ] Ensure the test suite passes (`nextflow run . -profile test,docker --outdir `). +- [ ] Ensure the test suite passes (`nf-test test main.nf.test -profile test,docker`). - [ ] Check for unexpected warnings in debug mode (`nextflow run . -profile debug,test,docker --outdir `). - [ ] Usage Documentation in `docs/usage.md` is updated. - [ ] Output Documentation in `docs/output.md` is updated. diff --git a/.github/include.yaml b/.github/include.yaml new file mode 100644 index 00000000..a3629f4c --- /dev/null +++ b/.github/include.yaml @@ -0,0 +1,10 @@ +".": + - ./.github/workflows/** + - ./nf-test.config + - ./nextflow.config +tests: + - ./assets/* + - ./bin/* + - ./conf/* + - ./main.nf + - ./nextflow_schema.json diff --git a/.github/python/find_changed_files.py b/.github/python/find_changed_files.py deleted file mode 100644 index 6dc73449..00000000 --- a/.github/python/find_changed_files.py +++ /dev/null @@ -1,152 +0,0 @@ -#!/usr/bin/env python - -## This script is used to generate scan *.nf.test files for function/process/workflow name and return as a JSON list -# It is functionally similar to nf-test list but fills a gap until feature https://github.com/askimed/nf-test/issues/196 is added - -import argparse -import json -import logging -import re - -from itertools import chain -from pathlib import Path - - -def parse_args() -> argparse.Namespace: - """ - Parse command line arguments and return an ArgumentParser object. - - Returns: - argparse.ArgumentParser: The ArgumentParser object with the parsed arguments. - """ - parser = argparse.ArgumentParser( - description="Scan *.nf.test files for function/process/workflow name and return as a JSON list" - ) - parser.add_argument( - "-p", - "--paths", - nargs="+", - default=["."], - help="List of directories or files to scan", - ) - parser.add_argument( - "-l", - "--log-level", - choices=["DEBUG", "INFO", "WARNING", "ERROR"], - default="INFO", - help="Logging level", - ) - parser.add_argument( - "-t", - "--types", - nargs="+", - choices=["function", "process", "workflow", "pipeline"], - default=["function", "process", "workflow", "pipeline"], - help="Types of tests to include.", - ) - return parser.parse_args() - - -def find_files(paths: list[str]) -> list[Path]: - """ - Find all files matching pattern *.nf.test recursively from a list of paths. - - Args: - paths (list): List of directories or files to scan. - - Returns: - list: List of files matching the pattern *.nf.test. - """ - # this is a bit clunky - result = [] - for path in paths: - path_obj = Path(path) - # If Path is the exact nf-test file add to list: - if path_obj.match("*.nf.test"): - result.append(path_obj) - # Else recursively search for nf-test files: - else: - for file in path_obj.rglob("*.nf.test"): - result.append(file) - return result - - -def process_files(files: list[Path]) -> list[str]: - """ - Process the files and return lines that begin with 'workflow', 'process', or 'function' and have a single string afterwards. - - Args: - files (list): List of files to process. - - Returns: - list: List of lines that match the criteria. - """ - result = [] - for file in files: - with open(file, "r") as f: - is_pipeline_test = True - lines = f.readlines() - for line in lines: - line = line.strip() - if line.startswith(("workflow", "process", "function")): - words = line.split() - if len(words) == 2 and re.match(r'^".*"$', words[1]): - result.append(line) - is_pipeline_test = False - - # If no results included workflow, process or function - # Add a dummy result to fill the 'pipeline' category - if is_pipeline_test: - result.append("pipeline 'PIPELINE'") - - return result - - -def generate( - lines: list[str], types: list[str] = ["function", "process", "workflow", "pipeline"] -) -> dict[str, list[str]]: - """ - Generate a dictionary of function, process and workflow lists from the lines. - - Args: - lines (list): List of lines to process. - types (list): List of types to include. - - Returns: - dict: Dictionary with function, process and workflow lists. - """ - result: dict[str, list[str]] = { - "function": [], - "process": [], - "workflow": [], - "pipeline": [], - } - for line in lines: - words = line.split() - if len(words) == 2: - keyword = words[0] - name = words[1].strip("'\"") # Strip both single and double quotes - if keyword in types: - result[keyword].append(name) - return result - - -if __name__ == "__main__": - - # Utility stuff - args = parse_args() - logging.basicConfig(level=args.log_level) - - # Parse nf-test files for targets of tests - files = find_files(args.paths) - lines = process_files(files) - result = generate(lines) - - # Get only relevant results (specified by -t) - # Unique using a set - target_results = list( - {item for sublist in map(result.get, args.types) for item in sublist} - ) - - # Print to stdout - print(json.dumps(target_results)) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa75579f..3b1c9c6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -27,72 +27,28 @@ jobs: name: Check for changes runs-on: ubuntu-latest outputs: - changes: ${{ steps.changed_files.outputs.any_modified }} - tags: ${{ steps.list.outputs.tags }} + nf_test_files: ${{ steps.list.outputs.components }} steps: - - uses: actions/setup-python@v4 - with: - python-version: "3.11" - architecture: "x64" - - uses: actions/checkout@v3 with: fetch-depth: 0 - - uses: tj-actions/changed-files@v42 - id: changed_files - with: - dir_names: "true" - output_renamed_files_as_deleted_and_added: "true" - # Define list of additional rules for testing paths - # Mostly, we define additional 'pipeline' or 'all' tests here - files_yaml: | - ".": - - .github/workflows/** - - nf-test.config - - nextflow.config - tests: - - assets/* - - bin/* - - conf/* - - main.nf - - nextflow_schema.json - - files_ignore: | - .git* - .gitpod.yml - .prettierignore - .prettierrc.yml - **.md - **.png - modules.json - pyproject.toml - tower.yml - - - name: debug - run: | - echo ${{ steps.changed_files.outputs.any_modified }} - echo ${{ steps.changed_files.outputs.all_changed_files }} - echo ${{ steps.changed_files.outputs.changed_keys }} - - - name: nf-test list tags + - name: List nf-test files id: list - if: ${{ steps.changed_files.outputs.any_modified }} - run: | - echo tags=$(python \ - .github/python/find_changed_files.py \ - -t pipeline workflow process \ - -p ${{ steps.changed_files.outputs.all_changed_files }} ${{ steps.changed_files.outputs.changed_keys }} \ - ) >> $GITHUB_OUTPUT + uses: adamrtalbot/detect-nf-test-changes@v0.0.3 + with: + head: ${{ github.sha }} + base: origin/${{ github.base_ref }} + include: .github/include.yaml - - name: debug2 + - name: print list of nf-test files run: | - echo ${{ steps.list.outputs.tags }} + echo ${{ steps.list.outputs.components }} test: - name: ${{ matrix.tags }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} + name: ${{ matrix.nf_test_files }} ${{ matrix.profile }} NF-${{ matrix.NXF_VER }} needs: [changes] - if: needs.changes.outputs.changes + if: needs.changes.outputs.nf_test_files != '[]' runs-on: ubuntu-latest strategy: fail-fast: false @@ -100,16 +56,16 @@ jobs: NXF_VER: - "latest-everything" - "23.04" - tags: ["${{ fromJson(needs.changes.outputs.tags) }}"] + nf_test_files: ["${{ fromJson(needs.changes.outputs.nf_test_files) }}"] profile: - "docker" steps: - name: Check out pipeline code - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4 + uses: actions/checkout@v4 - name: Install Nextflow - uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 + uses: nf-core/setup-nextflow@v2 with: version: "${{ matrix.NXF_VER }}" @@ -140,7 +96,7 @@ jobs: - name: Run nf-test run: | - nf-test test --verbose --tag ${{ matrix.tags }} --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap + nf-test test --verbose ${{ matrix.nf_test_files }} --profile "+${{ matrix.profile }}" --junitxml=test.xml --tap=test.tap - uses: pcolby/tap-summary@v1 with: diff --git a/.github/workflows/download_pipeline.yml b/.github/workflows/download_pipeline.yml index f823210d..08622fd5 100644 --- a/.github/workflows/download_pipeline.yml +++ b/.github/workflows/download_pipeline.yml @@ -28,7 +28,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Install Nextflow - uses: nf-core/setup-nextflow@b9f764e8ba5c76b712ace14ecbfcef0e40ae2dd8 # v1 + uses: nf-core/setup-nextflow@v1 - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 with: diff --git a/.github/workflows/release-announcements.yml b/.github/workflows/release-announcements.yml index c3674af2..d468aeaa 100644 --- a/.github/workflows/release-announcements.yml +++ b/.github/workflows/release-announcements.yml @@ -12,7 +12,7 @@ jobs: - name: get topics and convert to hashtags id: get_topics run: | - curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' > $GITHUB_OUTPUT + curl -s https://nf-co.re/pipelines.json | jq -r '.remote_workflows[] | select(.full_name == "${{ github.repository }}") | .topics[]' | awk '{print "#"$0}' | tr '\n' ' ' >> $GITHUB_OUTPUT - uses: rzr/fediverse-action@master with: diff --git a/.gitpod.yml b/.gitpod.yml index 363d5b1d..105a1821 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -10,13 +10,11 @@ tasks: vscode: extensions: # based on nf-core.nf-core-extensionpack - - codezombiech.gitignore # Language support for .gitignore files - # - cssho.vscode-svgviewer # SVG viewer - esbenp.prettier-vscode # Markdown/CommonMark linting and style checking for Visual Studio Code - - eamodio.gitlens # Quickly glimpse into whom, why, and when a line or code block was changed - EditorConfig.EditorConfig # override user/workspace settings with settings found in .editorconfig files - Gruntfuggly.todo-tree # Display TODO and FIXME in a tree view in the activity bar - mechatroner.rainbow-csv # Highlight columns in csv files in different colors - # - nextflow.nextflow # Nextflow syntax highlighting + # - nextflow.nextflow # Nextflow syntax highlighting - oderwat.indent-rainbow # Highlight indentation level - streetsidesoftware.code-spell-checker # Spelling checker for source code + - charliermarsh.ruff # Code linter Ruff diff --git a/CHANGELOG.md b/CHANGELOG.md index a1871ead..e69206d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Enhancements & fixes +- [PR #299](https://github.com/nf-core/fetchngs/pull/299) - Template update for nf-core/tools v2.13.1 +- [PR #300](https://github.com/nf-core/fetchngs/pull/300) - Use file paths instead of tags for testing matrix, should make matrices more efficient +- [PR #303](https://github.com/nf-core/fetchngs/pull/303) - Update wget container for SRA_FASTQ_FTP from 1.20.1 to 1.21.4 +- [PR #305](https://github.com/nf-core/fetchngs/pull/305) - Update module sratools/prefetch for reliable download integrity check - [PR #319](https://github.com/nf-core/fetchngs/pull/319) - Update subworkflows and modules to be TES compliance. ### :warning: Major enhancements @@ -15,11 +19,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Add support for tags - Updated `multiqc_mappings_config` to have different matching pattern for `config.yml` vs `versions.yml` -### Credits -Special thanks to the following for their contributions to the release: +### Software dependencies -- [Venkat Malladi](https://github.com/vsmalladi) +| Dependency | Old version | New version | +| ---------- | ----------- | ----------- | +| `wget` | 1.20.1 | 1.21.4 | +| `sratools` | 3.0.8 | 3.1.0 | ## [[1.12.0](https://github.com/nf-core/fetchngs/releases/tag/1.12.0)] - 2024-02-29 diff --git a/modules/local/sra_fastq_ftp/main.nf b/modules/local/sra_fastq_ftp/main.nf index e2274d46..017a08c9 100644 --- a/modules/local/sra_fastq_ftp/main.nf +++ b/modules/local/sra_fastq_ftp/main.nf @@ -4,10 +4,10 @@ process SRA_FASTQ_FTP { label 'process_low' label 'error_retry' - conda "conda-forge::wget=1.20.1" + conda "conda-forge::wget=1.21.4" container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? - 'https://depot.galaxyproject.org/singularity/wget:1.20.1' : - 'biocontainers/wget:1.20.1' }" + 'https://depot.galaxyproject.org/singularity/wget:1.21.4' : + 'biocontainers/wget:1.21.4' }" input: tuple val(meta), val(fastq) diff --git a/modules/local/sra_fastq_ftp/tests/main.nf.test.snap b/modules/local/sra_fastq_ftp/tests/main.nf.test.snap index 5b300f8c..229ad4fe 100644 --- a/modules/local/sra_fastq_ftp/tests/main.nf.test.snap +++ b/modules/local/sra_fastq_ftp/tests/main.nf.test.snap @@ -31,7 +31,7 @@ ] ], "2": [ - "versions.yml:md5,6b60ed6d5805271a1b97798e29c0635c" + "versions.yml:md5,7bfc86ca1f3e3236dbb91eb85c1d7af0" ], "fastq": [ [ @@ -62,7 +62,7 @@ ] ], "versions": [ - "versions.yml:md5,6b60ed6d5805271a1b97798e29c0635c" + "versions.yml:md5,7bfc86ca1f3e3236dbb91eb85c1d7af0" ] } ], @@ -72,4 +72,4 @@ }, "timestamp": "2024-02-28T11:51:51.301654" } -} \ No newline at end of file +} diff --git a/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap b/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap index 82a1969c..a1f82f45 100644 --- a/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap +++ b/modules/nf-core/sratools/prefetch/tests/main.nf.test.snap @@ -116,4 +116,4 @@ }, "timestamp": "2024-02-28T11:48:37.428307" } -} \ No newline at end of file +} diff --git a/nextflow.config b/nextflow.config index bb6b6b79..7f4f8ebf 100644 --- a/nextflow.config +++ b/nextflow.config @@ -221,7 +221,7 @@ manifest { description = """Pipeline to fetch metadata and raw FastQ files from public databases""" mainScript = 'main.nf' nextflowVersion = '!>=23.04.0' - version = '1.12.0' + version = '1.13.0dev' doi = 'https://doi.org/10.5281/zenodo.5070524' }