diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 13deb5393..04cc9b0b5 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -55,7 +55,7 @@ jobs: run: sudo usermod -c 'CI Runner' "$(whoami)" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.py-semver }} allow-prereleases: true @@ -104,7 +104,7 @@ jobs: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.py-semver }} cache: pip @@ -140,7 +140,7 @@ jobs: run: sudo usermod -c 'CI Runner' "$(whoami)" - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ env.py-semver }} cache: pip @@ -194,7 +194,7 @@ jobs: run: sudo rm -f /usr/bin/docker ; sudo apt-get install -y podman - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.12 cache: pip @@ -207,7 +207,7 @@ jobs: CWLTOOL_OPTIONS: ${{ matrix.cwl-version == 'v1.2' && '--relax-path-checks' || '' }} ${{ matrix.extras }} run: ./conformance-test.sh - name: Archive test results - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: cwl-${{ matrix.cwl-version }}-${{ matrix.container }}${{ matrix.extras }}-conformance-results path: | @@ -230,7 +230,7 @@ jobs: sudo apt-get install -y ./singularity-ce_3.10.4-jammy_amd64.deb - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.12 cache: pip @@ -272,7 +272,7 @@ jobs: with: fetch-depth: 0 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.12 cache: pip diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 8af9efe6f..3b84a185b 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -27,9 +27,9 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: python - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 diff --git a/conformance-test.sh b/conformance-test.sh index a1560a757..8b1e82f5b 100755 --- a/conformance-test.sh +++ b/conformance-test.sh @@ -76,7 +76,8 @@ else venv "${TMP_DIR}/cwl-conformance-venv" pip install -U setuptools wheel pip pip uninstall -y cwltool - pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt" + pip install -r"${SCRIPT_DIRECTORY}/mypy-requirements.txt" + CWLTOOL_USE_MYPYC=1 MYPYPATH="${SCRIPT_DIRECTORY}/mypy-stubs" pip install "${SCRIPT_DIRECTORY}" -r"${SCRIPT_DIRECTORY}/requirements.txt" pip install 'cwltest>=2.3' pytest-cov pytest-xdist fi diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index 0b0d7f3ec..5b2bea5b2 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -866,7 +866,7 @@ def calc_checksum(location: str) -> Optional[str]: and "checksum" in e and e["checksum"] != "sha1$hash" ): - return cast(Optional[str], e["checksum"]) + return cast(str, e["checksum"]) return None def remove_prefix(s: str, prefix: str) -> str: diff --git a/cwltool/process.py b/cwltool/process.py index c442bf249..4f046928b 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -1342,10 +1342,15 @@ def compute_checksums(fs_access: StdFsAccess, fileobj: CWLObjectType) -> None: if "checksum" not in fileobj: checksum = hashlib.sha1() # nosec location = cast(str, fileobj["location"]) - with fs_access.open(location, "rb") as f: - contents = f.read(1024 * 1024) - while contents != b"": - checksum.update(contents) + if "contents" in fileobj: + contents = cast(str, fileobj["contents"]).encode("utf-8") + checksum.update(contents) + fileobj["size"] = len(contents) + else: + with fs_access.open(location, "rb") as f: contents = f.read(1024 * 1024) + while contents != b"": + checksum.update(contents) + contents = f.read(1024 * 1024) + fileobj["size"] = fs_access.size(location) fileobj["checksum"] = "sha1$%s" % checksum.hexdigest() - fileobj["size"] = fs_access.size(location) diff --git a/lint-requirements.txt b/lint-requirements.txt index ef55b59e8..e33dfc2ab 100644 --- a/lint-requirements.txt +++ b/lint-requirements.txt @@ -1,3 +1,3 @@ flake8-bugbear<23.13 -black~=23.11 +black~=23.12 codespell diff --git a/setup.py b/setup.py index cc56c6889..78b4cd7e8 100644 --- a/setup.py +++ b/setup.py @@ -115,6 +115,7 @@ "argcomplete", "pyparsing != 3.0.2", # breaks --print-dot (pydot) https://github.com/pyparsing/pyparsing/issues/319 "cwl-utils >= 0.32", + "spython >= 0.3.0", ], extras_require={ "deps": [ diff --git a/tests/test_examples.py b/tests/test_examples.py index 1e4e939c6..4d479e313 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -1325,6 +1325,28 @@ def test_cache_relative_paths(tmp_path: Path, factor: str) -> None: assert (tmp_path / "cwltool_cache" / "27903451fc1ee10c148a0bdeb845b2cf").exists() +@pytest.mark.parametrize("factor", test_factors) +def test_cache_default_literal_file(tmp_path: Path, factor: str) -> None: + """Confirm that running a CLT with a default literal file with caching succeeds.""" + test_file = "tests/wf/extract_region_specs.cwl" + cache_dir = str(tmp_path / "cwltool_cache") + commands = factor.split() + commands.extend( + [ + "--out", + str(tmp_path / "out"), + "--cachedir", + cache_dir, + get_data(test_file), + ] + ) + error_code, _, stderr = get_main_output(commands) + + stderr = re.sub(r"\s\s+", " ", stderr) + assert "completed success" in stderr + assert error_code == 0 + + def test_write_summary(tmp_path: Path) -> None: """Test --write-summary.""" commands = [ diff --git a/tests/wf/extract_region_specs.cwl b/tests/wf/extract_region_specs.cwl new file mode 100644 index 000000000..279fa4400 --- /dev/null +++ b/tests/wf/extract_region_specs.cwl @@ -0,0 +1,21 @@ +{ +"cwlVersion": "v1.0", +"class": "CommandLineTool", +"inputs": [ + { + "type": "File", + "default": { + "class": "File", + "basename": "extract_regions.py", + "contents": "#!/usr/bin/env python3\n\nfrom __future__ import print_function, division\nimport sys\n\ninput_filename = sys.argv[1]\nif len(sys.argv) == 3:\n fuzz = int(sys.argv[2])\nelse:\n fuzz = 0\ninput_file = open(input_filename)\n\ncount = 0\nfor line in input_file:\n if not line.startswith(\">\"):\n continue\n count += 1\n contig_regions_file = open(\"contig_regions{}.txt\".format(count), \"w\")\n proteins_list_file = open(\"proteins{}.txt\".format(count), \"w\")\n fields = line.split(\"|\")\n protein_id = fields[0][1:]\n contig_id = fields[1]\n r_start = int(fields[6])\n if r_start > fuzz:\n r_start = r_start - fuzz\n r_end = int(fields[7]) + fuzz\n print(\"{}:{}-{}\".format(contig_id, r_start, r_end), file=contig_regions_file)\n print(protein_id, file=proteins_list_file)\n contig_regions_file.close()\n proteins_list_file.close()\n" + }, + "inputBinding": { + "position": 1 + }, + "id": "scripts" + } +], +"outputs": [ +], +"baseCommand": "cat" +}