MyPy stubs rename (#1699)

* rename our local type stubs dir & fix types

As per python/mypy#13155 (comment)

* upgrade minimum ruamel.yaml versions

.gitignore

@@ -28,10 +28,8 @@ venv3/
 # pycharm
 .idea/
 
-# typshed repo
-typeshed/2and3/schema_salad
-typeshed/2and3/ruamel/yaml
-
+# local stubs
+mypy-stubs/ruamel/yaml
 
 #mypy
 .mypy_cache/

MANIFEST.in

@@ -3,7 +3,7 @@ include MANIFEST.in
 include LICENSE.txt
 include *requirements.txt mypy.ini tox.ini
 include gittaggers.py Makefile cwltool.py
-recursive-include typeshed *.pyi
+recursive-include mypy-stubs *.pyi *.py
 include tests/*
 include tests/tmp1/tmp2/tmp3/.gitkeep
 include tests/tmp4/alpha/*

Makefile

@@ -90,7 +90,7 @@ clean: check-python3 FORCE
 
 # Linting and code style related targets
 ## sorting imports using isort: https://github.com/timothycrosley/isort
-sort_imports: $(PYSOURCES)
+sort_imports: $(PYSOURCES) mypy-stubs
 	isort $^
 
 remove_unused_imports: $(PYSOURCES)
@@ -110,14 +110,14 @@ diff_pydocstyle_report: pydocstyle_report.txt
 
 ## codespell   : check for common mispellings
 codespell:
-	codespell -w $(shell git ls-files | grep -v cwltool/schemas | grep -v cwltool/jshint/ | grep -v typeshed)
+	codespell -w $(shell git ls-files | grep -v cwltool/schemas | grep -v cwltool/jshint/ | grep -v mypy-stubs)
 
 ## format      : check/fix all code indentation and formatting (runs black)
 format:
-	black --exclude cwltool/schemas setup.py cwltool.py cwltool tests
+	black --exclude cwltool/schemas setup.py cwltool.py cwltool tests mypy-stubs
 
 format-check:
-	black --diff --check --exclude cwltool/schemas setup.py cwltool.py cwltool tests
+	black --diff --check --exclude cwltool/schemas setup.py cwltool.py cwltool tests mypy-stubs
 
 ## pylint      : run static code analysis on Python code
 pylint: $(PYSOURCES)
@@ -177,14 +177,14 @@ mypy3: mypy
 mypy: $(filter-out setup.py gittagger.py,$(PYSOURCES))
 	if ! test -f $(shell python -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))')/py.typed ; \
 	then \
-		rm -Rf typeshed/ruamel/yaml ; \
+		rm -Rf mypy-stubs/ruamel/yaml ; \
 		ln -s $(shell python -c 'import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
-			typeshed/ruamel/ ; \
+			mypy-stubs/ruamel/ ; \
 	fi  # if minimally required ruamel.yaml version is 0.15.99 or greater, than the above can be removed
-	MYPYPATH=$$MYPYPATH:typeshed mypy $^
+	MYPYPATH=$$MYPYPATH:mypy-stubs mypy $^
 
 mypyc: $(PYSOURCES)
-	MYPYPATH=typeshed CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \
+	MYPYPATH=mypy-stubs CWLTOOL_USE_MYPYC=1 pip install --verbose -e . \
 		 && pytest -rs -vv ${PYTEST_EXTRA}
 
 shellcheck: FORCE

cwltool.Dockerfile

@@ -9,10 +9,10 @@ RUN pip install toml -rmypy-requirements.txt "$(grep ruamel requirements.txt)" \
 	"$(grep schema.salad requirements.txt)"
 # schema-salad is needed to be installed (this time as pure Python) for
 # cwltool + mypyc
-RUN CWLTOOL_USE_MYPYC=1 MYPYPATH=typeshed pip wheel --no-binary schema-salad --wheel-dir=/wheels .[deps]
+RUN CWLTOOL_USE_MYPYC=1 MYPYPATH=mypy-stubs pip wheel --no-binary schema-salad --wheel-dir=/wheels .[deps]
 RUN rm /wheels/schema_salad*
 RUN pip install black
-RUN SCHEMA_SALAD_USE_MYPYC=1 MYPYPATH=typeshed pip wheel --no-binary schema-salad \
+RUN SCHEMA_SALAD_USE_MYPYC=1 MYPYPATH=mypy-stubs pip wheel --no-binary schema-salad \
 	$(grep schema.salad requirements.txt) black --wheel-dir=/wheels
 RUN pip install --force-reinstall --no-index --no-warn-script-location --root=/pythonroot/ /wheels/*.whl
 # --force-reinstall to install our new mypyc compiled schema-salad package

cwltool/cwlrdf.py

@@ -1,8 +1,9 @@
 import urllib
 from codecs import StreamWriter
-from typing import Any, Dict, Optional, TextIO, Union, cast
+from typing import Any, Dict, Iterator, Optional, TextIO, Union, cast
 
 from rdflib import Graph
+from rdflib.query import ResultRow
 from ruamel.yaml.comments import CommentedMap
 from schema_salad.jsonld_context import makerdf
 from schema_salad.utils import ContextType
@@ -26,7 +27,7 @@ def printrdf(wflow: Process, ctx: ContextType, style: str) -> str:
     rdf = gather(wflow, ctx).serialize(format=style, encoding="utf-8")
     if not rdf:
         return ""
-    return cast(str, rdf.decode("utf-8"))
+    return rdf.decode("utf-8")
 
 
 def lastpart(uri: Any) -> str:
@@ -37,28 +38,34 @@ def lastpart(uri: Any) -> str:
 
 
 def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None:
-    qres = g.query(
-        """SELECT ?step ?run ?runtype
+    qres = cast(
+        Iterator[ResultRow],
+        g.query(
+            """SELECT ?step ?run ?runtype
            WHERE {
               ?step cwl:run ?run .
               ?run rdf:type ?runtype .
            }"""
-    )
+        ),
+    )  # ResultRow because the query is of type SELECT
 
     for step, run, _ in qres:
         stdout.write(
             '"%s" [label="%s"]\n'
             % (lastpart(step), f"{lastpart(step)} ({lastpart(run)})")
         )
 
-    qres = g.query(
-        """SELECT ?step ?inp ?source
+    qres = cast(
+        Iterator[ResultRow],
+        g.query(
+            """SELECT ?step ?inp ?source
            WHERE {
               ?wf Workflow:steps ?step .
               ?step cwl:inputs ?inp .
               ?inp cwl:source ?source .
            }"""
-    )
+        ),
+    )  # ResultRow because the query is of type SELECT
 
     for step, inp, source in qres:
         stdout.write('"%s" [shape=box]\n' % (lastpart(inp)))
@@ -69,41 +76,50 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None:
             '"{}" -> "{}" [label="{}"]\n'.format(lastpart(inp), lastpart(step), "")
         )
 
-    qres = g.query(
-        """SELECT ?step ?out
+    qres = cast(
+        Iterator[ResultRow],
+        g.query(
+            """SELECT ?step ?out
            WHERE {
               ?wf Workflow:steps ?step .
               ?step cwl:outputs ?out .
            }"""
-    )
+        ),
+    )  # ResultRow because the query is of type SELECT
 
     for step, out in qres:
         stdout.write('"%s" [shape=box]\n' % (lastpart(out)))
         stdout.write(
             '"{}" -> "{}" [label="{}"]\n'.format(lastpart(step), lastpart(out), "")
         )
 
-    qres = g.query(
-        """SELECT ?out ?source
+    qres = cast(
+        Iterator[ResultRow],
+        g.query(
+            """SELECT ?out ?source
            WHERE {
               ?wf cwl:outputs ?out .
               ?out cwl:source ?source .
            }"""
-    )
+        ),
+    )  # ResultRow because the query is of type SELECT
 
     for out, source in qres:
         stdout.write('"%s" [shape=octagon]\n' % (lastpart(out)))
         stdout.write(
             '"{}" -> "{}" [label="{}"]\n'.format(lastpart(source), lastpart(out), "")
         )
 
-    qres = g.query(
-        """SELECT ?inp
+    qres = cast(
+        Iterator[ResultRow],
+        g.query(
+            """SELECT ?inp
            WHERE {
               ?wf rdf:type cwl:Workflow .
               ?wf cwl:inputs ?inp .
            }"""
-    )
+        ),
+    )  # ResultRow because the query is of type SELECT
 
     for (inp,) in qres:
         stdout.write('"%s" [shape=octagon]\n' % (lastpart(inp)))
@@ -116,27 +132,33 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non
     stdout.write("compound=true\n")
 
     subworkflows = set()
-    qres = g.query(
-        """SELECT ?run
+    qres = cast(
+        Iterator[ResultRow],
+        g.query(
+            """SELECT ?run
            WHERE {
               ?wf rdf:type cwl:Workflow .
               ?wf Workflow:steps ?step .
               ?step cwl:run ?run .
               ?run rdf:type cwl:Workflow .
            } ORDER BY ?wf"""
-    )
+        ),
+    )  # ResultRow because the query is of type SELECT
     for (run,) in qres:
         subworkflows.add(run)
 
-    qres = g.query(
-        """SELECT ?wf ?step ?run ?runtype
+    qres = cast(
+        Iterator[ResultRow],
+        g.query(
+            """SELECT ?wf ?step ?run ?runtype
            WHERE {
               ?wf rdf:type cwl:Workflow .
               ?wf Workflow:steps ?step .
               ?step cwl:run ?run .
               ?run rdf:type ?runtype .
            } ORDER BY ?wf"""
-    )
+        ),
+    )  # ResultRow because the query is of type SELECT
 
     currentwf = None  # type: Optional[str]
     for wf, step, _run, runtype in qres:
@@ -164,8 +186,10 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non
     if currentwf is not None:
         stdout.write("}\n")
 
-    qres = g.query(
-        """SELECT DISTINCT ?src ?sink ?srcrun ?sinkrun
+    qres = cast(
+        Iterator[ResultRow],
+        g.query(
+            """SELECT DISTINCT ?src ?sink ?srcrun ?sinkrun
            WHERE {
               ?wf1 Workflow:steps ?src .
               ?wf2 Workflow:steps ?sink .
@@ -175,7 +199,8 @@ def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> Non
               ?src cwl:run ?srcrun .
               ?sink cwl:run ?sinkrun .
            }"""
-    )
+        ),
+    )  # ResultRow because the query is of type SELECT
 
     for src, sink, srcrun, sinkrun in qres:
         attr = ""

cwltool/cwlviewer.py

@@ -1,5 +1,6 @@
 """Visualize a CWL workflow."""
 from pathlib import Path
+from typing import Iterator, List, cast
 from urllib.parse import urlparse
 
 import pydot
@@ -32,9 +33,12 @@ def _load_cwl_graph(self, rdf_description: str) -> rdflib.graph.Graph:
     def _set_inner_edges(self) -> None:
         with open(_get_inner_edges_query_path) as f:
             get_inner_edges_query = f.read()
-        inner_edges = self._rdf_graph.query(
-            get_inner_edges_query, initBindings={"root_graph": self._root_graph_uri}
-        )
+        inner_edges = cast(
+            Iterator[rdflib.query.ResultRow],
+            self._rdf_graph.query(
+                get_inner_edges_query, initBindings={"root_graph": self._root_graph_uri}
+            ),
+        )  # ResultRow because the query is of type SELECT
         for inner_edge_row in inner_edges:
             source_label = (
                 inner_edge_row["source_label"]
@@ -103,9 +107,12 @@ def _set_input_edges(self) -> None:
         inputs_subgraph.set("style", "dashed")
         inputs_subgraph.set("label", "Workflow Inputs")
 
-        input_edges = self._rdf_graph.query(
-            get_input_edges_query, initBindings={"root_graph": self._root_graph_uri}
-        )
+        input_edges = cast(
+            Iterator[rdflib.query.ResultRow],
+            self._rdf_graph.query(
+                get_input_edges_query, initBindings={"root_graph": self._root_graph_uri}
+            ),
+        )  # ResultRow because the query is of type SELECT
         for input_row in input_edges:
             n = pydot.Node(
                 "",
@@ -130,9 +137,12 @@ def _set_output_edges(self) -> None:
         outputs_graph.set("style", "dashed")
         outputs_graph.set("label", "Workflow Outputs")
         outputs_graph.set("labelloc", "b")
-        output_edges = self._rdf_graph.query(
-            get_output_edges, initBindings={"root_graph": self._root_graph_uri}
-        )
+        output_edges = cast(
+            Iterator[rdflib.query.ResultRow],
+            self._rdf_graph.query(
+                get_output_edges, initBindings={"root_graph": self._root_graph_uri}
+            ),
+        )  # ResultRow because the query is of type SELECT
         for output_edge_row in output_edges:
             n = pydot.Node(
                 "",
@@ -150,11 +160,14 @@ def _set_output_edges(self) -> None:
     def _get_root_graph_uri(self) -> rdflib.URIRef:
         with open(_get_root_query_path) as f:
             get_root_query = f.read()
-        root = list(
-            self._rdf_graph.query(
-                get_root_query,
-            )
-        )
+        root = cast(
+            List[rdflib.query.ResultRow],
+            list(
+                self._rdf_graph.query(
+                    get_root_query,
+                )
+            ),
+        )  # ResultRow because the query is of type SELECT
         if len(root) != 1:
             raise RuntimeError(
                 "Cannot identify root workflow! Notice that only Workflows can be visualized"