From b1e1b8b50756d2df94b61d7524078106a5198d30 Mon Sep 17 00:00:00 2001 From: patrykjarzyna Date: Sun, 13 Nov 2022 18:13:08 +0100 Subject: [PATCH 1/5] Add resolver --- .gitignore | 2 ++ README.md | 6 +++- extras/__init__.py | 0 extras/yaml/__init__.py | 0 extras/yaml/argo_templates/__init__.py | 0 pyproject.toml | 3 ++ requirements.txt | 0 requirements_dev.txt | 5 +++ setup.cfg | 45 ++++++++++++++++++++++++++ setup.py | 0 src/__init__.py | 0 src/enums/__init__.py | 0 src/enums/node_status.py | 9 ++++++ src/graph.py | 42 ++++++++++++++++++++++++ src/node.py | 13 ++++++++ tests/__init__.py | 0 tests/test_graph.py | 22 +++++++++++++ 17 files changed, 146 insertions(+), 1 deletion(-) create mode 100644 extras/__init__.py create mode 100644 extras/yaml/__init__.py create mode 100644 extras/yaml/argo_templates/__init__.py create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100644 requirements_dev.txt create mode 100644 setup.cfg create mode 100644 setup.py create mode 100644 src/__init__.py create mode 100644 src/enums/__init__.py create mode 100644 src/enums/node_status.py create mode 100644 src/graph.py create mode 100644 src/node.py create mode 100644 tests/__init__.py create mode 100644 tests/test_graph.py diff --git a/.gitignore b/.gitignore index b6e4761..b97131e 100644 --- a/.gitignore +++ b/.gitignore @@ -127,3 +127,5 @@ dmypy.json # Pyre type checker .pyre/ + +.idea/ diff --git a/README.md b/README.md index d717ad7..f94cf41 100644 --- a/README.md +++ b/README.md @@ -1 +1,5 @@ -# dag-dependency-resolver \ No newline at end of file +# dag-dependency-resolver + +[![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![Imports: isort](https://img.shields.io/badge/%20imports-isort-%231674b1?style=flat&labelColor=ef8336)](https://pycqa.github.io/isort/) +[![Static typing: mypy](https://img.shields.io/badge/%20static-mypy-%3167400?style=flat&labelColor=A09711)](http://mypy-lang.org/) diff --git a/extras/__init__.py b/extras/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/extras/yaml/__init__.py b/extras/yaml/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/extras/yaml/argo_templates/__init__.py b/extras/yaml/argo_templates/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..4e42573 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +line-length = 120 +target-version = ['py38'] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100644 index 0000000..f9c0bc3 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1,5 @@ +pytest==7.1.2 +unittest +black +isort +flake \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..968baba --- /dev/null +++ b/setup.cfg @@ -0,0 +1,45 @@ +[tool:isort] +line_length=120 +multi_line_output=3 +force_grid_wrap=True +indent=4 +use_parentheses=True +include_trailing_comma=True +lines_after_imports=2 +combine_as_imports=True +skip=venv + +[flake8] +ignore = C901, W503 +exclude = + .git, + venv, + .venv, + ./venv, + __pycache__ + +max-line-length = 120 +max-complexity = 10 +show-source = True +statistics = True + + +[mypy] +python_version = 3.8 +show_error_codes = True +show_error_context = True +pretty = True +warn_unused_configs = True +warn_unused_ignores = True +ignore_missing_imports = True +warn_unreachable = True +warn_return_any = True +warn_redundant_casts = True +no_implicit_optional = True +disallow_untyped_decorators = True +check_untyped_defs = True +disallow_incomplete_defs = True +disallow_untyped_defs = True +disallow_untyped_calls = True +disallow_subclassing_any = False +disallow_any_generics = True diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..e69de29 diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/enums/__init__.py b/src/enums/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/enums/node_status.py b/src/enums/node_status.py new file mode 100644 index 0000000..83a6405 --- /dev/null +++ b/src/enums/node_status.py @@ -0,0 +1,9 @@ +from enum import ( + Enum, + auto, +) + + +class NodeStatus(Enum): + VISITED = auto() + NOT_VISITED = auto() diff --git a/src/graph.py b/src/graph.py new file mode 100644 index 0000000..b43d639 --- /dev/null +++ b/src/graph.py @@ -0,0 +1,42 @@ +from dataclasses import ( + dataclass, + field, +) +from typing import ( + Dict, + List, + Text, +) + +from src.enums.node_status import NodeStatus +from src.node import Node + + +@dataclass +class DependencyGraph: + nodes: Dict[Text, Node] = field(default_factory=dict) + + def add_node(self, name: str, parents: List[Text]) -> None: + self.nodes[name] = Node(name=name, parents=parents) + + def get_dependencies_order(self) -> List[str]: + dependencies_order = [] + + for node in self.nodes.values(): + self._solve_deps(node, dependencies_order) + + self._reset_nodes_status() + return dependencies_order + + def _solve_deps(self, node: Node, dependencies_order: List[Text]) -> None: + if node.status != NodeStatus.VISITED: + for parent in node.parents: + self._solve_deps(self.nodes[parent], dependencies_order) + + node.status = NodeStatus.VISITED + + dependencies_order.append(node.name) + + def _reset_nodes_status(self) -> None: + for node in self.nodes.values(): + node.status = NodeStatus.NOT_VISITED diff --git a/src/node.py b/src/node.py new file mode 100644 index 0000000..da7b15c --- /dev/null +++ b/src/node.py @@ -0,0 +1,13 @@ +from dataclasses import ( + dataclass, + field, +) + +from src.enums.node_status import NodeStatus + + +@dataclass +class Node: + name: str + parents: list = field(default_factory=list) + status: NodeStatus = field(default=NodeStatus.NOT_VISITED) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_graph.py b/tests/test_graph.py new file mode 100644 index 0000000..d16d230 --- /dev/null +++ b/tests/test_graph.py @@ -0,0 +1,22 @@ +import unittest + +from src.graph import DependencyGraph +from src.node import Node + + +class TestStringMethods(unittest.TestCase): + def setUp(self): + self.dependency_graph = DependencyGraph() + + def test_add_node(self): + self.dependency_graph.add_node(name="test_node", parents=["a", "b"]) + assert self.dependency_graph.nodes == {"test_node": Node(name="test_node", parents=["a", "b"])} + + def test_get_dependencies_order(self): + self.dependency_graph.add_node(name="a", parents=["b", "c"]) + self.dependency_graph.add_node(name="b", parents=[]) + self.dependency_graph.add_node(name="c", parents=[]) + + dependencies_order = self.dependency_graph.get_dependencies_order() + + assert dependencies_order == ["b", "c", "a"] From 810cf779af5fb00e0737a1ba5d7fa3ce3903011d Mon Sep 17 00:00:00 2001 From: patrykjarzyna Date: Sun, 13 Nov 2022 18:33:35 +0100 Subject: [PATCH 2/5] fix static analysis --- requirements_dev.txt | 3 ++- src/graph.py | 2 +- src/node.py | 5 +++-- tests/test_graph.py | 6 +++--- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index f9c0bc3..ff09512 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -2,4 +2,5 @@ pytest==7.1.2 unittest black isort -flake \ No newline at end of file +flake +mypy diff --git a/src/graph.py b/src/graph.py index b43d639..dca608a 100644 --- a/src/graph.py +++ b/src/graph.py @@ -20,7 +20,7 @@ def add_node(self, name: str, parents: List[Text]) -> None: self.nodes[name] = Node(name=name, parents=parents) def get_dependencies_order(self) -> List[str]: - dependencies_order = [] + dependencies_order: List[str] = [] for node in self.nodes.values(): self._solve_deps(node, dependencies_order) diff --git a/src/node.py b/src/node.py index da7b15c..50da124 100644 --- a/src/node.py +++ b/src/node.py @@ -2,12 +2,13 @@ dataclass, field, ) +from typing import Text, List from src.enums.node_status import NodeStatus @dataclass class Node: - name: str - parents: list = field(default_factory=list) + name: Text + parents: List[str] = field(default_factory=list) status: NodeStatus = field(default=NodeStatus.NOT_VISITED) diff --git a/tests/test_graph.py b/tests/test_graph.py index d16d230..750792b 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -5,14 +5,14 @@ class TestStringMethods(unittest.TestCase): - def setUp(self): + def setUp(self) -> None: self.dependency_graph = DependencyGraph() - def test_add_node(self): + def test_add_node(self) -> None: self.dependency_graph.add_node(name="test_node", parents=["a", "b"]) assert self.dependency_graph.nodes == {"test_node": Node(name="test_node", parents=["a", "b"])} - def test_get_dependencies_order(self): + def test_get_dependencies_order(self) -> None: self.dependency_graph.add_node(name="a", parents=["b", "c"]) self.dependency_graph.add_node(name="b", parents=[]) self.dependency_graph.add_node(name="c", parents=[]) From 5f15c9ca620c777462d67ebd5466441586098eda Mon Sep 17 00:00:00 2001 From: patrykjarzyna Date: Sun, 13 Nov 2022 18:39:38 +0100 Subject: [PATCH 3/5] Fix deps --- requirements_dev.txt | 8 ++++---- src/enums/node_status.py | 2 +- src/graph.py | 2 +- src/node.py | 7 +++++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/requirements_dev.txt b/requirements_dev.txt index ff09512..3789e12 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,6 +1,6 @@ pytest==7.1.2 unittest -black -isort -flake -mypy +black==22.3.0 +isort==5.10.1 +flake8==5.0.4 +mypy==0.990 diff --git a/src/enums/node_status.py b/src/enums/node_status.py index 83a6405..16dd102 100644 --- a/src/enums/node_status.py +++ b/src/enums/node_status.py @@ -6,4 +6,4 @@ class NodeStatus(Enum): VISITED = auto() - NOT_VISITED = auto() + UNVISITED = auto() diff --git a/src/graph.py b/src/graph.py index dca608a..7542358 100644 --- a/src/graph.py +++ b/src/graph.py @@ -39,4 +39,4 @@ def _solve_deps(self, node: Node, dependencies_order: List[Text]) -> None: def _reset_nodes_status(self) -> None: for node in self.nodes.values(): - node.status = NodeStatus.NOT_VISITED + node.status = NodeStatus.UNVISITED diff --git a/src/node.py b/src/node.py index 50da124..39da03c 100644 --- a/src/node.py +++ b/src/node.py @@ -2,7 +2,10 @@ dataclass, field, ) -from typing import Text, List +from typing import ( + List, + Text, +) from src.enums.node_status import NodeStatus @@ -11,4 +14,4 @@ class Node: name: Text parents: List[str] = field(default_factory=list) - status: NodeStatus = field(default=NodeStatus.NOT_VISITED) + status: NodeStatus = field(default=NodeStatus.UNVISITED) From a67766057d2c4e72a532aa81d442c0a1fcf46f5b Mon Sep 17 00:00:00 2001 From: patrykjarzyna Date: Sun, 13 Nov 2022 19:11:23 +0100 Subject: [PATCH 4/5] Add tests --- tests/test_graph.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/test_graph.py b/tests/test_graph.py index 750792b..eac05ed 100644 --- a/tests/test_graph.py +++ b/tests/test_graph.py @@ -9,8 +9,9 @@ def setUp(self) -> None: self.dependency_graph = DependencyGraph() def test_add_node(self) -> None: - self.dependency_graph.add_node(name="test_node", parents=["a", "b"]) - assert self.dependency_graph.nodes == {"test_node": Node(name="test_node", parents=["a", "b"])} + self.dependency_graph.add_node(name="c", parents=["a", "b"]) + + self.assertDictEqual(self.dependency_graph.nodes, {"c": Node(name="c", parents=["a", "b"])}) def test_get_dependencies_order(self) -> None: self.dependency_graph.add_node(name="a", parents=["b", "c"]) @@ -19,4 +20,13 @@ def test_get_dependencies_order(self) -> None: dependencies_order = self.dependency_graph.get_dependencies_order() - assert dependencies_order == ["b", "c", "a"] + self.assertListEqual(dependencies_order, ["b", "c", "a"]) + + def test_get_dependencies_order_disconnected_graph(self) -> None: + self.dependency_graph.add_node(name="a", parents=[]) + self.dependency_graph.add_node(name="b", parents=[]) + self.dependency_graph.add_node(name="c", parents=[]) + + dependencies_order = self.dependency_graph.get_dependencies_order() + + self.assertListEqual(dependencies_order, ["a", "b", "c"]) From 0b7d7c2aef5dfe476cd0dc8fb481bc17b6aa370b Mon Sep 17 00:00:00 2001 From: patrykjarzyna Date: Sun, 13 Nov 2022 19:19:08 +0100 Subject: [PATCH 5/5] Update typing --- src/node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/node.py b/src/node.py index 39da03c..8935de1 100644 --- a/src/node.py +++ b/src/node.py @@ -13,5 +13,5 @@ @dataclass class Node: name: Text - parents: List[str] = field(default_factory=list) + parents: List[Text] = field(default_factory=list) status: NodeStatus = field(default=NodeStatus.UNVISITED)