From 78bb04b43145fc2b1d5b910f638784e4bfbffb3c Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Fri, 7 May 2021 13:21:41 -0700 Subject: [PATCH 01/10] Added a manager for runtime configurations, a yaml file with logging configurations and a globally scoped variable for configurations --- hatchet/__init__.py | 1 + hatchet/util/rcmanager.py | 62 +++++++++++++++++++++++++++++++++++++++ hatchetrc.yaml | 12 ++++++++ 3 files changed, 75 insertions(+) create mode 100644 hatchet/util/rcmanager.py create mode 100644 hatchetrc.yaml diff --git a/hatchet/__init__.py b/hatchet/__init__.py index 9f0a9eb2..d4e6682b 100644 --- a/hatchet/__init__.py +++ b/hatchet/__init__.py @@ -8,3 +8,4 @@ from .graphframe import GraphFrame from .query import QueryMatcher +from .util.rcmanager import RcParams \ No newline at end of file diff --git a/hatchet/util/rcmanager.py b/hatchet/util/rcmanager.py new file mode 100644 index 00000000..346d7ec5 --- /dev/null +++ b/hatchet/util/rcmanager.py @@ -0,0 +1,62 @@ +try: + from collections.abc import MutableMapping +except ImportError: + from collections import MutableMapping +import os.path as path +import yaml + +class RcManager(MutableMapping, dict): + ''' + A runtime configurations class; modeled after the RcParams class in matplotlib. + The benifit of this class over a dictonary is validation of set item and formatting + of output. + ''' + + def __init__(self, *args, **kwargs): + self.update(*args, **kwargs) + + def __setitem__(self, key, val): + ''' + We can put validation of configs here. + ''' + return dict.__setitem__(self, key, val) + + def __getitem__(self, key): + return dict.__getitem__(self, key) + + def __iter__(self): + for kv in sorted(dict.__iter__(self)): + yield kv + + def __str__(self): + return '\n'.join(map('{0[0]}: {0[1]}'.format, sorted(self.items()))) + + +def _resolve_conf_file(): + ''' + Determines which configuration file to load. + Uses the precendence order: + 1. $HOME/.config/hatchet/hatchetrc.yaml + 2. $HATCHET_BASE_DIR/hatchetrc.yaml + ''' + home = path.expanduser("~") + conf_dir = path.join(home, ".config", "hatchet", "hatchetrc.yaml") + if path.exists(conf_dir): + return conf_dir + else: + my_path = path.abspath(path.dirname(__file__)) + rel_path = path.join(my_path, "..", "..", "hatchetrc.yaml") + return rel_path + +def _read_config_from_file(filepath): + configs = {} + with open(filepath, 'r') as f: + configs = yaml.load(f, yaml.FullLoader) + return configs + +filepath = _resolve_conf_file() +configs = _read_config_from_file(filepath) + +# Global instance of configurations +RcParams = RcManager(configs) + diff --git a/hatchetrc.yaml b/hatchetrc.yaml new file mode 100644 index 00000000..71c195a0 --- /dev/null +++ b/hatchetrc.yaml @@ -0,0 +1,12 @@ +############################################## +## ## +## Global configuration file for hatchet. ## +## ## +############################################## + +## +## Logging configuration +## + +logging: True +log-directory: "." \ No newline at end of file From dc4204230c702ab611e1efe8e4f63720e5933966 Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Fri, 7 May 2021 13:33:27 -0700 Subject: [PATCH 02/10] Formatting fixes --- hatchet/__init__.py | 2 +- hatchet/util/rcmanager.py | 40 ++++++++++++++++++++------------------- 2 files changed, 22 insertions(+), 20 deletions(-) diff --git a/hatchet/__init__.py b/hatchet/__init__.py index d4e6682b..510d35f9 100644 --- a/hatchet/__init__.py +++ b/hatchet/__init__.py @@ -8,4 +8,4 @@ from .graphframe import GraphFrame from .query import QueryMatcher -from .util.rcmanager import RcParams \ No newline at end of file +from .util.rcmanager import RcParams diff --git a/hatchet/util/rcmanager.py b/hatchet/util/rcmanager.py index 346d7ec5..ae5d4581 100644 --- a/hatchet/util/rcmanager.py +++ b/hatchet/util/rcmanager.py @@ -5,40 +5,41 @@ import os.path as path import yaml + class RcManager(MutableMapping, dict): - ''' - A runtime configurations class; modeled after the RcParams class in matplotlib. - The benifit of this class over a dictonary is validation of set item and formatting - of output. - ''' + """ + A runtime configurations class; modeled after the RcParams class in matplotlib. + The benifit of this class over a dictonary is validation of set item and formatting + of output. + """ def __init__(self, *args, **kwargs): self.update(*args, **kwargs) def __setitem__(self, key, val): - ''' - We can put validation of configs here. - ''' + """ + We can put validation of configs here. + """ return dict.__setitem__(self, key, val) - + def __getitem__(self, key): return dict.__getitem__(self, key) - + def __iter__(self): for kv in sorted(dict.__iter__(self)): yield kv def __str__(self): - return '\n'.join(map('{0[0]}: {0[1]}'.format, sorted(self.items()))) + return "\n".join(map("{0[0]}: {0[1]}".format, sorted(self.items()))) def _resolve_conf_file(): - ''' - Determines which configuration file to load. - Uses the precendence order: - 1. $HOME/.config/hatchet/hatchetrc.yaml - 2. $HATCHET_BASE_DIR/hatchetrc.yaml - ''' + """ + Determines which configuration file to load. + Uses the precendence order: + 1. $HOME/.config/hatchet/hatchetrc.yaml + 2. $HATCHET_BASE_DIR/hatchetrc.yaml + """ home = path.expanduser("~") conf_dir = path.join(home, ".config", "hatchet", "hatchetrc.yaml") if path.exists(conf_dir): @@ -48,15 +49,16 @@ def _resolve_conf_file(): rel_path = path.join(my_path, "..", "..", "hatchetrc.yaml") return rel_path + def _read_config_from_file(filepath): configs = {} - with open(filepath, 'r') as f: + with open(filepath, "r") as f: configs = yaml.load(f, yaml.FullLoader) return configs + filepath = _resolve_conf_file() configs = _read_config_from_file(filepath) # Global instance of configurations RcParams = RcManager(configs) - From 71d51d186ea34374970793aca644fc590dd0f894 Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Fri, 7 May 2021 14:11:10 -0700 Subject: [PATCH 03/10] Added some additional sample configurations --- hatchetrc.yaml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/hatchetrc.yaml b/hatchetrc.yaml index 71c195a0..7cf05d07 100644 --- a/hatchetrc.yaml +++ b/hatchetrc.yaml @@ -8,5 +8,12 @@ ## Logging configuration ## -logging: True -log-directory: "." \ No newline at end of file +logging: False +log-directory: "/usr/WS2/asde/hatchet-logs/" + +## +## Tree Output Configuration +## + +highlight_name: False +invert_colormap: False \ No newline at end of file From 57f381c656c402a1a97a2f3f6678d350a6f2e494 Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Fri, 7 May 2021 15:34:52 -0700 Subject: [PATCH 04/10] added tests --- hatchet/tests/rctest.py | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 hatchet/tests/rctest.py diff --git a/hatchet/tests/rctest.py b/hatchet/tests/rctest.py new file mode 100644 index 00000000..13afa1a1 --- /dev/null +++ b/hatchet/tests/rctest.py @@ -0,0 +1,35 @@ +from hatchet.util.rcmanager import RcManager, _resolve_conf_file, _read_config_from_file + + +def test_creation(): + file = _resolve_conf_file() + config = _read_config_from_file(file) + RC = RcManager(config) + + assert RC is not None + + +def test_global_binding(): + import hatchet as ht + + assert hasattr(ht, "RcParams") + + +def test_change_value(): + import hatchet as ht + + assert "logging" in ht.RcParams + + ht.RcParams["logging"] = True + assert ht.RcParams["logging"] is True + + from hatchet.util.rcmanager import RcParams + + assert RcParams["logging"] is True + + ht.RcParams["logging"] = False + assert ht.RcParams["logging"] is False + + from hatchet.util.rcmanager import RcParams + + assert RcParams["logging"] is False From 3b9c38b8d09a7abb2c71457bc15c060e985a5a18 Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Fri, 7 May 2021 15:36:58 -0700 Subject: [PATCH 05/10] removed superflous imports from tests --- hatchet/tests/rctest.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/hatchet/tests/rctest.py b/hatchet/tests/rctest.py index 13afa1a1..dde4e781 100644 --- a/hatchet/tests/rctest.py +++ b/hatchet/tests/rctest.py @@ -17,19 +17,14 @@ def test_global_binding(): def test_change_value(): import hatchet as ht + from hatchet.util.rcmanager import RcParams assert "logging" in ht.RcParams ht.RcParams["logging"] = True assert ht.RcParams["logging"] is True - - from hatchet.util.rcmanager import RcParams - assert RcParams["logging"] is True ht.RcParams["logging"] = False assert ht.RcParams["logging"] is False - - from hatchet.util.rcmanager import RcParams - assert RcParams["logging"] is False From 9e1a59fd22b4b84dc9cae4c061967aa67b58f702 Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Mon, 10 May 2021 14:26:41 -0700 Subject: [PATCH 06/10] Added validation to configuration class --- hatchet/tests/rctest.py | 58 ++++++++++++++++++++++++- hatchet/util/rcmanager.py | 90 ++++++++++++++++++++++++++++++++++++++- hatchetrc.yaml | 8 +++- 3 files changed, 152 insertions(+), 4 deletions(-) diff --git a/hatchet/tests/rctest.py b/hatchet/tests/rctest.py index dde4e781..babebf0f 100644 --- a/hatchet/tests/rctest.py +++ b/hatchet/tests/rctest.py @@ -1,4 +1,9 @@ -from hatchet.util.rcmanager import RcManager, _resolve_conf_file, _read_config_from_file +from hatchet.util.rcmanager import ( + RcManager, + ConfigValidator, + _resolve_conf_file, + _read_config_from_file, +) def test_creation(): @@ -28,3 +33,54 @@ def test_change_value(): ht.RcParams["logging"] = False assert ht.RcParams["logging"] is False assert RcParams["logging"] is False + + +def test_validator(): + V = ConfigValidator() + + # Add validators for keys where we can input + # bad values + V._validations["bad_bool"] = V.bool_validator + V._validations["bad_string"] = V.str_validator + V._validations["bad_int"] = V.int_validator + V._validations["bad_float"] = V.float_validator + V._validations["bad_list"] = V.list_validator + V._validations["bad_dict"] = V.dict_validator + + # Test passes if exception is thrown + # Else: test fails + try: + V.validate("bad_bool", "True") + assert False + except TypeError: + assert True + + try: + V.validate("bad_string", True) + assert False + except TypeError: + assert True + + try: + V.validate("bad_int", "123") + assert False + except TypeError: + assert True + + try: + V.validate("bad_float", "1.2387") + assert False + except TypeError: + assert True + + try: + V.validate("bad_list", {}) + assert False + except TypeError: + assert True + + try: + V.validate("bad_dict", []) + assert False + except TypeError: + assert True diff --git a/hatchet/util/rcmanager.py b/hatchet/util/rcmanager.py index ae5d4581..9bb0de67 100644 --- a/hatchet/util/rcmanager.py +++ b/hatchet/util/rcmanager.py @@ -6,6 +6,87 @@ import yaml +class ConfigValidator: + def __init__(self): + self._validations = {} + self._set_validators_to_configs() + + def bool_validator(self, key, value): + if not isinstance(value, bool): + raise TypeError( + 'Error loading configuration: Configuration "{}" must be of type bool'.format( + key + ) + ) + else: + return value + + def str_validator(self, key, value): + if not isinstance(value, str): + raise TypeError( + 'Error loading configuration: Configuration "{}" must be of type string'.format( + key + ) + ) + else: + return value + + def int_validator(self, key, value): + if not isinstance(value, int): + raise TypeError( + 'Error loading configuration: Configuration "{}" must be of type int'.format( + key + ) + ) + else: + return value + + def float_validator(self, key, value): + if not isinstance(value, float): + raise TypeError( + 'Error loading configuration: Configuration "{}" must be of type float'.format( + key + ) + ) + else: + return value + + def list_validator(self, key, value): + if not isinstance(value, list): + raise TypeError( + 'Error loading configuration: Configuration "{}" must be of type list'.format( + key + ) + ) + else: + return value + + def dict_validator(self, key, value): + if not isinstance(value, dict): + raise TypeError( + 'Error loading configuration: Configuration "{}" must be of type dict'.format( + key + ) + ) + else: + return value + + def validate(self, key, value): + try: + return self._validations[key](key, value) + except TypeError as e: + raise e + except KeyError: + pass + + def _set_validators_to_configs(self): + # real configurations + self._validations["logging"] = self.bool_validator + self._validations["log_directory"] = self.str_validator + self._validations["highlight_name"] = self.bool_validator + self._validations["invert_colormap"] = self.bool_validator + + class RcManager(MutableMapping, dict): """ A runtime configurations class; modeled after the RcParams class in matplotlib. @@ -14,13 +95,18 @@ class RcManager(MutableMapping, dict): """ def __init__(self, *args, **kwargs): + self._validator = ConfigValidator() self.update(*args, **kwargs) def __setitem__(self, key, val): """ - We can put validation of configs here. + Function loads valid configurations and prints errors for invalid configs. """ - return dict.__setitem__(self, key, val) + try: + self._validator.validate(key, val) + return dict.__setitem__(self, key, val) + except TypeError as e: + print(e) def __getitem__(self, key): return dict.__getitem__(self, key) diff --git a/hatchetrc.yaml b/hatchetrc.yaml index 7cf05d07..a71688fa 100644 --- a/hatchetrc.yaml +++ b/hatchetrc.yaml @@ -4,12 +4,18 @@ ## ## ############################################## +## If you would like to customize these global configurations +## please copy it to the following directory: +## $HOME/.config/hatchet/ +## +## The configurations in this file will take precedence + ## ## Logging configuration ## logging: False -log-directory: "/usr/WS2/asde/hatchet-logs/" +log_directory: "~" ## ## Tree Output Configuration From 46bcb933fd83393292d6e2e7aebfbd20329af999 Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Tue, 1 Jun 2021 12:49:44 -0700 Subject: [PATCH 07/10] Made changes per Ian's reccomendations --- hatchet/tests/rctest.py | 39 +++++++++++++++++++++++++++++++++++++++ hatchet/util/rcmanager.py | 26 ++++++++++++++++---------- 2 files changed, 55 insertions(+), 10 deletions(-) diff --git a/hatchet/tests/rctest.py b/hatchet/tests/rctest.py index babebf0f..3704b9f4 100644 --- a/hatchet/tests/rctest.py +++ b/hatchet/tests/rctest.py @@ -84,3 +84,42 @@ def test_validator(): assert False except TypeError: assert True + + #Testing valid inputs + # Goes through to true assertion if + # validation works + try: + V.validate("bad_bool", True) + assert True + except TypeError: + assert False + + try: + V.validate("bad_string", "string") + assert True + except TypeError: + assert False + + try: + V.validate("bad_int", 1) + assert True + except TypeError: + assert False + + try: + V.validate("bad_float", 1.2387) + assert True + except TypeError: + assert False + + try: + V.validate("bad_list", []) + assert True + except TypeError: + assert False + + try: + V.validate("bad_dict", {}) + assert True + except TypeError: + assert False \ No newline at end of file diff --git a/hatchet/util/rcmanager.py b/hatchet/util/rcmanager.py index 9bb0de67..d410bd5b 100644 --- a/hatchet/util/rcmanager.py +++ b/hatchet/util/rcmanager.py @@ -76,18 +76,17 @@ def validate(self, key, value): return self._validations[key](key, value) except TypeError as e: raise e - except KeyError: - pass + except KeyError as e: + raise e def _set_validators_to_configs(self): - # real configurations self._validations["logging"] = self.bool_validator self._validations["log_directory"] = self.str_validator self._validations["highlight_name"] = self.bool_validator self._validations["invert_colormap"] = self.bool_validator -class RcManager(MutableMapping, dict): +class RcManager(MutableMapping): """ A runtime configurations class; modeled after the RcParams class in matplotlib. The benifit of this class over a dictonary is validation of set item and formatting @@ -95,8 +94,9 @@ class RcManager(MutableMapping, dict): """ def __init__(self, *args, **kwargs): + self._data = {} self._validator = ConfigValidator() - self.update(*args, **kwargs) + self._data.update(*args, **kwargs) def __setitem__(self, key, val): """ @@ -104,19 +104,25 @@ def __setitem__(self, key, val): """ try: self._validator.validate(key, val) - return dict.__setitem__(self, key, val) + return self._data.__setitem__(key, val) except TypeError as e: - print(e) + raise e def __getitem__(self, key): - return dict.__getitem__(self, key) + return self._data[key] def __iter__(self): - for kv in sorted(dict.__iter__(self)): + for kv in sorted(self._data.__iter__(self)): yield kv def __str__(self): return "\n".join(map("{0[0]}: {0[1]}".format, sorted(self.items()))) + + def __len__(self): + return len(self._data) + + def __delitem__(self, item): + del self._data[item] def _resolve_conf_file(): @@ -131,7 +137,7 @@ def _resolve_conf_file(): if path.exists(conf_dir): return conf_dir else: - my_path = path.abspath(path.dirname(__file__)) + my_path = path.abspath(path.dirname(path.abspath(__file__))) rel_path = path.join(my_path, "..", "..", "hatchetrc.yaml") return rel_path From 51d223e71e39c6289cfdab8d6f1825999a0b061c Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Tue, 1 Jun 2021 12:52:27 -0700 Subject: [PATCH 08/10] FOrmatting fixes --- hatchet/tests/rctest.py | 4 ++-- hatchet/util/rcmanager.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/hatchet/tests/rctest.py b/hatchet/tests/rctest.py index 3704b9f4..01f92b78 100644 --- a/hatchet/tests/rctest.py +++ b/hatchet/tests/rctest.py @@ -85,7 +85,7 @@ def test_validator(): except TypeError: assert True - #Testing valid inputs + # Testing valid inputs # Goes through to true assertion if # validation works try: @@ -122,4 +122,4 @@ def test_validator(): V.validate("bad_dict", {}) assert True except TypeError: - assert False \ No newline at end of file + assert False diff --git a/hatchet/util/rcmanager.py b/hatchet/util/rcmanager.py index d410bd5b..79426784 100644 --- a/hatchet/util/rcmanager.py +++ b/hatchet/util/rcmanager.py @@ -117,7 +117,7 @@ def __iter__(self): def __str__(self): return "\n".join(map("{0[0]}: {0[1]}".format, sorted(self.items()))) - + def __len__(self): return len(self._data) From 44d492fe2f5e2ab1dd231eb9190db3820126a0d3 Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Mon, 7 Jun 2021 14:13:10 -0700 Subject: [PATCH 09/10] Made changes to expected root directory where rc file can be found and updated comments. --- hatchet/util/rcmanager.py | 13 +++++++++---- hatchetrc.yaml | 11 ++++++++--- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/hatchet/util/rcmanager.py b/hatchet/util/rcmanager.py index 79426784..01ae5581 100644 --- a/hatchet/util/rcmanager.py +++ b/hatchet/util/rcmanager.py @@ -1,3 +1,8 @@ +# Copyright 2021 The University of Arizona and other Hatchet Project +# Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: MIT + try: from collections.abc import MutableMapping except ImportError: @@ -129,16 +134,16 @@ def _resolve_conf_file(): """ Determines which configuration file to load. Uses the precendence order: - 1. $HOME/.config/hatchet/hatchetrc.yaml + 1. $HOME/.hatchet/hatchetrc.yaml 2. $HATCHET_BASE_DIR/hatchetrc.yaml """ home = path.expanduser("~") - conf_dir = path.join(home, ".config", "hatchet", "hatchetrc.yaml") + conf_dir = path.join(home, ".hatchet", "hatchetrc.yaml") if path.exists(conf_dir): return conf_dir else: - my_path = path.abspath(path.dirname(path.abspath(__file__))) - rel_path = path.join(my_path, "..", "..", "hatchetrc.yaml") + hatchet_path = path.abspath(path.dirname(path.abspath(__file__))) + rel_path = path.join(hatchet_path, "..", "..", "hatchetrc.yaml") return rel_path diff --git a/hatchetrc.yaml b/hatchetrc.yaml index a71688fa..e4676edd 100644 --- a/hatchetrc.yaml +++ b/hatchetrc.yaml @@ -5,10 +5,15 @@ ############################################## ## If you would like to customize these global configurations -## please copy it to the following directory: -## $HOME/.config/hatchet/ +## please copy it to the following directory and edit there: +## $HOME/.hatchet/ ## -## The configurations in this file will take precedence +## This configurations file is considered in the following order of precedence: +## 1. $HOME/.hatchet/hatchetrc.yaml (optional) +## 2. $HATCHET_BASE_DIR/hatchetrc.yaml +## +## If no file can be found at $HOME/.hatchet then configurations are drawn from +## the hatchetrc.yaml file at the root of hatchet's installation location ## ## Logging configuration From 4b1f8f3b2b808613f91ca6e4a3b539e5c15db9ad Mon Sep 17 00:00:00 2001 From: Connor Scully-Allison Date: Mon, 7 Jun 2021 14:16:48 -0700 Subject: [PATCH 10/10] Removed unecessary try catch per Ian's suggestion. --- hatchet/util/rcmanager.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/hatchet/util/rcmanager.py b/hatchet/util/rcmanager.py index 01ae5581..6f0bf547 100644 --- a/hatchet/util/rcmanager.py +++ b/hatchet/util/rcmanager.py @@ -77,12 +77,7 @@ def dict_validator(self, key, value): return value def validate(self, key, value): - try: - return self._validations[key](key, value) - except TypeError as e: - raise e - except KeyError as e: - raise e + return self._validations[key](key, value) def _set_validators_to_configs(self): self._validations["logging"] = self.bool_validator