fix: save mongo configuration to values.yaml

fix: change directory for temp files

fix: add test for edited values.yaml file

fix: typos and description of yaml_escape_list function

backend/SC4SNMP_UI_backend/__init__.py

@@ -14,9 +14,16 @@
 mongo_client = MongoClient(MONGO_URI)
 CELERY_BROKER_URL = os.getenv("CELERY_BROKER_URL", "amqp://guest:guest@localhost:5672//")
 REDIS_URL = os.getenv("REDIS_URL")
+VALUES_DIRECTORY = os.getenv("VALUES_DIRECTORY", "")
+KEEP_TEMP_FILES = os.getenv("KEEP_TEMP_FILES", "false")
 
+class NoValuesDirectoryException(Exception):
+    pass
 
 def create_app():
+    if len(VALUES_DIRECTORY) == 0:
+        raise NoValuesDirectoryException
+
     app = Flask(__name__)
 
     app.config.from_mapping(

backend/SC4SNMP_UI_backend/apply_changes/config_to_yaml_utils.py

@@ -16,8 +16,21 @@ def bool_to_str(value):
 
 
 class MongoToYamlDictConversion:
+    """
+    MongoToYamlDictConversion is an abstract class. Implementations of this class converts
+    appropriate mongo collections to dictionaries in such a way, that configurations from those collections can be
+    dumped to yaml file with appropriate formatting.
+    """
     @classmethod
     def yaml_escape_list(cls, *l):
+        """
+        This function is used to parse an example list [yaml_escape_list(el1, el2, el3)] like this:
+        - [el1, el2, el3]
+        and not like this:
+        - el1
+        - el2
+        - el3
+        """
         ret = ruamel.yaml.comments.CommentedSeq(l)
         ret.fa.set_flow_style()
         return ret
@@ -28,6 +41,12 @@ def convert(self, documents: list) -> dict:
 
 class ProfilesToYamlDictConversion(MongoToYamlDictConversion):
     def convert(self, documents: list) -> dict:
+        """
+        ProfilesToYamlDictConversion converts profiles from mongo collection to
+        format that can be dumped to yaml file
+        :param documents: list of profiles from mongo
+        :return: dictionary that can be dumped to yaml
+        """
         result = {}
         for profile in documents:
             profile_name = get_group_or_profile_name_from_backend(profile)
@@ -79,6 +98,12 @@ def convert(self, documents: list) -> dict:
 
 class GroupsToYamlDictConversion(MongoToYamlDictConversion):
     def convert(self, documents: list) -> dict:
+        """
+        GroupsToYamlDictConversion converts groups from mongo collection to
+        format that can be dumped to yaml file
+        :param documents: list of groups from mongo
+        :return: dictionary that can be dumped to yaml
+        """
         result = {}
         for group in documents:
             group_name = get_group_or_profile_name_from_backend(group)
@@ -99,6 +124,12 @@ def convert(self, documents: list) -> dict:
 
 class InventoryToYamlDictConversion(MongoToYamlDictConversion):
     def convert(self, documents: list) -> dict:
+        """
+        InventoryToYamlDictConversion converts inventory from mongo collection to
+        format that can be dumped to yaml file
+        :param documents: inventory from mongo
+        :return: dictionary that can be dumped to yaml
+        """
         inventory_string = "address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete"
         for inv in documents:
             smart_profiles = bool_to_str(inv['smart_profiles'])
@@ -112,6 +143,12 @@ def convert(self, documents: list) -> dict:
 
 
 class TempFileHandling:
+    """
+    After converting configurations from mongo to dictionaries ready to be dumped to yaml file, those dictionaries
+    must be dumped to temporary files. This is because those configurations must be parsed before they are inserted
+    to values.yaml file. TempFileHandling is an abstract class whose implementations parse dictionaries and return
+    ready configuration that can be saved in values.yaml
+    """
     def __init__(self, file_path: str):
         self._file_path = file_path
 
@@ -124,7 +161,7 @@ def _delete_temp(self):
         if os.path.exists(self._file_path):
             os.remove(self._file_path)
         else:
-            current_app.logger.info(f"Pod directory {self._file_path} doesn't exist. File wasn't removed.")
+            current_app.logger.info(f"Directory {self._file_path} doesn't exist inside a Pod. File wasn't removed.")
 
     @abstractmethod
     def parse_dict_to_yaml(self, document: dict, delete_tmp: bool = True):
@@ -136,45 +173,60 @@ def __init__(self, file_path: str):
         super().__init__(file_path)
 
     def parse_dict_to_yaml(self, document: dict, delete_tmp: bool = True):
+        """
+        :param document: dictionary with profiles configuration
+        :param delete_tmp: whether to delete temporary file after parsing
+        :return: parsed configuration ready to be saved to values.yaml
+        """
         self._save_temp(document)
         lines = ""
         with open(self._file_path, "r") as file:
             line = file.readline()
             while line != "":
-                lines += f"{line}"
+                lines += line
                 line = file.readline()
         if delete_tmp:
             self._delete_temp()
         return literal_string(lines)
 
 
-class GroupsTempHandling(TempFileHandling):
+class InventoryTempHandling(TempFileHandling):
     def __init__(self, file_path: str):
         super().__init__(file_path)
 
     def parse_dict_to_yaml(self, document: dict, delete_tmp: bool = True):
+        """
+        :param document: dictionary with inventory configuration
+        :param delete_tmp: whether to delete temporary file after parsing
+        :return: parsed configuration ready to be saved to values.yaml
+        """
         self._save_temp(document)
-        lines = ""
+        yaml = ruamel.yaml.YAML()
         with open(self._file_path, "r") as file:
-            line = file.readline()
-            while line != "":
-                lines += f"{line}"
-                line = file.readline()
+            inventory = yaml.load(file)
+            result = inventory["inventory"]
         if delete_tmp:
             self._delete_temp()
-        return literal_string(lines)
+        return literal_string(result)
 
 
-class InventoryTempHandling(TempFileHandling):
+class GroupsTempHandling(TempFileHandling):
     def __init__(self, file_path: str):
         super().__init__(file_path)
 
     def parse_dict_to_yaml(self, document: dict, delete_tmp: bool = True):
+        """
+        :param document: dictionary with groups configuration
+        :param delete_tmp: whether to delete temporary file after parsing
+        :return: parsed configuration ready to be saved to values.yaml
+        """
         self._save_temp(document)
-        yaml = ruamel.yaml.YAML()
+        lines = ""
         with open(self._file_path, "r") as file:
-            inventory = yaml.load(file)
-            result = inventory["inventory"]
+            line = file.readline()
+            while line != "":
+                lines += line
+                line = file.readline()
         if delete_tmp:
             self._delete_temp()
-        return literal_string(result)
+        return literal_string(lines)

backend/SC4SNMP_UI_backend/apply_changes/handling_chain.py

@@ -8,10 +8,11 @@
 
 
 CHANGES_INTERVAL_SECONDS = 300
-TMP_FILE_PREFIX = "sc4snmp_ui"
+TMP_FILE_PREFIX = "sc4snmp_ui_"
 TMP_DIR = "/tmp"
 VALUES_DIRECTORY = os.getenv("VALUES_DIRECTORY", "")
 VALUES_FILE = os.getenv("VALUES_FILE", "")
+KEEP_TEMP_FILES = os.getenv("KEEP_TEMP_FILES", "false")
 mongo_config_collection = mongo_client.sc4snmp.config_collection
 mongo_groups = mongo_client.sc4snmp.groups_ui
 mongo_inventory = mongo_client.sc4snmp.inventory_ui
@@ -44,28 +45,43 @@ def handle(self, request: dict):
 class SaveConfigToFileHandler(AbstractHandler):
     def handle(self, request: dict):
         """
+        SaveConfigToFileHandler saves current configuration of profiles, groups and inventory from mongo
+        to files on the host machine.
 
-        :yaml_sections = {
-            "<values.yaml.key>": (mongo_collection, MongoToYamlDictConversion, TempFileHandling)
+        :param request: dictionary with at least one key "yaml_sections". Under this key there should be dictionary
+        with the following structure
+        {
+            "key.to.section": (mongo_collection, MongoToYamlDictConversion, TempFileHandling)
         }
+        where:
+            - "key.to.section": a key to section of values.yaml file that should be updated (e.g. "scheduler.profiles")
+            - mongo_collection: mongo collection with configuration of given section
+            - MongoToYamlDictConversion: implementation of this abstract class
+            - TempFileHandling: implementation of this abstract class
         """
-        if len(VALUES_DIRECTORY) == 0:
-            raise ValueError("VALUES_DIRECTORY must be provided.")
 
         yaml = ruamel.yaml.YAML()
         values_file_resolved = True
         values_file_path = os.path.join(VALUES_DIRECTORY, VALUES_FILE)
+
         if len(VALUES_FILE) == 0 or (VALUES_FILE.split(".")[1] != "yaml" and VALUES_FILE.split(".")[1] != "yml") or \
                 not os.path.exists(os.path.join(VALUES_DIRECTORY, VALUES_FILE)):
+            # If VALUES_FILE can't be found or wasn't provided, it won't be updated. In this case separate files
+            # with configuration of specific section will be saved in the hosts machine.
             values_file_resolved = False
         values = {}
         if values_file_resolved:
             with open(values_file_path, "r") as file:
                 values = yaml.load(file)
 
+        if not values_file_resolved or KEEP_TEMP_FILES.lower() in ["t", "true", "y", "yes", "1"]:
+            delete_temp_files = False
+        else:
+            delete_temp_files = True
+
         for key, value in request["yaml_sections"].items():
             tmp_file_name = TMP_FILE_PREFIX + key.replace(".", "_") + ".yaml"
-            directory = VALUES_DIRECTORY if not values_file_resolved else TMP_DIR
+            directory = VALUES_DIRECTORY if not delete_temp_files else TMP_DIR
             tmp_file_path = os.path.join(directory, tmp_file_name)
 
             mongo_collection = value[0]
@@ -74,8 +90,9 @@ def handle(self, request: dict):
 
             documents = list(mongo_collection.find())
             converted = mongo_to_yaml_conversion.convert(documents)
-            parsed_values = tmp_file_handling.parse_dict_to_yaml(converted, values_file_resolved)
+            parsed_values = tmp_file_handling.parse_dict_to_yaml(converted, delete_temp_files)
 
+            # update appropriate section values dictionary
             values_keys = key.split(".")
             sub_dict = values
             for value_index, value_key in enumerate(values_keys):