From 8a6cb0134b08256e4bd81b972e5941ed894ad60b Mon Sep 17 00:00:00 2001 From: Wojciech Zyla Date: Wed, 19 Jul 2023 15:07:10 +0200 Subject: [PATCH] fix: add warnings when deleting groups or profiles which are configured in the inventory. No frequency configuration for walk profiles. Default number of presented items changed to 20. fix: change message while deleting profiles. Change what is displayed in Frequency column for walk profiles fix: update tests fix: remove unnecessary print fix: improve logging in handle_changes.py, update config_collection in job insted of in CheckIfPreviousJobFailed fix: fix error in celery job fix: update request message in apply_changes fix: typo in return message fix: fix problem with editing walk profile fix: refactor files and write config from mongo to yaml files on host machine fix: change message in error thrown by SaveConfigToFileHandler --- backend/Dockerfile | 2 + .../apply_changes/apply_changes.py | 69 ++++++ .../apply_changes/config_to_yaml_utils.py | 180 ++++++++++++++++ .../apply_changes/handle_changes.py | 197 ------------------ .../apply_changes/handling_chain.py | 152 ++++++++++++++ .../apply_changes/routes.py | 15 +- .../SC4SNMP_UI_backend/apply_changes/tasks.py | 55 +++++ ...nversions.py => backend_ui_conversions.py} | 94 +++++---- .../common/{helpers.py => inventory_utils.py} | 3 +- backend/SC4SNMP_UI_backend/groups/routes.py | 33 +-- .../SC4SNMP_UI_backend/inventory/routes.py | 4 +- backend/SC4SNMP_UI_backend/profiles/routes.py | 23 +- backend/requirements.txt | 3 +- backend/tests/common/test_conversions.py | 27 ++- .../get_endpoints/test_get_endpoints.py | 21 +- .../post_endpoints/test_post_groups.py | 27 ++- .../post_endpoints/test_post_profiles.py | 3 +- .../splunk-app/appserver/templates/demo.html | 2 +- .../manager/demo/standalone/index.html | 2 +- .../manager/src/components/DeleteModal.jsx | 5 + .../src/components/groups/GroupsList.jsx | 19 +- .../components/inventory/InventoryList.jsx | 7 +- .../components/profiles/AddProfileModal.jsx | 5 +- .../src/components/profiles/ProfilesList.jsx | 18 +- .../manager/src/store/group-contxt.jsx | 3 + .../manager/src/store/profile-contxt.jsx | 5 +- 26 files changed, 658 insertions(+), 316 deletions(-) create mode 100644 backend/SC4SNMP_UI_backend/apply_changes/apply_changes.py create mode 100644 backend/SC4SNMP_UI_backend/apply_changes/config_to_yaml_utils.py delete mode 100644 backend/SC4SNMP_UI_backend/apply_changes/handle_changes.py create mode 100644 backend/SC4SNMP_UI_backend/apply_changes/handling_chain.py create mode 100644 backend/SC4SNMP_UI_backend/apply_changes/tasks.py rename backend/SC4SNMP_UI_backend/common/{conversions.py => backend_ui_conversions.py} (82%) rename backend/SC4SNMP_UI_backend/common/{helpers.py => inventory_utils.py} (99%) diff --git a/backend/Dockerfile b/backend/Dockerfile index 68e3a2f..d1d734c 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -13,5 +13,7 @@ RUN chmod +x /flask_start.sh COPY ./celery_start.sh /celery_start.sh RUN chmod +x /celery_start.sh +USER 10000:10000 + EXPOSE 5000 CMD ["gunicorn", "-b", ":5000", "app:flask_app", "--log-level", "INFO"] \ No newline at end of file diff --git a/backend/SC4SNMP_UI_backend/apply_changes/apply_changes.py b/backend/SC4SNMP_UI_backend/apply_changes/apply_changes.py new file mode 100644 index 0000000..b220f9d --- /dev/null +++ b/backend/SC4SNMP_UI_backend/apply_changes/apply_changes.py @@ -0,0 +1,69 @@ +from threading import Lock +import os +from SC4SNMP_UI_backend import mongo_client +from SC4SNMP_UI_backend.apply_changes.handling_chain import CheckJobHandler, ScheduleHandler, SaveConfigToFileHandler +from SC4SNMP_UI_backend.apply_changes.config_to_yaml_utils import ProfilesToYamlDictConversion, ProfilesTempHandling, \ + GroupsToYamlDictConversion, GroupsTempHandling, InventoryToYamlDictConversion, InventoryTempHandling + + +MONGO_URI = os.getenv("MONGO_URI") +JOB_CREATION_RETRIES = int(os.getenv("JOB_CREATION_RETRIES", 10)) +mongo_config_collection = mongo_client.sc4snmp.config_collection +mongo_groups = mongo_client.sc4snmp.groups_ui +mongo_inventory = mongo_client.sc4snmp.inventory_ui +mongo_profiles = mongo_client.sc4snmp.profiles_ui + + + +class SingletonMeta(type): + _instances = {} + _lock: Lock = Lock() + + def __call__(cls, *args, **kwargs): + with cls._lock: + if cls not in cls._instances: + instance = super().__call__(*args, **kwargs) + cls._instances[cls] = instance + return cls._instances[cls] + +class ApplyChanges(metaclass=SingletonMeta): + def __init__(self) -> None: + """ + ApplyChanges is a singleton responsible for creating mongo record with a current state of kubernetes job. + Structure of the record: + { + "previous_job_start_time": datetime.datetime or None if job hasn't been scheduled yet, + "currently_scheduled": bool + } + """ + self.__handling_chain = SaveConfigToFileHandler() + check_job_handler = CheckJobHandler() + schedule_handler = ScheduleHandler() + self.__handling_chain.set_next(check_job_handler).set_next(schedule_handler) + mongo_config_collection.update_one( + { + "previous_job_start_time": {"$exists": True}, + "currently_scheduled": {"$exists": True}} + ,{ + "$set":{ + "previous_job_start_time": None, + "currently_scheduled": False + } + }, + upsert=True + ) + + + def apply_changes(self): + """ + Run chain of actions to schedule new kubernetes job. + """ + yaml_sections = { + "scheduler.groups": (mongo_groups, GroupsToYamlDictConversion, GroupsTempHandling), + "scheduler.profiles": (mongo_profiles, ProfilesToYamlDictConversion, ProfilesTempHandling), + "poller.inventory": (mongo_inventory, InventoryToYamlDictConversion, InventoryTempHandling) + } + return self.__handling_chain.handle({ + "yaml_sections": yaml_sections + }) + diff --git a/backend/SC4SNMP_UI_backend/apply_changes/config_to_yaml_utils.py b/backend/SC4SNMP_UI_backend/apply_changes/config_to_yaml_utils.py new file mode 100644 index 0000000..7c188ee --- /dev/null +++ b/backend/SC4SNMP_UI_backend/apply_changes/config_to_yaml_utils.py @@ -0,0 +1,180 @@ +from abc import abstractmethod +import ruamel +from ruamel.yaml.scalarstring import SingleQuotedScalarString as single_quote +from ruamel.yaml.scalarstring import DoubleQuotedScalarString as double_quote +from SC4SNMP_UI_backend.common.backend_ui_conversions import get_group_or_profile_name_from_backend +from ruamel.yaml.scalarstring import LiteralScalarString as literal_string +import os +from flask import current_app + + +def bool_to_str(value): + if value: + return "t" + else: + return "f" + + +class MongoToYamlDictConversion: + @classmethod + def yaml_escape_list(cls, *l): + ret = ruamel.yaml.comments.CommentedSeq(l) + ret.fa.set_flow_style() + return ret + @abstractmethod + def convert(self, documents: list) -> dict: + pass + + +class ProfilesToYamlDictConversion(MongoToYamlDictConversion): + def convert(self, documents: list) -> dict: + result = {} + for profile in documents: + profile_name = get_group_or_profile_name_from_backend(profile) + prof = profile[profile_name] + var_binds = [] + condition = None + conditions = None + is_walk_profile = False + + for var_bind in prof["varBinds"]: + var_binds.append(self.yaml_escape_list(*[single_quote(vb) for vb in var_bind])) + + if "condition" in prof: + backend_condition = prof["condition"] + condition_type = backend_condition["type"] + is_walk_profile = True if backend_condition["type"] == "walk" else False + condition = { + "type": condition_type + } + if condition_type == "field": + condition["field"] = backend_condition["field"] + condition["patterns"] = [single_quote(pattern) for pattern in backend_condition["patterns"]] + + if "conditions" in prof: + backend_conditions = prof["conditions"] + conditions = [] + for cond in backend_conditions: + if cond["operation"] == "in": + value = [double_quote(v) if type(v) == str else v for v in cond["value"]] + else: + value = double_quote(cond["value"]) if type(cond["value"]) == str else cond["value"] + conditions.append({ + "field": cond["field"], + "operation": double_quote(cond["operation"]), + "value": value + }) + + result[profile_name] = {} + if not is_walk_profile: + result[profile_name]["frequency"] = prof['frequency'] + if condition is not None: + result[profile_name]["condition"] = condition + if conditions is not None: + result[profile_name]["conditions"] = conditions + result[profile_name]["varBinds"] = var_binds + + return result + + +class GroupsToYamlDictConversion(MongoToYamlDictConversion): + def convert(self, documents: list) -> dict: + result = {} + for group in documents: + group_name = get_group_or_profile_name_from_backend(group) + gr = group[group_name] + hosts = [] + for host in gr: + host_config = host + if "community" in host: + host_config["community"] = single_quote(host["community"]) + if "secret" in host: + host_config["secret"] = single_quote(host["secret"]) + if "version" in host: + host_config["version"] = single_quote(host["version"]) + hosts.append(host_config) + result[group_name] = hosts + return result + + +class InventoryToYamlDictConversion(MongoToYamlDictConversion): + def convert(self, documents: list) -> dict: + inventory_string = "address,port,version,community,secret,security_engine,walk_interval,profiles,smart_profiles,delete" + for inv in documents: + smart_profiles = bool_to_str(inv['smart_profiles']) + inv_delete = bool_to_str(inv['delete']) + inventory_string += f"\n{inv['address']},{inv['port']},{inv['version']},{inv['community']}," \ + f"{inv['secret']},{inv['security_engine']},{inv['walk_interval']},{inv['profiles']}," \ + f"{smart_profiles},{inv_delete}" + return { + "inventory": literal_string(inventory_string) + } + + +class TempFileHandling: + def __init__(self, file_path: str): + self._file_path = file_path + + def _save_temp(self, content): + yaml = ruamel.yaml.YAML() + with open(self._file_path, "w") as file: + yaml.dump(content, file) + + def _delete_temp(self): + if os.path.exists(self._file_path): + os.remove(self._file_path) + else: + current_app.logger.info(f"Pod directory {self._file_path} doesn't exist. File wasn't removed.") + + @abstractmethod + def parse_dict_to_yaml(self, document: dict, delete_tmp: bool = True): + pass + + +class ProfilesTempHandling(TempFileHandling): + def __init__(self, file_path: str): + super().__init__(file_path) + + def parse_dict_to_yaml(self, document: dict, delete_tmp: bool = True): + self._save_temp(document) + lines = "" + with open(self._file_path, "r") as file: + line = file.readline() + while line != "": + lines += f"{line}" + line = file.readline() + if delete_tmp: + self._delete_temp() + return literal_string(lines) + + +class GroupsTempHandling(TempFileHandling): + def __init__(self, file_path: str): + super().__init__(file_path) + + def parse_dict_to_yaml(self, document: dict, delete_tmp: bool = True): + self._save_temp(document) + lines = "" + with open(self._file_path, "r") as file: + line = file.readline() + while line != "": + lines += f"{line}" + line = file.readline() + if delete_tmp: + self._delete_temp() + return literal_string(lines) + + +class InventoryTempHandling(TempFileHandling): + def __init__(self, file_path: str): + super().__init__(file_path) + + def parse_dict_to_yaml(self, document: dict, delete_tmp: bool = True): + self._save_temp(document) + yaml = ruamel.yaml.YAML() + with open(self._file_path, "r") as file: + inventory = yaml.load(file) + result = inventory["inventory"] + if delete_tmp: + self._delete_temp() + return literal_string(result) \ No newline at end of file diff --git a/backend/SC4SNMP_UI_backend/apply_changes/handle_changes.py b/backend/SC4SNMP_UI_backend/apply_changes/handle_changes.py deleted file mode 100644 index 0269e72..0000000 --- a/backend/SC4SNMP_UI_backend/apply_changes/handle_changes.py +++ /dev/null @@ -1,197 +0,0 @@ -import time -from abc import abstractmethod, ABC -from celery import shared_task -from threading import Lock -import datetime -import os -from kubernetes import client, config -import yaml -from kubernetes.client import ApiException -from SC4SNMP_UI_backend import mongo_client -from SC4SNMP_UI_backend.apply_changes.kubernetes_job import create_job_object, create_job -from pymongo import MongoClient -from celery.utils.log import get_task_logger - -CHANGES_INTERVAL_SECONDS = 300 -MONGO_URI = os.getenv("MONGO_URI") -JOB_CONFIG_PATH = os.getenv("JOB_CONFIG_PATH", "/config/job_config.yaml") -JOB_NAMESPACE = os.getenv("JOB_NAMESPACE", "sc4snmp") -JOB_CREATION_RETRIES = int(os.getenv("JOB_CREATION_RETRIES", 10)) -mongo_config_collection = mongo_client.sc4snmp.config_collection -logger = get_task_logger(__name__) - -class Handler(ABC): - @abstractmethod - def set_next(self, handler): - pass - - @abstractmethod - def handle(self, request): - pass - - -class AbstractHandler(Handler): - _next_handler: Handler = None - def set_next(self, handler: Handler) -> Handler: - self._next_handler = handler - return handler - - @abstractmethod - def handle(self, request: dict): - if self._next_handler: - return self._next_handler.handle(request) - return None - - -class CheckJobHandler(AbstractHandler): - def handle(self, request: dict=None): - """ - CheckJobHandler checks whether a new kubernetes job with updated sc4snmp configuration can be run immediately - or should it be scheduled for the future. - - :return: pass dictionary with job_delay in seconds to the next handler - """ - record = list(mongo_config_collection.find())[0] - last_update = record["previous_job_start_time"] - if last_update is None: - # If it's the first time that the job is run (record in mongo_config_collection has been created - # in ApplyChanges class and last_update attribute is None) then job delay should be equal to - # CHANGES_INTERVAL_SECONDS. Update the mongo record with job state accordingly. - job_delay = CHANGES_INTERVAL_SECONDS - mongo_config_collection.update_one({"_id": record["_id"]}, - {"$set": {"previous_job_start_time": datetime.datetime.utcnow()}}) - # time from the last update - time_difference = 0 - else: - # Check how many seconds have elapsed since the last time that the job was run. If the time difference - # is greater than CHANGES_INTERVAL_SECONDS then job can be run immediately. Otherwise, calculate how - # many seconds are left until minimum time difference between updates (CHANGES_INTERVAL_SECONDS). - current_time = datetime.datetime.utcnow() - delta = current_time - last_update - time_difference = delta.total_seconds() - if time_difference > CHANGES_INTERVAL_SECONDS: - job_delay = 1 - else: - job_delay = int(CHANGES_INTERVAL_SECONDS-time_difference) - - result = { - "job_delay": job_delay, - "time_from_last_update": time_difference - } - - return super().handle(result) - -class CheckIfPreviousJobFailed(AbstractHandler): - def handle(self, request: dict): - """ - If previously scheduled task had failed to create the kubernetes job, then currently_scheduled parameter in mongo - would still be set to True. In this scenario the new job will never be scheduled. CheckIfPreviousJobFailed checks - whether this situation happened and if so, updates currently_scheduled to False. - :param request: - :return: - """ - record = list(mongo_config_collection.find())[0] - time_from_last_update = request["time_from_last_update"] - if time_from_last_update > CHANGES_INTERVAL_SECONDS+10*JOB_CREATION_RETRIES and record["currently_scheduled"]: - # if currently_scheduled is set to True and time_from_last_update is greater than CHANGES_INTERVAL_SECONDS - # plus JOB_CREATION_RETRIES times 10s of wait time between retries, then previous task failed to create the job. - mongo_config_collection.update_one({"_id": record["_id"]}, - {"$set": {"currently_scheduled": False}}) - return super().handle(request) - - -class ScheduleHandler(AbstractHandler): - def handle(self, request: dict): - """ - ScheduleHandler schedules the kubernetes job with updated sc4snmp configuration - """ - record = list(mongo_config_collection.find())[0] - if not record["currently_scheduled"]: - # If the task isn't currently scheduled, schedule it and update its state in mongo. - mongo_config_collection.update_one({"_id": record["_id"]}, - {"$set": {"currently_scheduled": True}}) - run_job.apply_async(countdown=request["job_delay"], queue='apply_changes') - return request["job_delay"] - - -class SingletonMeta(type): - _instances = {} - _lock: Lock = Lock() - - def __call__(cls, *args, **kwargs): - with cls._lock: - if cls not in cls._instances: - instance = super().__call__(*args, **kwargs) - cls._instances[cls] = instance - return cls._instances[cls] - -class ApplyChanges(metaclass=SingletonMeta): - def __init__(self) -> None: - """ - ApplyChanges is a singleton responsible for creating mongo record with a current state of kubernetes job. - Structure of the record: - { - "previous_job_start_time": datetime.datetime or None if job hasn't been scheduled yet, - "currently_scheduled": bool - } - """ - self.__handling_chain = CheckJobHandler() - previous_job_failure = CheckIfPreviousJobFailed() - schedule_handler = ScheduleHandler() - self.__handling_chain.set_next(previous_job_failure).set_next(schedule_handler) - mongo_config_collection.update_one( - { - "previous_job_start_time": {"$exists": True}, - "currently_scheduled": {"$exists": True}} - ,{ - "$set":{ - "previous_job_start_time": None, - "currently_scheduled": False - } - }, - upsert=True - ) - - - def apply_changes(self): - """ - Run chain of actions to schedule new kubernetes job. - """ - job_delay = self.__handling_chain.handle() - return job_delay - -@shared_task() -def run_job(): - job = None - batch_v1 = None - with open(JOB_CONFIG_PATH, encoding="utf-8") as file: - config_file = yaml.safe_load(file) - if config_file["apiVersion"] != "batch/v1": - raise ValueError("api version is different from batch/v1") - config.load_incluster_config() - batch_v1 = client.BatchV1Api() - job = create_job_object(config_file) - - with MongoClient(MONGO_URI) as connection: - try_creating = True - iteration = 0 - while try_creating and iteration < JOB_CREATION_RETRIES: - # Try creating a new job. If the previous job is still present in the namespace, - # ApiException will we be raised. In that happens wait for 10 seconds and try creating the job again - try: - create_job(batch_v1, job, JOB_NAMESPACE) - try_creating = False - try: - record = list(connection.sc4snmp.config_collection.find())[0] - connection.sc4snmp.config_collection.update_one({"_id": record["_id"]}, - {"$set": {"previous_job_start_time": datetime.datetime.utcnow(), - "currently_scheduled": False}}) - except Exception as e: - logger.info(f"Error occurred while updating job state after job creation: {str(e)}") - except ApiException: - iteration += 1 - if iteration == JOB_CREATION_RETRIES: - logger.info(f"Kubernetes job was not created. Max retries ({JOB_CREATION_RETRIES}) exceeded.") - else: - time.sleep(10) - diff --git a/backend/SC4SNMP_UI_backend/apply_changes/handling_chain.py b/backend/SC4SNMP_UI_backend/apply_changes/handling_chain.py new file mode 100644 index 0000000..7093d47 --- /dev/null +++ b/backend/SC4SNMP_UI_backend/apply_changes/handling_chain.py @@ -0,0 +1,152 @@ +from abc import abstractmethod, ABC +import ruamel.yaml +from flask import current_app +from SC4SNMP_UI_backend import mongo_client +from SC4SNMP_UI_backend.apply_changes.tasks import run_job +import datetime +import os + + +CHANGES_INTERVAL_SECONDS = 300 +TMP_FILE_PREFIX = "sc4snmp_ui" +TMP_DIR = "/tmp" +VALUES_DIRECTORY = os.getenv("VALUES_DIRECTORY", "") +VALUES_FILE = os.getenv("VALUES_FILE", "") +mongo_config_collection = mongo_client.sc4snmp.config_collection +mongo_groups = mongo_client.sc4snmp.groups_ui +mongo_inventory = mongo_client.sc4snmp.inventory_ui +mongo_profiles = mongo_client.sc4snmp.profiles_ui + +class Handler(ABC): + @abstractmethod + def set_next(self, handler): + pass + + @abstractmethod + def handle(self, request): + pass + + +class AbstractHandler(Handler): + _next_handler: Handler = None + + def set_next(self, handler: Handler) -> Handler: + self._next_handler = handler + return handler + + @abstractmethod + def handle(self, request: dict): + if self._next_handler: + return self._next_handler.handle(request) + return None + + +class SaveConfigToFileHandler(AbstractHandler): + def handle(self, request: dict): + """ + + :yaml_sections = { + "": (mongo_collection, MongoToYamlDictConversion, TempFileHandling) + } + """ + if len(VALUES_DIRECTORY) == 0: + raise ValueError("VALUES_DIRECTORY must be provided.") + + yaml = ruamel.yaml.YAML() + values_file_resolved = True + values_file_path = os.path.join(VALUES_DIRECTORY, VALUES_FILE) + if len(VALUES_FILE) == 0 or (VALUES_FILE.split(".")[1] != "yaml" and VALUES_FILE.split(".")[1] != "yml") or \ + not os.path.exists(os.path.join(VALUES_DIRECTORY, VALUES_FILE)): + values_file_resolved = False + values = {} + if values_file_resolved: + with open(values_file_path, "r") as file: + values = yaml.load(file) + + for key, value in request["yaml_sections"].items(): + tmp_file_name = TMP_FILE_PREFIX + key.replace(".", "_") + ".yaml" + directory = VALUES_DIRECTORY if not values_file_resolved else TMP_DIR + tmp_file_path = os.path.join(directory, tmp_file_name) + + mongo_collection = value[0] + mongo_to_yaml_conversion = value[1]() + tmp_file_handling = value[2](tmp_file_path) + + documents = list(mongo_collection.find()) + converted = mongo_to_yaml_conversion.convert(documents) + parsed_values = tmp_file_handling.parse_dict_to_yaml(converted, values_file_resolved) + + values_keys = key.split(".") + sub_dict = values + for value_index, value_key in enumerate(values_keys): + if value_index == len(values_keys)-1: + sub_dict[value_key] = parsed_values + else: + sub_dict = sub_dict.get(value_key, {}) + + if values_file_resolved: + with open(values_file_path, "w") as file: + yaml.dump(values, file) + + next_chain_request = {} + if "next" in request: + next_chain_request = request["next"] + return super().handle(next_chain_request) + + +class CheckJobHandler(AbstractHandler): + def handle(self, request: dict = None): + """ + CheckJobHandler checks whether a new kubernetes job with updated sc4snmp configuration can be run immediately + or should it be scheduled for the future. + + :return: pass dictionary with job_delay in seconds to the next handler + """ + record = list(mongo_config_collection.find())[0] + last_update = record["previous_job_start_time"] + if last_update is None: + # If it's the first time that the job is run (record in mongo_config_collection has been created + # in ApplyChanges class and last_update attribute is None) then job delay should be equal to + # CHANGES_INTERVAL_SECONDS. Update the mongo record with job state accordingly. + job_delay = CHANGES_INTERVAL_SECONDS + mongo_config_collection.update_one({"_id": record["_id"]}, + {"$set": {"previous_job_start_time": datetime.datetime.utcnow()}}) + # time from the last update + time_difference = 0 + else: + # Check how many seconds have elapsed since the last time that the job was run. If the time difference + # is greater than CHANGES_INTERVAL_SECONDS then job can be run immediately. Otherwise, calculate how + # many seconds are left until minimum time difference between updates (CHANGES_INTERVAL_SECONDS). + current_time = datetime.datetime.utcnow() + delta = current_time - last_update + time_difference = delta.total_seconds() + if time_difference > CHANGES_INTERVAL_SECONDS: + job_delay = 1 + else: + job_delay = int(CHANGES_INTERVAL_SECONDS - time_difference) + + result = { + "job_delay": job_delay, + "time_from_last_update": time_difference + } + + current_app.logger.info(f"CheckJobHandler: {result}") + return super().handle(result) + + +class ScheduleHandler(AbstractHandler): + def handle(self, request: dict): + """ + ScheduleHandler schedules the kubernetes job with updated sc4snmp configuration + """ + record = list(mongo_config_collection.find())[0] + if not record["currently_scheduled"]: + # If the task isn't currently scheduled, schedule it and update its state in mongo. + mongo_config_collection.update_one({"_id": record["_id"]}, + {"$set": {"currently_scheduled": True}}) + run_job.apply_async(countdown=request["job_delay"], queue='apply_changes') + current_app.logger.info( + f"ScheduleHandler: scheduling new task with the delay of {request['job_delay']} seconds.") + else: + current_app.logger.info("ScheduleHandler: new job wasn't scheduled.") + return request["job_delay"], record["currently_scheduled"] \ No newline at end of file diff --git a/backend/SC4SNMP_UI_backend/apply_changes/routes.py b/backend/SC4SNMP_UI_backend/apply_changes/routes.py index eea10d2..1fdf712 100644 --- a/backend/SC4SNMP_UI_backend/apply_changes/routes.py +++ b/backend/SC4SNMP_UI_backend/apply_changes/routes.py @@ -1,13 +1,22 @@ from flask import Blueprint, jsonify from flask_cors import cross_origin -from SC4SNMP_UI_backend.apply_changes.handle_changes import ApplyChanges +from SC4SNMP_UI_backend.apply_changes.apply_changes import ApplyChanges +import os apply_changes_blueprint = Blueprint('common_blueprint', __name__) +JOB_CREATION_RETRIES = int(os.getenv("JOB_CREATION_RETRIES", 10)) @apply_changes_blueprint.route("/apply-changes", methods=['POST']) @cross_origin() def apply_changes(): changes = ApplyChanges() - job_delay = changes.apply_changes() - result = jsonify({"message": f"Configuration will be updated in approximately {job_delay} seconds"}) + job_delay, currently_scheduled = changes.apply_changes() + if job_delay <= 1 and currently_scheduled: + message = "There might be previous kubernetes job still present in the namespace. Configuration update will be" \ + f"retried {JOB_CREATION_RETRIES} times. If your configuration won't be updated in a few minutes, make sure that " \ + f"snmp-splunk-connect-for-snmp-inventory job isn't present in your kubernetes deployment namespace and " \ + f"click 'Apply changes' button once again." + else: + message = f"Configuration will be updated in approximately {job_delay} seconds." + result = jsonify({"message": message}) return result, 200 \ No newline at end of file diff --git a/backend/SC4SNMP_UI_backend/apply_changes/tasks.py b/backend/SC4SNMP_UI_backend/apply_changes/tasks.py new file mode 100644 index 0000000..2e5bfed --- /dev/null +++ b/backend/SC4SNMP_UI_backend/apply_changes/tasks.py @@ -0,0 +1,55 @@ +import time +from celery import shared_task +import datetime +from kubernetes import client, config +import yaml +from kubernetes.client import ApiException +from SC4SNMP_UI_backend.apply_changes.kubernetes_job import create_job_object, create_job +from pymongo import MongoClient +import os +from celery.utils.log import get_task_logger + +MONGO_URI = os.getenv("MONGO_URI") +JOB_NAMESPACE = os.getenv("JOB_NAMESPACE", "sc4snmp") +JOB_CREATION_RETRIES = int(os.getenv("JOB_CREATION_RETRIES", 10)) +JOB_CONFIG_PATH = os.getenv("JOB_CONFIG_PATH", "/config/job_config.yaml") +celery_logger = get_task_logger(__name__) + +@shared_task() +def run_job(): + job = None + batch_v1 = None + with open(JOB_CONFIG_PATH, encoding="utf-8") as file: + config_file = yaml.safe_load(file) + if config_file["apiVersion"] != "batch/v1": + raise ValueError("api version is different from batch/v1") + config.load_incluster_config() + batch_v1 = client.BatchV1Api() + job = create_job_object(config_file) + + with MongoClient(MONGO_URI) as connection: + try_creating = True + iteration = 0 + while try_creating and iteration < JOB_CREATION_RETRIES: + # Try creating a new job. If the previous job is still present in the namespace, + # ApiException will we be raised. In that happens wait for 10 seconds and try creating the job again + try: + create_job(batch_v1, job, JOB_NAMESPACE) + try_creating = False + try: + record = list(connection.sc4snmp.config_collection.find())[0] + connection.sc4snmp.config_collection.update_one({"_id": record["_id"]}, + {"$set": {"previous_job_start_time": datetime.datetime.utcnow(), + "currently_scheduled": False}}) + except Exception as e: + celery_logger.info(f"Error occurred while updating job state after job creation: {str(e)}") + except ApiException: + iteration += 1 + if iteration == JOB_CREATION_RETRIES: + try_creating = False + celery_logger.info(f"Kubernetes job was not created. Max retries ({JOB_CREATION_RETRIES}) exceeded.") + record = list(connection.sc4snmp.config_collection.find())[0] + connection.sc4snmp.config_collection.update_one({"_id": record["_id"]}, + {"$set": {"currently_scheduled": False}}) + else: + time.sleep(10) \ No newline at end of file diff --git a/backend/SC4SNMP_UI_backend/common/conversions.py b/backend/SC4SNMP_UI_backend/common/backend_ui_conversions.py similarity index 82% rename from backend/SC4SNMP_UI_backend/common/conversions.py rename to backend/SC4SNMP_UI_backend/common/backend_ui_conversions.py index 5b0ac69..ced9a20 100644 --- a/backend/SC4SNMP_UI_backend/common/conversions.py +++ b/backend/SC4SNMP_UI_backend/common/backend_ui_conversions.py @@ -19,28 +19,35 @@ def snake_case2camel_case(txt): return ''.join(result) -def get_group_name_from_backend(document: dict): - group_name = None +def get_group_or_profile_name_from_backend(document: dict): + group_or_profile_name = None for key in document.keys(): if key != "_id": - group_name = key - return group_name + group_or_profile_name = key + return group_or_profile_name class Conversion: + @abstractmethod - def _ui2backend_map(self, document: dict, **kwargs): + def backend2ui(self, document: dict, **kwargs): pass @abstractmethod - def _backend2ui_map(self, document: dict, **kwargs): + def ui2backend(self, document: dict, **kwargs): pass - def backend2ui(self, document: dict, **kwargs): - return self._backend2ui_map(document, **kwargs) - def ui2backend(self, document: dict, **kwargs): - return self._ui2backend_map(document, **kwargs) +def string_value_to_numeric(value: str): + try: + if value.isnumeric(): + return int(value) + elif value.replace(".", "").isnumeric(): + return float(value) + else: + return value + except ValueError: + return value class ProfileConversion(Conversion): @@ -65,25 +72,14 @@ def __init__(self, *args, **kwargs): for key, value in self.__backend2ui_profile_types.items(): self.__ui2backend_profile_types[value] = key - def __string_value_to_numeric(self, value: str): - try: - if value.isnumeric(): - return int(value) - elif value.replace(".", "").isnumeric(): - return float(value) - else: - return value - except ValueError: - return value - - def _backend2ui_map(self, document: dict, **kwargs): - profile_name = None - for key in document.keys(): - if key != "_id": - profile_name = key - if profile_name is None: + def backend2ui(self, document: dict, **kwargs): + profile_name = get_group_or_profile_name_from_backend(document) + if "profile_in_inventory" not in kwargs.keys(): + raise ValueError("No profile_in_inventory provided") + elif profile_name is None: raise ValueError("No profile name detected") else: + profile_in_inventory = kwargs["profile_in_inventory"] backend_var_binds = document[profile_name]["varBinds"] var_binds = [] for vb in backend_var_binds: @@ -136,13 +132,14 @@ def _backend2ui_map(self, document: dict, **kwargs): result = { "_id": str(document["_id"]), "profileName": profile_name, - "frequency": document[profile_name].get("frequency", 0), + "frequency": document[profile_name].get("frequency", 1), "conditions": conditions, - "varBinds": var_binds + "varBinds": var_binds, + "profileInInventory": profile_in_inventory } return result - def _ui2backend_map(self, document: dict, **kwargs): + def ui2backend(self, document: dict, **kwargs): conditions = None condition = None if document['conditions']['condition'] == "smart": @@ -159,9 +156,9 @@ def _ui2backend_map(self, document: dict, **kwargs): if operation == "in": value = [] for v in ui_condition["value"]: - value.append(self.__string_value_to_numeric(v)) + value.append(string_value_to_numeric(v)) else: - value = self.__string_value_to_numeric(ui_condition["value"][0]) + value = string_value_to_numeric(ui_condition["value"][0]) conditions.append( {"field": field, "operation": operation, "value": value} ) @@ -180,10 +177,11 @@ def _ui2backend_map(self, document: dict, **kwargs): item = { document['profileName']: { - 'frequency': int(document['frequency']), 'varBinds': var_binds } } + if document['conditions']['condition'] != "walk": + item[document['profileName']].update({'frequency': int(document['frequency'])}) if condition is not None: item[document['profileName']].update({'condition': condition}) if conditions is not None: @@ -195,15 +193,19 @@ class GroupConversion(Conversion): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def _backend2ui_map(self, document: dict, **kwargs): - group_name = get_group_name_from_backend(document) - result = { - "_id": str(document["_id"]), - "groupName": group_name - } - return result + def backend2ui(self, document: dict, **kwargs): + if "group_in_inventory" in kwargs.keys(): + group_name = get_group_or_profile_name_from_backend(document) + result = { + "_id": str(document["_id"]), + "groupName": group_name, + "groupInInventory": kwargs["group_in_inventory"] + } + return result + else: + raise ValueError("No group_in_inventory provided") - def _ui2backend_map(self, document: dict, **kwargs): + def ui2backend(self, document: dict, **kwargs): result = { document["groupName"]: [] } @@ -215,7 +217,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.optional_fields = ["port", "version", "community", "secret", "security_engine"] - def _backend2ui_map(self, document: dict, **kwargs): + def backend2ui(self, document: dict, **kwargs): if "group_id" in kwargs.keys() and "device_id" in kwargs.keys(): group_id = kwargs["group_id"] device_id = kwargs["device_id"] @@ -234,7 +236,7 @@ def _backend2ui_map(self, document: dict, **kwargs): else: raise ValueError("No group_id or device_id provided") - def _ui2backend_map(self, document: dict, **kwargs): + def ui2backend(self, document: dict, **kwargs): result = { "address": document["address"] } @@ -251,7 +253,7 @@ class InventoryConversion(Conversion): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - def _ui2backend_map(self, document: dict, **kwargs): + def ui2backend(self, document: dict, **kwargs): if "delete" in kwargs.keys(): profiles = "" for i in range(len(document['profiles'])): @@ -274,7 +276,7 @@ def _ui2backend_map(self, document: dict, **kwargs): else: raise ValueError("No delete provided") - def _backend2ui_map(self, document: dict, **kwargs): + def backend2ui(self, document: dict, **kwargs): profiles_mongo = document['profiles'] if len(profiles_mongo) > 0: profiles = profiles_mongo.split(";") @@ -292,4 +294,4 @@ def _backend2ui_map(self, document: dict, **kwargs): 'profiles': profiles, 'smartProfiles': document['smart_profiles'] } - return result + return result \ No newline at end of file diff --git a/backend/SC4SNMP_UI_backend/common/helpers.py b/backend/SC4SNMP_UI_backend/common/inventory_utils.py similarity index 99% rename from backend/SC4SNMP_UI_backend/common/helpers.py rename to backend/SC4SNMP_UI_backend/common/inventory_utils.py index 56fb63b..2f7e882 100644 --- a/backend/SC4SNMP_UI_backend/common/helpers.py +++ b/backend/SC4SNMP_UI_backend/common/inventory_utils.py @@ -2,7 +2,7 @@ from enum import Enum from typing import Callable from bson import ObjectId -from SC4SNMP_UI_backend.common.conversions import InventoryConversion +from SC4SNMP_UI_backend.common.backend_ui_conversions import InventoryConversion mongo_groups = mongo_client.sc4snmp.groups_ui mongo_inventory = mongo_client.sc4snmp.inventory_ui @@ -30,6 +30,7 @@ def update_profiles_in_inventory(profile_to_search: str, process_record: Callabl record_updated = process_record(index_to_update, record_updated, kwargs) record_updated = inventory_conversion.ui2backend(record_updated, delete=False) mongo_inventory.update_one({"_id": ObjectId(record_id)}, {"$set": record_updated}) + return inventory_records class HandleNewDevice: diff --git a/backend/SC4SNMP_UI_backend/groups/routes.py b/backend/SC4SNMP_UI_backend/groups/routes.py index 6074e26..5c95049 100644 --- a/backend/SC4SNMP_UI_backend/groups/routes.py +++ b/backend/SC4SNMP_UI_backend/groups/routes.py @@ -2,10 +2,10 @@ from flask import request, Blueprint, jsonify from flask_cors import cross_origin from SC4SNMP_UI_backend import mongo_client -from SC4SNMP_UI_backend.common.conversions import GroupConversion, GroupDeviceConversion, InventoryConversion, \ - get_group_name_from_backend +from SC4SNMP_UI_backend.common.backend_ui_conversions import GroupConversion, GroupDeviceConversion, InventoryConversion, \ + get_group_or_profile_name_from_backend from copy import copy -from SC4SNMP_UI_backend.common.helpers import HandleNewDevice +from SC4SNMP_UI_backend.common.inventory_utils import HandleNewDevice groups_blueprint = Blueprint('groups_blueprint', __name__) @@ -21,7 +21,9 @@ def get_groups_list(): groups = mongo_groups.find() groups_list = [] for gr in list(groups): - groups_list.append(group_conversion.backend2ui(gr)) + group_name = get_group_or_profile_name_from_backend(gr) + group_in_inventory = True if list(mongo_inventory.find({"address": group_name, "delete": False})) else False + groups_list.append(group_conversion.backend2ui(gr, group_in_inventory=group_in_inventory)) return jsonify(groups_list) @@ -50,7 +52,7 @@ def update_group(group_id): {"message": f"Group with name {group_obj['groupName']} already exists. Group was not edited."}), 400 else: old_group = list(mongo_groups.find({'_id': ObjectId(group_id)}))[0] - old_group_name = get_group_name_from_backend(old_group) + old_group_name = get_group_or_profile_name_from_backend(old_group) mongo_groups.update_one({'_id': old_group['_id']}, {"$rename": {f"{old_group_name}": f"{group_obj['groupName']}"}}) # Rename corresponding group in the inventory @@ -63,19 +65,26 @@ def update_group(group_id): @cross_origin() def delete_group_and_devices(group_id): group = list(mongo_groups.find({'_id': ObjectId(group_id)}))[0] - group_name = get_group_name_from_backend(group) + group_name = get_group_or_profile_name_from_backend(group) + configured_in_inventory = False with mongo_client.start_session() as session: with session.start_transaction(): mongo_groups.delete_one({'_id': ObjectId(group_id)}) + if list(mongo_inventory.find({"address": group_name})): + configured_in_inventory = True mongo_inventory.update_one({"address": group_name}, {"$set": {"delete": True}}) - return jsonify({"message": f"Group {group_name} was deleted. If {group_name} was configured in the inventory, it was deleted from there."}), 200 + if configured_in_inventory: + message = f"Group {group_name} was deleted. It was also deleted from the inventory." + else: + message = f"Group {group_name} was deleted." + return jsonify({"message": message}), 200 @groups_blueprint.route('/group//devices/count') @cross_origin() def get_devices_count_for_group(group_id): group = list(mongo_groups.find({"_id": ObjectId(group_id)}))[0] - group_name = get_group_name_from_backend(group) + group_name = get_group_or_profile_name_from_backend(group) total_count = len(group[group_name]) return jsonify(total_count) @@ -88,7 +97,7 @@ def get_devices_of_group(group_id, page_num, dev_per_page): skips = dev_per_page * (page_num - 1) group = list(mongo_groups.find({"_id": ObjectId(group_id)}))[0] - group_name = get_group_name_from_backend(group) + group_name = get_group_or_profile_name_from_backend(group) devices_list = [] for i, device in enumerate(group[group_name]): devices_list.append(group_device_conversion.backend2ui(device, group_id=group_id, device_id=copy(i))) @@ -113,7 +122,7 @@ def add_device_to_group(): device_obj = request.json group_id = device_obj["groupId"] group = list(mongo_groups.find({'_id': ObjectId(group_id)}, {"_id": 0}))[0] - group_name = get_group_name_from_backend(group) + group_name = get_group_or_profile_name_from_backend(group) device_obj = group_device_conversion.ui2backend(device_obj) handler = HandleNewDevice(mongo_groups, mongo_inventory) host_added, message = handler.add_group_host(group_name, ObjectId(group_id), device_obj) @@ -132,7 +141,7 @@ def update_device_from_group(device_id): device_id = device_id.split("-")[1] group = list(mongo_groups.find({'_id': ObjectId(group_id)}, {"_id": 0}))[0] device_obj = group_device_conversion.ui2backend(device_obj) - group_name = get_group_name_from_backend(group) + group_name = get_group_or_profile_name_from_backend(group) handler = HandleNewDevice(mongo_groups, mongo_inventory) host_edited, message = handler.edit_group_host(group_name, ObjectId(group_id), device_id, device_obj, ) @@ -149,7 +158,7 @@ def delete_device_from_group_record(device_id: str): group_id = device_id.split("-")[0] device_id = device_id.split("-")[1] group = list(mongo_groups.find({'_id': ObjectId(group_id)}, {"_id": 0}))[0] - group_name = get_group_name_from_backend(group) + group_name = get_group_or_profile_name_from_backend(group) removed_device = group[group_name].pop(int(device_id)) device_name = f"{removed_device['address']}:{removed_device.get('port','')}" new_values = {"$set": group} diff --git a/backend/SC4SNMP_UI_backend/inventory/routes.py b/backend/SC4SNMP_UI_backend/inventory/routes.py index a47724c..959e021 100644 --- a/backend/SC4SNMP_UI_backend/inventory/routes.py +++ b/backend/SC4SNMP_UI_backend/inventory/routes.py @@ -2,8 +2,8 @@ from flask import request, Blueprint, jsonify from flask_cors import cross_origin from SC4SNMP_UI_backend import mongo_client -from SC4SNMP_UI_backend.common.conversions import InventoryConversion -from SC4SNMP_UI_backend.common.helpers import HandleNewDevice +from SC4SNMP_UI_backend.common.backend_ui_conversions import InventoryConversion +from SC4SNMP_UI_backend.common.inventory_utils import HandleNewDevice inventory_blueprint = Blueprint('inventory_blueprint', __name__) diff --git a/backend/SC4SNMP_UI_backend/profiles/routes.py b/backend/SC4SNMP_UI_backend/profiles/routes.py index 861e35a..cdc0e4c 100644 --- a/backend/SC4SNMP_UI_backend/profiles/routes.py +++ b/backend/SC4SNMP_UI_backend/profiles/routes.py @@ -2,13 +2,14 @@ from flask import request, Blueprint, jsonify from flask_cors import cross_origin from SC4SNMP_UI_backend import mongo_client -from SC4SNMP_UI_backend.common.conversions import ProfileConversion -from SC4SNMP_UI_backend.common.helpers import update_profiles_in_inventory +from SC4SNMP_UI_backend.common.backend_ui_conversions import ProfileConversion, get_group_or_profile_name_from_backend +from SC4SNMP_UI_backend.common.inventory_utils import update_profiles_in_inventory profiles_blueprint = Blueprint('profiles_blueprint', __name__) profile_conversion = ProfileConversion() mongo_profiles = mongo_client.sc4snmp.profiles_ui +mongo_inventory = mongo_client.sc4snmp.inventory_ui # @cross_origin(origins='*', headers=['access-control-allow-origin', 'Content-Type']) @profiles_blueprint.route('/profiles/names') @@ -17,7 +18,7 @@ def get_profile_names(): profiles = list(mongo_profiles.find()) profiles_list = [] for pr in profiles: - converted = profile_conversion.backend2ui(pr) + converted = profile_conversion.backend2ui(pr, profile_in_inventory=True) if converted['conditions']['condition'] not in ['mandatory', 'base']: profiles_list.append(converted) return jsonify([el["profileName"] for el in profiles_list]) @@ -38,7 +39,10 @@ def get_profiles_list(page_num, prof_per_page): profiles = list(mongo_profiles.find().skip(skips).limit(prof_per_page)) profiles_list = [] for pr in profiles: - converted = profile_conversion.backend2ui(pr) + profile_name = get_group_or_profile_name_from_backend(pr) + profile_in_inventory = True if list(mongo_inventory.find({"profiles": {"$regex": f'.*{profile_name}.*'}, + "delete": False})) else False + converted = profile_conversion.backend2ui(pr, profile_in_inventory=profile_in_inventory) if converted['conditions']['condition'] not in ['mandatory']: profiles_list.append(converted) return jsonify(profiles_list) @@ -50,7 +54,7 @@ def get_all_profiles_list(): profiles = list(mongo_profiles.find()) profiles_list = [] for pr in profiles: - converted = profile_conversion.backend2ui(pr) + converted = profile_conversion.backend2ui(pr, profile_in_inventory=True) if converted['conditions']['condition'] not in ['mandatory']: profiles_list.append(converted) return jsonify(profiles_list) @@ -80,11 +84,14 @@ def delete_profile_record(profile_id): def delete_profile(index, record_to_update, kwargs): record_to_update["profiles"].pop(index) return record_to_update - update_profiles_in_inventory(profile_name, delete_profile) + inventory_records = update_profiles_in_inventory(profile_name, delete_profile) + if inventory_records: + message = f"Profile {profile_name} was deleted. It was also deleted from some inventory records." + else: + message = f"Profile {profile_name} was deleted." mongo_profiles.delete_one({'_id': ObjectId(profile_id)}) - return jsonify({"message": f"Profile {profile_name} was deleted. If {profile_name} was used in some records in the inventory," - f" those records were updated."}), 200 + return jsonify({"message": message}), 200 @profiles_blueprint.route('/profiles/update/', methods=['POST']) diff --git a/backend/requirements.txt b/backend/requirements.txt index 4fd1105..620a31d 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -13,4 +13,5 @@ kubernetes~=26.1.0 python-dotenv~=0.21.0 PyYAML~=6.0 celery==5.2.7 -redis==4.5.5 \ No newline at end of file +redis==4.5.5 +ruamel.yaml===0.17.32 \ No newline at end of file diff --git a/backend/tests/common/test_conversions.py b/backend/tests/common/test_conversions.py index fe2ebfd..33d790d 100644 --- a/backend/tests/common/test_conversions.py +++ b/backend/tests/common/test_conversions.py @@ -1,5 +1,5 @@ from unittest import TestCase -from SC4SNMP_UI_backend.common.conversions import ProfileConversion, GroupConversion, GroupDeviceConversion, \ +from SC4SNMP_UI_backend.common.backend_ui_conversions import ProfileConversion, GroupConversion, GroupDeviceConversion, \ InventoryConversion from bson import ObjectId @@ -29,7 +29,8 @@ def setUpClass(cls): "varBinds": [{"family": "IF-MIB", "category": "ifInDiscards", "index": "1.test.2"}, {"family": "IF-MIB", "category": "ifInDiscards", "index": "1"}, {"family": "IF-MIB", "category": "", "index": ""}, - {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}] + {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}], + "profileInInventory": True } cls.ui_prof_2 = { @@ -44,7 +45,8 @@ def setUpClass(cls): }, "varBinds": [{"family": "IF-MIB", "category": "ifInDiscards", "index": "1"}, {"family": "IF-MIB", "category": "", "index": ""}, - {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}] + {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}], + "profileInInventory": False } cls.ui_prof_3 = { @@ -59,7 +61,8 @@ def setUpClass(cls): }, "varBinds": [{"family": "IF-MIB", "category": "ifInDiscards", "index": "1"}, {"family": "IF-MIB", "category": "", "index": ""}, - {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}] + {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}], + "profileInInventory": True } cls.ui_prof_4 = { @@ -79,7 +82,8 @@ def setUpClass(cls): }, "varBinds": [{"family": "IF-MIB", "category": "ifInDiscards", "index": "1"}, {"family": "IF-MIB", "category": "", "index": ""}, - {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}] + {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}], + "profileInInventory": False } cls.backend_prof_1 = { @@ -139,7 +143,8 @@ def setUpClass(cls): cls.ui_group = { "_id": common_id, - "groupName": "group_1" + "groupName": "group_1", + "groupInInventory": False } cls.ui_group_device_1 = { @@ -241,10 +246,10 @@ def setUpClass(cls): } def test_profile_backend_to_ui(self): - self.assertDictEqual(profile_conversion.backend2ui(self.backend_prof_1), self.ui_prof_1) - self.assertDictEqual(profile_conversion.backend2ui(self.backend_prof_2), self.ui_prof_2) - self.assertDictEqual(profile_conversion.backend2ui(self.backend_prof_3), self.ui_prof_3) - self.assertDictEqual(profile_conversion.backend2ui(self.backend_prof_4), self.ui_prof_4) + self.assertDictEqual(profile_conversion.backend2ui(self.backend_prof_1, profile_in_inventory=True), self.ui_prof_1) + self.assertDictEqual(profile_conversion.backend2ui(self.backend_prof_2, profile_in_inventory=False), self.ui_prof_2) + self.assertDictEqual(profile_conversion.backend2ui(self.backend_prof_3, profile_in_inventory=True), self.ui_prof_3) + self.assertDictEqual(profile_conversion.backend2ui(self.backend_prof_4, profile_in_inventory=False), self.ui_prof_4) def test_profile_ui_to_backend(self): back_pr1 = self.backend_prof_1 @@ -265,7 +270,7 @@ def test_profile_ui_to_backend(self): self.assertDictEqual(profile_conversion.ui2backend(self.ui_prof_4), back_pr4) def test_group_backend_to_ui(self): - self.assertDictEqual(group_conversion.backend2ui(self.backend_group), self.ui_group) + self.assertDictEqual(group_conversion.backend2ui(self.backend_group, group_in_inventory=False), self.ui_group) def test_group_ui_to_backend(self): new_group_from_ui = { diff --git a/backend/tests/ui_handling/get_endpoints/test_get_endpoints.py b/backend/tests/ui_handling/get_endpoints/test_get_endpoints.py index b1d7fff..cc7b53b 100644 --- a/backend/tests/ui_handling/get_endpoints/test_get_endpoints.py +++ b/backend/tests/ui_handling/get_endpoints/test_get_endpoints.py @@ -60,7 +60,8 @@ def test_get_all_profiles_list(m_client, client): }, "varBinds": [{"family": "IF-MIB", "category": "ifInDiscards", "index": "1"}, {"family": "IF-MIB", "category": "", "index": ""}, - {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}] + {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}], + 'profileInInventory': True, } ui_prof_2 = { @@ -75,7 +76,8 @@ def test_get_all_profiles_list(m_client, client): }, "varBinds": [{"family": "IF-MIB", "category": "ifInDiscards", "index": "1"}, {"family": "IF-MIB", "category": "", "index": ""}, - {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}] + {"family": "IF-MIB", "category": "ifOutErrors", "index": ""}], + 'profileInInventory': True, } response = client.get('/profiles') @@ -86,8 +88,9 @@ def test_get_all_profiles_list(m_client, client): @mock.patch("pymongo.collection.Collection.find") def test_get_groups_list(m_client, client): common_id = "635916b2c8cb7a15f28af40a" - m_client.return_value = [ - { + + m_client.side_effect = [ + [{ "_id": common_id, "group_1": [ {"address": "1.2.3.4"} @@ -98,17 +101,21 @@ def test_get_groups_list(m_client, client): "group_2": [ {"address": "1.2.3.4"} ] - } + }], + [], + [{"address": "group_2"}] ] expected_groups = [ { "_id": common_id, - "groupName": "group_1" + "groupName": "group_1", + "groupInInventory": False }, { "_id": common_id, - "groupName": "group_2" + "groupName": "group_2", + "groupInInventory": True } ] diff --git a/backend/tests/ui_handling/post_endpoints/test_post_groups.py b/backend/tests/ui_handling/post_endpoints/test_post_groups.py index 8eb8657..85d2d69 100644 --- a/backend/tests/ui_handling/post_endpoints/test_post_groups.py +++ b/backend/tests/ui_handling/post_endpoints/test_post_groups.py @@ -119,16 +119,37 @@ def test_delete_group_and_devices(m_session, m_update, m_delete, m_find, client) } m_session.return_value.__enter__.return_value.start_transaction.__enter__ = Mock() - m_find.return_value = [backend_group] + m_find.side_effect = [ + [backend_group], + [] + ] + + calls_find = [ + call({'_id': ObjectId(common_id)}), + call({"address": "group_1"}) + ] + m_delete.return_value = None m_update.return_value = None response = client.post(f"/groups/delete/{common_id}") - assert m_find.call_args == call({'_id': ObjectId(common_id)}) + m_find.assert_has_calls(calls_find) + assert m_delete.call_args == call({'_id': ObjectId(common_id)}) + assert m_update.call_args == call({"address": "group_1"}, {"$set": {"delete": True}}) + assert response.json == { + "message": "Group group_1 was deleted."} + + m_find.side_effect = [ + [backend_group], + [{}] + ] + + response = client.post(f"/groups/delete/{common_id}") + m_find.assert_has_calls(calls_find) assert m_delete.call_args == call({'_id': ObjectId(common_id)}) assert m_update.call_args == call({"address": "group_1"}, {"$set": {"delete": True}}) assert response.json == { - "message": "Group group_1 was deleted. If group_1 was configured in the inventory, it was deleted from there."} + "message": "Group group_1 was deleted. It was also deleted from the inventory."} # TEST ADDING DEVICE diff --git a/backend/tests/ui_handling/post_endpoints/test_post_profiles.py b/backend/tests/ui_handling/post_endpoints/test_post_profiles.py index 9d8d747..b9bfd84 100644 --- a/backend/tests/ui_handling/post_endpoints/test_post_profiles.py +++ b/backend/tests/ui_handling/post_endpoints/test_post_profiles.py @@ -117,8 +117,7 @@ def test_delete_profile_record(m_update, m_delete, m_find, client): m_find.assert_has_calls(calls) assert m_delete.call_args == call({"_id": ObjectId(common_id)}) assert m_update.call_args == call({"_id": ObjectId(common_id)}, {"$set": backend_inventory_update}) - assert response.json == {"message": f"Profile profile_1 was deleted. If profile_1 was used in some records in the inventory," - f" those records were updated."} + assert response.json == {"message": f"Profile profile_1 was deleted. It was also deleted from some inventory records."} # TEST UPDATING PROFILE diff --git a/frontend/packages/manager/demo/splunk-app/appserver/templates/demo.html b/frontend/packages/manager/demo/splunk-app/appserver/templates/demo.html index 1ff433a..1e49b85 100644 --- a/frontend/packages/manager/demo/splunk-app/appserver/templates/demo.html +++ b/frontend/packages/manager/demo/splunk-app/appserver/templates/demo.html @@ -5,7 +5,7 @@ - Manager Demo App + SC4SNMP Manager diff --git a/frontend/packages/manager/demo/standalone/index.html b/frontend/packages/manager/demo/standalone/index.html index 8912646..dc7ac45 100644 --- a/frontend/packages/manager/demo/standalone/index.html +++ b/frontend/packages/manager/demo/standalone/index.html @@ -3,7 +3,7 @@ - Manager + SC4SNMP Manager diff --git a/frontend/packages/manager/src/components/DeleteModal.jsx b/frontend/packages/manager/src/components/DeleteModal.jsx index ac7ce3d..440134b 100644 --- a/frontend/packages/manager/src/components/DeleteModal.jsx +++ b/frontend/packages/manager/src/components/DeleteModal.jsx @@ -2,6 +2,7 @@ import React, { useCallback, useState, useContext } from 'react'; import Button from '@splunk/react-ui/Button'; import Modal from '@splunk/react-ui/Modal'; import P from '@splunk/react-ui/Paragraph'; +import Message from "@splunk/react-ui/Message"; import ButtonsContext from "../store/buttons-contx"; function DeleteModal(props) { @@ -25,6 +26,10 @@ function DeleteModal(props) {

Are you sure you want to delete {props.deleteName} ?

+ {("customWarning" in props && props["customWarning"] != null) ? + ( + {props["customWarning"]} + ) : null}