Merge pull request #56 from shashank-boyapally/cmd-templating

Cmd templating

Author: vishnuchalla

orion.py

@@ -11,7 +11,7 @@
 import uvicorn
 from fmatch.logrus import SingletonLogger
 from pkg.runTest import run
-from pkg.utils import load_config
+from pkg.config import load_config
 import pkg.constants as cnsts
 
 warnings.filterwarnings("ignore", message="Unverified HTTPS request.*")

pkg/config.py

@@ -0,0 +1,63 @@
+# pylint: disable = line-too-long
+"""
+Module file for config reading and loading
+"""
+
+
+import os
+import sys
+from typing import Any, Dict, Set
+
+from jinja2 import Environment, Template, meta
+from fmatch.logrus import SingletonLogger
+import yaml
+
+
+def load_config(config: str, parameters: Dict= None) -> Dict[str, Any]:
+    """Loads config file
+
+    Args:
+        config (str): path to config file
+        logger (Logger): logger
+
+    Returns:
+        dict: dictionary of the config file
+    """
+    env_vars = {k.lower(): v for k, v in os.environ.items()}
+    merged_parameters = {}
+    merged_parameters.update(env_vars)
+    if parameters:
+        merged_parameters.update(parameters)
+    logger_instance = SingletonLogger.getLogger("Orion")
+    try:
+        with open(config, "r", encoding="utf-8") as template_file:
+            template_content = template_file.read()
+            logger_instance.debug("The %s file has successfully loaded", config)
+    except FileNotFoundError as e:
+        logger_instance.error("Config file not found: %s", e)
+        sys.exit(1)
+    except Exception as e:  # pylint: disable=broad-exception-caught
+        logger_instance.error("An error occurred: %s", e)
+        sys.exit(1)
+
+    required_parameters = get_template_variables(template_content)
+    logger_instance.debug(f"Variables required by the template: {required_parameters}")
+
+    # Check for missing variables
+    missing_vars = required_parameters - merged_parameters.keys()
+    if missing_vars:
+        logger_instance.error(f"Missing required parameters: {missing_vars}, use environment variables to set")
+        sys.exit(1)
+
+    template = Template(template_content)
+    rendered_config_yaml = template.render(merged_parameters)
+    rendered_config = yaml.safe_load(rendered_config_yaml)
+    return rendered_config
+
+
+def get_template_variables(template_content: str) -> Set[str]:
+    """Extracts all variables from the Jinja2 template content."""
+    env = Environment()
+    parsed_content = env.parse(template_content)
+    variables = meta.find_undeclared_variables(parsed_content)
+    return variables

pkg/daemon.py

@@ -6,10 +6,9 @@
 import os
 from typing import Any
 from fastapi import FastAPI, HTTPException
-from jinja2 import Template
 import pkg_resources
-import yaml
 from fmatch.logrus import SingletonLogger
+from pkg.config import load_config
 import pkg.constants as cnsts
 
 from . import runTest
@@ -37,7 +36,8 @@ async def daemon_changepoint( # pylint: disable = R0913
         json: json object of the changepoints and metrics
     """
     parameters = {"version": version}
-    config_file_name=test_name+".yml"
+    config_file_name=f"{test_name}.yml"
+    config_path = pkg_resources.resource_filename("configs", config_file_name)
     option_arguments = {
         "config": config_file_name,
         "save_data_path": "output.csv",
@@ -47,7 +47,7 @@ async def daemon_changepoint( # pylint: disable = R0913
         "uuid": uuid,
         "lookback":lookback,
         "baseline": baseline,
-        "configMap": render_template(config_file_name, parameters),
+        "configMap": load_config(config_path, parameters),
         "convert_tinyurl": convert_tinyurl.lower() not in "false",
     }
     filter_changepoints = (
@@ -90,7 +90,7 @@ async def get_options() -> Any:
         raise HTTPException(status_code=500, detail=str(e)) from e
 
 @app.get("/daemon/anomaly")
-async def daemon_anomaly( # pylint: disable = R0913
+async def daemon_anomaly( # pylint: disable = R0913, R0914
     version: str = "4.17",
     uuid: str = "",
     baseline: str = "",
@@ -111,6 +111,7 @@ async def daemon_anomaly( # pylint: disable = R0913
     """
     parameters = {"version": version}
     config_file_name=test_name+".yml"
+    config_path = pkg_resources.resource_filename("configs", config_file_name)
     option_arguments = {
         "config": config_file_name,
         "save_data_path": "output.csv",
@@ -120,7 +121,7 @@ async def daemon_anomaly( # pylint: disable = R0913
         "uuid": uuid,
         "lookback":lookback,
         "baseline": baseline,
-        "configMap": render_template(config_file_name, parameters),
+        "configMap": load_config(config_path, parameters),
         "anomaly_window": int(anomaly_window),
         "min_anomaly_percent":int(min_anomaly_percent),
         "convert_tinyurl": convert_tinyurl.lower() not in "false",
@@ -136,22 +137,3 @@ async def daemon_anomaly( # pylint: disable = R0913
         for key, value in result.items():
             result[key] = list(filter(lambda x: x.get("is_changepoint", False), value))
     return result
-
-
-def render_template(test_name: str, parameters: dict[str,Any]) -> Any:
-    """replace parameters in the config file
-
-    Args:
-        file_name (str): the config file
-        parameters (dict): parameters to be replaces
-
-    Returns:
-        dict: configMap in dict
-    """
-    config_path = pkg_resources.resource_filename("configs", test_name)
-    with open(config_path, "r", encoding="utf-8") as template_file:
-        template_content = template_file.read()
-    template = Template(template_content)
-    rendered_config_yaml = template.render(parameters)
-    rendered_config = yaml.safe_load(rendered_config_yaml)
-    return rendered_config

pkg/utils.py

@@ -16,10 +16,7 @@
 from tabulate import tabulate
 from fmatch.matcher import Matcher
 from fmatch.logrus import SingletonLogger
-
-import yaml
 import pandas as pd
-
 import pyshorteners
 
 
@@ -132,28 +129,7 @@ def extract_metadata_from_test(test: Dict[str, Any]) -> Dict[Any, Any]:
     return metadata
 
 
-def load_config(config: str) -> Dict[str, Any]:
-    """Loads config file
-
-    Args:
-        config (str): path to config file
-        logger (Logger): logger
 
-    Returns:
-        dict: dictionary of the config file
-    """
-    logger_instance = SingletonLogger.getLogger("Orion")
-    try:
-        with open(config, "r", encoding="utf-8") as file:
-            data = yaml.safe_load(file)
-            logger_instance.debug("The %s file has successfully loaded", config)
-    except FileNotFoundError as e:
-        logger_instance.error("Config file not found: %s", e)
-        sys.exit(1)
-    except Exception as e:  # pylint: disable=broad-exception-caught
-        logger_instance.error("An error occurred: %s", e)
-        sys.exit(1)
-    return data
 
 
 def get_datasource(data: Dict[Any, Any]) -> str: