Merge pull request #71 from debiai/Feature/refactor_data_provider

0.19.0 Feature/refactor data provider

.dockerignore

@@ -4,7 +4,6 @@ images/
 TODO
 README.md
 kubernetes.yaml
-debiai.log
 LICENSE
 
 # Back

.github/workflows/black-format-check.yml

@@ -0,0 +1,31 @@
+# Check that the code is formatted with black
+
+name: Black Format Check
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Install dependencies
+        run: |
+          cd backend
+          python -m pip install --upgrade pip
+          pip install black
+      - name: Check that the code is formatted with black
+        run: |
+          black backend/ --check --diff

.github/workflows/python-tests.yml

@@ -0,0 +1,45 @@
+# Install dependencies and run tests with pytest
+
+name: Python tests
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.8", "3.9", "3.10"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install dependencies
+        run: |
+          cd backend
+          python -m pip install --upgrade pip
+          pip install flake8 pytest
+          pip install -r requirements.txt
+      - name: Lint with flake8
+        run: |
+          # Syntax verification : Will do later
+          # stop the build if there are Python syntax errors or undefined names
+          # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+          # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      - name: Test with pytest
+        run: |
+          cd backend
+          python websrv.py & sleep 3 && pytest tests/

.gitignore

@@ -1,11 +1,11 @@
 TODO
-debiai.log
 dist/
 deployment/
+.vscode/
 
 # Back
 backend/data
-backend/logs/
+.coverage
 __pycache__/
 
 # Front

backend/config/config.ini

@@ -1,16 +1,34 @@
-[LOG]
-operationTraceLog = light
-#   full, light or none
-#   full : Trace log funtion parameters and return (may be heavy)
-#   light : Trace log funtion parameters
-#   none : Nothing is traced
+# Debiai configuration file
 
 [DATA_PROVIDERS]
-# List all the data providers you want to use
-# My_data_provider1 = http://localhost:5000/debiai/
-# My_data_provider2 = http://localhost:6000/
+# Configure what a user is allowed to do with data providers
+# Allow or not the creation of data providers from the web interface
+# (true/false), default is true
+creation = true 
+# Allow or not the deletion of data providers from the web interface
+# (true/false), default is true
+deletion = true
+
+[PYTHON_MODULE_DATA_PROVIDER]
+# Options for the python module data provider
+# Enable or disable the python module data provider
+# This will make DebiAI a read only application
+# (true/false), default is true
+enabled = true
+
+[WEB_DATA_PROVIDERS]
+# List all the web data providers you want to use
+#My_data_provider1 = http://localhost:3010/debiai
+#My_data_provider2 = http://localhost:3011/debiai
 
 [EXPORT_METHODS]
 # List all the application export methods according to the available export types
-# My_PostHTTP_export_method = post, http://localhost:5000/export/
-# My_Kafka_export_method = kafka, kafka.svc.local:9092, My_Kafka_topic
+# Available export types: Kafka and HTTP
+
+# HTTP export method:
+# Format: export_method_name = post, <HTTP URL>
+#My_PostHTTP_export_method = post, http://localhost:5000/export/
+
+# Kafka export method:
+# Format: export_method_name = kafka, <Kafka broker URL>, <Kafka topic>
+#My_Kafka_export_method = kafka, kafka.svc.local:9092, My_Kafka_topic

backend/config/init_config.py

@@ -0,0 +1,183 @@
+from configparser import ConfigParser
+import os, json
+
+
+config_path = "config/config.ini"
+config_parser = ConfigParser()
+config_parser.optionxform = str  # To preserve the case
+
+config = {}
+
+
+def init_config():
+    global config
+
+    # Expected sections:
+    # - PYTHON_MODULE_DATA_PROVIDER
+    # - WEB_DATA_PROVIDERS
+    # - EXPORT_METHODS
+
+    print("===================== CONFIG =======================")
+
+    # First, read the config file
+    config_parser.read(config_path)
+    config = {
+        "DATA_PROVIDERS": {"creation": True, "deletion": True},
+        "PYTHON_MODULE_DATA_PROVIDER": {"enabled": True},
+        "WEB_DATA_PROVIDERS": {},
+        "EXPORT_METHODS": {},
+    }
+
+    for section in config_parser.sections():
+        if section == "DATA_PROVIDERS":
+            if "creation" in config_parser[section]:
+                if str.lower(config_parser[section]["creation"]) == "false":
+                    print("Config file: Data Providers creation disabled")
+                    config["DATA_PROVIDERS"]["creation"] = False
+
+            if "deletion" in config_parser[section]:
+                if str.lower(config_parser[section]["deletion"]) == "false":
+                    print("Config file: Data Providers deletion disabled")
+                    config["DATA_PROVIDERS"]["deletion"] = False
+            continue
+
+        if section == "PYTHON_MODULE_DATA_PROVIDER":
+            if "enabled" in config_parser[section]:
+                if str.lower(config_parser[section]["enabled"]) == "false":
+                    print("Config file: Python Module Data Provider disabled")
+                    config["PYTHON_MODULE_DATA_PROVIDER"]["enabled"] = False
+                elif str.lower(config_parser[section]["enabled"]) == "true":
+                    print("Config file: Python Module Data Provider enabled")
+                    config["PYTHON_MODULE_DATA_PROVIDER"]["enabled"] = True
+            continue
+
+        if section == "WEB_DATA_PROVIDERS":
+            for data_provider in config_parser[section]:
+                print(
+                    "Config file: detected data provider '"
+                    + data_provider
+                    + "' from config file"
+                )
+                config["WEB_DATA_PROVIDERS"][data_provider] = config_parser[section][
+                    data_provider
+                ]
+            continue
+
+        if section == "EXPORT_METHODS":
+            for export_method in config_parser[section]:
+                print(
+                    "Config file: detected export method '"
+                    + export_method
+                    + "' from config file"
+                )
+                config["EXPORT_METHODS"][export_method] = config_parser[section][
+                    export_method
+                ]
+            continue
+
+        print("Config section '" + section + "' not recognized, skipping")
+
+    # Then deal with environment variables
+    for env_var in os.environ:
+        # Deal with DATA_PROVIDERS in env variables
+        if env_var == "DEBIAI_DATA_PROVIDERS_CREATION_ENABLED":
+            # Env var format: DEBIAI_DATA_PROVIDERS_CREATION_ENABLED=<True|False>
+            if str.lower(os.environ[env_var]) == "false":
+                print("Environment variables: Data Providers creation disabled")
+                config["DATA_PROVIDERS"]["creation"] = False
+
+        if env_var == "DEBIAI_DATA_PROVIDERS_DELETION_ENABLED":
+            # Env var format: DEBIAI_DATA_PROVIDERS_DELETION_ENABLED=<True|False>
+            if str.lower(os.environ[env_var]) == "false":
+                print("Environment variables: Data Providers deletion disabled")
+                config["DATA_PROVIDERS"]["deletion"] = False
+
+        # Deal with PYTHON_MODULE_DATA_PROVIDER in env variables
+        if env_var == "DEBIAI_PYTHON_MODULE_DATA_PROVIDER_ENABLED":
+            # Env var format: DEBIAI_PYTHON_MODULE_DATA_PROVIDER_ENABLED=<True|False>
+            if str.lower(os.environ[env_var]) == "false":
+                print("Environment variables: Python Module Data Provider disabled")
+                config["PYTHON_MODULE_DATA_PROVIDER"]["enabled"] = False
+            elif str.lower(os.environ[env_var]) == "true":
+                print("Environment variables: Python Module Data Provider enabled")
+                config["PYTHON_MODULE_DATA_PROVIDER"]["enabled"] = True
+
+        # Deal with Data Providers in env variables
+        if "DEBIAI_WEB_DATA_PROVIDER" in env_var:
+            # Env var format: DEBIAI_DATA_PROVIDER_<name>=<url>
+            if len(env_var.split("_")) != 5:
+                print(
+                    "Environment variables: invalid environment variable '"
+                    + env_var
+                    + "', skipping"
+                )
+                print("Expected format: DEBIAI_WEB_DATA_PROVIDER_<name>=<url>")
+                continue
+
+            data_provider_name = env_var.split("_")[3]
+            data_provider_url = os.environ[env_var]
+
+            if len(data_provider_name) == 0:
+                print(
+                    "Environment variables: invalid data provider name '"
+                    + env_var
+                    + "', skipping"
+                )
+                print("Expected format: DEBIAI_WEB_DATA_PROVIDER_<name>=<url>")
+                continue
+
+            print(
+                "Environment variables: detected Web data provider '"
+                + data_provider_name
+                + "' from environment variables"
+            )
+
+            config["WEB_DATA_PROVIDERS"][data_provider_name] = data_provider_url
+
+        # Deal with Export Methods
+        if "DEBIAI_EXPORT_METHOD" in env_var:
+            # Env var format: DEBIAI_EXPORT_METHOD_<name>=<type>, <param1>, <param2>, ..."
+            if len(env_var.split("_")) != 4:
+                print(
+                    "Environment variables: invalid environment variable '"
+                    + env_var
+                    + "', skipping"
+                )
+                print(
+                    "Expected format: DEBIAI_EXPORT_METHOD_<name>=<type>, <param1>, <param2>, ..."
+                )
+                continue
+
+            export_method_name = env_var.split("_")[3]
+            export_method_type_and_parameters = os.environ[env_var]
+
+            if len(export_method_name) == 0:
+                print(
+                    "Environment variables: Invalid export method name "
+                    + env_var
+                    + ", skipping"
+                )
+                print(
+                    "Expected format: DEBIAI_EXPORT_METHOD_<name>=<type>, <param1>, <param2>, ..."
+                )
+                continue
+
+            print(
+                "Environment variables: Detected export method '"
+                + export_method_name
+                + "' from environment variables"
+            )
+
+            if "EXPORT_METHODS" not in config:
+                config["EXPORT_METHODS"] = {}
+
+            config["EXPORT_METHODS"][
+                export_method_name
+            ] = export_method_type_and_parameters
+
+    print("Config loaded")
+    print(json.dumps(config, sort_keys=True, indent=4))
+
+
+def get_config():
+    return config