Merge pull request #14 from JustinFrizzell/updated-yaml-structure

Updated yaml structure

docker-compose.yaml

@@ -3,15 +3,6 @@ version: '3.8'
 services:
   app:
     build: .
-    depends_on:
-      postgres_db:
-        condition: service_started
-      mssql_db:
-        condition: service_started        
-      oracle_db:
-        condition: service_started        
-      mysql_db:
-        condition: service_started        
     environment:
       - RUNNING_IN_DOCKER=true        
     networks:
@@ -20,7 +11,7 @@ services:
   postgres_db:
     image: postgres
     environment:
-      POSTGRES_PASSWORD: "mysecretpassword"
+      POSTGRES_PASSWORD: "test_p@ssword_f0r_CI!"
     ports:
       - "5432:5432"
     networks:
@@ -29,7 +20,7 @@ services:
   mssql_db:
     image: mcr.microsoft.com/mssql/server:2019-latest
     environment:
-      SA_PASSWORD: "MySecretPassw0rd!"
+      SA_PASSWORD: "test_p@ssword_f0r_CI!"
       ACCEPT_EULA: "Y"
     ports:
       - "1433:1433"
@@ -39,7 +30,7 @@ services:
   oracle_db:
     image: container-registry.oracle.com/database/express:latest
     environment:
-      ORACLE_PWD: "MySecretPassw0rd!"
+      ORACLE_PWD: "test_p@ssword_f0r_CI!"
     ports:
       - "1521:1521"
     networks:
@@ -48,7 +39,7 @@ services:
   mysql_db:
     image: mysql
     environment:
-      MYSQL_ROOT_PASSWORD: "MySecretPassw0rd!"
+      MYSQL_ROOT_PASSWORD: "test_p@ssword_f0r_CI!"
     ports:
       - "3306:3306"
     networks:

sqlconnect/config.py

@@ -36,6 +36,7 @@
 from pathlib import Path
 import yaml
 from dotenv import load_dotenv
+from sqlalchemy import URL
 
 
 def get_connection_config(connection_name: str, config_path: str = None) -> dict:
@@ -102,7 +103,7 @@ def get_connection_config(connection_name: str, config_path: str = None) -> dict
                 )
 
             # Check if all required keys are present
-            required_keys = ["sqlalchemy_driver", "odbc_driver", "server", "database"]
+            required_keys = ["dialect", "dbapi", "host"]
             missing_keys = [
                 key for key in required_keys if key not in connection_config
             ]
@@ -119,11 +120,11 @@ def get_connection_config(connection_name: str, config_path: str = None) -> dict
     )
 
 
-def get_db_url(connection_config: dict) -> str:
+def get_db_url(connection_config: dict) -> URL:
     """
-    Constructs and returns a database connection string from the given configuration dictionary.
+    Constructs and returns a database connection URL from the given configuration dictionary.
 
-    This function builds a database connection string suitable for SQLAlchemy, using
+    This function builds a database connection URL for SQLAlchemy, using
     details provided in a configuration dictionary. It handles the inclusion of
     authentication details securely by retrieving them from environment variables if
     necessary.
@@ -132,73 +133,82 @@ def get_db_url(connection_config: dict) -> str:
     ----------
     connection_config : dict
         A dictionary containing the database connection parameters. Expected keys include
-        'sqlalchemy_driver', 'odbc_driver', 'server', 'database', and optionally 'username',
-        'password', and 'options'. The 'username' and 'password' can be environment variable
-        keys enclosed in curly braces (e.g., "${ENV_VAR}").
+        'dialect', 'dbapi', 'host' and optionally 'username', 'password', and 'options'.
+        The 'username' and 'password' can be environment variable keys enclosed in
+        curly braces (e.g., "${ENV_VAR}").
 
     Returns
     -------
-    str
-        The constructed database connection string.
+    URL
+        The constructed database connection URL.
 
     Raises
     ------
     EnvironmentError
         If 'username' and/or 'password' are specified as environment variables in the configuration
         and these variables are not found in either the current directory's 'sqlconnect.env' file or the
         user's home directory 'sqlconnect.env' file.
+    """
 
-    Examples
-    --------
-    >>> connection_config = {
-    ...     "sqlalchemy_driver": "mssql+pyodbc",
-    ...     "odbc_driver": "ODBC Driver 17 for SQL Server",
-    ...     "server": "my_server",
-    ...     "database": "my_database",
-    ...     "username": "${DB_USER}",
-    ...     "password": "${DB_PASS}"
-    ... }
-    >>> get_db_url(connection_config)
-    'mssql+pyodbc://username:password@my_server/my_database?driver=ODBC Driver 17 for SQL Server'
+    # Required
+    dialect = connection_config["dialect"]
+    dbapi = connection_config["dbapi"]
+    host = connection_config["host"]
+
+    # Optional
+    database = connection_config.get("database")
+    username, password = get_credentials(connection_config)
+    query = connection_config.get("options")
+
+    return URL.create(
+        f"{dialect}+{dbapi}",
+        host=host,
+        database=database,
+        username=username,
+        password=password,
+        query=query,
+    )
 
-    Notes
-    -----
-    The function checks for 'sqlconnect.env' files in the current directory and the user's home directory
-    for environment variables if 'username' and 'password' are provided as environment variable keys.
-    It uses `os.getenv` to retrieve these environment variables.
+
+def load_environment_file(file_paths: list[Path]):
+    """Load environment variables from the first existing .env file in the provided list of file paths."""
+    for file_path in file_paths:
+        if file_path.exists():
+            load_dotenv(file_path)
+            return True
+    return False
+
+
+def get_credentials(connection_config: dict) -> tuple:
     """
+    Retrieves credentials from environment variables based on the provided connection configuration.
+    If 'username' or 'password' keys are not present in connection_config, returns None for them.
 
-    sqlalchemy_driver = connection_config["sqlalchemy_driver"]
-    odbc_driver = connection_config["odbc_driver"]
-    server = connection_config["server"]
-    database = connection_config["database"]
-    options = "&".join([f"driver={odbc_driver}"] + connection_config.get("options", []))
-
-    # Check for username and password
-    auth_details = ""
-    if "username" in connection_config and "password" in connection_config:
-        if Path("sqlconnect.env").exists():
-            load_dotenv(Path("sqlconnect.env"))
-        else:
-            home_env_path = Path.home() / "sqlconnect.env"
-            if home_env_path.exists():
-                load_dotenv(home_env_path)
-
-        env_username_key = connection_config["username"].strip("${}")
-        env_password_key = connection_config["password"].strip("${}")
-        username = os.getenv(env_username_key)
-        password = os.getenv(env_password_key)
-
-        if username is None or password is None:
-            raise EnvironmentError(
-                f"Environment variables '{env_username_key}' and/or '{env_password_key}' not "
-                f"found in {Path('sqlconnect.env').absolute()} or {Path.home() / 'sqlconnect.env'}"
-            )
-
-        auth_details = f"{username}:{password}@"
-
-    # Construct the connection string
-    connection_string = (
-        f"{sqlalchemy_driver}://{auth_details}{server}/{database}?{options}"
-    )
-    return connection_string
+    Parameters
+    ----------
+    connection_config (dict): A dictionary possibly containing keys 'username' and 'password' with environment variable names.
+
+    Returns
+    -------
+    tuple: A tuple containing the username and password, or None for each if not found.
+    """
+    load_environment_file([Path("sqlconnect.env"), Path.home() / "sqlconnect.env"])
+
+    env_username_key = connection_config.get("username")
+    env_password_key = connection_config.get("password")
+    if env_username_key:
+        env_username_key = env_username_key.strip("${}")
+    if env_password_key:
+        env_password_key = env_password_key.strip("${}")
+
+    # Get username and password from environment variables, or default to None.
+    username = os.getenv(env_username_key) if env_username_key else None
+    password = os.getenv(env_password_key) if env_password_key else None
+
+    if (env_username_key and not username) or (env_password_key and not password):
+        raise EnvironmentError(
+            f"Environment variables '{env_username_key}' and/or '{env_password_key}' not "
+            f"found in {Path('sqlconnect.env').absolute()} or {Path.home() / 'sqlconnect.env'}"
+        )
+
+    return username, password

tests/integration/inputs/mssql_sqlconnect.env

@@ -1,2 +1,2 @@
 MSSQL_USERNAME=sa
-MSSQL_PASSWORD=MySecretPassw0rd!
+MSSQL_PASSWORD=test_p@ssword_f0r_CI!

tests/integration/inputs/mssql_sqlconnect.yaml

@@ -1,8 +1,10 @@
 connections:
   Mssql:
-    sqlalchemy_driver: 'mssql+pyodbc'
-    server: 'mssql_db:1433'
-    odbc_driver: 'ODBC+Driver+17+for+SQL+Server'
+    dialect: 'mssql'
+    dbapi: 'pyodbc'
+    host: 'mssql_db'
     database: 'master'
     username: '${MSSQL_USERNAME}'
     password: '${MSSQL_PASSWORD}'
+    options: 
+      driver: 'ODBC Driver 17 for SQL Server'

tests/integration/inputs/mysql_sqlconnect.env

@@ -1,2 +1,2 @@
 MYSQL_USERNAME=root
-MYSQL_PASSWORD=MySecretPassw0rd!
+MYSQL_PASSWORD=test_p@ssword_f0r_CI!

tests/integration/inputs/mysql_sqlconnect.yaml

@@ -1,8 +1,8 @@
 connections:
   Mysql:
-    sqlalchemy_driver: 'mysql+pymysql'
-    server: 'mysql_db:3306'
-    odbc_driver: ''
+    dialect: 'mysql'
+    dbapi: 'pymysql'
+    host: 'mysql_db'
     database: 'sys'
     username: '${MYSQL_USERNAME}'
     password: '${MYSQL_PASSWORD}'

tests/integration/inputs/oracle_sqlconnect.env

@@ -1,2 +1,2 @@
 ORACLE_USERNAME=system
-ORACLE_PASSWORD=MySecretPassw0rd!
+ORACLE_PASSWORD=test_p@ssword_f0r_CI!

tests/integration/inputs/oracle_sqlconnect.yaml

@@ -1,10 +1,9 @@
 connections:
   Oracle:
-    sqlalchemy_driver: 'oracle+oracledb'
-    server: 'oracle_db:1521'
-    odbc_driver: ''
-    database: ''
+    dialect: 'oracle'
+    dbapi: 'oracledb'
+    host: 'oracle_db'
     username: '${ORACLE_USERNAME}'
     password: '${ORACLE_PASSWORD}'
     options:
-      - 'service_name=XE'
+      service_name: 'XE'

tests/integration/inputs/postgres_sqlconnect.env

@@ -1,2 +1,2 @@
 POSTGRES_USERNAME=postgres
-POSTGRES_PASSWORD=mysecretpassword
+POSTGRES_PASSWORD=test_p@ssword_f0r_CI!

tests/integration/inputs/postgres_sqlconnect.yaml

@@ -1,8 +1,8 @@
 connections:
   Postgres:
-    sqlalchemy_driver: 'postgresql+psycopg2'
-    server: 'postgres_db:5432'
-    odbc_driver: ''
+    dialect: 'postgresql'
+    dbapi: 'psycopg2'
+    host: 'postgres_db'
     database: 'postgres'
     username: '${POSTGRES_USERNAME}'
     password: '${POSTGRES_PASSWORD}'

tests/integration/setup/migration.py

@@ -11,28 +11,28 @@
         host="postgres_db",
         database="postgres",
         username="postgres",
-        password="mysecretpassword",
+        password="test_p@ssword_f0r_CI!",
     ),
     URL.create(
         "mssql+pyodbc",
         host="mssql_db",
         database="master",
         username="sa",
-        password="MySecretPassw0rd!",
+        password="test_p@ssword_f0r_CI!",
         query={"driver": "ODBC Driver 17 for SQL Server"},
     ),
     URL.create(
         "mysql+pymysql",
         host="mysql_db",
         database="sys",
         username="root",
-        password="MySecretPassw0rd!",
+        password="test_p@ssword_f0r_CI!",
     ),
     URL.create(
         "oracle+oracledb",
         host="oracle_db",
         username="system",
-        password="MySecretPassw0rd!",
+        password="test_p@ssword_f0r_CI!",
         query={"service_name": "XE"},
     ),
 ]
@@ -54,7 +54,7 @@ def insert_data(engine, employees, data):
         with engine.connect() as connection:
             connection.execute(employees.insert(), data)
             connection.commit()
-            print(f"Data inserted successfully into {connection.engine} ")
+            print(f"Test data inserted successfully into {connection.engine} ")
     except SQLAlchemyError as e:
         print(f"An error occurred during data insertion: {e}")
 

tests/integration/test_mysql.py

@@ -38,7 +38,7 @@ def setup_connections():
         target_env.unlink()
 
 
-def test_sql_to_df_str_postgres(setup_env, setup_connections):
+def test_sql_to_df_str_mysql(setup_env, setup_connections):
     conn = sc.Sqlconnector("Mysql")
 
     df = conn.sql_to_df_str(

tests/integration/test_oracle.py

@@ -38,7 +38,7 @@ def setup_connections():
         target_env.unlink()
 
 
-def test_sql_to_df_str_postgres(setup_env, setup_connections):
+def test_sql_to_df_str_oracle(setup_env, setup_connections):
     conn = sc.Sqlconnector("Oracle")
 
     df = conn.sql_to_df_str(