Chore/1

.github/workflows/demigrate.yml

@@ -24,46 +24,36 @@ jobs:
       - name: Deploy to EC2
         env:
           SSH_KEY: ${{ secrets.EC2_SSH_KEY }}
-          HOST: ${{ secrets.EC2_STG_HOST }}
-          DB_URL: ${{ secrets.DB_URL }}
+          HOST: ${{ secrets.EC2_STG_HOST }} # Assuming STG host is for RDS
+          DB_URL: ${{ secrets.DB_URL }} # DB_URL secret now holds the DB host
+          DB_USERNAME: ${{ secrets.DB_USERNAME }}
+          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
         run: |
           # Setup SSH
           mkdir -p ~/.ssh
           ssh-keyscan -H $HOST >> ~/.ssh/known_hosts
           echo "$SSH_KEY" > key.pem
           chmod 600 key.pem
 
-          # Validate DB_URL secret
-          if [ -z "$DB_URL" ]; then
-            echo "Error: DB_URL secret is not set or is empty. Please configure the DB_URL secret in your repository settings."
+          # Validate DB secrets
+          if [ -z "$DB_URL" ] || [ -z "$DB_USERNAME" ] || [ -z "$DB_PASSWORD" ]; then
+            echo "Error: One or more DB secrets (DB_URL, DB_USERNAME, DB_PASSWORD) are not set or are empty. Please configure them in your repository settings."
             exit 1
           fi
 
           # Define project path on EC2
           PROJECT_PATH="/home/ubuntu/BeyondU-Data"
 
-          # SSH to EC2 and run demigration script
-          ssh -i key.pem ubuntu@$HOST "
-            # Navigate to the project directory
-            cd $PROJECT_PATH
-
-            # Pull the latest changes
-            git pull
-
-            # Set up python virtual environment and install dependencies
-            python3 -m venv venv
-            source venv/bin/activate
-            pip install -r requirements.txt
+                              # Construct the full DATABASE_URL for MySQL
 
-                        if [ "${{ github.event.inputs.confirm_drop_db }}" != "true" ]; then
+                              # Assuming default MySQL port 3306 and a database named 'beyondu' (or your application DB) on RDS
 
-                          echo "Database drop not confirmed. Exiting."
+                              FULL_DATABASE_URL="mysql+mysqlconnector://${DB_USERNAME}:${DB_PASSWORD}@${DB_URL}:3306/beyondu"
 
-                          exit 1
-
-                        fi
-
-                        # Run the ETL script with --drop-db to clear all data, passing DATABASE_URL as environment variable
+          # SSH to EC2 and run demigration script
+          ssh -i key.pem ubuntu@$HOST "
+            # ... (inside SSH block) ...
 
-                        DATABASE_URL="$DB_URL" python scripts/run_etl.py --drop-db
+            # Run the ETL script with --drop-db to clear all data, passing DATABASE_URL as environment variable
+            DATABASE_URL=\"$FULL_DATABASE_URL\" python scripts/run_etl.py --drop-db
           "

.github/workflows/migrate.yml

@@ -21,42 +21,115 @@ jobs:
       - name: Deploy to EC2
         env:
           SSH_KEY: ${{ secrets.EC2_SSH_KEY }}
-          HOST: ${{ secrets.EC2_STG_HOST }}
-          DB_URL: ${{ secrets.DB_URL }}
+          HOST: ${{ secrets.EC2_STG_HOST }} # Assuming STG host is for RDS
+          DB_URL: ${{ secrets.DB_URL }} # DB_URL secret now holds the DB host
+          DB_USERNAME: ${{ secrets.DB_USERNAME }}
+          DB_PASSWORD: ${{ secrets.DB_PASSWORD }}
         run: |
           # Setup SSH
           mkdir -p ~/.ssh
           ssh-keyscan -H $HOST >> ~/.ssh/known_hosts
           echo "$SSH_KEY" > key.pem
           chmod 600 key.pem
 
-          # Validate DB_URL secret
-          if [ -z "$DB_URL" ]; then
-            echo "Error: DB_URL secret is not set or is empty. Please configure the DB_URL secret in your repository settings."
-            exit 1
-          fi
+                    # Validate DB secrets
 
-          # Define project path on EC2
-          PROJECT_PATH="/home/ubuntu/BeyondU-Data"
+                    if [ -z "$DB_URL" ] || [ -z "$DB_USERNAME" ] || [ -z "$DB_PASSWORD" ]; then
 
-          # SSH to EC2 and run migration script
-          ssh -i key.pem ubuntu@$HOST "
+                      echo "Error: One or more DB secrets (DB_URL, DB_USERNAME, DB_PASSWORD) are not set or are empty. Please configure them in your repository settings."
 
-          # Navigate to the project directory
-          cd $PROJECT_PATH
+                      exit 1
 
-          # Pull the latest changes
-          git pull
+                    fi
 
+
 
-          # Set up python virtual environment and install dependencies
-          python3 -m venv venv
-          source venv/bin/activate
-          pip install -r requirements.txt
-          if [ "${{ github.event.inputs.confirm_migration }}" != "true" ]; then
-            echo "Database migration not confirmed. Exiting."
-            exit 1
-          fi
-          # Run the ETL script with DATABASE_URL as environment variable
-          DATABASE_URL="$DB_URL" python scripts/run_etl.py --input data/raw --latest-only --init-db
-          "
+                    # Define project path on EC2
+
+                    PROJECT_PATH="/home/ubuntu/BeyondU-Data"
+
+
+
+                    # Construct the full DATABASE_URL for MySQL
+                    # Assuming default MySQL port 3306 and a database named 'beyondu' (or your application DB) on RDS
+                    FULL_DATABASE_URL="mysql+mysqlconnector://${DB_USERNAME}:${DB_PASSWORD}@${DB_URL}:3306/beyondu"
+
+
+
+                    # SSH to EC2 and run migration script
+
+                    ssh -i key.pem ubuntu@$HOST "
+
+                      # Check if project directory exists, if not clone it
+
+                      if [ ! -d \"$PROJECT_PATH\" ]; then
+
+                        echo \"Cloning repository to $PROJECT_PATH\"
+
+                        git clone https://github.com/${{ github.repository }}.git \"$PROJECT_PATH\"
+
+                      fi
+
+
+
+                      # Navigate to the project directory
+
+                      cd \"$PROJECT_PATH\"
+
+
+
+                      # Pull the latest changes and ensure the correct branch is checked out
+
+                      git fetch origin
+
+                      git checkout "${{ github.ref_name }}" # Checkout the branch this workflow is running on
+
+                      git reset --hard "origin/${{ github.ref_name }}" # Force update to the latest remote commit of the current branch
+
+                      git clean -fdx # Clean up untracked files and directories, just in case
+
+
+
+                      # Debug: Remove .pyc files and inspect src/config.py on EC2
+
+                      find src -name "*.pyc" -delete
+
+                      ls -la src/config.py
+
+                      cat src/config.py
+
+
+
+                      # Install python3-venv if not already installed (idempotent)
+
+                      sudo apt-get update && sudo apt-get install -y python3-venv
+
+
+
+                      # Set up python virtual environment and install dependencies
+
+                      python3 -m venv venv
+
+                      source venv/bin/activate
+
+                      pip install -r requirements.txt
+
+
+
+                      if [ \"${{ github.event.inputs.confirm_migration }}\" != \"true\" ]; then
+
+                        echo \"Database migration not confirmed. Exiting.\"
+
+                        exit 1
+
+                      fi
+
+
+
+                      # Run the ETL script with DATABASE_URL as environment variable
+
+                      DATABASE_URL=\"$FULL_DATABASE_URL\" python scripts/run_etl.py --input data/raw --latest-only --init-db
+
+                    "
+
+

pyproject.toml

@@ -6,14 +6,14 @@ requires-python = ">=3.11"
 
 [tool.ruff]
 line-length = 100
-target-version = "py311"
+target-version = "py38"
 
 [tool.ruff.lint]
 select = ["E", "F", "I", "N", "W", "UP"]
 ignore = ["E501"]
 
 [tool.mypy]
-python_version = "3.11"
+python_version = "3.8"
 strict = true
 ignore_missing_imports = true
 

requirements.txt

@@ -4,7 +4,7 @@ openpyxl>=3.1.0
 numpy>=1.24.0
 
 # Database
-psycopg2-binary>=2.9.0
+mysql-connector-python>=8.0.0
 sqlalchemy>=2.0.0
 alembic>=1.12.0
 

scripts/run_etl.py

@@ -3,6 +3,7 @@
 import argparse
 import sys
 from pathlib import Path
+from typing import Dict, List, Optional, Union, Tuple
 
 # Add project root to path
 sys.path.insert(0, str(Path(__file__).parent.parent))
@@ -20,7 +21,7 @@ def process_file(
     file_path: Path,
     loader: DatabaseLoader,
     dry_run: bool = False,
-) -> dict[str, int]:
+) -> Dict[str, int]:
     """Process a single Excel file through the ETL pipeline."""
     logger.info(f"Processing: {file_path.name}")