Nu dev

.github/workflows/backend-docker-image-build.yml

@@ -4,7 +4,8 @@ on:
   push:
     branches:
       - "**"
-
+    paths:
+      - 'services/backend/**'
 jobs:
   backend_build:
     runs-on: ubuntu-latest
@@ -31,3 +32,14 @@ jobs:
           docker build -t $IMAGE .
           docker push $IMAGE
           cd ../..
+
+      - name: Deploy to Cloud Run
+        run: |
+          gcloud run deploy backend-service \
+            --source services/backend \
+            --region ${{ secrets.GCP_REGION }} \
+            --platform managed \
+            --allow-unauthenticated \
+            --memory 4Gi \
+            --timeout 3600s \
+            --set-env-vars "AIRFLOW_UID=5000,BASE_URL=https://www.khoury.northeastern.edu/,MAX_DEPTH=3,CONCURRENT_REQUESTS=10,DATA_FOLDER=scraped_data,MISTRAL_API_KEY=${{ secrets.MISTRAL_API_KEY }},MLFLOW_TRACKING_URI=${{ secrets.MLFLOW_TRACKING_URI }},BUCKET_NAME=${{ secrets.BUCKET_NAME }},RAW_DATA_FOLDER=raw_data,FAISS_INDEX_FOLDER=faiss_index,URLS_LIST= "https://www.khoury.northeastern.edu/""

.github/workflows/frontend-docker-image-build.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - "**"
+    paths:
+      - 'services/frontend/**'
 
 jobs:
   frontend_build:
@@ -31,3 +33,14 @@ jobs:
           docker build -t $IMAGE .
           docker push $IMAGE
           cd ../..
+
+      - name: Deploy to Cloud Run
+        run: |
+          gcloud run deploy frontend-service \
+            --source services/frontend \
+            --region ${{ secrets.GCP_REGION }} \
+            --platform managed \
+            --allow-unauthenticated \
+            --memory 1Gi \
+            --timeout 1800s \
+            --set-env-vars "API_URL=https://backend-service-273412-default.run.app/NuBot/"

.github/workflows/frontend1-docker-image-build.yml

@@ -0,0 +1,45 @@
+name: "build_reactfrontned_image"
+
+on:
+  push:
+    branches:
+      - "**"
+    paths:
+      - "services/frontend1/**"
+jobs:
+  backend_build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: GCP Authentication
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: "${{ secrets.GCP_KEY }}"
+
+      - name: Setup gcloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Docker login for Artifact Registry
+        run: |
+          gcloud auth configure-docker ${{ secrets.GCP_ARTIFACT_REGISTRY_REGION }}-docker.pkg.dev
+
+      - name: Build and Push Backend Image
+        run: |
+          cd services/frontend1
+          IMAGE=${{ secrets.GCP_ARTIFACT_REGISTRY_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/backend-nubot/react-service:latest
+          docker build -t $IMAGE .
+          docker push $IMAGE
+          cd ../..
+
+      - name: Deploy to Cloud Run
+        run: |
+          gcloud run deploy react-service \
+            --source services/frontend1 \
+            --region ${{ secrets.GCP_REGION }} \
+            --platform managed \
+            --allow-unauthenticated \
+            --memory 1Gi \
+            --timeout 3600s \
+            --set-env-vars "REACT_APP_API_URL=${{secrets.REACT_APP_API_URL}}"

.github/workflows/prefect_orchestraiton.yml

@@ -0,0 +1,59 @@
+name: Deploy Prefect Flow to Cloud Run
+
+on:
+  push:
+    branches: ["**"] # Trigger on push to main (adjust as needed)
+
+jobs:
+  build-and-deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+
+      # Authenticate to Google Cloud using the service account JSON key
+      - name: GCP Authentication
+        uses: google-github-actions/auth@v2
+        with:
+          credentials_json: "${{ secrets.GCP_KEY }}"
+
+      - name: Setup gcloud SDK
+        uses: google-github-actions/setup-gcloud@v2
+
+      - name: Docker login for Artifact Registry
+        run: |
+          gcloud auth configure-docker ${{ secrets.GCP_ARTIFACT_REGISTRY_REGION }}-docker.pkg.dev
+
+      - name: Build Docker image
+        run: |
+          cd prefectWorkflows
+          IMAGE_URI=${{ secrets.GCP_ARTIFACT_REGISTRY_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/backend-nubot/prefect-scraper:latest
+          echo "Building image $IMAGE_URI"
+          docker build -t "$IMAGE_URI"   .
+          # Note: The context is the repository root (.), adjust path to Dockerfile if needed.
+
+      - name: Push Docker image to Artifact Registry
+        run: |
+          IMAGE_URI=${{ secrets.GCP_ARTIFACT_REGISTRY_REGION }}-docker.pkg.dev/${{ secrets.GCP_PROJECT_ID }}/backend-nubot/prefect-scraper:latest
+          docker push "$IMAGE_URI"
+        # After this step, the image is available in Artifact Registry for Cloud Run to use.
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+
+      - name: Install Prefect3
+        run: pip install --no-cache-dir "prefect>=3.2.4"
+
+      - name: Deploy via prefect deploy
+        run: |
+          cd prefectWorkflows
+          prefect deploy -n scraper-cron-deployment           # tell pool to use latest image
+
+      - name: Deploy Prefect flow
+        run: |
+          cd prefectWorkflows   # navigate to the folder containing prefect.yaml
+          prefect deploy -n scraperflow-deployment
+        # The -n flag ensures we deploy the specific deployment by name (optional if only one deployment in YAML).
+        # This command reads prefect.yaml and registers/updates the deployment in Prefect Cloud.

.gitignore

@@ -177,6 +177,7 @@ mlflow-artifacts/
 # PyPI configuration file
 .pypirc
 
-*.html
+
 
 *.json
+!package.json

airflow/dags/dataflow/chunk_data.py

@@ -11,7 +11,8 @@
 from dataflow.store_data import upload_faiss_index_to_bucket
 load_dotenv(override=True)
 BUCKET_NAME= os.getenv('BUCKET_NAME')
-GOOGLE_APPLICATION_CREDENTIALS=os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
+from google.auth import default
+credentials, project = default()
 RAW_DATA_FOLDER= os.getenv('RAW_DATA_FOLDER')
 def chunk_data():
     # Load all JSON files from a directory

airflow/dags/dataflow/scraper.py

@@ -13,7 +13,8 @@
 BASE_URL = os.getenv('BASE_URL')
 MAX_DEPTH = int(os.getenv('MAX_DEPTH'))             # Maximum recursion depth (base URL is depth 0)
 CONCURRENT_REQUESTS = int(os.getenv('CONCURRENT_REQUESTS'))  # Maximum number of concurrent requests
-GOOGLE_APPLICATION_CREDENTIALS =os.getenv('GOOGLE_APPLICATION_CREDENTIALS ')
+from google.auth import default
+credentials, project = default()
 # Create folder for JSON data
 DATA_FOLDER = "scraped_data"
 if not os.path.exists(DATA_FOLDER):

airflow/dags/dataflow/store_data.py

@@ -5,7 +5,8 @@
 BUCKET_NAME= os.getenv('BUCKET_NAME')
 RAW_DATA_FOLDER= os.getenv('RAW_DATA_FOLDER')
 FAISS_INDEX_FOLDER= os.getenv('FAISS_INDEX_FOLDER')
-GOOGLE_APPLICATION_CREDENTIALS=os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
+from google.auth import default
+credentials, project = default()
 
 def get_blob_from_bucket():
     storage_client = Client()

prefectWorkflows/.dockerignore

@@ -0,0 +1,3 @@
+*.env
+scraped_data/
+faiss_index/

prefectWorkflows/.env

@@ -6,4 +6,6 @@ DATA_FOLDER = "scraped_data"
 BUCKET_NAME=scraped_raw_data_nubot
 RAW_DATA_FOLDER=raw_data
 FAISS_INDEX_FOLDER=faiss_index
-GOOGLE_APPLICATION_CREDENTIALS="E:/gcpkeys/nubot/nubot-nikhil-6adeee091d55.json"
+GOOGLE_APPLICATION_CREDENTIALS="E:/gcpkeys/nubot/nubot-nikhil-6adeee091d55.json"
+PREFECT_API_KEY=pnu_mRGcrBkC9qyFbwGfgrVbjbOoL7WIZ411TKYp
+PREFECT_API_URL="https://api.prefect.cloud/api/accounts/806f2e07-5063-4fbe-9b46-0545ad5de2d1/workspaces/acdf9e9e-8a55-446a-ac46-80a3f843d8b6"

prefectWorkflows/Dockerfile

@@ -0,0 +1,25 @@
+# Start from a lightweight Python image (use the appropriate Python version)
+FROM python:3.10-slim
+
+# Set working directory in container
+WORKDIR /app
+
+# Install Python dependencies.
+# If you have a requirements.txt, copy and install it:
+COPY requirements.txt . 
+
+RUN pip install --no-cache-dir -r requirements.txt
+
+# (Alternatively, directly install Prefect and any needed libraries)
+# RUN pip install prefect==3.1.10
+
+# Copy the Prefect flow code and the dataflow module into the image
+COPY . .
+
+
+# Ensure Python can find the 'dataflow' module (add /app to PYTHONPATH)
+ENV PYTHONPATH="/app:${PYTHONPATH}"
+
+# (Optional) Set a default command (Prefect Cloud will override this when submitting the flow run)
+# By default, do nothing or use a generic command. Prefect Cloud's work pool will specify the entrypoint at runtime.
+CMD ["python", "-c", "print('Container built for Prefect flow execution')"]

prefectWorkflows/dataflow/chunk_data.py

@@ -11,7 +11,20 @@
 from dataflow.store_data import upload_faiss_index_to_bucket
 load_dotenv(override=True)
 BUCKET_NAME= os.getenv('BUCKET_NAME')
-GOOGLE_APPLICATION_CREDENTIALS=os.getenv('GOOGLE_APPLICATION_CREDENTIALS')
+from google.auth import default
+from google.oauth2 import service_account
+
+# Try to get credentials - works in both Docker and Cloud Run
+try:
+    # First try Application Default Credentials (works in Cloud Run)
+    credentials, project = default()
+except Exception:
+    # Fall back to explicit credentials file (for Docker)
+    credentials_path = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")
+    if credentials_path:
+        credentials = service_account.Credentials.from_service_account_file(credentials_path)
+    else:
+        raise Exception("No credentials available")
 RAW_DATA_FOLDER= os.getenv('RAW_DATA_FOLDER')
 def chunk_data():
     # Load all JSON files from a directory
@@ -52,4 +65,4 @@ def chunk_data():
 
 if __name__=="__main__":
     chunk_data()
-    
+    upload_faiss_index_to_bucket()