Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/infrastructure/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ services:
- server

server:
build:
build:
context: ../server
dockerfile: ../infrastructure/Dockerfile.server
ports:
Expand Down
57 changes: 1 addition & 56 deletions src/server/controllers/example_controller.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import os
from schemas.schema import SampleRequest
from services.service import sample_service,AppError
from services.example_service import sample_service,AppError
from fastapi import HTTPException


Expand All @@ -18,57 +17,3 @@ def sample_controller(request:SampleRequest)->type:
)


import hashlib
from langchain_community.document_loaders import DirectoryLoader, JSONLoader
import json
from langchain_community.vectorstores.upstash import UpstashVectorStore
import getpass
from langchain_openai import OpenAIEmbeddings

#input should be a folder with json or a singular json file
def update_Database(path:str)->str: #return number of added docs

os.environ["UPSTASH_VECTOR_REST_URL"] = "https://loving-kingfish-56853-us1-vector.upstash.io"
if not os.environ.get("UPSTASH_VECTOR_REST_TOKEN"):
os.environ["UPSTASH_VECTOR_REST_TOKEN"] = getpass.getpass("Enter API key for Upstash: ")

if not os.environ.get("OPENAI_API_KEY"):
os.environ["OPENAI_API_KEY"] = getpass.getpass("Enter API key for OpenAI: ")
loader_kwargs = {
"jq_schema": ".[]", #iterate over question objects.
"text_content": False
}
loader = DirectoryLoader(
path=path,
glob="**/*.json", #ensures json
loader_cls=JSONLoader,
loader_kwargs=loader_kwargs
)
docs = loader.load()
sanitized_docs = []
doc_ids = [] # Standardized name

for doc in docs:
raw_data = json.loads(doc.page_content)
question_text = raw_data.get("question")
unique_id = hashlib.md5(question_text.encode('utf-8')).hexdigest()

doc.metadata = {
"choices": raw_data.get("choices"),
"category": raw_data.get("category"),
"answer": raw_data.get("answer"),
"difficulty": raw_data.get("difficulty"),
}
doc.page_content = question_text
doc_ids.append(unique_id)
sanitized_docs.append(doc)

embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
store = UpstashVectorStore(
embedding=embeddings
)
store.add_documents(documents=sanitized_docs, ids=doc_ids)
return f"Successfully processed {len(sanitized_docs)} documents."



File renamed without changes.
4 changes: 2 additions & 2 deletions src/server/server.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
from fastapi import FastAPI
from jwt_auth.services.auth_services import router as auth_router
from routes.example_route import router as business_router
from jwt_auth.auth_routes import router as auth_router
from routes.example_router import router as business_router
from fastapi.middleware.cors import CORSMiddleware

app = FastAPI()
Expand Down
File renamed without changes.
63 changes: 63 additions & 0 deletions src/server/services/services.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
'''IMPLEMENT SERVICES HERE'''



class AppError(Exception):
def __init__(self, message: str, status_code: int = 400):
self.message = message
self.status_code = status_code
super().__init__(message)

import hashlib,os,json,getpass
from langchain_community.document_loaders import DirectoryLoader, JSONLoader
from langchain_community.vectorstores.upstash import UpstashVectorStore
from langchain_openai import OpenAIEmbeddings

#input should be a folder with json or a singular json file
def update_Database(path:str)->str: #return number of added docs

os.environ["UPSTASH_VECTOR_REST_URL"] = "https://loving-kingfish-56853-us1-vector.upstash.io"
if not os.environ.get("UPSTASH_VECTOR_REST_TOKEN"):
os.environ["UPSTASH_VECTOR_REST_TOKEN"] = getpass.getpass("Enter API key for Upstash: ")

if not os.environ.get("OPENAI_API_KEY"):
os.environ["OPENAI_API_KEY"] = getpass.getpass("Enter API key for OpenAI: ")
loader_kwargs = {
"jq_schema": ".[]", #iterate over question objects.
"text_content": False
}
loader = DirectoryLoader(
path=path,
glob="**/*.json", #ensures json
loader_cls=JSONLoader,
loader_kwargs=loader_kwargs
)
docs = loader.load()
sanitized_docs = []
doc_ids = [] # Standardized name

for doc in docs:
raw_data = json.loads(doc.page_content)
question_text = raw_data.get("question")
unique_id = hashlib.md5(question_text.encode('utf-8')).hexdigest()

doc.metadata = {
"choices": raw_data.get("choices"),
"category": raw_data.get("category"),
"answer": raw_data.get("answer"),
"difficulty": raw_data.get("difficulty"),
}
doc.page_content = question_text
doc_ids.append(unique_id)
sanitized_docs.append(doc)

embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
store = UpstashVectorStore(
embedding=embeddings
)
store.add_documents(documents=sanitized_docs, ids=doc_ids)
return f"Successfully processed {len(sanitized_docs)} documents."