Merge pull request #5 from IDinsight/pull-main-update-docs

Pull main update docs

.secrets.baseline

@@ -484,13 +484,6 @@
       }
     ],
     "deployment/docker-compose/template.core_backend.env": [
-      {
-        "type": "Secret Keyword",
-        "filename": "deployment/docker-compose/template.core_backend.env",
-        "hashed_secret": "1c8d549f8ff68a7790ba6825cedf89735c115972",
-        "is_verified": false,
-        "line_number": 12
-      },
       {
         "type": "Secret Keyword",
         "filename": "deployment/docker-compose/template.core_backend.env",
@@ -521,13 +514,6 @@
         "is_verified": false,
         "line_number": 18
       },
-      {
-        "type": "Secret Keyword",
-        "filename": "deployment/docker-compose/template.litellm_proxy.env",
-        "hashed_secret": "8586c76a0032202d1a68b119f9d0ffe9cae81220",
-        "is_verified": false,
-        "line_number": 22
-      },
       {
         "type": "Secret Keyword",
         "filename": "deployment/docker-compose/template.litellm_proxy.env",
@@ -560,9 +546,9 @@
         "filename": "docs/develop/setup.md",
         "hashed_secret": "1c8d549f8ff68a7790ba6825cedf89735c115972",
         "is_verified": false,
-        "line_number": 278
+        "line_number": 277
       }
     ]
   },
-  "generated_at": "2024-08-14T13:31:04Z"
+  "generated_at": "2024-08-14T18:24:15Z"
 }

Makefile

@@ -37,7 +37,7 @@ teardown-dev: teardown-db teardown-redis teardown-embeddings
 add-users-to-db:
 	$(CONDA_ACTIVATE) $(PROJECT_NAME); \
 	set -a && \
-        source $(CURDIR)/deployment/docker-compose/.core_backend.env && \
+        source "$(CURDIR)/deployment/docker-compose/.core_backend.env" && \
         set +a && \
 	python core_backend/add_users_to_db.py
 
@@ -52,8 +52,8 @@ setup-db:
 		-p 5432:5432 \
 		-d pgvector/pgvector:pg16
 	set -a && \
-        source $(CURDIR)/deployment/docker-compose/.base.env && \
-        source $(CURDIR)/deployment/docker-compose/.core_backend.env && \
+        source "$(CURDIR)/deployment/docker-compose/.base.env" && \
+        source "$(CURDIR)/deployment/docker-compose/.core_backend.env" && \
         set +a && \
 	cd core_backend && \
 	python -m alembic upgrade head
@@ -113,7 +113,7 @@ setup-embeddings-arm:
 	@sleep 2
 	@docker run \
 		--name huggingface-embeddings \
-        -p 8080:80 \
+        -p 8081:80 \
         -v "$(PWD)/data:/data" \
         -d text-embeddings-inference-arm \
         --model-id $(HUGGINGFACE_MODEL) \
@@ -126,7 +126,7 @@ setup-embeddings:
 	@sleep 2
 	@docker run \
 		--name huggingface-embeddings \
-		-p 8080:80 \
+		-p 8081:80 \
 		-v "$(PWD)/data:/data" \
 		--pull always ghcr.io/huggingface/text-embeddings-inference:cpu-1.5 \
 		--model-id $(HUGGINGFACE_MODEL) \

admin_app/src/app/playground/components/PlaygroundComponents.tsx

@@ -145,9 +145,9 @@ const PersistentSearchBar = ({
                   <MenuItem value="embeddings-search" autoFocus={true}>
                     <Typography variant="body2">Content Search</Typography>
                   </MenuItem>
-                  <MenuItem value="llm-response">
+                  {/* <MenuItem value="llm-response">
                     <Typography variant="body2">AI Response</Typography>
-                  </MenuItem>
+                  </MenuItem>*/}
                   <MenuItem value="urgency-detection">
                     <Typography variant="body2">Urgency Detection</Typography>
                   </MenuItem>

core_backend/add_dummy_data_to_db.py

@@ -23,7 +23,7 @@
         sys.path.append(PACKAGE_PATH)
 
 
-from app.contents.config import PGVECTOR_VECTOR_SIZE
+from app.config import PGVECTOR_VECTOR_SIZE
 from app.contents.models import ContentDB
 from app.database import get_session
 from app.question_answer.models import (

core_backend/app/config.py

@@ -12,6 +12,12 @@
 POSTGRES_PORT = os.environ.get("POSTGRES_PORT", "5432")
 POSTGRES_DB = os.environ.get("POSTGRES_DB", "postgres")
 
+# PGVector variables
+PGVECTOR_VECTOR_SIZE = os.environ.get("PGVECTOR_VECTOR_SIZE", "1024")
+PGVECTOR_M = os.environ.get("PGVECTOR_M", "16")
+PGVECTOR_EF_CONSTRUCTION = os.environ.get("PGVECTOR_EF_CONSTRUCTION", "64")
+PGVECTOR_DISTANCE = os.environ.get("PGVECTOR_DISTANCE", "vector_cosine_ops")
+
 # LiteLLM proxy variables
 # Endpoint
 LITELLM_ENDPOINT = os.environ.get("LITELLM_ENDPOINT", "http://localhost:8080")

core_backend/app/contents/models.py

@@ -22,16 +22,16 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Mapped, mapped_column, relationship, selectinload
 
-from ..models import Base, JSONDict
-from ..schemas import FeedbackSentiment, QuerySearchResult
-from ..tags.models import content_tags_table
-from ..utils import embedding
-from .config import (
+from ..config import (
     PGVECTOR_DISTANCE,
     PGVECTOR_EF_CONSTRUCTION,
     PGVECTOR_M,
     PGVECTOR_VECTOR_SIZE,
 )
+from ..models import Base, JSONDict
+from ..schemas import FeedbackSentiment, QuerySearchResult
+from ..tags.models import content_tags_table
+from ..utils import embedding
 from .schemas import ContentCreate, ContentUpdate
 
 

core_backend/app/urgency_rules/models.py

@@ -18,7 +18,7 @@
 from sqlalchemy.ext.asyncio import AsyncSession
 from sqlalchemy.orm import Mapped, mapped_column
 
-from ..contents.config import PGVECTOR_VECTOR_SIZE
+from ..config import PGVECTOR_VECTOR_SIZE
 from ..models import Base, JSONDict
 from ..utils import embedding
 from .schemas import UrgencyRuleCosineDistance, UrgencyRuleCreate

core_backend/migrations/versions/2024_02_18_f269c75dbf69_create_content_table.py

@@ -11,7 +11,7 @@
 import pgvector
 import sqlalchemy as sa
 from alembic import op
-from app.contents.config import (
+from app.config import (
     PGVECTOR_DISTANCE,
     PGVECTOR_EF_CONSTRUCTION,
     PGVECTOR_M,

core_backend/migrations/versions/2024_04_30_72d3851d44a6_create_ud_tables.py

@@ -11,7 +11,7 @@
 import pgvector
 import sqlalchemy as sa
 from alembic import op
-from app.contents.config import PGVECTOR_VECTOR_SIZE
+from app.config import PGVECTOR_VECTOR_SIZE
 
 # revision identifiers, used by Alembic.
 revision: str = "72d3851d44a6"

core_backend/tests/api/conftest.py

@@ -18,8 +18,8 @@
     LITELLM_API_KEY,
     LITELLM_ENDPOINT,
     LITELLM_MODEL_EMBEDDING,
+    PGVECTOR_VECTOR_SIZE,
 )
-from core_backend.app.contents.config import PGVECTOR_VECTOR_SIZE
 from core_backend.app.contents.models import ContentDB
 from core_backend.app.database import (
     SYNC_DB_API,

core_backend/tests/api/test_dashboard.py

@@ -7,7 +7,7 @@
 from sqlalchemy import delete
 from sqlalchemy.ext.asyncio import AsyncSession
 
-from core_backend.app.contents.config import PGVECTOR_VECTOR_SIZE
+from core_backend.app.config import PGVECTOR_VECTOR_SIZE
 from core_backend.app.contents.models import ContentDB
 from core_backend.app.dashboard.models import (
     get_content_feedback_stats,

deployment/docker-compose/docker-compose.dev.yml

@@ -12,8 +12,8 @@ services:
       - .core_backend.env
     volumes:
       - db_volume:/var/lib/postgresql/data
-    ports: # Expose the port to port 5434 on the host machine for debugging
-      - 5434:5432
+    ports:
+      - 5432:5432
 
 volumes:
   db_volume:

deployment/docker-compose/docker-compose.yml

@@ -50,27 +50,28 @@ services:
     env_file:
       - .base.env
 
-  local-embeddings:
+  huggingface-embeddings:
     # image either refers to locally built image or defaults to the one from the registry
     image: ${EMBEDDINGS_IMAGE_NAME:-ghcr.io/huggingface/text-embeddings-inference:cpu-1.5}
     profiles:
-      - local-embeddings
+      - huggingface-embeddings
       - optional-components
     volumes:
       - $PWD/data:/data
     command:
       ["--model-id", "${HUGGINGFACE_MODEL}", "--api-key", "${LITELLM_API_KEY}"]
     restart: always
     env_file:
-      - .litellm_proxy.env
+      - .core_backend.env
     develop:
       watch:
         - action: rebuild
           path: ../../optional_components/embeddings
 
   redis:
     image: "redis:6.0-alpine"
-    ports: # Expose the port to port 6380 on the host machine for debugging
+    ports:
+      # Expose the port to port 6380 on the host machine for debugging
       - "6380:6379"
     restart: always
 

deployment/docker-compose/template.base.env

@@ -8,11 +8,6 @@ BACKEND_ROOT_PATH="/api"
 # This is the path that core_backend will be hosted on.
 # Only change if you want to use a different backend root path.
 
-#### Google OAuth Client ID ###################################################
-# NEXT_PUBLIC_GOOGLE_LOGIN_CLIENT_ID="update-me"
-# If you want to use Google OAuth, set the correct value for your production. This value
-# is used by core_backend and admin_app.
-
 #### Backend URL ##############################################################
 NEXT_PUBLIC_BACKEND_URL="https://${DOMAIN}${BACKEND_ROOT_PATH}"
 # Do not change this value. This value is used by admin_app.

deployment/docker-compose/template.core_backend.env

@@ -9,7 +9,7 @@ POSTGRES_DB=postgres
 
 #### 🔒 Admin user -- change for production ####################################
 ADMIN_USERNAME="admin"
-ADMIN_PASSWORD="fullaccess"
+ADMIN_PASSWORD="fullaccess" #pragma: allowlist secret
 ADMIN_API_KEY="admin-key"
 
 #### Admin user rate limits ###################################################
@@ -24,36 +24,29 @@ REDIS_HOST="redis://localhost:6379"
 # For docker compose, use "redis://redis:6379"
 
 #### 🔒 LiteLLM Proxy Server -- change for production ##########################
-LITELLM_ENDPOINT="http://local-embeddings"
+LITELLM_ENDPOINT="http://localhost:8081"
 LITELLM_API_KEY="update-embeddings-api-key"
 HUGGINGFACE_MODEL="Alibaba-NLP/gte-large-en-v1.5"
-# For docker compose, use "http://local-embeddings"
+# For docker compose, use "http://huggingface-embeddings"
 
 #### Variables for Huggingface embeddings container ###########################
 # If on ARM, you need to build the embeddings image manually using
 # `make build-embeddings-arm` from repository root and set the following variables
-# EMBEDDINGS_IMAGE_NAME=text-embeddings-inference-arm
-# PGVECTOR_VECTOR_SIZE=1024
+#EMBEDDINGS_IMAGE_NAME=text-embeddings-inference-arm
+PGVECTOR_VECTOR_SIZE=1024
 
 #### Temporary folder for prometheus gunicorn multiprocess ####################
 PROMETHEUS_MULTIPROC_DIR="/tmp"
 
 #### Application-wide content limits ##########################################
-# CHECK_CONTENT_LIMIT=True
+# CHECK_CONTENT_LIMIT=False
 # DEFAULT_CONTENT_QUOTA=50
 
 #### Number of top content to return for /search. #############################
 # N_TOP_CONTENT=5
 
 #### Urgency detection variables ##############################################
-# URGENCY_CLASSIFIER="cosine_distance_classifier"
-# Choose between `cosine_distance_classifier` and `llm_entailment_classifier`
-
-# URGENCY_DETECTION_MAX_DISTANCE=0.5
-# Only used if URGENCY_CLASSIFIER=cosine_distance_classifier
-
-# URGENCY_DETECTION_MIN_PROBABILITY=0.5
-# Only used if URGENCY_CLASSIFIER=llm_entailment_classifier
+URGENCY_CLASSIFIER="cosine_distance_classifier"
 
 #### LiteLLM tracing ##########################################################
 LANGFUSE=False

deployment/docker-compose/template.litellm_proxy.env

@@ -19,8 +19,8 @@ OPENAI_API_KEY="sk-..."
 
 #### 🔒 Huggingface embeddings -- change for production ########################
 # HUGGINGFACE_MODEL="Alibaba-NLP/gte-large-en-v1.5"
-# CUSTOM_EMBEDDINGS_API_KEY="embeddings"
-# CUSTOM_EMBEDDINGS_ENDPOINT="http://local-embeddings"
+# HUGGINGFACE_EMBEDDINGS_API_KEY="embeddings"  #pragma: allowlist secret
+# HUGGINGFACE_EMBEDDINGS_ENDPOINT="http://huggingface-embeddings"
 # This default endpoint value should work with docker compose.
 
 #### 🔒 LiteLLM Proxy UI -- change for production ##############################