From b8af136a9887144c9412be4115fe1b5f52cd0640 Mon Sep 17 00:00:00 2001 From: Carlos Samey Date: Sun, 11 Aug 2024 20:54:31 +0300 Subject: [PATCH 1/5] add test fix and remove query language check in /search --- .secrets.baseline | 45 ++++++++++++++----- core_backend/app/contents/models.py | 17 ++++--- core_backend/app/question_answer/routers.py | 3 -- core_backend/tests/api/conftest.py | 1 + .../tests/api/test_question_answer.py | 17 +++++++ 5 files changed, 60 insertions(+), 23 deletions(-) diff --git a/.secrets.baseline b/.secrets.baseline index bca8c75..66114d6 100644 --- a/.secrets.baseline +++ b/.secrets.baseline @@ -414,7 +414,7 @@ "filename": "core_backend/tests/api/conftest.py", "hashed_secret": "3465834d516797458465ae4ed2c62e7020032c4e", "is_verified": false, - "line_number": 310 + "line_number": 311 } ], "core_backend/tests/api/test.env": [ @@ -457,14 +457,14 @@ "filename": "core_backend/tests/api/test_question_answer.py", "hashed_secret": "1d2be5ef28a76e2207456e7eceabe1219305e43d", "is_verified": false, - "line_number": 284 + "line_number": 285 }, { "type": "Secret Keyword", "filename": "core_backend/tests/api/test_question_answer.py", "hashed_secret": "6367c48dd193d56ea7b0baad25b19455e529f5ee", "is_verified": false, - "line_number": 638 + "line_number": 639 } ], "core_backend/tests/api/test_user_tools.py": [ @@ -555,24 +555,24 @@ "line_number": 5 } ], - "deployment/docker-compose/template.core_backend.env": [ + "deployment\\docker-compose\\template.env": [ { "type": "Secret Keyword", - "filename": "deployment/docker-compose/template.core_backend.env", + "filename": "deployment\\docker-compose\\template.env", "hashed_secret": "1c8d549f8ff68a7790ba6825cedf89735c115972", "is_verified": false, "line_number": 12 }, { "type": "Secret Keyword", - "filename": "deployment/docker-compose/template.core_backend.env", + "filename": "deployment\\docker-compose\\template.env", "hashed_secret": "4402394e878f7a467f57e5e02d450190b511f7a8", "is_verified": false, "line_number": 13 }, { "type": "Secret Keyword", - "filename": "deployment/docker-compose/template.core_backend.env", + "filename": "deployment\\docker-compose\\template.env", "hashed_secret": "b83efb89100a5f11b5cb4418471326f6c5b669c7", "is_verified": false, "line_number": 20 @@ -624,10 +624,19 @@ "line_number": 78 } ], - "docs/components/litellm-proxy/index.md": [ + "docs\\blog\\posts\\search-endpoint.md": [ + { + "type": "Secret Keyword", + "filename": "docs\\blog\\posts\\search-endpoint.md", + "hashed_secret": "5b8b7a620e54e681c584f5b5c89152773c10c253", + "is_verified": false, + "line_number": 47 + } + ], + "docs\\components\\litellm-proxy\\index.md": [ { "type": "Secret Keyword", - "filename": "docs/components/litellm-proxy/index.md", + "filename": "docs\\components\\litellm-proxy\\index.md", "hashed_secret": "07db03b4eef371b55ee656a334d69857b58a09da", "is_verified": false, "line_number": 21 @@ -639,7 +648,21 @@ "filename": "docs/develop/setup.md", "hashed_secret": "1c8d549f8ff68a7790ba6825cedf89735c115972", "is_verified": false, - "line_number": 278 + "line_number": 178 + }, + { + "type": "Secret Keyword", + "filename": "docs\\develop\\setup.md", + "hashed_secret": "2e0300945cd913be92e5626ce4f3f8c9c15457cb", + "is_verified": false, + "line_number": 179 + }, + { + "type": "Secret Keyword", + "filename": "docs\\develop\\setup.md", + "hashed_secret": "f6d49a2cab4f8cb42ab444cad067560a23b5cc7e", + "is_verified": false, + "line_number": 184 } ], "scripts/add_dummy_data_to_db.py": [ @@ -652,5 +675,5 @@ } ] }, - "generated_at": "2024-08-12T06:41:51Z" + "generated_at": "2024-08-08T09:37:57Z" } diff --git a/core_backend/app/contents/models.py b/core_backend/app/contents/models.py index 48f6452..fea2e72 100644 --- a/core_backend/app/contents/models.py +++ b/core_backend/app/contents/models.py @@ -505,17 +505,16 @@ async def get_search_results( dictionary """ - query = select( - ContentDB, - ContentDB.content_embedding.cosine_distance(question_embedding).label( - "distance" - ), - ).where(ContentDB.user_id == user_id) + distance = ContentDB.content_embedding.cosine_distance(question_embedding).label( + "distance" + ) + + query = select(ContentDB, distance).where(ContentDB.user_id == user_id) + if exclude_archived: query = query.where(ContentDB.is_archived == false()) - query = query.order_by( - ContentDB.content_embedding.cosine_distance(question_embedding) - ).limit(n_similar) + + query = query.order_by(distance).limit(n_similar) search_result = (await asession.execute(query)).all() diff --git a/core_backend/app/question_answer/routers.py b/core_backend/app/question_answer/routers.py index 308bed1..811cdd6 100644 --- a/core_backend/app/question_answer/routers.py +++ b/core_backend/app/question_answer/routers.py @@ -276,9 +276,6 @@ async def search_base( If the question language is not identified. """ - if query_refined.original_language is None: - raise ValueError(("Identify language before calling this function.")) - # always do the embeddings search even if some guardrails have failed metadata = create_langfuse_metadata(query_id=response.query_id, user_id=user_id) search_results = await get_similar_content_async( diff --git a/core_backend/tests/api/conftest.py b/core_backend/tests/api/conftest.py index a01eb50..851304c 100644 --- a/core_backend/tests/api/conftest.py +++ b/core_backend/tests/api/conftest.py @@ -187,6 +187,7 @@ async def faq_contents( await asession.execute(deleteFeedback) await asession.execute(content_query) await asession.delete(content) + await asession.commit() diff --git a/core_backend/tests/api/test_question_answer.py b/core_backend/tests/api/test_question_answer.py index e5332f2..3c19744 100644 --- a/core_backend/tests/api/test_question_answer.py +++ b/core_backend/tests/api/test_question_answer.py @@ -198,8 +198,17 @@ def test_search_results( expected_status_code: int, client: TestClient, api_key_user1: str, + fullaccess_token: str, faq_contents: pytest.FixtureRequest, ) -> None: + while True: + response = client.get( + "/content", + headers={"Authorization": f"Bearer {fullaccess_token}"}, + ) + if len(response.json()) == 9: + break + request_token = api_key_user1 if token == "api_key_correct" else token response = client.post( "/search", @@ -387,9 +396,17 @@ def test_user2_access_user1_content( api_key_user1: str, api_key_user2: str, expect_found: bool, + fullaccess_token: str, faq_contents: List[int], ) -> None: token = api_key_user1 if username == TEST_USERNAME else api_key_user2 + while True: + response = client.get( + "/content", + headers={"Authorization": f"Bearer {fullaccess_token}"}, + ) + if len(response.json()) == 9: + break response = client.post( "/search", json={ From f010f249d3e699a80cb57bfabc8c531fc7a7f0d4 Mon Sep 17 00:00:00 2001 From: Mustafa Akolawala <112169542+MustafaAkolawala@users.noreply.github.com> Date: Mon, 12 Aug 2024 21:16:10 +0530 Subject: [PATCH 2/5] HOTFIX for alembic history issue (#390) --- ...65368ca2bac_rename_tables_and_add_user_id_column.py | 4 ++-- ...62a0b83_add_generate_tts_column_to_query_table_.py} | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) rename core_backend/migrations/versions/{2024_08_06_7192d62a0b83_add_generate_tts_column_to_query_table_.py => 2024_08_12_7192d62a0b83_add_generate_tts_column_to_query_table_.py} (78%) diff --git a/core_backend/migrations/versions/2024_08_06_465368ca2bac_rename_tables_and_add_user_id_column.py b/core_backend/migrations/versions/2024_08_06_465368ca2bac_rename_tables_and_add_user_id_column.py index fcea2f6..e8f2fdc 100644 --- a/core_backend/migrations/versions/2024_08_06_465368ca2bac_rename_tables_and_add_user_id_column.py +++ b/core_backend/migrations/versions/2024_08_06_465368ca2bac_rename_tables_and_add_user_id_column.py @@ -1,7 +1,7 @@ """Rename tables and add user_id column Revision ID: 465368ca2bac -Revises: 7192d62a0b83 +Revises: b87b336df479 Create Date: 2024-08-01 20:36:52.629572 """ @@ -13,7 +13,7 @@ # revision identifiers, used by Alembic. revision: str = "465368ca2bac" # pragma: allowlist secret -down_revision: Union[str, None] = "7192d62a0b83" +down_revision: Union[str, None] = "b87b336df479" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None diff --git a/core_backend/migrations/versions/2024_08_06_7192d62a0b83_add_generate_tts_column_to_query_table_.py b/core_backend/migrations/versions/2024_08_12_7192d62a0b83_add_generate_tts_column_to_query_table_.py similarity index 78% rename from core_backend/migrations/versions/2024_08_06_7192d62a0b83_add_generate_tts_column_to_query_table_.py rename to core_backend/migrations/versions/2024_08_12_7192d62a0b83_add_generate_tts_column_to_query_table_.py index b305c7b..478b081 100644 --- a/core_backend/migrations/versions/2024_08_06_7192d62a0b83_add_generate_tts_column_to_query_table_.py +++ b/core_backend/migrations/versions/2024_08_12_7192d62a0b83_add_generate_tts_column_to_query_table_.py @@ -1,8 +1,8 @@ """add generate_tts column to query table and tts_file column to query response table Revision ID: 7192d62a0b83 -Revises: b87b336df479 -Create Date: 2024-08-07 18:50:40.834060 +Revises: 358588881e01 +Create Date: 2024-08-12 18:50:40.834060 """ @@ -13,7 +13,7 @@ # revision identifiers, used by Alembic. revision: str = "7192d62a0b83" # pragma: allowlist secret -down_revision: Union[str, None] = "b87b336df479" +down_revision: Union[str, None] = "358588881e01" branch_labels: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None @@ -21,12 +21,12 @@ def upgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### op.add_column("query", sa.Column("generate_tts", sa.Boolean(), nullable=True)) - op.add_column("query-response", sa.Column("tts_file", sa.String(), nullable=True)) + op.add_column("query_response", sa.Column("tts_file", sa.String(), nullable=True)) # ### end Alembic commands ### def downgrade() -> None: # ### commands auto generated by Alembic - please adjust! ### - op.drop_column("query-response", "tts_file") + op.drop_column("query_response", "tts_file") op.drop_column("query", "generate_tts") # ### end Alembic commands ### From ca3c8c281715ccc8aa64c374c28578af03f2a894 Mon Sep 17 00:00:00 2001 From: Suzin You <7042047+suzinyou@users.noreply.github.com> Date: Mon, 12 Aug 2024 21:36:10 +0530 Subject: [PATCH 3/5] Overwrite CNAME not append --- .github/workflows/docs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index b76e942..e1c8271 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -42,7 +42,7 @@ jobs: run: | git fetch origin git switch gh-pages - echo "docs.ask-a-question.com" >> CNAME + echo "docs.ask-a-question.com" > CNAME git add CNAME git commit -m "Add CNAME" git push origin gh-pages From f969ce750611b66109f1fb1adcd3f86156d9e94a Mon Sep 17 00:00:00 2001 From: Suzin You <7042047+suzinyou@users.noreply.github.com> Date: Mon, 12 Aug 2024 22:02:32 +0530 Subject: [PATCH 4/5] Set default version for docs --- .github/workflows/docs.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index e1c8271..9e2ea3e 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -21,6 +21,7 @@ jobs: path: .cache restore-keys: | mkdocs-material- + - name: Install dependencies run: pip install mkdocs-material==9.5.3 \ pymdown-extensions==10.3 \ @@ -34,6 +35,7 @@ jobs: git config user.name github-actions-runner git config user.email aaq@idinsight.org git fetch origin gh-pages + mike set-default latest mike deploy \ --push \ --update-aliases ${{ github.ref_name }} latest From 3da37817436d47d6b5c247ec6d0134be081411f3 Mon Sep 17 00:00:00 2001 From: Suzin You <7042047+suzinyou@users.noreply.github.com> Date: Mon, 12 Aug 2024 22:24:14 +0530 Subject: [PATCH 5/5] pass new env vars to CI/CD scripts --- .github/workflows/deploy_gcp_admin_app.yaml | 2 +- .github/workflows/deploy_gcp_caddy.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/deploy_gcp_admin_app.yaml b/.github/workflows/deploy_gcp_admin_app.yaml index 07b8e16..1c56822 100644 --- a/.github/workflows/deploy_gcp_admin_app.yaml +++ b/.github/workflows/deploy_gcp_admin_app.yaml @@ -101,7 +101,7 @@ jobs: --network aaq-network \ --name admin_app \ -e NEXT_PUBLIC_BACKEND_URL="https://${{ steps.secrets.outputs.domain }}/api" \ - -e NEXT_PUBLIC_GOOGLE_LOGIN_CLIENT_ID="${{ steps.secrets.outputs.google-login-client-id }}" \ + -e NEXT_PUBLIC_GOOGLE_LOGIN_CLIENT_ID="${{ steps.secrets.outputs.google_login_client_id }}" \ ${{ env.REPO }}/admin_app:latest docker system prune -f || true diff --git a/.github/workflows/deploy_gcp_caddy.yaml b/.github/workflows/deploy_gcp_caddy.yaml index 284cb41..58fa28b 100644 --- a/.github/workflows/deploy_gcp_caddy.yaml +++ b/.github/workflows/deploy_gcp_caddy.yaml @@ -83,6 +83,7 @@ jobs: -v caddy_data:/data \ -v caddy_config:/config \ -e DOMAIN=${{ steps.secrets.outputs.domain }} \ + -e BACKEND_ROOT_PATH=/api \ -p 80:80 \ -p 443:443 \ -p 443:443/udp \