From 5483f92111163946096ef94653b2bc4ee536fc5e Mon Sep 17 00:00:00 2001 From: Chinedum Echeta <60179183+cecheta@users.noreply.github.com> Date: Wed, 22 May 2024 18:13:51 +0100 Subject: [PATCH] fix: Add `vector_fields` to Azure OpenAI On Your Data (#966) --- code/create_app.py | 3 +++ .../with_byod/test_conversation_flow.py | 5 +++++ code/tests/test_app.py | 5 +++++ infra/main.bicep | 8 ++++++++ infra/main.json | 17 ++++++++++++++++- 5 files changed, 37 insertions(+), 1 deletion(-) diff --git a/code/create_app.py b/code/create_app.py index 22016b8a5..0d27e8149 100644 --- a/code/create_app.py +++ b/code/create_app.py @@ -129,6 +129,9 @@ def conversation_with_data(conversation: Request, env_helper: EnvHelper): if env_helper.AZURE_SEARCH_CONTENT_COLUMNS else [] ), + "vector_fields": [ + env_helper.AZURE_SEARCH_CONTENT_VECTOR_COLUMNS + ], "title_field": env_helper.AZURE_SEARCH_TITLE_COLUMN or None, "url_field": env_helper.AZURE_SEARCH_URL_COLUMN or None, "filepath_field": ( diff --git a/code/tests/functional/tests/backend_api/with_byod/test_conversation_flow.py b/code/tests/functional/tests/backend_api/with_byod/test_conversation_flow.py index 17e196b9e..66f1c8b96 100644 --- a/code/tests/functional/tests/backend_api/with_byod/test_conversation_flow.py +++ b/code/tests/functional/tests/backend_api/with_byod/test_conversation_flow.py @@ -112,6 +112,11 @@ def test_post_makes_correct_call_to_azure_openai( "index_name": app_config.get("AZURE_SEARCH_INDEX"), "fields_mapping": { "content_fields": ["content"], + "vector_fields": [ + app_config.get( + "AZURE_SEARCH_CONTENT_VECTOR_COLUMNS" + ) + ], "title_field": "title", "url_field": "url", "filepath_field": "filepath", diff --git a/code/tests/test_app.py b/code/tests/test_app.py index 3918825d9..c25a2b690 100644 --- a/code/tests/test_app.py +++ b/code/tests/test_app.py @@ -21,6 +21,7 @@ AZURE_SEARCH_INDEX = "mock-search-index" AZURE_SEARCH_SERVICE = "mock-search-service" AZURE_SEARCH_CONTENT_COLUMNS = "field1|field2" +AZURE_SEARCH_CONTENT_VECTOR_COLUMNS = "vector-column" AZURE_SEARCH_TITLE_COLUMN = "title" AZURE_SEARCH_FILENAME_COLUMN = "filename" AZURE_SEARCH_URL_COLUMN = "url" @@ -66,6 +67,9 @@ def env_helper_mock(): env_helper.AZURE_SEARCH_INDEX = AZURE_SEARCH_INDEX env_helper.AZURE_SEARCH_SERVICE = AZURE_SEARCH_SERVICE env_helper.AZURE_SEARCH_CONTENT_COLUMNS = AZURE_SEARCH_CONTENT_COLUMNS + env_helper.AZURE_SEARCH_CONTENT_VECTOR_COLUMNS = ( + AZURE_SEARCH_CONTENT_VECTOR_COLUMNS + ) env_helper.AZURE_SEARCH_TITLE_COLUMN = AZURE_SEARCH_TITLE_COLUMN env_helper.AZURE_SEARCH_FILENAME_COLUMN = AZURE_SEARCH_FILENAME_COLUMN env_helper.AZURE_SEARCH_URL_COLUMN = AZURE_SEARCH_URL_COLUMN @@ -547,6 +551,7 @@ def test_conversation_azure_byod_returns_correct_response_when_streaming_with_da "index_name": AZURE_SEARCH_INDEX, "fields_mapping": { "content_fields": ["field1", "field2"], + "vector_fields": [AZURE_SEARCH_CONTENT_VECTOR_COLUMNS], "title_field": AZURE_SEARCH_TITLE_COLUMN, "url_field": AZURE_SEARCH_URL_COLUMN, "filepath_field": AZURE_SEARCH_FILENAME_COLUMN, diff --git a/infra/main.bicep b/infra/main.bicep index 5608f5f09..aefaa9c27 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -72,6 +72,9 @@ param azureSearchEnableInDomain string = 'false' @description('Content columns') param azureSearchContentColumns string = 'content' +@description('Vector columns') +param azureSearchVectorColumns string = 'content_vector' + @description('Filename column') param azureSearchFilenameColumn string = 'filename' @@ -541,6 +544,7 @@ module web './app/web.bicep' = if (hostingModel == 'code') { AZURE_SEARCH_TOP_K: azureSearchTopK AZURE_SEARCH_ENABLE_IN_DOMAIN: azureSearchEnableInDomain AZURE_SEARCH_CONTENT_COLUMNS: azureSearchContentColumns + AZURE_SEARCH_CONTENT_VECTOR_COLUMNS: azureSearchVectorColumns AZURE_SEARCH_FILENAME_COLUMN: azureSearchFilenameColumn AZURE_SEARCH_FILTER: azureSearchFilter AZURE_SEARCH_TITLE_COLUMN: azureSearchTitleColumn @@ -613,6 +617,7 @@ module web_docker './app/web.bicep' = if (hostingModel == 'container') { AZURE_SEARCH_TOP_K: azureSearchTopK AZURE_SEARCH_ENABLE_IN_DOMAIN: azureSearchEnableInDomain AZURE_SEARCH_CONTENT_COLUMNS: azureSearchContentColumns + AZURE_SEARCH_CONTENT_VECTOR_COLUMNS: azureSearchVectorColumns AZURE_SEARCH_FILENAME_COLUMN: azureSearchFilenameColumn AZURE_SEARCH_FILTER: azureSearchFilter AZURE_SEARCH_TITLE_COLUMN: azureSearchTitleColumn @@ -684,6 +689,7 @@ module adminweb './app/adminweb.bicep' = if (hostingModel == 'code') { AZURE_SEARCH_TOP_K: azureSearchTopK AZURE_SEARCH_ENABLE_IN_DOMAIN: azureSearchEnableInDomain AZURE_SEARCH_CONTENT_COLUMNS: azureSearchContentColumns + AZURE_SEARCH_CONTENT_VECTOR_COLUMNS: azureSearchVectorColumns AZURE_SEARCH_FILENAME_COLUMN: azureSearchFilenameColumn AZURE_SEARCH_FILTER: azureSearchFilter AZURE_SEARCH_TITLE_COLUMN: azureSearchTitleColumn @@ -755,6 +761,7 @@ module adminweb_docker './app/adminweb.bicep' = if (hostingModel == 'container') AZURE_SEARCH_TOP_K: azureSearchTopK AZURE_SEARCH_ENABLE_IN_DOMAIN: azureSearchEnableInDomain AZURE_SEARCH_CONTENT_COLUMNS: azureSearchContentColumns + AZURE_SEARCH_CONTENT_VECTOR_COLUMNS: azureSearchVectorColumns AZURE_SEARCH_FILENAME_COLUMN: azureSearchFilenameColumn AZURE_SEARCH_FILTER: azureSearchFilter AZURE_SEARCH_TITLE_COLUMN: azureSearchTitleColumn @@ -1065,6 +1072,7 @@ output AZURE_SEARCH_INDEX_IS_PRECHUNKED string = azureSearchIndexIsPrechunked output AZURE_SEARCH_TOP_K string = azureSearchTopK output AZURE_SEARCH_ENABLE_IN_DOMAIN string = azureSearchEnableInDomain output AZURE_SEARCH_CONTENT_COLUMNS string = azureSearchContentColumns +output AZURE_SEARCH_CONTENT_VECTOR_COLUMNS string = azureSearchVectorColumns output AZURE_SEARCH_FILENAME_COLUMN string = azureSearchFilenameColumn output AZURE_SEARCH_FILTER string = azureSearchFilter output AZURE_SEARCH_TITLE_COLUMN string = azureSearchTitleColumn diff --git a/infra/main.json b/infra/main.json index 693064cdb..7973062ff 100644 --- a/infra/main.json +++ b/infra/main.json @@ -5,7 +5,7 @@ "_generator": { "name": "bicep", "version": "0.27.1.19265", - "templateHash": "6490847733397864992" + "templateHash": "6027201902589320671" } }, "parameters": { @@ -141,6 +141,13 @@ "description": "Content columns" } }, + "azureSearchVectorColumns": { + "type": "string", + "defaultValue": "content_vector", + "metadata": { + "description": "Vector columns" + } + }, "azureSearchFilenameColumn": { "type": "string", "defaultValue": "filename", @@ -2014,6 +2021,7 @@ "AZURE_SEARCH_TOP_K": "[parameters('azureSearchTopK')]", "AZURE_SEARCH_ENABLE_IN_DOMAIN": "[parameters('azureSearchEnableInDomain')]", "AZURE_SEARCH_CONTENT_COLUMNS": "[parameters('azureSearchContentColumns')]", + "AZURE_SEARCH_CONTENT_VECTOR_COLUMNS": "[parameters('azureSearchVectorColumns')]", "AZURE_SEARCH_FILENAME_COLUMN": "[parameters('azureSearchFilenameColumn')]", "AZURE_SEARCH_FILTER": "[parameters('azureSearchFilter')]", "AZURE_SEARCH_TITLE_COLUMN": "[parameters('azureSearchTitleColumn')]", @@ -2966,6 +2974,7 @@ "AZURE_SEARCH_TOP_K": "[parameters('azureSearchTopK')]", "AZURE_SEARCH_ENABLE_IN_DOMAIN": "[parameters('azureSearchEnableInDomain')]", "AZURE_SEARCH_CONTENT_COLUMNS": "[parameters('azureSearchContentColumns')]", + "AZURE_SEARCH_CONTENT_VECTOR_COLUMNS": "[parameters('azureSearchVectorColumns')]", "AZURE_SEARCH_FILENAME_COLUMN": "[parameters('azureSearchFilenameColumn')]", "AZURE_SEARCH_FILTER": "[parameters('azureSearchFilter')]", "AZURE_SEARCH_TITLE_COLUMN": "[parameters('azureSearchTitleColumn')]", @@ -3917,6 +3926,7 @@ "AZURE_SEARCH_TOP_K": "[parameters('azureSearchTopK')]", "AZURE_SEARCH_ENABLE_IN_DOMAIN": "[parameters('azureSearchEnableInDomain')]", "AZURE_SEARCH_CONTENT_COLUMNS": "[parameters('azureSearchContentColumns')]", + "AZURE_SEARCH_CONTENT_VECTOR_COLUMNS": "[parameters('azureSearchVectorColumns')]", "AZURE_SEARCH_FILENAME_COLUMN": "[parameters('azureSearchFilenameColumn')]", "AZURE_SEARCH_FILTER": "[parameters('azureSearchFilter')]", "AZURE_SEARCH_TITLE_COLUMN": "[parameters('azureSearchTitleColumn')]", @@ -4859,6 +4869,7 @@ "AZURE_SEARCH_TOP_K": "[parameters('azureSearchTopK')]", "AZURE_SEARCH_ENABLE_IN_DOMAIN": "[parameters('azureSearchEnableInDomain')]", "AZURE_SEARCH_CONTENT_COLUMNS": "[parameters('azureSearchContentColumns')]", + "AZURE_SEARCH_CONTENT_VECTOR_COLUMNS": "[parameters('azureSearchVectorColumns')]", "AZURE_SEARCH_FILENAME_COLUMN": "[parameters('azureSearchFilenameColumn')]", "AZURE_SEARCH_FILTER": "[parameters('azureSearchFilter')]", "AZURE_SEARCH_TITLE_COLUMN": "[parameters('azureSearchTitleColumn')]", @@ -11008,6 +11019,10 @@ "type": "string", "value": "[parameters('azureSearchContentColumns')]" }, + "AZURE_SEARCH_CONTENT_VECTOR_COLUMNS": { + "type": "string", + "value": "[parameters('azureSearchVectorColumns')]" + }, "AZURE_SEARCH_FILENAME_COLUMN": { "type": "string", "value": "[parameters('azureSearchFilenameColumn')]"