From 320f3335d79b7f1e4c33c95ff5641a677e36903b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 11:49:58 +1100 Subject: [PATCH 01/17] Add a bing search grounding option --- app/backend/app.py | 4 ++++ app/backend/config.py | 1 + infra/main.bicep | 4 ++++ infra/main.parameters.json | 3 +++ 4 files changed, 12 insertions(+) diff --git a/app/backend/app.py b/app/backend/app.py index 039bde7369..3d591c2763 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -81,6 +81,7 @@ CONFIG_USER_BLOB_CONTAINER_CLIENT, CONFIG_USER_UPLOAD_ENABLED, CONFIG_VECTOR_SEARCH_ENABLED, + CONFIG_BING_SEARCH_ENABLED, ) from core.authentication import AuthenticationHelper from core.sessionhelper import create_session_id @@ -299,6 +300,7 @@ def config(): "showSpeechOutputAzure": current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED], "showChatHistoryBrowser": current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED], "showChatHistoryCosmos": current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED], + "showBingSearch": current_app.config[CONFIG_BING_SEARCH_ENABLED], } ) @@ -466,6 +468,7 @@ async def setup_clients(): USE_SPEECH_OUTPUT_AZURE = os.getenv("USE_SPEECH_OUTPUT_AZURE", "").lower() == "true" USE_CHAT_HISTORY_BROWSER = os.getenv("USE_CHAT_HISTORY_BROWSER", "").lower() == "true" USE_CHAT_HISTORY_COSMOS = os.getenv("USE_CHAT_HISTORY_COSMOS", "").lower() == "true" + USE_BING_SEARCH = os.getenv("USE_BING_SEARCH", "").lower() == "true" # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None @@ -642,6 +645,7 @@ async def setup_clients(): current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED] = USE_SPEECH_OUTPUT_AZURE current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED] = USE_CHAT_HISTORY_BROWSER current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED] = USE_CHAT_HISTORY_COSMOS + current_app.config[CONFIG_BING_SEARCH_ENABLED] = USE_BING_SEARCH prompt_manager = PromptyManager() diff --git a/app/backend/config.py b/app/backend/config.py index eaba154116..0d4495228b 100644 --- a/app/backend/config.py +++ b/app/backend/config.py @@ -26,3 +26,4 @@ CONFIG_CHAT_HISTORY_COSMOS_ENABLED = "chat_history_cosmos_enabled" CONFIG_COSMOS_HISTORY_CLIENT = "cosmos_history_client" CONFIG_COSMOS_HISTORY_CONTAINER = "cosmos_history_container" +CONFIG_BING_SEARCH_ENABLED = "bing_search_enabled" diff --git a/infra/main.bicep b/infra/main.bicep index cb337e3d80..0708d58390 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -230,6 +230,9 @@ param useUserUpload bool = false param useLocalPdfParser bool = false param useLocalHtmlParser bool = false +@description('Use Bing search for web search grounding') +param useBingSearch bool = false + var abbrs = loadJsonContent('abbreviations.json') var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) var tags = { 'azd-env-name': environmentName } @@ -407,6 +410,7 @@ var appEnvVariables = { USE_VECTORS: useVectors USE_GPT4V: useGPT4V USE_USER_UPLOAD: useUserUpload + USE_BING_SEARCH: useBingSearch AZURE_USERSTORAGE_ACCOUNT: useUserUpload ? userStorage.outputs.name : '' AZURE_USERSTORAGE_CONTAINER: useUserUpload ? userStorageContainerName : '' AZURE_DOCUMENTINTELLIGENCE_SERVICE: documentIntelligence.outputs.name diff --git a/infra/main.parameters.json b/infra/main.parameters.json index e1444fcec1..8dfc0194e3 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -278,6 +278,9 @@ "useUserUpload": { "value": "${USE_USER_UPLOAD}" }, + "useBingSearch": { + "value": "${USE_BING_SEARCH}" + }, "useLocalPdfParser": { "value": "${USE_LOCAL_PDF_PARSER}" }, From 175b7c55b2f6a2b30ee06c591553d8370eba8785 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 12:59:19 +1100 Subject: [PATCH 02/17] Add a bing client and some configuration options --- app/backend/app.py | 16 +++++++++++ app/backend/bing_client.py | 57 ++++++++++++++++++++++++++++++++++++++ app/backend/config.py | 1 + infra/main.bicep | 4 +++ infra/main.parameters.json | 6 ++++ 5 files changed, 84 insertions(+) create mode 100644 app/backend/bing_client.py diff --git a/app/backend/app.py b/app/backend/app.py index 3d591c2763..fc84916d32 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -54,6 +54,7 @@ from approaches.promptmanager import PromptyManager from approaches.retrievethenread import RetrieveThenReadApproach from approaches.retrievethenreadvision import RetrieveThenReadVisionApproach +from bing_client import AsyncBingClient from chat_history.cosmosdb import chat_history_cosmosdb_bp from config import ( CONFIG_ASK_APPROACH, @@ -82,6 +83,7 @@ CONFIG_USER_UPLOAD_ENABLED, CONFIG_VECTOR_SEARCH_ENABLED, CONFIG_BING_SEARCH_ENABLED, + CONFIG_BING_SEARCH_CLIENT, ) from core.authentication import AuthenticationHelper from core.sessionhelper import create_session_id @@ -469,6 +471,8 @@ async def setup_clients(): USE_CHAT_HISTORY_BROWSER = os.getenv("USE_CHAT_HISTORY_BROWSER", "").lower() == "true" USE_CHAT_HISTORY_COSMOS = os.getenv("USE_CHAT_HISTORY_COSMOS", "").lower() == "true" USE_BING_SEARCH = os.getenv("USE_BING_SEARCH", "").lower() == "true" + BING_SEARCH_API_KEY = os.getenv("BING_SEARCH_API_KEY") + BING_SEARCH_ENDPOINT = os.getenv("BING_SEARCH_ENDPOINT") # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None @@ -591,6 +595,18 @@ async def setup_clients(): # Wait until token is needed to fetch for the first time current_app.config[CONFIG_SPEECH_SERVICE_TOKEN] = None + if USE_BING_SEARCH: + current_app.logger.info("USE_BING_SEARCH is true, setting up Bing search client") + if not BING_SEARCH_API_KEY: + raise ValueError("BING_SEARCH_API_KEY must be set when USE_BING_SEARCH is true") + if not BING_SEARCH_ENDPOINT: + raise ValueError("BING_SEARCH_ENDPOINT must be set when USE_BING_SEARCH is true") + bing_search_client = AsyncBingClient(BING_SEARCH_API_KEY, BING_SEARCH_ENDPOINT) + current_app.config[CONFIG_BING_SEARCH_CLIENT] = bing_search_client + else: + current_app.logger.info("USE_BING_SEARCH is false, Bing search client not set up") + bing_search_client = None + if OPENAI_HOST.startswith("azure"): if OPENAI_HOST == "azure_custom": current_app.logger.info("OPENAI_HOST is azure_custom, setting up Azure OpenAI custom client") diff --git a/app/backend/bing_client.py b/app/backend/bing_client.py new file mode 100644 index 0000000000..48a61e925a --- /dev/null +++ b/app/backend/bing_client.py @@ -0,0 +1,57 @@ +""" +An async client for Bing Web Search API +""" + +import httpx +from pydantic import BaseModel +from typing import Any + + +class WebPage(BaseModel): + about: Any + dateLastCrawled: str + datePublished: str + datePublishedDisplayText: str + contractualRules: Any + deepLinks: Any + displayUrl: str + id: str + isFamilyFriendly: bool + isNavigational: bool + language: str + malware: Any + name: str + mentions: Any + searchTags: Any + snippet: str + url: str + + +class WebAnswer(BaseModel): + id: str + someResultsRemoved: bool + totalEstimatedMatches: int + value: list[WebPage] + webSearchUrl: str + + +class AsyncBingClient: + def __init__(self, api_key: str, bing_endpoint: Optional[str] = "api.bing.microsoft.com"): + self.api_key = api_key + self.base_url = f"https://{bing_endpoint}/v7.0/search" + self.headers = { + "Ocp-Apim-Subscription-Key": self.api_key, + "User-Agent": "azure-search-openai-demo", + # "X-Search-Location": "" # this would be useful in future + } + + async def search(self, query: str) -> WebAnswer: + params = { + "q": query, + "textDecorations": True, + "textFormat": "HTML", + } + async with httpx.AsyncClient() as client: + response = await client.get(self.base_url, headers=self.headers, params=params) + response.raise_for_status() + return WebAnswer.model_validate(response.json()['webPages']) diff --git a/app/backend/config.py b/app/backend/config.py index 0d4495228b..cb20329f6b 100644 --- a/app/backend/config.py +++ b/app/backend/config.py @@ -27,3 +27,4 @@ CONFIG_COSMOS_HISTORY_CLIENT = "cosmos_history_client" CONFIG_COSMOS_HISTORY_CONTAINER = "cosmos_history_container" CONFIG_BING_SEARCH_ENABLED = "bing_search_enabled" +CONFIG_BING_SEARCH_CLIENT = "bing_search_client" diff --git a/infra/main.bicep b/infra/main.bicep index 0708d58390..1df94a7d8e 100644 --- a/infra/main.bicep +++ b/infra/main.bicep @@ -232,6 +232,8 @@ param useLocalHtmlParser bool = false @description('Use Bing search for web search grounding') param useBingSearch bool = false +param bingSearchApiKey string = '' +param bingSearchEndpoint string = '' var abbrs = loadJsonContent('abbreviations.json') var resourceToken = toLower(uniqueString(subscription().id, environmentName, location)) @@ -411,6 +413,8 @@ var appEnvVariables = { USE_GPT4V: useGPT4V USE_USER_UPLOAD: useUserUpload USE_BING_SEARCH: useBingSearch + BING_SEARCH_API_KEY: bingSearchApiKey + BING_SEARCH_ENDPOINT: bingSearchEndpoint AZURE_USERSTORAGE_ACCOUNT: useUserUpload ? userStorage.outputs.name : '' AZURE_USERSTORAGE_CONTAINER: useUserUpload ? userStorageContainerName : '' AZURE_DOCUMENTINTELLIGENCE_SERVICE: documentIntelligence.outputs.name diff --git a/infra/main.parameters.json b/infra/main.parameters.json index 8dfc0194e3..2d4c3da259 100644 --- a/infra/main.parameters.json +++ b/infra/main.parameters.json @@ -281,6 +281,12 @@ "useBingSearch": { "value": "${USE_BING_SEARCH}" }, + "bingSearchApiKey": { + "value": "${BING_SEARCH_API_KEY}" + }, + "bingSearchEndpoint": { + "value": "${BING_SEARCH_ENDPOINT}" + }, "useLocalPdfParser": { "value": "${USE_LOCAL_PDF_PARSER}" }, From 98c49a05b8bc3b7e2261eb3077f7fe5f77291ab0 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 14:12:54 +1100 Subject: [PATCH 03/17] Add some setting init --- app/backend/app.py | 5 +++-- app/backend/bing_client.py | 4 ++-- docs/deploy_features.md | 18 ++++++++++++++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index fc84916d32..a47dfd3b2e 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -473,6 +473,8 @@ async def setup_clients(): USE_BING_SEARCH = os.getenv("USE_BING_SEARCH", "").lower() == "true" BING_SEARCH_API_KEY = os.getenv("BING_SEARCH_API_KEY") BING_SEARCH_ENDPOINT = os.getenv("BING_SEARCH_ENDPOINT") + if BING_SEARCH_ENDPOINT is not None and BING_SEARCH_ENDPOINT.trim() == "": + BING_SEARCH_ENDPOINT = None # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None @@ -599,8 +601,7 @@ async def setup_clients(): current_app.logger.info("USE_BING_SEARCH is true, setting up Bing search client") if not BING_SEARCH_API_KEY: raise ValueError("BING_SEARCH_API_KEY must be set when USE_BING_SEARCH is true") - if not BING_SEARCH_ENDPOINT: - raise ValueError("BING_SEARCH_ENDPOINT must be set when USE_BING_SEARCH is true") + bing_search_client = AsyncBingClient(BING_SEARCH_API_KEY, BING_SEARCH_ENDPOINT) current_app.config[CONFIG_BING_SEARCH_CLIENT] = bing_search_client else: diff --git a/app/backend/bing_client.py b/app/backend/bing_client.py index 48a61e925a..33fedb3735 100644 --- a/app/backend/bing_client.py +++ b/app/backend/bing_client.py @@ -1,10 +1,10 @@ """ -An async client for Bing Web Search API +An async client for Bing Web Search API. """ import httpx from pydantic import BaseModel -from typing import Any +from typing import Any, Optional class WebPage(BaseModel): diff --git a/docs/deploy_features.md b/docs/deploy_features.md index 7c5f2a4038..82d77060ce 100644 --- a/docs/deploy_features.md +++ b/docs/deploy_features.md @@ -337,3 +337,21 @@ If you want to decrease the charges by using local parsers instead of Azure Docu 1. Run `azd env set USE_LOCAL_HTML_PARSER true` to use the local HTML parser. The local parsers will be used the next time you run the data ingestion script. To use these parsers for the user document upload system, you'll need to run `azd provision` to update the web app to use the local parsers. + +## Using Bing Search Grounding + +To enable Bing Search, first provision a Bing.Search API endpoint in the [Azure Portal](https://portal.azure.com/#create/Microsoft.BingSearch). + +Then enable the feature: + +```console +azd env set USE_BING_SEARCH true +``` + +Set the API key via the BING_SEARCH_API_KEY command: + +```console +azd env set BING_SEARCH_API_KEY +``` + +Note that Managed Identity is not available in Bing Search API. \ No newline at end of file From 013a430f042843ebfec3ebeb985fd04eb55f2786 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 14:42:51 +1100 Subject: [PATCH 04/17] Enable a toggle and propagate all the way to the approach class --- app/backend/app.py | 2 +- app/backend/approaches/approach.py | 1 + app/backend/approaches/chatreadretrieveread.py | 2 ++ .../approaches/chatreadretrievereadvision.py | 2 ++ app/backend/approaches/retrievethenread.py | 2 ++ app/backend/approaches/retrievethenreadvision.py | 2 ++ .../src/components/Settings/Settings.tsx | 16 ++++++++++++++++ app/frontend/src/locales/en/translation.json | 6 ++++-- app/frontend/src/pages/ask/Ask.tsx | 9 +++++++++ app/frontend/src/pages/chat/Chat.tsx | 9 +++++++++ 10 files changed, 48 insertions(+), 3 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index a47dfd3b2e..779888de18 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -302,7 +302,7 @@ def config(): "showSpeechOutputAzure": current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED], "showChatHistoryBrowser": current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED], "showChatHistoryCosmos": current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED], - "showBingSearch": current_app.config[CONFIG_BING_SEARCH_ENABLED], + "showBingSearchOption": current_app.config[CONFIG_BING_SEARCH_ENABLED], } ) diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index 44a1d6380a..4132854ec3 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -149,6 +149,7 @@ async def search( use_semantic_captions: bool, minimum_search_score: Optional[float], minimum_reranker_score: Optional[float], + use_bing_search: bool = False, ) -> List[Document]: search_text = query_text if use_text_search else "" search_vectors = vectors if use_vector_search else [] diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 7777b9a741..0a01edf7af 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -89,6 +89,7 @@ async def run_until_final_call( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False + use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) @@ -147,6 +148,7 @@ async def run_until_final_call( use_semantic_captions, minimum_search_score, minimum_reranker_score, + use_bing_search, ) # STEP 3: Generate a contextual and content specific answer using the search results and chat history diff --git a/app/backend/approaches/chatreadretrievereadvision.py b/app/backend/approaches/chatreadretrievereadvision.py index 3c05d22180..2e3dc4e6f7 100644 --- a/app/backend/approaches/chatreadretrievereadvision.py +++ b/app/backend/approaches/chatreadretrievereadvision.py @@ -82,6 +82,7 @@ async def run_until_final_call( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False + use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) @@ -151,6 +152,7 @@ async def run_until_final_call( use_semantic_captions, minimum_search_score, minimum_reranker_score, + use_bing_search, ) # STEP 3: Generate a contextual and content specific answer using the search results and chat history diff --git a/app/backend/approaches/retrievethenread.py b/app/backend/approaches/retrievethenread.py index f3c9331e36..e2bfe74afe 100644 --- a/app/backend/approaches/retrievethenread.py +++ b/app/backend/approaches/retrievethenread.py @@ -68,6 +68,7 @@ async def run( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False + use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) @@ -89,6 +90,7 @@ async def run( use_semantic_captions, minimum_search_score, minimum_reranker_score, + use_bing_search, ) # Process results diff --git a/app/backend/approaches/retrievethenreadvision.py b/app/backend/approaches/retrievethenreadvision.py index 14318d83fa..84803d34dc 100644 --- a/app/backend/approaches/retrievethenreadvision.py +++ b/app/backend/approaches/retrievethenreadvision.py @@ -77,6 +77,7 @@ async def run( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False + use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) @@ -108,6 +109,7 @@ async def run( use_semantic_captions, minimum_search_score, minimum_reranker_score, + use_bing_search, ) # Process results diff --git a/app/frontend/src/components/Settings/Settings.tsx b/app/frontend/src/components/Settings/Settings.tsx index de404297ab..cca8deeeba 100644 --- a/app/frontend/src/components/Settings/Settings.tsx +++ b/app/frontend/src/components/Settings/Settings.tsx @@ -56,10 +56,12 @@ export const Settings = ({ retrievalMode, useGPT4V, gpt4vInput, + useBingSearch, vectorFieldList, showSemanticRankerOption, showGPT4VOptions, showVectorOption, + showBingSearchOption, useOidSecurityFilter, useGroupsSecurityFilter, useLogin, @@ -104,6 +106,8 @@ export const Settings = ({ const shouldStreamFieldId = useId("shouldStreamField"); const suggestFollowupQuestionsId = useId("suggestFollowupQuestions"); const suggestFollowupQuestionsFieldId = useId("suggestFollowupQuestionsField"); + const useBingSearchId = useId("useBingSearch"); + const useBingSearchFieldId = useId("useBingSearchField"); const renderLabel = (props: RenderLabelType | undefined, labelId: string, fieldId: string, helpText: string) => ( @@ -311,6 +315,18 @@ export const Settings = ({ } /> )} + + {showBingSearchOption && ( + onChange("useBingSearch", !!checked)} + aria-labelledby={useBingSearchId} + onRenderLabel={props => renderLabel(props, useBingSearchId, useBingSearchFieldId, t("helpTexts.useBingSearch"))} + /> + )} ); }; diff --git a/app/frontend/src/locales/en/translation.json b/app/frontend/src/locales/en/translation.json index 07f657da8b..56b0785796 100644 --- a/app/frontend/src/locales/en/translation.json +++ b/app/frontend/src/locales/en/translation.json @@ -118,7 +118,8 @@ }, "useOidSecurityFilter": "Use oid security filter", "useGroupsSecurityFilter": "Use groups security filter", - "shouldStream": "Stream chat completion responses" + "shouldStream": "Stream chat completion responses", + "useBingSearch": "Use Bing search" }, "helpTexts": { @@ -151,6 +152,7 @@ "Sets the retrieval mode for the Azure AI Search query. `Vectors + Text (Hybrid)` uses a combination of vector search and full text search, `Vectors` uses only vector search, and `Text` uses only full text search. Hybrid is generally optimal.", "streamChat": "Continuously streams the response to the chat UI as it is generated.", "useOidSecurityFilter": "Filter search results based on the authenticated user's OID.", - "useGroupsSecurityFilter": "Filter search results based on the authenticated user's groups." + "useGroupsSecurityFilter": "Filter search results based on the authenticated user's groups.", + "useBingSearch": "Use Bing search to ground search results." } } diff --git a/app/frontend/src/pages/ask/Ask.tsx b/app/frontend/src/pages/ask/Ask.tsx index 35137afb3d..9f4d43bc5e 100644 --- a/app/frontend/src/pages/ask/Ask.tsx +++ b/app/frontend/src/pages/ask/Ask.tsx @@ -33,6 +33,7 @@ export function Component(): JSX.Element { const [useSemanticRanker, setUseSemanticRanker] = useState(true); const [useSemanticCaptions, setUseSemanticCaptions] = useState(false); const [useGPT4V, setUseGPT4V] = useState(false); + const [useBingSearch, setUseBingSearch] = useState(false); const [gpt4vInput, setGPT4VInput] = useState(GPT4VInput.TextAndImages); const [includeCategory, setIncludeCategory] = useState(""); const [excludeCategory, setExcludeCategory] = useState(""); @@ -48,6 +49,7 @@ export function Component(): JSX.Element { const [showSpeechInput, setShowSpeechInput] = useState(false); const [showSpeechOutputBrowser, setShowSpeechOutputBrowser] = useState(false); const [showSpeechOutputAzure, setShowSpeechOutputAzure] = useState(false); + const [showBingSearchOption, setShowBingSearchOption] = useState(false); const audio = useRef(new Audio()).current; const [isPlaying, setIsPlaying] = useState(false); @@ -79,6 +81,7 @@ export function Component(): JSX.Element { setUseSemanticRanker(config.showSemanticRankerOption); setShowSemanticRankerOption(config.showSemanticRankerOption); setShowVectorOption(config.showVectorOption); + setShowBingSearchOption(config.showBingSearchOption); if (!config.showVectorOption) { setRetrievalMode(RetrievalMode.Text); } @@ -132,6 +135,7 @@ export function Component(): JSX.Element { use_gpt4v: useGPT4V, gpt4v_input: gpt4vInput, language: i18n.language, + use_bing_search: useBingSearch, ...(seed !== null ? { seed: seed } : {}) } }, @@ -204,6 +208,9 @@ export function Component(): JSX.Element { case "retrievalMode": setRetrievalMode(value); break; + case "useBingSearch": + setUseBingSearch(value); + break; } }; @@ -325,11 +332,13 @@ export function Component(): JSX.Element { includeCategory={includeCategory} retrievalMode={retrievalMode} useGPT4V={useGPT4V} + useBingSearch={useBingSearch} gpt4vInput={gpt4vInput} vectorFieldList={vectorFieldList} showSemanticRankerOption={showSemanticRankerOption} showGPT4VOptions={showGPT4VOptions} showVectorOption={showVectorOption} + showBingSearchOption={showBingSearchOption} useOidSecurityFilter={useOidSecurityFilter} useGroupsSecurityFilter={useGroupsSecurityFilter} useLogin={!!useLogin} diff --git a/app/frontend/src/pages/chat/Chat.tsx b/app/frontend/src/pages/chat/Chat.tsx index 7dabff8c7f..03cde2ff56 100644 --- a/app/frontend/src/pages/chat/Chat.tsx +++ b/app/frontend/src/pages/chat/Chat.tsx @@ -58,6 +58,7 @@ const Chat = () => { const [useGroupsSecurityFilter, setUseGroupsSecurityFilter] = useState(false); const [gpt4vInput, setGPT4VInput] = useState(GPT4VInput.TextAndImages); const [useGPT4V, setUseGPT4V] = useState(false); + const [useBingSearch, setUseBingSearch] = useState(false); const lastQuestionRef = useRef(""); const chatMessageStreamEnd = useRef(null); @@ -77,6 +78,7 @@ const Chat = () => { const [showGPT4VOptions, setShowGPT4VOptions] = useState(false); const [showSemanticRankerOption, setShowSemanticRankerOption] = useState(false); const [showVectorOption, setShowVectorOption] = useState(false); + const [showBingSearchOption, setShowBingSearchOption] = useState(false); const [showUserUpload, setShowUserUpload] = useState(false); const [showLanguagePicker, setshowLanguagePicker] = useState(false); const [showSpeechInput, setShowSpeechInput] = useState(false); @@ -111,6 +113,7 @@ const Chat = () => { setShowSpeechOutputAzure(config.showSpeechOutputAzure); setShowChatHistoryBrowser(config.showChatHistoryBrowser); setShowChatHistoryCosmos(config.showChatHistoryCosmos); + setShowBingSearchOption(config.showBingSearchOption); }); }; @@ -204,6 +207,7 @@ const Chat = () => { use_gpt4v: useGPT4V, gpt4v_input: gpt4vInput, language: i18n.language, + use_bing_search: useBingSearch, ...(seed !== null ? { seed: seed } : {}) } }, @@ -318,6 +322,9 @@ const Chat = () => { case "retrievalMode": setRetrievalMode(value); break; + case "useBingSearch": + setUseBingSearch(value); + break; } }; @@ -503,11 +510,13 @@ const Chat = () => { includeCategory={includeCategory} retrievalMode={retrievalMode} useGPT4V={useGPT4V} + useBingSearch={useBingSearch} gpt4vInput={gpt4vInput} vectorFieldList={vectorFieldList} showSemanticRankerOption={showSemanticRankerOption} showGPT4VOptions={showGPT4VOptions} showVectorOption={showVectorOption} + showBingSearchOption={showBingSearchOption} useOidSecurityFilter={useOidSecurityFilter} useGroupsSecurityFilter={useGroupsSecurityFilter} useLogin={!!useLogin} From 067e1f4db6ac31fe1730291cb874f922b4b144d0 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 16:08:44 +1100 Subject: [PATCH 05/17] Implement additional prompts for merging results. --- app/backend/app.py | 8 +- app/backend/approaches/approach.py | 1 - .../approaches/chatreadretrieveread.py | 120 ++++++--- .../approaches/chatreadretrievereadvision.py | 4 +- .../prompts/chat_bing_answer_question.prompty | 57 ++++ .../prompts/chat_bing_ground_rewrite.prompty | 51 ++++ .../chat_bing_ground_rewrite_tools.json | 17 ++ app/backend/approaches/retrievethenread.py | 1 - .../approaches/retrievethenreadvision.py | 1 - app/backend/bing_client.py | 34 +-- docs/deploy_features.md | 2 +- tests/bing/example_result.json | 249 ++++++++++++++++++ 12 files changed, 481 insertions(+), 64 deletions(-) create mode 100644 app/backend/approaches/prompts/chat_bing_answer_question.prompty create mode 100644 app/backend/approaches/prompts/chat_bing_ground_rewrite.prompty create mode 100644 app/backend/approaches/prompts/chat_bing_ground_rewrite_tools.json create mode 100644 tests/bing/example_result.json diff --git a/app/backend/app.py b/app/backend/app.py index 779888de18..90b1b525ef 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -601,8 +601,10 @@ async def setup_clients(): current_app.logger.info("USE_BING_SEARCH is true, setting up Bing search client") if not BING_SEARCH_API_KEY: raise ValueError("BING_SEARCH_API_KEY must be set when USE_BING_SEARCH is true") - - bing_search_client = AsyncBingClient(BING_SEARCH_API_KEY, BING_SEARCH_ENDPOINT) + if BING_SEARCH_ENDPOINT: + bing_search_client = AsyncBingClient(BING_SEARCH_API_KEY, BING_SEARCH_ENDPOINT) + else: + bing_search_client = AsyncBingClient(BING_SEARCH_API_KEY) current_app.config[CONFIG_BING_SEARCH_CLIENT] = bing_search_client else: current_app.logger.info("USE_BING_SEARCH is false, Bing search client not set up") @@ -699,6 +701,7 @@ async def setup_clients(): query_language=AZURE_SEARCH_QUERY_LANGUAGE, query_speller=AZURE_SEARCH_QUERY_SPELLER, prompt_manager=prompt_manager, + bing_client=bing_search_client, ) if USE_GPT4V: @@ -745,6 +748,7 @@ async def setup_clients(): query_language=AZURE_SEARCH_QUERY_LANGUAGE, query_speller=AZURE_SEARCH_QUERY_SPELLER, prompt_manager=prompt_manager, + bing_client=bing_search_client, ) diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index 4132854ec3..44a1d6380a 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -149,7 +149,6 @@ async def search( use_semantic_captions: bool, minimum_search_score: Optional[float], minimum_reranker_score: Optional[float], - use_bing_search: bool = False, ) -> List[Document]: search_text = query_text if use_text_search else "" search_vectors = vectors if use_vector_search else [] diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 0a01edf7af..824296605c 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -11,10 +11,11 @@ ) from openai_messages_token_helper import build_messages, get_token_limit -from approaches.approach import ThoughtStep +from approaches.approach import Document, ThoughtStep from approaches.chatapproach import ChatApproach from approaches.promptmanager import PromptManager from core.authentication import AuthenticationHelper +from bing_client import AsyncBingClient class ChatReadRetrieveReadApproach(ChatApproach): @@ -39,10 +40,12 @@ def __init__( content_field: str, query_language: str, query_speller: str, - prompt_manager: PromptManager + prompt_manager: PromptManager, + bing_client: Optional[AsyncBingClient] = None, ): self.search_client = search_client self.openai_client = openai_client + self.bing_client = bing_client self.auth_helper = auth_helper self.chatgpt_model = chatgpt_model self.chatgpt_deployment = chatgpt_deployment @@ -58,6 +61,9 @@ def __init__( self.query_rewrite_prompt = self.prompt_manager.load_prompt("chat_query_rewrite.prompty") self.query_rewrite_tools = self.prompt_manager.load_tools("chat_query_rewrite_tools.json") self.answer_prompt = self.prompt_manager.load_prompt("chat_answer_question.prompty") + self.bing_answer_prompt = self.prompt_manager.load_prompt("chat_bing_answer_question.prompty") + self.bing_ground_rewrite_prompt = self.prompt_manager.load_prompt("chat_bing_ground_rewrite.prompty") + self.bing_ground_rewrite_tools = self.prompt_manager.load_tools("chat_bing_ground_rewrite_tools.json") @overload async def run_until_final_call( @@ -99,36 +105,45 @@ async def run_until_final_call( if not isinstance(original_user_query, str): raise ValueError("The most recent message content must be a string.") + # STEP 1: Generate an optimized keyword search query based on the chat history and the last question + async def keyword_rewrite(rendered_prompt, tools): + query_response_token_limit = 100 + query_messages = build_messages( + model=self.chatgpt_model, + system_prompt=rendered_prompt.system_content, + few_shots=rendered_prompt.few_shot_messages, + past_messages=rendered_prompt.past_messages, + new_user_content=rendered_prompt.new_user_content, + tools=tools, + max_tokens=self.chatgpt_token_limit - query_response_token_limit, + fallback_to_default=self.ALLOW_NON_GPT_MODELS, + ) + + chat_completion: ChatCompletion = await self.openai_client.chat.completions.create( + messages=query_messages, # type: ignore + # Azure OpenAI takes the deployment name as the model name + model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model, + temperature=0.0, # Minimize creativity for search query generation + max_tokens=query_response_token_limit, # Setting too low risks malformed JSON, setting too high may affect performance + n=1, + tools=tools, + seed=seed, + ) + + return query_messages, self.get_search_query(chat_completion, original_user_query) + rendered_query_prompt = self.prompt_manager.render_prompt( self.query_rewrite_prompt, {"user_query": original_user_query, "past_messages": messages[:-1]} ) tools: List[ChatCompletionToolParam] = self.query_rewrite_tools - - # STEP 1: Generate an optimized keyword search query based on the chat history and the last question - query_response_token_limit = 100 - query_messages = build_messages( - model=self.chatgpt_model, - system_prompt=rendered_query_prompt.system_content, - few_shots=rendered_query_prompt.few_shot_messages, - past_messages=rendered_query_prompt.past_messages, - new_user_content=rendered_query_prompt.new_user_content, - tools=tools, - max_tokens=self.chatgpt_token_limit - query_response_token_limit, - fallback_to_default=self.ALLOW_NON_GPT_MODELS, - ) - - chat_completion: ChatCompletion = await self.openai_client.chat.completions.create( - messages=query_messages, # type: ignore - # Azure OpenAI takes the deployment name as the model name - model=self.chatgpt_deployment if self.chatgpt_deployment else self.chatgpt_model, - temperature=0.0, # Minimize creativity for search query generation - max_tokens=query_response_token_limit, # Setting too low risks malformed JSON, setting too high may affect performance - n=1, - tools=tools, - seed=seed, - ) - - query_text = self.get_search_query(chat_completion, original_user_query) + query_messages, query_text = await keyword_rewrite(rendered_query_prompt, tools) + if use_bing_search: + bing_search_prompt = self.prompt_manager.render_prompt( + self.bing_ground_rewrite_prompt, + {"user_query": original_user_query, "past_messages": messages[:-1]}, + ) + _, bing_query_text = await keyword_rewrite(bing_search_prompt, self.bing_ground_rewrite_tools) + bing_results = await self.bing_client.search(bing_query_text) # STEP 2: Retrieve relevant documents from the search index with the GPT optimized query @@ -137,7 +152,7 @@ async def run_until_final_call( if use_vector_search: vectors.append(await self.compute_text_embedding(query_text)) - results = await self.search( + results: list[Document] = await self.search( top, query_text, filter, @@ -148,21 +163,36 @@ async def run_until_final_call( use_semantic_captions, minimum_search_score, minimum_reranker_score, - use_bing_search, ) # STEP 3: Generate a contextual and content specific answer using the search results and chat history text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=False) - rendered_answer_prompt = self.prompt_manager.render_prompt( - self.answer_prompt, - self.get_system_prompt_variables(overrides.get("prompt_template")) - | { - "include_follow_up_questions": bool(overrides.get("suggest_followup_questions")), - "past_messages": messages[:-1], - "user_query": original_user_query, - "text_sources": text_sources, - }, - ) + + if use_bing_search and bing_results.totalEstimatedMatches > 0: + web_sources = [hit.snippet for hit in bing_results.value[:2]] + + rendered_answer_prompt = self.prompt_manager.render_prompt( + self.bing_answer_prompt, + self.get_system_prompt_variables(overrides.get("prompt_template")) + | { + "include_follow_up_questions": bool(overrides.get("suggest_followup_questions")), + "past_messages": messages[:-1], + "user_query": original_user_query, + "text_sources": text_sources, + "web_search_snippets": web_sources, + }, + ) + else: + rendered_answer_prompt = self.prompt_manager.render_prompt( + self.answer_prompt, + self.get_system_prompt_variables(overrides.get("prompt_template")) + | { + "include_follow_up_questions": bool(overrides.get("suggest_followup_questions")), + "past_messages": messages[:-1], + "user_query": original_user_query, + "text_sources": text_sources, + }, + ) response_token_limit = 1024 messages = build_messages( @@ -186,6 +216,16 @@ async def run_until_final_call( else {"model": self.chatgpt_model} ), ), + ThoughtStep( + "Bing search query", + bing_query_text if use_bing_search else None, + { + } + ), + ThoughtStep( + "Bing search results", + [result.snippet for result in bing_results.value[:2]] if use_bing_search else None, + ), ThoughtStep( "Search using generated search query", query_text, diff --git a/app/backend/approaches/chatreadretrievereadvision.py b/app/backend/approaches/chatreadretrievereadvision.py index 2e3dc4e6f7..e7f165a225 100644 --- a/app/backend/approaches/chatreadretrievereadvision.py +++ b/app/backend/approaches/chatreadretrievereadvision.py @@ -16,6 +16,7 @@ from approaches.promptmanager import PromptManager from core.authentication import AuthenticationHelper from core.imageshelper import fetch_image +from bing_client import AsyncBingClient class ChatReadRetrieveReadVisionApproach(ChatApproach): @@ -46,10 +47,12 @@ def __init__( vision_endpoint: str, vision_token_provider: Callable[[], Awaitable[str]], prompt_manager: PromptManager, + bing_client: Optional[AsyncBingClient] = None, ): self.search_client = search_client self.blob_container_client = blob_container_client self.openai_client = openai_client + self.bing_client = bing_client self.auth_helper = auth_helper self.chatgpt_model = chatgpt_model self.chatgpt_deployment = chatgpt_deployment @@ -152,7 +155,6 @@ async def run_until_final_call( use_semantic_captions, minimum_search_score, minimum_reranker_score, - use_bing_search, ) # STEP 3: Generate a contextual and content specific answer using the search results and chat history diff --git a/app/backend/approaches/prompts/chat_bing_answer_question.prompty b/app/backend/approaches/prompts/chat_bing_answer_question.prompty new file mode 100644 index 0000000000..7dfecdea7c --- /dev/null +++ b/app/backend/approaches/prompts/chat_bing_answer_question.prompty @@ -0,0 +1,57 @@ +--- +name: Chat +description: Answer a question (with chat history) using text sources and web searches. +model: + api: chat +sample: + user_query: What does a product manager do that a CEO doesn't? + include_follow_up_questions: true + past_messages: + - role: user + content: "What does a CEO do?" + - role: assistant + content: "A CEO, or Chief Executive Officer, is responsible for providing strategic direction and oversight to a company to ensure its long-term success and profitability. They develop and implement strategies and objectives for financial success and growth, provide guidance to the executive team, manage day-to-day operations, ensure compliance with laws and regulations, develop and maintain relationships with stakeholders, monitor industry trends, and represent the company in public events 12. [role_library.pdf#page=1][role_library.pdf#page=3]" + text_sources: + - "role_library.pdf#page=29: The Manager of Product Management will collaborate with internal teams, such as engineering, sales, marketing, and finance, as well as external partners, suppliers, and customers to ensure successful product execution. Responsibilities: · Lead the product management team and provide guidance on product strategy, design, development, and launch. · Develop and implement product life-cycle management processes. · Monitor and analyze industry trends to identify opportunities for new products. · Develop product marketing plans and go-to-market strategies. · Research customer needs and develop customer-centric product roadmaps. · Collaborate with internal teams to ensure product execution and successful launch. · Develop pricing strategies and cost models. · Oversee product portfolio and performance metrics. · Manage product development budget. · Analyze product performance and customer feedback to identify areas for improvement. Qualifications: · Bachelor's degree in business, engineering, or a related field. · At least 5 years of experience in product management. · Proven track record of successful product launches." + - "role_library.pdf#page=23: Company: Contoso Electronics Location: Anywhere Job Type: Full-Time Salary: Competitive, commensurate with experience Job Summary: The Senior Manager of Product Management will be responsible for leading the product management team at Contoso Electronics. This role includes developing strategies, plans and objectives for the product management team and managing the day-to-day operations. The Senior Manager of Product Management will be responsible for the successful launch of new products and the optimization of existing products. Responsibilities: · Develop and implement product management strategies, plans and objectives to maximize team performance. · Analyze competitive landscape and market trends to develop product strategies. · Lead the product management team in the development of product plans, roadmaps and launch plans. · Monitor the performance of product management team, analyze results and implement corrective action as needed. · Manage the product lifecycle, including product development, launch, and end of life. · Ensure product features and benefits meet customer requirements. · Establish and maintain relationships with key customers, partners, and vendors." + - "role_library.pdf#page=28: · 7+ years of experience in research and development in the electronics sector. · Proven track record of successfully designing, testing, and optimizing products. · Experience leading a team of researchers and engineers. · Excellent problem-solving and analytical skills. · Ability to work in a fast-paced environment and meet tight deadlines.· Knowledge of industry trends, technologies, and regulations. · Excellent communication and presentation skills. Manager of Product Management Job Title: Manager of Product Management, Contoso Electronics Job Summary: The Manager of Product Management is responsible for overseeing the product management team, driving product development and marketing strategy for Contoso Electronics. This individual will be accountable for the successful launch of new products and the implementation of product life-cycle management processes. The Manager of Product Management will collaborate with internal teams, such as engineering, sales, marketing, and finance, as well as external partners, suppliers, and customers to ensure successful product execution." +--- +system: +{% if override_prompt %} +{{ override_prompt }} +{% else %} +Assistant helps the company employees with their healthcare plan questions, and questions about the employee handbook. Be brief in your answers. +Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question. +If the question is not in English, answer in the language used in the question. +Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]. +Additional web search snippets are included. If the sources do not answer the question, you can use the web search snippets to find the answer. The original sources take precedence over the web search snippets. +{{ injected_prompt }} +{% endif %} + +{% if include_follow_up_questions %} +Generate 3 very brief follow-up questions that the user would likely ask next. +Enclose the follow-up questions in double angle brackets. Example: +<> +<> +<> +Do not repeat questions that have already been asked. +Make sure the last question ends with ">>". +{% endif %} + +{% for message in past_messages %} +{{ message["role"] }}: +{{ message["content"] }} +{% endfor %} + +user: +{{ user_query }} + +Sources: +{% for text_source in text_sources %} +{{ text_source }} +{% endfor %} + +Web Search Snippets: +{% for snippet in web_search_snippets %} +{{ snippet }} +{% endfor %} \ No newline at end of file diff --git a/app/backend/approaches/prompts/chat_bing_ground_rewrite.prompty b/app/backend/approaches/prompts/chat_bing_ground_rewrite.prompty new file mode 100644 index 0000000000..54094a3c6d --- /dev/null +++ b/app/backend/approaches/prompts/chat_bing_ground_rewrite.prompty @@ -0,0 +1,51 @@ +--- +name: Rewrite RAG query for web search +description: Suggest the optimal search query based on the user's query, examples, and chat history. +model: + api: chat + parameters: + tools: ${file:chat_query_rewrite_tools.json} +sample: + user_query: Does it include hearing? + past_messages: + - role: user + content: "What is included in my Northwind Health Plus plan that is not in standard?" + - role: assistant + content: "The Northwind Health Plus plan includes coverage for emergency services, mental health and substance abuse coverage, and out-of-network services, which are not included in the Northwind Standard plan. [Benefit_Options.pdf#page=3]" +--- +system: +Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base and/or a web search. +If the user question requires grounding from an Internet search, generate a Bing search query based on the conversation and the new question. +If you cannot generate a search query, return just the number 0. + +user: +(EXAMPLE) Who won the 2020 NBA championship? + +assistant: +2020 NBA championship winner + +user: +(EXAMPLE) What are my health plans? + +assistant: +0 + +user: +(EXAMPLE) What is the corporate pay scheme? + +assistant: +0 + +user: +(EXAMPLE) What is the address of Microsoft headquarters? + +assistant: +Microsoft headquarters address + +{% for message in past_messages %} +{{ message["role"] }}: +{{ message["content"] }} +{% endfor %} + +user: +Generate Bing search query for: {{ user_query }} diff --git a/app/backend/approaches/prompts/chat_bing_ground_rewrite_tools.json b/app/backend/approaches/prompts/chat_bing_ground_rewrite_tools.json new file mode 100644 index 0000000000..8dc57ee083 --- /dev/null +++ b/app/backend/approaches/prompts/chat_bing_ground_rewrite_tools.json @@ -0,0 +1,17 @@ +[{ + "type": "function", + "function": { + "name": "search_bing", + "description": "Retrieve sources from Bing", + "parameters": { + "type": "object", + "properties": { + "bing_search_query": { + "type": "string", + "description": "Query string to retrieve documents from bing e.g.: 'Microsoft headquarters address'" + } + }, + "required": ["bing_search_query"] + } + } +}] diff --git a/app/backend/approaches/retrievethenread.py b/app/backend/approaches/retrievethenread.py index e2bfe74afe..7acd15e52b 100644 --- a/app/backend/approaches/retrievethenread.py +++ b/app/backend/approaches/retrievethenread.py @@ -90,7 +90,6 @@ async def run( use_semantic_captions, minimum_search_score, minimum_reranker_score, - use_bing_search, ) # Process results diff --git a/app/backend/approaches/retrievethenreadvision.py b/app/backend/approaches/retrievethenreadvision.py index 84803d34dc..469f75c168 100644 --- a/app/backend/approaches/retrievethenreadvision.py +++ b/app/backend/approaches/retrievethenreadvision.py @@ -109,7 +109,6 @@ async def run( use_semantic_captions, minimum_search_score, minimum_reranker_score, - use_bing_search, ) # Process results diff --git a/app/backend/bing_client.py b/app/backend/bing_client.py index 33fedb3735..a8e9405ba5 100644 --- a/app/backend/bing_client.py +++ b/app/backend/bing_client.py @@ -3,37 +3,37 @@ """ import httpx -from pydantic import BaseModel +from pydantic import BaseModel, ConfigDict from typing import Any, Optional class WebPage(BaseModel): - about: Any - dateLastCrawled: str - datePublished: str - datePublishedDisplayText: str - contractualRules: Any - deepLinks: Any - displayUrl: str id: str - isFamilyFriendly: bool - isNavigational: bool - language: str - malware: Any name: str - mentions: Any - searchTags: Any - snippet: str url: str + displayUrl: str + dateLastCrawled: str + language: str + snippet: Optional[str] = None + isFamilyFriendly: Optional[bool] = True + siteName: Optional[str] = None + + # There are more fields in the response, but we only care about these for now. + model_config = ConfigDict( + extra='allow', + ) class WebAnswer(BaseModel): - id: str - someResultsRemoved: bool totalEstimatedMatches: int value: list[WebPage] webSearchUrl: str + # There are more fields in the response, but we only care about these for now. + model_config = ConfigDict( + extra='allow', + ) + class AsyncBingClient: def __init__(self, api_key: str, bing_endpoint: Optional[str] = "api.bing.microsoft.com"): diff --git a/docs/deploy_features.md b/docs/deploy_features.md index 82d77060ce..cc60369bb7 100644 --- a/docs/deploy_features.md +++ b/docs/deploy_features.md @@ -340,7 +340,7 @@ The local parsers will be used the next time you run the data ingestion script. ## Using Bing Search Grounding -To enable Bing Search, first provision a Bing.Search API endpoint in the [Azure Portal](https://portal.azure.com/#create/Microsoft.BingSearch). +To enable Bing Search, first provision a Bing.Search API endpoint in the [Azure Portal](https://portal.azure.com/#create/Microsoft.BingSearch). You will need access to the Web Search API. The [F1 (free), S1, or S2 SKUS will be suitable](https://www.microsoft.com/bing/apis/pricing). Then enable the feature: diff --git a/tests/bing/example_result.json b/tests/bing/example_result.json new file mode 100644 index 0000000000..a2fca7361b --- /dev/null +++ b/tests/bing/example_result.json @@ -0,0 +1,249 @@ +{ + "_type": "SearchResponse", + "queryContext": { + "originalQuery": "Who won the 2024 NBA northern division?" + }, + "webPages": { + "webSearchUrl": "https:\\/\\/www.bing.com\\/search?q=Who+won+the+2024+NBA+northern+division%3f", + "totalEstimatedMatches": 237000, + "value": [ + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.0", + "name": "NBA Team Standings & Stats | NBA.com", + "url": "https:\\/\\/www.nba.com\\/standings?Season=2024-25&os=vb..&ref=app", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/www.nba.com\\/standings?Season=2024-25&os=vb..&ref=app", + "snippet": "NBA 2024-25 Regular Season Standings. Season ... Head-to-head won-lost percentage (2) Division leader wins tie from team not leading a division (3) Division won-lost percentage for teams in the ...", + "dateLastCrawled": "2025-01-21T07:08:00.0000000Z", + "language": "en", + "isNavigational": true, + "noCache": false, + "siteName": "NBA" + }, + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.1", + "name": "2024-25 NBA Standings: Division - FOX Sports", + "url": "https:\\/\\/www.foxsports.com\\/nba\\/standings?type=division", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/www.foxsports.com\\/nba\\/standings?type=division", + "snippet": "2024-25 nba division standings. atlantic. w-l pct gb pf pa home away conf div l10 strk 1. celtics 30-13 .698 - 117.7 108.1 ...", + "dateLastCrawled": "2025-01-20T16:38:00.0000000Z", + "language": "en", + "isNavigational": false, + "noCache": false, + "siteName": "Fox Sports" + }, + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.2", + "contractualRules": [ + { + "_type": "ContractualRules\\/LicenseAttribution", + "targetPropertyName": "snippet", + "targetPropertyIndex": 2, + "mustBeCloseToContent": true, + "license": { + "name": "CC-BY-SA", + "url": "http:\\/\\/creativecommons.org\\/licenses\\/by-sa\\/3.0\\/" + }, + "licenseNotice": "Text under CC-BY-SA licence" + } + ], + "name": "2024 NBA playoffs - Wikipedia", + "url": "https:\\/\\/en.wikipedia.org\\/wiki\\/2024_NBA_playoffs", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/en.wikipedia.org\\/wiki\\/2024_NBA_playoffs", + "snippet": "The 2024 NBA playoffs was the postseason tournament of the National Basketball Association's (NBA) 2023–24 season. ... and the numbers to the right indicate the number of games the team won in that round. The division champions are marked by an asterisk. Teams with home court advantage, the higher seeded team, are shown in italics.", + "dateLastCrawled": "2025-01-22T16:35:00.0000000Z", + "language": "en", + "isNavigational": false, + "noCache": false, + "siteName": "Wikipedia" + }, + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.3", + "name": "2024 NBA Playoffs | Official Bracket, Schedule and Series Matchups", + "url": "https:\\/\\/www.nba.com\\/playoffs\\/2024?os=win&ref=app", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/www.nba.com\\/playoffs\\/2024?os=win&ref=app", + "snippet": "The official site of the 2024 NBA Playoffs. Latest news, schedules, matchups, highlights, bracket and more. ... clinching title with a 4-1 series win. Horford finally champ after key sacrifice.", + "dateLastCrawled": "2025-01-17T08:53:00.0000000Z", + "language": "en", + "isNavigational": false, + "noCache": false, + "siteName": "NBA" + }, + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.4", + "name": "2024-2025 NBA Schedule and Scores: Full List - LandOfBasketball.com", + "url": "https:\\/\\/www.landofbasketball.com\\/results\\/2024_2025_scores_full.htm", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/www.landofbasketball.com\\/results\\/2024_2025_scores_full.htm", + "snippet": "All the results of NBA games played in the 2024-25 Season. Complete list with date, points scored, location and other information. October \\/ November \\/ December \\/ January \\/ February \\/ March \\/ April \\/ May \\/ June. 2024-25 NBA Schedule and Scores: Full List Full List - NBA Regular Season Oct. 22, 2024:", + "dateLastCrawled": "2025-01-22T07:16:00.0000000Z", + "language": "en", + "isNavigational": false, + "noCache": false, + "siteName": "LandOfBasketball.com" + }, + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.5", + "name": "2024-2025 NBA Standings - plaintextsports.com", + "url": "https:\\/\\/plaintextsports.com\\/nba\\/2024-2025\\/standings", + "thumbnailUrl": "https:\\/\\/www.bing.com\\/th?id=OIP.XG7Vso_hVPpfTXTNNgr67gHaHa&w=80&h=80&c=1&pid=5.1", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/plaintextsports.com\\/nba\\/2024-2025\\/standings", + "snippet": "2024-2025 NBA Standings. Division Conference Overall NBA Cup. Eastern Conference. Atlantic: W-L PCT GB STRK L10 HOME AWAY. 1:BOS 30-13 .698 - W1 6-4 15-8 15-5. 2:NYK 28-16 .636 2.5 W1 4-6 14-8 14-8. ... What is the NBA Cup? Home games are in italics. Eastern Conference. East A:", + "dateLastCrawled": "2025-01-21T23:11:00.0000000Z", + "primaryImageOfPage": { + "thumbnailUrl": "https:\\/\\/www.bing.com\\/th?id=OIP.XG7Vso_hVPpfTXTNNgr67gHaHa&w=80&h=80&c=1&pid=5.1", + "width": 80, + "height": 80, + "sourceWidth": 474, + "sourceHeight": 474, + "imageId": "OIP.XG7Vso_hVPpfTXTNNgr67gHaHa" + }, + "language": "en", + "isNavigational": false, + "noCache": false + }, + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.6", + "name": "2023-24 NBA Standings - Basketball-Reference.com", + "url": "https:\\/\\/www.basketball-reference.com\\/leagues\\/NBA_2024_standings.html", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/www.basketball-reference.com\\/leagues\\/NBA_2024_standings.html", + "snippet": "Checkout the latest 2023-24 NBA Standing including Conference and Division Standings, Expanding Standings, Team vs Team Stats and more on Basketball-Reference.com. ... 2022-23 Standings 2024-25 Standings. League Champion: Boston Celtics.", + "dateLastCrawled": "2025-01-22T02:07:00.0000000Z", + "language": "en", + "isNavigational": false, + "noCache": false, + "siteName": "Basketball-Reference.com" + }, + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.7", + "name": "2024 NBA Playoffs Summary - Basketball-Reference.com", + "url": "https:\\/\\/www.basketball-reference.com\\/playoffs\\/NBA_2024.html", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/www.basketball-reference.com\\/playoffs\\/NBA_2024.html", + "snippet": "2024 NBA Playoffs Summary 2023 Playoffs Summary. League Champion: Boston Celtics. Finals MVP: Jaylen Brown (20.8 \\/ 5.4 \\/ 5.0) 2024 Playoff Leaders:", + "dateLastCrawled": "2025-01-21T08:09:00.0000000Z", + "language": "en", + "isNavigational": false, + "noCache": false, + "siteName": "Basketball-Reference.com" + }, + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.8", + "name": "NBA 2023-24 Regular Season Standings - LandOfBasketball.com", + "url": "https:\\/\\/www.landofbasketball.com\\/yearbyyear\\/2023_2024_standings.htm", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/www.landofbasketball.com\\/yearbyyear\\/2023_2024_standings.htm", + "snippet": "Complete 2023-2024 NBA season standings, with conference and division rank and teams that qualified for the playoffs. Land Of Basketball.com. Teams, players profiles, awards, stats, records and championships. Menu. NBA Seasons: 2023-2024 Season Final Standings. ...", + "dateLastCrawled": "2025-01-21T00:40:00.0000000Z", + "language": "en", + "isNavigational": false, + "noCache": false, + "siteName": "LandOfBasketball.com" + }, + { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.9", + "name": "2024 NBA Finals | NBA.com", + "url": "https:\\/\\/www.nba.com\\/playoffs\\/2024\\/nba-finals?os=wtmbTQtAJk9ya&ref=app", + "thumbnailUrl": "https:\\/\\/www.bing.com\\/th?id=OIP.DoilHjZHaPBaU4LsW342SgHaEK&w=80&h=80&c=1&pid=5.1", + "isFamilyFriendly": true, + "displayUrl": "https:\\/\\/www.nba.com\\/playoffs\\/2024\\/nba-finals?os=wtmbTQtAJk9ya&ref=app", + "snippet": "Even with Kristaps Porzingis missing much of the 2024 NBA playoffs, the Celtics prove deep enough to emerge as champions. ... Tatum: Title win is '10x better than imagined' 00:58. Porzingis on ...", + "dateLastCrawled": "2025-01-22T11:02:00.0000000Z", + "primaryImageOfPage": { + "thumbnailUrl": "https:\\/\\/www.bing.com\\/th?id=OIP.DoilHjZHaPBaU4LsW342SgHaEK&w=80&h=80&c=1&pid=5.1", + "width": 80, + "height": 80, + "sourceWidth": 474, + "sourceHeight": 266, + "imageId": "OIP.DoilHjZHaPBaU4LsW342SgHaEK" + }, + "language": "en", + "isNavigational": false, + "noCache": false, + "siteName": "NBA" + } + ], + "someResultsRemoved": true + }, + "rankingResponse": { + "mainline": { + "items": [ + { + "answerType": "WebPages", + "resultIndex": 0, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.0" + } + }, + { + "answerType": "WebPages", + "resultIndex": 1, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.1" + } + }, + { + "answerType": "WebPages", + "resultIndex": 2, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.2" + } + }, + { + "answerType": "WebPages", + "resultIndex": 3, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.3" + } + }, + { + "answerType": "WebPages", + "resultIndex": 4, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.4" + } + }, + { + "answerType": "WebPages", + "resultIndex": 5, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.5" + } + }, + { + "answerType": "WebPages", + "resultIndex": 6, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.6" + } + }, + { + "answerType": "WebPages", + "resultIndex": 7, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.7" + } + }, + { + "answerType": "WebPages", + "resultIndex": 8, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.8" + } + }, + { + "answerType": "WebPages", + "resultIndex": 9, + "value": { + "id": "https:\\/\\/api.bing.microsoft.com\\/api\\/v7\\/#WebPages.9" + } + } + ] + } + } +} \ No newline at end of file From 93cbdbc8a7930e230474e7b4432e52a4ad2e5bdf Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 16:31:01 +1100 Subject: [PATCH 06/17] Add missing ts fields --- app/frontend/src/api/models.ts | 2 ++ app/frontend/src/components/Settings/Settings.tsx | 2 ++ 2 files changed, 4 insertions(+) diff --git a/app/frontend/src/api/models.ts b/app/frontend/src/api/models.ts index ef1fa154b0..6d38fb1962 100644 --- a/app/frontend/src/api/models.ts +++ b/app/frontend/src/api/models.ts @@ -37,6 +37,7 @@ export type ChatAppRequestOverrides = { gpt4v_input?: GPT4VInput; vector_fields: VectorFieldOptions[]; language: string; + use_bing_search?: boolean; }; export type ResponseMessage = { @@ -85,6 +86,7 @@ export type Config = { showGPT4VOptions: boolean; showSemanticRankerOption: boolean; showVectorOption: boolean; + showBingSearchOption: boolean; showUserUpload: boolean; showLanguagePicker: boolean; showSpeechInput: boolean; diff --git a/app/frontend/src/components/Settings/Settings.tsx b/app/frontend/src/components/Settings/Settings.tsx index cca8deeeba..36d4c050da 100644 --- a/app/frontend/src/components/Settings/Settings.tsx +++ b/app/frontend/src/components/Settings/Settings.tsx @@ -24,6 +24,7 @@ export interface SettingsProps { retrievalMode: RetrievalMode; useGPT4V: boolean; gpt4vInput: GPT4VInput; + useBingSearch: boolean; vectorFieldList: VectorFieldOptions[]; showSemanticRankerOption: boolean; showGPT4VOptions: boolean; @@ -40,6 +41,7 @@ export interface SettingsProps { promptTemplatePrefix?: string; promptTemplateSuffix?: string; showSuggestFollowupQuestions?: boolean; + showBingSearchOption: boolean; } export const Settings = ({ From d80e7bed03bef65854766b023fefbc00e329cb03 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 16:34:58 +1100 Subject: [PATCH 07/17] Formatting --- app/backend/app.py | 4 ++-- app/backend/approaches/chatreadretrieveread.py | 2 +- app/backend/approaches/chatreadretrievereadvision.py | 4 ++-- app/backend/approaches/retrievethenread.py | 2 +- app/backend/approaches/retrievethenreadvision.py | 2 +- app/backend/bing_client.py | 3 ++- 6 files changed, 9 insertions(+), 8 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index 90b1b525ef..58a79eeebc 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -60,6 +60,8 @@ CONFIG_ASK_APPROACH, CONFIG_ASK_VISION_APPROACH, CONFIG_AUTH_CLIENT, + CONFIG_BING_SEARCH_CLIENT, + CONFIG_BING_SEARCH_ENABLED, CONFIG_BLOB_CONTAINER_CLIENT, CONFIG_CHAT_APPROACH, CONFIG_CHAT_HISTORY_BROWSER_ENABLED, @@ -82,8 +84,6 @@ CONFIG_USER_BLOB_CONTAINER_CLIENT, CONFIG_USER_UPLOAD_ENABLED, CONFIG_VECTOR_SEARCH_ENABLED, - CONFIG_BING_SEARCH_ENABLED, - CONFIG_BING_SEARCH_CLIENT, ) from core.authentication import AuthenticationHelper from core.sessionhelper import create_session_id diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 824296605c..50536b1b7a 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -14,8 +14,8 @@ from approaches.approach import Document, ThoughtStep from approaches.chatapproach import ChatApproach from approaches.promptmanager import PromptManager -from core.authentication import AuthenticationHelper from bing_client import AsyncBingClient +from core.authentication import AuthenticationHelper class ChatReadRetrieveReadApproach(ChatApproach): diff --git a/app/backend/approaches/chatreadretrievereadvision.py b/app/backend/approaches/chatreadretrievereadvision.py index e7f165a225..c2e6f0e41d 100644 --- a/app/backend/approaches/chatreadretrievereadvision.py +++ b/app/backend/approaches/chatreadretrievereadvision.py @@ -14,9 +14,9 @@ from approaches.approach import ThoughtStep from approaches.chatapproach import ChatApproach from approaches.promptmanager import PromptManager +from bing_client import AsyncBingClient from core.authentication import AuthenticationHelper from core.imageshelper import fetch_image -from bing_client import AsyncBingClient class ChatReadRetrieveReadVisionApproach(ChatApproach): @@ -85,7 +85,7 @@ async def run_until_final_call( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False - use_bing_search = True if overrides.get("use_bing_search") else False + # use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) diff --git a/app/backend/approaches/retrievethenread.py b/app/backend/approaches/retrievethenread.py index 7acd15e52b..1bc808afe9 100644 --- a/app/backend/approaches/retrievethenread.py +++ b/app/backend/approaches/retrievethenread.py @@ -68,7 +68,7 @@ async def run( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False - use_bing_search = True if overrides.get("use_bing_search") else False + # use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) diff --git a/app/backend/approaches/retrievethenreadvision.py b/app/backend/approaches/retrievethenreadvision.py index 469f75c168..cb1905f2bc 100644 --- a/app/backend/approaches/retrievethenreadvision.py +++ b/app/backend/approaches/retrievethenreadvision.py @@ -77,7 +77,7 @@ async def run( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False - use_bing_search = True if overrides.get("use_bing_search") else False + # use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) diff --git a/app/backend/bing_client.py b/app/backend/bing_client.py index a8e9405ba5..7c54a11b51 100644 --- a/app/backend/bing_client.py +++ b/app/backend/bing_client.py @@ -2,9 +2,10 @@ An async client for Bing Web Search API. """ +from typing import Optional + import httpx from pydantic import BaseModel, ConfigDict -from typing import Any, Optional class WebPage(BaseModel): From e697aaecee5acb557ae9385e59feaa129e37b753 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 16:35:20 +1100 Subject: [PATCH 08/17] Black formatting --- app/backend/approaches/chatreadretrieveread.py | 7 +------ app/backend/bing_client.py | 8 ++++---- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 50536b1b7a..72f481315c 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -216,12 +216,7 @@ async def keyword_rewrite(rendered_prompt, tools): else {"model": self.chatgpt_model} ), ), - ThoughtStep( - "Bing search query", - bing_query_text if use_bing_search else None, - { - } - ), + ThoughtStep("Bing search query", bing_query_text if use_bing_search else None, {}), ThoughtStep( "Bing search results", [result.snippet for result in bing_results.value[:2]] if use_bing_search else None, diff --git a/app/backend/bing_client.py b/app/backend/bing_client.py index 7c54a11b51..3fa8da3b62 100644 --- a/app/backend/bing_client.py +++ b/app/backend/bing_client.py @@ -21,7 +21,7 @@ class WebPage(BaseModel): # There are more fields in the response, but we only care about these for now. model_config = ConfigDict( - extra='allow', + extra="allow", ) @@ -32,7 +32,7 @@ class WebAnswer(BaseModel): # There are more fields in the response, but we only care about these for now. model_config = ConfigDict( - extra='allow', + extra="allow", ) @@ -45,7 +45,7 @@ def __init__(self, api_key: str, bing_endpoint: Optional[str] = "api.bing.micros "User-Agent": "azure-search-openai-demo", # "X-Search-Location": "" # this would be useful in future } - + async def search(self, query: str) -> WebAnswer: params = { "q": query, @@ -55,4 +55,4 @@ async def search(self, query: str) -> WebAnswer: async with httpx.AsyncClient() as client: response = await client.get(self.base_url, headers=self.headers, params=params) response.raise_for_status() - return WebAnswer.model_validate(response.json()['webPages']) + return WebAnswer.model_validate(response.json()["webPages"]) From e68619b386d55a51faa8d2e9e111607bacac4018 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 16:44:07 +1100 Subject: [PATCH 09/17] mypy updates --- app/backend/app.py | 2 -- app/backend/approaches/chatreadretrieveread.py | 2 +- app/backend/bing_client.py | 4 ++-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/app/backend/app.py b/app/backend/app.py index 58a79eeebc..1f320390ee 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -473,8 +473,6 @@ async def setup_clients(): USE_BING_SEARCH = os.getenv("USE_BING_SEARCH", "").lower() == "true" BING_SEARCH_API_KEY = os.getenv("BING_SEARCH_API_KEY") BING_SEARCH_ENDPOINT = os.getenv("BING_SEARCH_ENDPOINT") - if BING_SEARCH_ENDPOINT is not None and BING_SEARCH_ENDPOINT.trim() == "": - BING_SEARCH_ENDPOINT = None # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 72f481315c..4ac7c6ecd3 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -137,7 +137,7 @@ async def keyword_rewrite(rendered_prompt, tools): ) tools: List[ChatCompletionToolParam] = self.query_rewrite_tools query_messages, query_text = await keyword_rewrite(rendered_query_prompt, tools) - if use_bing_search: + if use_bing_search and self.bing_client: bing_search_prompt = self.prompt_manager.render_prompt( self.bing_ground_rewrite_prompt, {"user_query": original_user_query, "past_messages": messages[:-1]}, diff --git a/app/backend/bing_client.py b/app/backend/bing_client.py index 3fa8da3b62..88d13122a0 100644 --- a/app/backend/bing_client.py +++ b/app/backend/bing_client.py @@ -2,7 +2,7 @@ An async client for Bing Web Search API. """ -from typing import Optional +from typing import Optional, Union import httpx from pydantic import BaseModel, ConfigDict @@ -47,7 +47,7 @@ def __init__(self, api_key: str, bing_endpoint: Optional[str] = "api.bing.micros } async def search(self, query: str) -> WebAnswer: - params = { + params: dict[str, Union[str,bool]] = { "q": query, "textDecorations": True, "textFormat": "HTML", From 902ac4aeb168a0898cf29017382b856d25df7a55 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 16:51:46 +1100 Subject: [PATCH 10/17] Add extra bing params --- app/backend/bing_client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/app/backend/bing_client.py b/app/backend/bing_client.py index 88d13122a0..c9c6e88e34 100644 --- a/app/backend/bing_client.py +++ b/app/backend/bing_client.py @@ -46,11 +46,15 @@ def __init__(self, api_key: str, bing_endpoint: Optional[str] = "api.bing.micros # "X-Search-Location": "" # this would be useful in future } - async def search(self, query: str) -> WebAnswer: - params: dict[str, Union[str,bool]] = { + async def search(self, query: str, lang="en-US") -> WebAnswer: + params: dict[str, Union[str, bool, int]] = { "q": query, + "mkt": lang, "textDecorations": True, "textFormat": "HTML", + "responseFilter": "Webpages", + "safeSearch": "Strict", + "setLang": lang, } async with httpx.AsyncClient() as client: response = await client.get(self.base_url, headers=self.headers, params=params) From 72d9879529e0858f367def9d5978fdfb01c85ac8 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 16:53:10 +1100 Subject: [PATCH 11/17] Use the configured language in Bing --- app/backend/approaches/chatreadretrieveread.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 4ac7c6ecd3..7e8c08ef9e 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -143,7 +143,7 @@ async def keyword_rewrite(rendered_prompt, tools): {"user_query": original_user_query, "past_messages": messages[:-1]}, ) _, bing_query_text = await keyword_rewrite(bing_search_prompt, self.bing_ground_rewrite_tools) - bing_results = await self.bing_client.search(bing_query_text) + bing_results = await self.bing_client.search(bing_query_text, lang=self.query_language) # STEP 2: Retrieve relevant documents from the search index with the GPT optimized query From 05754e23476ce29afaf74f106bfa0509d58bcb45 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 16:54:15 +1100 Subject: [PATCH 12/17] Update markdown --- docs/deploy_features.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deploy_features.md b/docs/deploy_features.md index cc60369bb7..10f7f48cdf 100644 --- a/docs/deploy_features.md +++ b/docs/deploy_features.md @@ -354,4 +354,4 @@ Set the API key via the BING_SEARCH_API_KEY command: azd env set BING_SEARCH_API_KEY ``` -Note that Managed Identity is not available in Bing Search API. \ No newline at end of file +Note that Managed Identity is not available in Bing Search API. From 705538c072817ca39dfe6e79df8767f3993a5c4b Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 20:08:37 +1100 Subject: [PATCH 13/17] Include web result used in answer in the response. --- app/backend/approaches/approach.py | 4 +++ .../approaches/chatreadretrieveread.py | 11 ++++---- .../prompts/chat_bing_answer_question.prompty | 3 ++- .../SupportingContent/SupportingContent.tsx | 26 ++++++++++++++++++- 4 files changed, 37 insertions(+), 7 deletions(-) diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index 44a1d6380a..e36748e31c 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -26,6 +26,7 @@ from approaches.promptmanager import PromptManager from core.authentication import AuthenticationHelper +from bing_client import WebPage @dataclass @@ -236,6 +237,9 @@ def get_citation(self, sourcepage: str, use_image_citation: bool) -> str: return sourcepage + def get_links(self, webpages: list[WebPage]) -> list[str]: + return [f"{page.id}: {page.snippet}" for page in webpages] + async def compute_text_embedding(self, q: str): SUPPORTED_DIMENSIONS_MODEL = { "text-embedding-ada-002": False, diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 7e8c08ef9e..5e16081520 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -14,7 +14,7 @@ from approaches.approach import Document, ThoughtStep from approaches.chatapproach import ChatApproach from approaches.promptmanager import PromptManager -from bing_client import AsyncBingClient +from bing_client import AsyncBingClient, WebPage from core.authentication import AuthenticationHelper @@ -167,9 +167,10 @@ async def keyword_rewrite(rendered_prompt, tools): # STEP 3: Generate a contextual and content specific answer using the search results and chat history text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=False) - + web_sources: list[WebPage] = [] if use_bing_search and bing_results.totalEstimatedMatches > 0: - web_sources = [hit.snippet for hit in bing_results.value[:2]] + web_sources = bing_results.value[:2] + web_sources_text = self.get_links(web_sources) rendered_answer_prompt = self.prompt_manager.render_prompt( self.bing_answer_prompt, @@ -179,7 +180,7 @@ async def keyword_rewrite(rendered_prompt, tools): "past_messages": messages[:-1], "user_query": original_user_query, "text_sources": text_sources, - "web_search_snippets": web_sources, + "web_search_snippets": web_sources_text, }, ) else: @@ -205,7 +206,7 @@ async def keyword_rewrite(rendered_prompt, tools): ) extra_info = { - "data_points": {"text": text_sources}, + "data_points": {"text": text_sources, "web_search": [hit.model_dump() for hit in web_sources]}, "thoughts": [ ThoughtStep( "Prompt to generate search query", diff --git a/app/backend/approaches/prompts/chat_bing_answer_question.prompty b/app/backend/approaches/prompts/chat_bing_answer_question.prompty index 7dfecdea7c..309c0313cf 100644 --- a/app/backend/approaches/prompts/chat_bing_answer_question.prompty +++ b/app/backend/approaches/prompts/chat_bing_answer_question.prompty @@ -24,7 +24,8 @@ Assistant helps the company employees with their healthcare plan questions, and Answer ONLY with the facts listed in the list of sources below. If there isn't enough information below, say you don't know. Do not generate answers that don't use the sources below. If asking a clarifying question to the user would help, ask the question. If the question is not in English, answer in the language used in the question. Each source has a name followed by colon and the actual information, always include the source name for each fact you use in the response. Use square brackets to reference the source, for example [info1.txt]. Don't combine sources, list each source separately, for example [info1.txt][info2.pdf]. -Additional web search snippets are included. If the sources do not answer the question, you can use the web search snippets to find the answer. The original sources take precedence over the web search snippets. +Additional "web search snippets" are included, each with a unique URI then a colon and the snippet text in HTML. If the sources do not answer the question, you can use the web search snippets to find the answer. The original sources take precedence over the web search snippets. +If you use a web search snippet in your response, always include the URI of the snippet in square brackets, for example [https://api.bing.microsoft.com/api/v7/#WebPages.0]. {{ injected_prompt }} {% endif %} diff --git a/app/frontend/src/components/SupportingContent/SupportingContent.tsx b/app/frontend/src/components/SupportingContent/SupportingContent.tsx index 94df0ecca2..fd53ccc20d 100644 --- a/app/frontend/src/components/SupportingContent/SupportingContent.tsx +++ b/app/frontend/src/components/SupportingContent/SupportingContent.tsx @@ -2,13 +2,26 @@ import { parseSupportingContentItem } from "./SupportingContentParser"; import styles from "./SupportingContent.module.css"; +interface WebPage { + id: string; + name: string; + url: string; + displayUrl: string; + dateLastCrawled: string; + language: string; + snippet?: string; + isFamilyFriendly?: boolean; + siteName?: string; +} + interface Props { - supportingContent: string[] | { text: string[]; images?: string[] }; + supportingContent: string[] | { text: string[]; images?: string[]; web_search?: WebPage[] }; } export const SupportingContent = ({ supportingContent }: Props) => { const textItems = Array.isArray(supportingContent) ? supportingContent : supportingContent.text; const imageItems = !Array.isArray(supportingContent) ? supportingContent?.images : []; + const webSearchItems = !Array.isArray(supportingContent) ? supportingContent?.web_search : []; return (
    {textItems.map((c, ind) => { @@ -27,6 +40,17 @@ export const SupportingContent = ({ supportingContent }: Props) => { ); })} + {webSearchItems?.map((webPage, ind) => { + return ( +
  • +

    {webPage.name}

    + + {webPage.url} + +

    {webPage.snippet}

    +
    • + ); + })}
); }; From bbb73b2fa4a31b265284fcdb6261446a58206bc5 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Thu, 23 Jan 2025 20:11:10 +1100 Subject: [PATCH 14/17] Fix import order --- app/backend/approaches/approach.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index e36748e31c..894ddcece7 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -25,8 +25,8 @@ from openai.types.chat import ChatCompletionMessageParam from approaches.promptmanager import PromptManager -from core.authentication import AuthenticationHelper from bing_client import WebPage +from core.authentication import AuthenticationHelper @dataclass From 7c93eb32a7faf7468758f786929614e9f83125c4 Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Fri, 24 Jan 2025 13:08:09 +1100 Subject: [PATCH 15/17] render web citations --- app/frontend/src/components/Answer/Answer.tsx | 25 ++++++++++--- .../src/components/Answer/AnswerParser.tsx | 37 +++++++++++++++++++ .../SupportingContent/SupportingContent.tsx | 2 +- 3 files changed, 57 insertions(+), 7 deletions(-) diff --git a/app/frontend/src/components/Answer/Answer.tsx b/app/frontend/src/components/Answer/Answer.tsx index 75b0a03504..7c2a652409 100644 --- a/app/frontend/src/components/Answer/Answer.tsx +++ b/app/frontend/src/components/Answer/Answer.tsx @@ -8,10 +8,11 @@ import rehypeRaw from "rehype-raw"; import styles from "./Answer.module.css"; import { ChatAppResponse, getCitationFilePath, SpeechConfig } from "../../api"; -import { parseAnswerToHtml } from "./AnswerParser"; +import { citationIdToCitation, parseAnswerToHtml } from "./AnswerParser"; import { AnswerIcon } from "./AnswerIcon"; import { SpeechOutputBrowser } from "./SpeechOutputBrowser"; import { SpeechOutputAzure } from "./SpeechOutputAzure"; +import { WebPage } from "../SupportingContent"; interface Props { answer: ChatAppResponse; @@ -110,11 +111,23 @@ export const Answer = ({ {t("citationWithColon")} {parsedAnswer.citations.map((x, i) => { const path = getCitationFilePath(x); - return ( - onCitationClicked(path)}> - {`${++i}. ${x}`} - - ); + const citation = citationIdToCitation(x, answer.context.data_points); + + if (citation.type === "document") + return ( + onCitationClicked(path)}> + {`${++i}. ${x}`} + + ); + else if (citation.type === "web") { + const webPage = citation.citation as WebPage; + const label = webPage.siteName ? webPage.siteName : webPage.url; + return ( + + {`${++i}. ${label}`} + + ); + } })} diff --git a/app/frontend/src/components/Answer/AnswerParser.tsx b/app/frontend/src/components/Answer/AnswerParser.tsx index 3807592f6d..108d011189 100644 --- a/app/frontend/src/components/Answer/AnswerParser.tsx +++ b/app/frontend/src/components/Answer/AnswerParser.tsx @@ -1,11 +1,40 @@ import { renderToStaticMarkup } from "react-dom/server"; import { ChatAppResponse, getCitationFilePath } from "../../api"; +import { WebPage } from "../SupportingContent"; type HtmlParsedAnswer = { answerHtml: string; citations: string[]; }; +type Citation = { + id: string; + type: "document" | "web"; + citation: string | WebPage; +}; + +export function citationIdToCitation(citationId: string, contextDataPoints: any): Citation { + // See if this is a web page citation + const webSearch = contextDataPoints.web_search; + if (Array.isArray(webSearch)) { + const webPage = webSearch.find((page: WebPage) => page.id === citationId); + if (webPage) { + return { + id: citationId, + type: "web", + citation: webPage + }; + } + } + + // Otherwise, assume it's a document citation + return { + id: citationId, + type: "document", + citation: citationId + }; +} + // Function to validate citation format and check if dataPoint starts with possible citation function isCitationValid(contextDataPoints: any, citationCandidate: string): boolean { const regex = /.+\.\w{1,}(?:#\S*)?$/; @@ -22,6 +51,14 @@ function isCitationValid(contextDataPoints: any, citationCandidate: string): boo } else { return false; } + // If there are web_sources, add those to the list of identifiers + if (Array.isArray(contextDataPoints.web_search)) { + contextDataPoints.web_search.forEach((source: any) => { + if (source.id) { + dataPointsArray.push(source.id); + } + }); + } const isValidCitation = dataPointsArray.some(dataPoint => { return dataPoint.startsWith(citationCandidate); diff --git a/app/frontend/src/components/SupportingContent/SupportingContent.tsx b/app/frontend/src/components/SupportingContent/SupportingContent.tsx index fd53ccc20d..aadb32284c 100644 --- a/app/frontend/src/components/SupportingContent/SupportingContent.tsx +++ b/app/frontend/src/components/SupportingContent/SupportingContent.tsx @@ -2,7 +2,7 @@ import { parseSupportingContentItem } from "./SupportingContentParser"; import styles from "./SupportingContent.module.css"; -interface WebPage { +export interface WebPage { id: string; name: string; url: string; From 6595fdf6d0d63d99aaeeb844fbca5f8c6e97020d Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 17 Feb 2025 11:17:35 +1100 Subject: [PATCH 16/17] Make the grounding backend generic --- app/backend/app.py | 40 +++++++++---------- app/backend/approaches/approach.py | 2 +- .../approaches/chatreadretrieveread.py | 36 ++++++++--------- .../approaches/chatreadretrievereadvision.py | 7 ++-- ...ty => chat_ground_answer_question.prompty} | 0 ...te.prompty => chat_ground_rewrite.prompty} | 6 +-- ...ls.json => chat_ground_rewrite_tools.json} | 10 ++--- app/backend/approaches/retrievethenread.py | 1 - .../approaches/retrievethenreadvision.py | 1 - app/backend/config.py | 4 +- app/backend/requirements.in | 1 + .../{bing_client.py => search_client.py} | 10 ++++- app/frontend/src/api/models.ts | 4 +- .../src/components/Settings/Settings.tsx | 26 ++++++------ app/frontend/src/locales/en/translation.json | 4 +- app/frontend/src/pages/ask/Ask.tsx | 16 ++++---- app/frontend/src/pages/chat/Chat.tsx | 16 ++++---- 17 files changed, 95 insertions(+), 89 deletions(-) rename app/backend/approaches/prompts/{chat_bing_answer_question.prompty => chat_ground_answer_question.prompty} (100%) rename app/backend/approaches/prompts/{chat_bing_ground_rewrite.prompty => chat_ground_rewrite.prompty} (87%) rename app/backend/approaches/prompts/{chat_bing_ground_rewrite_tools.json => chat_ground_rewrite_tools.json} (51%) rename app/backend/{bing_client.py => search_client.py} (87%) diff --git a/app/backend/app.py b/app/backend/app.py index 1f320390ee..2226de2637 100644 --- a/app/backend/app.py +++ b/app/backend/app.py @@ -54,14 +54,12 @@ from approaches.promptmanager import PromptyManager from approaches.retrievethenread import RetrieveThenReadApproach from approaches.retrievethenreadvision import RetrieveThenReadVisionApproach -from bing_client import AsyncBingClient +from search_client import AsyncGroundingSearchClient from chat_history.cosmosdb import chat_history_cosmosdb_bp from config import ( CONFIG_ASK_APPROACH, CONFIG_ASK_VISION_APPROACH, CONFIG_AUTH_CLIENT, - CONFIG_BING_SEARCH_CLIENT, - CONFIG_BING_SEARCH_ENABLED, CONFIG_BLOB_CONTAINER_CLIENT, CONFIG_CHAT_APPROACH, CONFIG_CHAT_HISTORY_BROWSER_ENABLED, @@ -69,6 +67,8 @@ CONFIG_CHAT_VISION_APPROACH, CONFIG_CREDENTIAL, CONFIG_GPT4V_DEPLOYED, + CONFIG_GROUNDING_SEARCH_CLIENT, + CONFIG_GROUNDING_SEARCH_ENABLED, CONFIG_INGESTER, CONFIG_LANGUAGE_PICKER_ENABLED, CONFIG_OPENAI_CLIENT, @@ -302,7 +302,7 @@ def config(): "showSpeechOutputAzure": current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED], "showChatHistoryBrowser": current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED], "showChatHistoryCosmos": current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED], - "showBingSearchOption": current_app.config[CONFIG_BING_SEARCH_ENABLED], + "showGroundingSearchOption": current_app.config[CONFIG_GROUNDING_SEARCH_ENABLED], } ) @@ -470,9 +470,9 @@ async def setup_clients(): USE_SPEECH_OUTPUT_AZURE = os.getenv("USE_SPEECH_OUTPUT_AZURE", "").lower() == "true" USE_CHAT_HISTORY_BROWSER = os.getenv("USE_CHAT_HISTORY_BROWSER", "").lower() == "true" USE_CHAT_HISTORY_COSMOS = os.getenv("USE_CHAT_HISTORY_COSMOS", "").lower() == "true" - USE_BING_SEARCH = os.getenv("USE_BING_SEARCH", "").lower() == "true" - BING_SEARCH_API_KEY = os.getenv("BING_SEARCH_API_KEY") - BING_SEARCH_ENDPOINT = os.getenv("BING_SEARCH_ENDPOINT") + USE_GROUNDING_SEARCH = os.getenv("USE_GROUNDING_SEARCH", "").lower() == "true" + GROUNDING_SEARCH_API_KEY = os.getenv("GROUNDING_SEARCH_API_KEY") + GROUNDING_SEARCH_ENDPOINT = os.getenv("GROUNDING_SEARCH_ENDPOINT") # WEBSITE_HOSTNAME is always set by App Service, RUNNING_IN_PRODUCTION is set in main.bicep RUNNING_ON_AZURE = os.getenv("WEBSITE_HOSTNAME") is not None or os.getenv("RUNNING_IN_PRODUCTION") is not None @@ -595,18 +595,18 @@ async def setup_clients(): # Wait until token is needed to fetch for the first time current_app.config[CONFIG_SPEECH_SERVICE_TOKEN] = None - if USE_BING_SEARCH: - current_app.logger.info("USE_BING_SEARCH is true, setting up Bing search client") - if not BING_SEARCH_API_KEY: - raise ValueError("BING_SEARCH_API_KEY must be set when USE_BING_SEARCH is true") - if BING_SEARCH_ENDPOINT: - bing_search_client = AsyncBingClient(BING_SEARCH_API_KEY, BING_SEARCH_ENDPOINT) + if USE_GROUNDING_SEARCH: + current_app.logger.info("USE_GROUNDING_SEARCH is true, setting up search client") + if not GROUNDING_SEARCH_API_KEY: + raise ValueError("GROUNDING_SEARCH_API_KEY must be set when USE_GROUNDING_SEARCH is true") + if GROUNDING_SEARCH_ENDPOINT: + grounding_search_client = AsyncGroundingSearchClient(GROUNDING_SEARCH_API_KEY, GROUNDING_SEARCH_ENDPOINT) else: - bing_search_client = AsyncBingClient(BING_SEARCH_API_KEY) - current_app.config[CONFIG_BING_SEARCH_CLIENT] = bing_search_client + grounding_search_client = AsyncGroundingSearchClient(GROUNDING_SEARCH_API_KEY) + current_app.config[CONFIG_GROUNDING_SEARCH_CLIENT] = grounding_search_client else: - current_app.logger.info("USE_BING_SEARCH is false, Bing search client not set up") - bing_search_client = None + current_app.logger.info("USE_GROUNDING_SEARCH is false, search client not set up") + grounding_search_client = None if OPENAI_HOST.startswith("azure"): if OPENAI_HOST == "azure_custom": @@ -662,7 +662,7 @@ async def setup_clients(): current_app.config[CONFIG_SPEECH_OUTPUT_AZURE_ENABLED] = USE_SPEECH_OUTPUT_AZURE current_app.config[CONFIG_CHAT_HISTORY_BROWSER_ENABLED] = USE_CHAT_HISTORY_BROWSER current_app.config[CONFIG_CHAT_HISTORY_COSMOS_ENABLED] = USE_CHAT_HISTORY_COSMOS - current_app.config[CONFIG_BING_SEARCH_ENABLED] = USE_BING_SEARCH + current_app.config[CONFIG_GROUNDING_SEARCH_ENABLED] = USE_GROUNDING_SEARCH prompt_manager = PromptyManager() @@ -699,7 +699,7 @@ async def setup_clients(): query_language=AZURE_SEARCH_QUERY_LANGUAGE, query_speller=AZURE_SEARCH_QUERY_SPELLER, prompt_manager=prompt_manager, - bing_client=bing_search_client, + grounding_search_client=grounding_search_client, ) if USE_GPT4V: @@ -746,7 +746,7 @@ async def setup_clients(): query_language=AZURE_SEARCH_QUERY_LANGUAGE, query_speller=AZURE_SEARCH_QUERY_SPELLER, prompt_manager=prompt_manager, - bing_client=bing_search_client, + grounding_search_client=grounding_search_client, ) diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index 894ddcece7..b55f3975be 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -25,7 +25,7 @@ from openai.types.chat import ChatCompletionMessageParam from approaches.promptmanager import PromptManager -from bing_client import WebPage +from app.backend.search_client import WebPage from core.authentication import AuthenticationHelper diff --git a/app/backend/approaches/chatreadretrieveread.py b/app/backend/approaches/chatreadretrieveread.py index 5e16081520..e8a481fbb1 100644 --- a/app/backend/approaches/chatreadretrieveread.py +++ b/app/backend/approaches/chatreadretrieveread.py @@ -14,7 +14,7 @@ from approaches.approach import Document, ThoughtStep from approaches.chatapproach import ChatApproach from approaches.promptmanager import PromptManager -from bing_client import AsyncBingClient, WebPage +from search_client import AsyncGroundingSearchClient, WebPage from core.authentication import AuthenticationHelper @@ -41,11 +41,11 @@ def __init__( query_language: str, query_speller: str, prompt_manager: PromptManager, - bing_client: Optional[AsyncBingClient] = None, + grounding_search_client: Optional[AsyncGroundingSearchClient] = None, ): self.search_client = search_client self.openai_client = openai_client - self.bing_client = bing_client + self.grounding_search_client = grounding_search_client self.auth_helper = auth_helper self.chatgpt_model = chatgpt_model self.chatgpt_deployment = chatgpt_deployment @@ -61,9 +61,9 @@ def __init__( self.query_rewrite_prompt = self.prompt_manager.load_prompt("chat_query_rewrite.prompty") self.query_rewrite_tools = self.prompt_manager.load_tools("chat_query_rewrite_tools.json") self.answer_prompt = self.prompt_manager.load_prompt("chat_answer_question.prompty") - self.bing_answer_prompt = self.prompt_manager.load_prompt("chat_bing_answer_question.prompty") - self.bing_ground_rewrite_prompt = self.prompt_manager.load_prompt("chat_bing_ground_rewrite.prompty") - self.bing_ground_rewrite_tools = self.prompt_manager.load_tools("chat_bing_ground_rewrite_tools.json") + self.ground_answer_prompt = self.prompt_manager.load_prompt("chat_ground_answer_question.prompty") + self.ground_rewrite_prompt = self.prompt_manager.load_prompt("chat_ground_rewrite.prompty") + self.ground_rewrite_tools = self.prompt_manager.load_tools("chat_ground_rewrite_tools.json") @overload async def run_until_final_call( @@ -95,7 +95,7 @@ async def run_until_final_call( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False - use_bing_search = True if overrides.get("use_bing_search") else False + use_grounding_search = True if overrides.get("use_grounding_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) @@ -137,13 +137,13 @@ async def keyword_rewrite(rendered_prompt, tools): ) tools: List[ChatCompletionToolParam] = self.query_rewrite_tools query_messages, query_text = await keyword_rewrite(rendered_query_prompt, tools) - if use_bing_search and self.bing_client: - bing_search_prompt = self.prompt_manager.render_prompt( - self.bing_ground_rewrite_prompt, + if use_grounding_search and self.grounding_search_client: + ground_search_prompt = self.prompt_manager.render_prompt( + self.ground_rewrite_prompt, {"user_query": original_user_query, "past_messages": messages[:-1]}, ) - _, bing_query_text = await keyword_rewrite(bing_search_prompt, self.bing_ground_rewrite_tools) - bing_results = await self.bing_client.search(bing_query_text, lang=self.query_language) + _, ground_query_text = await keyword_rewrite(ground_search_prompt, self.ground_rewrite_tools) + ground_results = await self.grounding_search_client.search(ground_query_text, lang=self.query_language) # STEP 2: Retrieve relevant documents from the search index with the GPT optimized query @@ -168,12 +168,12 @@ async def keyword_rewrite(rendered_prompt, tools): # STEP 3: Generate a contextual and content specific answer using the search results and chat history text_sources = self.get_sources_content(results, use_semantic_captions, use_image_citation=False) web_sources: list[WebPage] = [] - if use_bing_search and bing_results.totalEstimatedMatches > 0: - web_sources = bing_results.value[:2] + if use_grounding_search and ground_results.totalEstimatedMatches > 0: + web_sources = ground_results.value[:2] web_sources_text = self.get_links(web_sources) rendered_answer_prompt = self.prompt_manager.render_prompt( - self.bing_answer_prompt, + self.ground_answer_prompt, self.get_system_prompt_variables(overrides.get("prompt_template")) | { "include_follow_up_questions": bool(overrides.get("suggest_followup_questions")), @@ -217,10 +217,10 @@ async def keyword_rewrite(rendered_prompt, tools): else {"model": self.chatgpt_model} ), ), - ThoughtStep("Bing search query", bing_query_text if use_bing_search else None, {}), + ThoughtStep("Grounding search query", ground_query_text if use_grounding_search else None, {}), ThoughtStep( - "Bing search results", - [result.snippet for result in bing_results.value[:2]] if use_bing_search else None, + "Grounding search results", + [result.snippet for result in ground_results.value[:2]] if use_grounding_search else None, ), ThoughtStep( "Search using generated search query", diff --git a/app/backend/approaches/chatreadretrievereadvision.py b/app/backend/approaches/chatreadretrievereadvision.py index c2e6f0e41d..dd02198428 100644 --- a/app/backend/approaches/chatreadretrievereadvision.py +++ b/app/backend/approaches/chatreadretrievereadvision.py @@ -14,7 +14,7 @@ from approaches.approach import ThoughtStep from approaches.chatapproach import ChatApproach from approaches.promptmanager import PromptManager -from bing_client import AsyncBingClient +from search_client import AsyncGroundingSearchClient from core.authentication import AuthenticationHelper from core.imageshelper import fetch_image @@ -47,12 +47,12 @@ def __init__( vision_endpoint: str, vision_token_provider: Callable[[], Awaitable[str]], prompt_manager: PromptManager, - bing_client: Optional[AsyncBingClient] = None, + grounding_search_client: Optional[AsyncGroundingSearchClient] = None, ): self.search_client = search_client self.blob_container_client = blob_container_client self.openai_client = openai_client - self.bing_client = bing_client + self.grounding_search_client = grounding_search_client self.auth_helper = auth_helper self.chatgpt_model = chatgpt_model self.chatgpt_deployment = chatgpt_deployment @@ -85,7 +85,6 @@ async def run_until_final_call( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False - # use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) diff --git a/app/backend/approaches/prompts/chat_bing_answer_question.prompty b/app/backend/approaches/prompts/chat_ground_answer_question.prompty similarity index 100% rename from app/backend/approaches/prompts/chat_bing_answer_question.prompty rename to app/backend/approaches/prompts/chat_ground_answer_question.prompty diff --git a/app/backend/approaches/prompts/chat_bing_ground_rewrite.prompty b/app/backend/approaches/prompts/chat_ground_rewrite.prompty similarity index 87% rename from app/backend/approaches/prompts/chat_bing_ground_rewrite.prompty rename to app/backend/approaches/prompts/chat_ground_rewrite.prompty index 54094a3c6d..ce3ebd7089 100644 --- a/app/backend/approaches/prompts/chat_bing_ground_rewrite.prompty +++ b/app/backend/approaches/prompts/chat_ground_rewrite.prompty @@ -4,7 +4,7 @@ description: Suggest the optimal search query based on the user's query, example model: api: chat parameters: - tools: ${file:chat_query_rewrite_tools.json} + tools: ${file:chat_ground_rewrite_tools.json} sample: user_query: Does it include hearing? past_messages: @@ -15,7 +15,7 @@ sample: --- system: Below is a history of the conversation so far, and a new question asked by the user that needs to be answered by searching in a knowledge base and/or a web search. -If the user question requires grounding from an Internet search, generate a Bing search query based on the conversation and the new question. +If the user question requires grounding from an Internet search, generate an Internet search query based on the conversation and the new question. If you cannot generate a search query, return just the number 0. user: @@ -48,4 +48,4 @@ Microsoft headquarters address {% endfor %} user: -Generate Bing search query for: {{ user_query }} +Generate Grounding search query for: {{ user_query }} diff --git a/app/backend/approaches/prompts/chat_bing_ground_rewrite_tools.json b/app/backend/approaches/prompts/chat_ground_rewrite_tools.json similarity index 51% rename from app/backend/approaches/prompts/chat_bing_ground_rewrite_tools.json rename to app/backend/approaches/prompts/chat_ground_rewrite_tools.json index 8dc57ee083..eef5214962 100644 --- a/app/backend/approaches/prompts/chat_bing_ground_rewrite_tools.json +++ b/app/backend/approaches/prompts/chat_ground_rewrite_tools.json @@ -1,17 +1,17 @@ [{ "type": "function", "function": { - "name": "search_bing", - "description": "Retrieve sources from Bing", + "name": "search_internet", + "description": "Retrieve sources from the Internet", "parameters": { "type": "object", "properties": { - "bing_search_query": { + "internet_search_query": { "type": "string", - "description": "Query string to retrieve documents from bing e.g.: 'Microsoft headquarters address'" + "description": "Query string to retrieve documents from an Internet search e.g.: 'Microsoft headquarters address'" } }, - "required": ["bing_search_query"] + "required": ["internet_search_query"] } } }] diff --git a/app/backend/approaches/retrievethenread.py b/app/backend/approaches/retrievethenread.py index 1bc808afe9..f3c9331e36 100644 --- a/app/backend/approaches/retrievethenread.py +++ b/app/backend/approaches/retrievethenread.py @@ -68,7 +68,6 @@ async def run( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False - # use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) diff --git a/app/backend/approaches/retrievethenreadvision.py b/app/backend/approaches/retrievethenreadvision.py index cb1905f2bc..14318d83fa 100644 --- a/app/backend/approaches/retrievethenreadvision.py +++ b/app/backend/approaches/retrievethenreadvision.py @@ -77,7 +77,6 @@ async def run( use_vector_search = overrides.get("retrieval_mode") in ["vectors", "hybrid", None] use_semantic_ranker = True if overrides.get("semantic_ranker") else False use_semantic_captions = True if overrides.get("semantic_captions") else False - # use_bing_search = True if overrides.get("use_bing_search") else False top = overrides.get("top", 3) minimum_search_score = overrides.get("minimum_search_score", 0.0) minimum_reranker_score = overrides.get("minimum_reranker_score", 0.0) diff --git a/app/backend/config.py b/app/backend/config.py index cb20329f6b..251315be26 100644 --- a/app/backend/config.py +++ b/app/backend/config.py @@ -26,5 +26,5 @@ CONFIG_CHAT_HISTORY_COSMOS_ENABLED = "chat_history_cosmos_enabled" CONFIG_COSMOS_HISTORY_CLIENT = "cosmos_history_client" CONFIG_COSMOS_HISTORY_CONTAINER = "cosmos_history_container" -CONFIG_BING_SEARCH_ENABLED = "bing_search_enabled" -CONFIG_BING_SEARCH_CLIENT = "bing_search_client" +CONFIG_GROUNDING_SEARCH_ENABLED = "grounding_search_enabled" +CONFIG_GROUNDING_SEARCH_CLIENT = "grounding_search_client" diff --git a/app/backend/requirements.in b/app/backend/requirements.in index e2b005154a..c44d9216ce 100644 --- a/app/backend/requirements.in +++ b/app/backend/requirements.in @@ -33,3 +33,4 @@ python-dotenv prompty rich typing-extensions +langchain-community \ No newline at end of file diff --git a/app/backend/bing_client.py b/app/backend/search_client.py similarity index 87% rename from app/backend/bing_client.py rename to app/backend/search_client.py index c9c6e88e34..7e4bf62d55 100644 --- a/app/backend/bing_client.py +++ b/app/backend/search_client.py @@ -36,7 +36,15 @@ class WebAnswer(BaseModel): ) -class AsyncBingClient: +class AsyncGroundingSearchClient: + def __init__(self, api_key: str, endpoint: Optional[str] = None): + ... + + async def search(self, query: str, lang="en-US") -> WebAnswer: + ... + + +class AsyncBingSearchClient(AsyncGroundingSearchClient): def __init__(self, api_key: str, bing_endpoint: Optional[str] = "api.bing.microsoft.com"): self.api_key = api_key self.base_url = f"https://{bing_endpoint}/v7.0/search" diff --git a/app/frontend/src/api/models.ts b/app/frontend/src/api/models.ts index 6d38fb1962..f742f5d2f7 100644 --- a/app/frontend/src/api/models.ts +++ b/app/frontend/src/api/models.ts @@ -37,7 +37,7 @@ export type ChatAppRequestOverrides = { gpt4v_input?: GPT4VInput; vector_fields: VectorFieldOptions[]; language: string; - use_bing_search?: boolean; + use_grounding_search?: boolean; }; export type ResponseMessage = { @@ -86,7 +86,7 @@ export type Config = { showGPT4VOptions: boolean; showSemanticRankerOption: boolean; showVectorOption: boolean; - showBingSearchOption: boolean; + showGroundingSearchOption: boolean; showUserUpload: boolean; showLanguagePicker: boolean; showSpeechInput: boolean; diff --git a/app/frontend/src/components/Settings/Settings.tsx b/app/frontend/src/components/Settings/Settings.tsx index 36d4c050da..571930a97c 100644 --- a/app/frontend/src/components/Settings/Settings.tsx +++ b/app/frontend/src/components/Settings/Settings.tsx @@ -24,7 +24,7 @@ export interface SettingsProps { retrievalMode: RetrievalMode; useGPT4V: boolean; gpt4vInput: GPT4VInput; - useBingSearch: boolean; + useGroundingSearch: boolean; vectorFieldList: VectorFieldOptions[]; showSemanticRankerOption: boolean; showGPT4VOptions: boolean; @@ -41,7 +41,7 @@ export interface SettingsProps { promptTemplatePrefix?: string; promptTemplateSuffix?: string; showSuggestFollowupQuestions?: boolean; - showBingSearchOption: boolean; + showGroundingSearchOption: boolean; } export const Settings = ({ @@ -58,12 +58,12 @@ export const Settings = ({ retrievalMode, useGPT4V, gpt4vInput, - useBingSearch, + useGroundingSearch, vectorFieldList, showSemanticRankerOption, showGPT4VOptions, showVectorOption, - showBingSearchOption, + showGroundingSearchOption, useOidSecurityFilter, useGroupsSecurityFilter, useLogin, @@ -108,8 +108,8 @@ export const Settings = ({ const shouldStreamFieldId = useId("shouldStreamField"); const suggestFollowupQuestionsId = useId("suggestFollowupQuestions"); const suggestFollowupQuestionsFieldId = useId("suggestFollowupQuestionsField"); - const useBingSearchId = useId("useBingSearch"); - const useBingSearchFieldId = useId("useBingSearchField"); + const useGroundingSearchId = useId("useGroundingSearch"); + const useGroundingSearchFieldId = useId("useGroundingSearchField"); const renderLabel = (props: RenderLabelType | undefined, labelId: string, fieldId: string, helpText: string) => ( @@ -318,15 +318,15 @@ export const Settings = ({ /> )} - {showBingSearchOption && ( + {showGroundingSearchOption && ( onChange("useBingSearch", !!checked)} - aria-labelledby={useBingSearchId} - onRenderLabel={props => renderLabel(props, useBingSearchId, useBingSearchFieldId, t("helpTexts.useBingSearch"))} + checked={useGroundingSearch} + label={t("labels.useGroundingSearch")} + onChange={(_ev, checked) => onChange("useGroundingSearch", !!checked)} + aria-labelledby={useGroundingSearchId} + onRenderLabel={props => renderLabel(props, useGroundingSearchId, useGroundingSearchFieldId, t("helpTexts.useGroundingSearch"))} /> )} diff --git a/app/frontend/src/locales/en/translation.json b/app/frontend/src/locales/en/translation.json index 56b0785796..8222f030a6 100644 --- a/app/frontend/src/locales/en/translation.json +++ b/app/frontend/src/locales/en/translation.json @@ -119,7 +119,7 @@ "useOidSecurityFilter": "Use oid security filter", "useGroupsSecurityFilter": "Use groups security filter", "shouldStream": "Stream chat completion responses", - "useBingSearch": "Use Bing search" + "useGroundingSearch": "Use Grounding search" }, "helpTexts": { @@ -153,6 +153,6 @@ "streamChat": "Continuously streams the response to the chat UI as it is generated.", "useOidSecurityFilter": "Filter search results based on the authenticated user's OID.", "useGroupsSecurityFilter": "Filter search results based on the authenticated user's groups.", - "useBingSearch": "Use Bing search to ground search results." + "useGroundingSearch": "Use Grounding search to ground search results." } } diff --git a/app/frontend/src/pages/ask/Ask.tsx b/app/frontend/src/pages/ask/Ask.tsx index 9f4d43bc5e..6a023d5131 100644 --- a/app/frontend/src/pages/ask/Ask.tsx +++ b/app/frontend/src/pages/ask/Ask.tsx @@ -33,7 +33,7 @@ export function Component(): JSX.Element { const [useSemanticRanker, setUseSemanticRanker] = useState(true); const [useSemanticCaptions, setUseSemanticCaptions] = useState(false); const [useGPT4V, setUseGPT4V] = useState(false); - const [useBingSearch, setUseBingSearch] = useState(false); + const [useGroundingSearch, setUseGroundingSearch] = useState(false); const [gpt4vInput, setGPT4VInput] = useState(GPT4VInput.TextAndImages); const [includeCategory, setIncludeCategory] = useState(""); const [excludeCategory, setExcludeCategory] = useState(""); @@ -49,7 +49,7 @@ export function Component(): JSX.Element { const [showSpeechInput, setShowSpeechInput] = useState(false); const [showSpeechOutputBrowser, setShowSpeechOutputBrowser] = useState(false); const [showSpeechOutputAzure, setShowSpeechOutputAzure] = useState(false); - const [showBingSearchOption, setShowBingSearchOption] = useState(false); + const [showGroundingSearchOption, setShowGroundingSearchOption] = useState(false); const audio = useRef(new Audio()).current; const [isPlaying, setIsPlaying] = useState(false); @@ -81,7 +81,7 @@ export function Component(): JSX.Element { setUseSemanticRanker(config.showSemanticRankerOption); setShowSemanticRankerOption(config.showSemanticRankerOption); setShowVectorOption(config.showVectorOption); - setShowBingSearchOption(config.showBingSearchOption); + setShowGroundingSearchOption(config.showGroundingSearchOption); if (!config.showVectorOption) { setRetrievalMode(RetrievalMode.Text); } @@ -135,7 +135,7 @@ export function Component(): JSX.Element { use_gpt4v: useGPT4V, gpt4v_input: gpt4vInput, language: i18n.language, - use_bing_search: useBingSearch, + use_grounding_search: useGroundingSearch, ...(seed !== null ? { seed: seed } : {}) } }, @@ -208,8 +208,8 @@ export function Component(): JSX.Element { case "retrievalMode": setRetrievalMode(value); break; - case "useBingSearch": - setUseBingSearch(value); + case "useGroundingSearch": + setUseGroundingSearch(value); break; } }; @@ -332,13 +332,13 @@ export function Component(): JSX.Element { includeCategory={includeCategory} retrievalMode={retrievalMode} useGPT4V={useGPT4V} - useBingSearch={useBingSearch} + useGroundingSearch={useGroundingSearch} gpt4vInput={gpt4vInput} vectorFieldList={vectorFieldList} showSemanticRankerOption={showSemanticRankerOption} showGPT4VOptions={showGPT4VOptions} showVectorOption={showVectorOption} - showBingSearchOption={showBingSearchOption} + showGroundingSearchOption={showGroundingSearchOption} useOidSecurityFilter={useOidSecurityFilter} useGroupsSecurityFilter={useGroupsSecurityFilter} useLogin={!!useLogin} diff --git a/app/frontend/src/pages/chat/Chat.tsx b/app/frontend/src/pages/chat/Chat.tsx index 03cde2ff56..7a6f7cf9ba 100644 --- a/app/frontend/src/pages/chat/Chat.tsx +++ b/app/frontend/src/pages/chat/Chat.tsx @@ -58,7 +58,7 @@ const Chat = () => { const [useGroupsSecurityFilter, setUseGroupsSecurityFilter] = useState(false); const [gpt4vInput, setGPT4VInput] = useState(GPT4VInput.TextAndImages); const [useGPT4V, setUseGPT4V] = useState(false); - const [useBingSearch, setUseBingSearch] = useState(false); + const [useGroundingSearch, setUseGroundingSearch] = useState(false); const lastQuestionRef = useRef(""); const chatMessageStreamEnd = useRef(null); @@ -78,7 +78,7 @@ const Chat = () => { const [showGPT4VOptions, setShowGPT4VOptions] = useState(false); const [showSemanticRankerOption, setShowSemanticRankerOption] = useState(false); const [showVectorOption, setShowVectorOption] = useState(false); - const [showBingSearchOption, setShowBingSearchOption] = useState(false); + const [showGroundingSearchOption, setShowGroundingSearchOption] = useState(false); const [showUserUpload, setShowUserUpload] = useState(false); const [showLanguagePicker, setshowLanguagePicker] = useState(false); const [showSpeechInput, setShowSpeechInput] = useState(false); @@ -113,7 +113,7 @@ const Chat = () => { setShowSpeechOutputAzure(config.showSpeechOutputAzure); setShowChatHistoryBrowser(config.showChatHistoryBrowser); setShowChatHistoryCosmos(config.showChatHistoryCosmos); - setShowBingSearchOption(config.showBingSearchOption); + setShowGroundingSearchOption(config.showGroundingSearchOption); }); }; @@ -207,7 +207,7 @@ const Chat = () => { use_gpt4v: useGPT4V, gpt4v_input: gpt4vInput, language: i18n.language, - use_bing_search: useBingSearch, + use_grounding_search: useGroundingSearch, ...(seed !== null ? { seed: seed } : {}) } }, @@ -322,8 +322,8 @@ const Chat = () => { case "retrievalMode": setRetrievalMode(value); break; - case "useBingSearch": - setUseBingSearch(value); + case "useGroundingSearch": + setUseGroundingSearch(value); break; } }; @@ -510,13 +510,13 @@ const Chat = () => { includeCategory={includeCategory} retrievalMode={retrievalMode} useGPT4V={useGPT4V} - useBingSearch={useBingSearch} + useGroundingSearch={useGroundingSearch} gpt4vInput={gpt4vInput} vectorFieldList={vectorFieldList} showSemanticRankerOption={showSemanticRankerOption} showGPT4VOptions={showGPT4VOptions} showVectorOption={showVectorOption} - showBingSearchOption={showBingSearchOption} + showGroundingSearchOption={showGroundingSearchOption} useOidSecurityFilter={useOidSecurityFilter} useGroupsSecurityFilter={useGroupsSecurityFilter} useLogin={!!useLogin} From 7d3f9110099d5cf27ac09ed841ea7b31256ddf2e Mon Sep 17 00:00:00 2001 From: Anthony Shaw Date: Mon, 17 Feb 2025 13:38:35 +1100 Subject: [PATCH 17/17] Use agents SDK --- app/backend/approaches/approach.py | 2 +- app/backend/requirements.in | 2 +- app/backend/search_client.py | 64 +++++++++++++++++++++++++++++- 3 files changed, 64 insertions(+), 4 deletions(-) diff --git a/app/backend/approaches/approach.py b/app/backend/approaches/approach.py index b55f3975be..3eee3b3c1f 100644 --- a/app/backend/approaches/approach.py +++ b/app/backend/approaches/approach.py @@ -25,7 +25,7 @@ from openai.types.chat import ChatCompletionMessageParam from approaches.promptmanager import PromptManager -from app.backend.search_client import WebPage +from search_client import WebPage from core.authentication import AuthenticationHelper diff --git a/app/backend/requirements.in b/app/backend/requirements.in index c44d9216ce..1e560a931a 100644 --- a/app/backend/requirements.in +++ b/app/backend/requirements.in @@ -33,4 +33,4 @@ python-dotenv prompty rich typing-extensions -langchain-community \ No newline at end of file +azure-ai-projects \ No newline at end of file diff --git a/app/backend/search_client.py b/app/backend/search_client.py index 7e4bf62d55..c1fc8b2d33 100644 --- a/app/backend/search_client.py +++ b/app/backend/search_client.py @@ -6,8 +6,13 @@ import httpx from pydantic import BaseModel, ConfigDict +from azure.ai.projects.aio import AIProjectClient +from azure.identity import DefaultAzureCredential +from azure.ai.projects.models import BingGroundingTool +BING_CONNECTION_NAME = 'antbingtesting' + class WebPage(BaseModel): id: str name: str @@ -37,12 +42,67 @@ class WebAnswer(BaseModel): class AsyncGroundingSearchClient: + project_client: AIProjectClient + bing_tool: BingGroundingTool + agent_id: str = "asst_u8x2Hb9c9stVQwQMbostJ8JK" + def __init__(self, api_key: str, endpoint: Optional[str] = None): - ... + self.connection_string = endpoint + async def search(self, query: str, lang="en-US") -> WebAnswer: - ... + cred = DefaultAzureCredential() + # endpoint is the connection string + self.project_client = AIProjectClient.from_connection_string(self.connection_string, cred) + + async with self.project_client: + # Create thread for communication + thread = await self.project_client.agents.create_thread() + + # Create message to thread + message = await self.project_client.agents.create_message( + thread_id=thread.id, + role="user", + content=query, + ) + + # Create and process agent run in thread with tools + run = await self.project_client.agents.create_and_process_run(thread_id=thread.id, assistant_id=self.agent_id) + + if run.status == "failed": + raise Exception(run.last_error) + + # Fetch and log all messages + messages = await self.project_client.agents.list_messages(thread_id=thread.id) + + print(f"Messages: {messages}") + + run_steps = await self.project_client.agents.list_run_steps(run_id=run.id, thread_id=thread.id) + run_steps_data = run_steps['data'] + print(run_steps_data) + + url = messages.data[0].content[0].text.annotations[0].as_dict()['url_citation']['url'] + title = messages.data[0].content[0].text.annotations[0].as_dict()['url_citation']['title'] + snippet = messages.data[0].content[0].text.value + + return WebAnswer( + totalEstimatedMatches=1, + webSearchUrl="https://www.bing.com", + value=[ + WebPage( + id="1", + name=title, + url=url, + displayUrl=url, + dateLastCrawled="2021-10-01", + language="en", + snippet=snippet, + isFamilyFriendly=True, + siteName="Bing" + ) + ], + ) class AsyncBingSearchClient(AsyncGroundingSearchClient): def __init__(self, api_key: str, bing_endpoint: Optional[str] = "api.bing.microsoft.com"):