Merge branch 'main' into dependabotchanges

.env.sample

@@ -36,12 +36,9 @@ AzureWebJobsStorage=
 BACKEND_URL=http://localhost:7071
 DOCUMENT_PROCESSING_QUEUE_NAME=
 # Azure Blob Storage for storing the original documents to be processed
-AZURE_BLOB_ACCOUNT_NAME=
-AZURE_BLOB_ACCOUNT_KEY=
-AZURE_BLOB_CONTAINER_NAME=
+AZURE_BLOB_STORAGE_INFO="{\"containerName\":\"documents\",\"accountName\":\"\",\"accountKey\":\"\"}"
 # Azure Form Recognizer for extracting the text from the documents
-AZURE_FORM_RECOGNIZER_ENDPOINT=
-AZURE_FORM_RECOGNIZER_KEY=
+AZURE_FORM_RECOGNIZER_INFO="{\"endpoint\":\"\",\"key\":\"\"}"
 # Azure AI Content Safety for filtering out the inappropriate questions or answers
 AZURE_CONTENT_SAFETY_ENDPOINT=
 AZURE_CONTENT_SAFETY_KEY=
@@ -66,3 +63,5 @@ CONVERSATION_FLOW=
 AZURE_COSMOSDB_INFO="{\"accountName\":\"cosmos-abc123\",\"databaseName\":\"db_conversation_history\",\"containerName\":\"conversations\"}"
 AZURE_COSMOSDB_ACCOUNT_KEY=
 AZURE_COSMOSDB_ENABLE_FEEDBACK=
+AZURE_POSTGRESQL_INFO="{\"user\":\"\",\"dbname\":\"postgres\",\"host\":\"\"}"
+DATABASE_TYPE="CosmosDB"

.github/dependabot.yml

@@ -7,31 +7,42 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
+
     commit-message:
       prefix: "build"
+    target-branch: "dependabotchanges"
   - package-ecosystem: "pip"
     directory: "/"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
+
+
     commit-message:
       prefix: "build"
     groups:
       langchain:
         patterns:
           - "langchain*"
     open-pull-requests-limit: 50
+    target-branch: "dependabotchanges"
   - package-ecosystem: "npm"
     directory: "/code/frontend"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
+
+
     commit-message:
       prefix: "build"
     open-pull-requests-limit: 50
+    target-branch: "dependabotchanges"
   - package-ecosystem: "npm"
     directory: "/tests/integration/ui"
     schedule:
-      interval: "weekly"
+      interval: "monthly"
+
+
     commit-message:
       prefix: "build"
     open-pull-requests-limit: 50
+    target-branch: "dependabotchanges"

.github/workflows/build-docker-images.yml

@@ -4,9 +4,15 @@ on:
   workflow_run:
     workflows: [Tests]
     types: [completed]
-    branches: [main]
+    branches:
+      - main
+      - dev
+      - demo
   pull_request:
-    branches: [main]
+    branches:
+      - main
+      - dev
+      - demo
     types:
     - opened
     - ready_for_review
@@ -28,9 +34,9 @@ jobs:
             dockerfile: docker/Frontend.Dockerfile
     uses: ./.github/workflows/build-docker.yml
     with:
-      registry: fruoccopublic.azurecr.io
-      username: fruoccopublic
+      registry: ${{ github.event.workflow_run.head_branch == 'main' && 'fruoccopublic.azurecr.io' || 'cwydcontainerreg.azurecr.io'}}
+      username: ${{ github.event.workflow_run.head_branch == 'main' && 'fruoccopublic' || 'cwydcontainerreg'}}
       app_name: ${{ matrix.app_name }}
       dockerfile: ${{ matrix.dockerfile }}
-      push: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_branch == 'main' }}
+      push: ${{ github.event.workflow_run.head_branch == 'main' || github.event.workflow_run.head_branch == 'dev' || github.event.workflow_run.head_branch == 'demo' }}
     secrets: inherit

.github/workflows/build-docker.yml

@@ -21,6 +21,8 @@ on:
     secrets:
       DOCKER_PASSWORD:
         required: false
+      DEV_DOCKER_PASSWORD:
+        required: false
 
 jobs:
   docker-build:
@@ -31,13 +33,21 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Docker Login
-      if: ${{ inputs.push }}
+      if: ${{ inputs.push == true && github.ref_name == 'main' }}
       uses: docker/login-action@v3
       with:
         registry: ${{ inputs.registry }}
         username: ${{ inputs.username }}
         password: ${{ secrets.DOCKER_PASSWORD }}
 
+    - name: Dev Docker Login
+      if: ${{ inputs.push == true && (github.ref_name == 'dev' || github.ref_name == 'demo') }}
+      uses: docker/login-action@v3
+      with:
+        registry: ${{ inputs.registry }}
+        username: ${{ inputs.username }}
+        password: ${{ secrets.DEV_DOCKER_PASSWORD }}
+
     - name: Set up Docker Buildx
       uses: docker/setup-buildx-action@v3
 
@@ -51,7 +61,7 @@ jobs:
         context: .
         file: ${{ inputs.dockerfile }}
         push: ${{ inputs.push }}
-        cache-from: type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name}}:latest
+        cache-from: type=registry,ref=${{ inputs.registry }}/${{ inputs.app_name}}:${{ github.ref_name == 'main' && 'latest' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo' || 'latest' }}
         tags: |
-          ${{ inputs.registry }}/${{ inputs.app_name}}:latest
+          ${{ inputs.registry }}/${{ inputs.app_name}}:${{ github.ref_name == 'main' && 'latest' || github.ref_name == 'dev' && 'dev' || github.ref_name == 'demo' && 'demo' || 'latest' }}
           ${{ inputs.registry }}/${{ inputs.app_name}}:${{ steps.date.outputs.date }}_${{ github.run_number }}

.github/workflows/ci.yml

@@ -4,6 +4,8 @@ on:
   push:
     branches:
       - main
+      - dev
+      - demo
   schedule:
     - cron: '0 6,18 * * *'  # Runs at 6:00 AM and 6:00 PM GMT
 
@@ -34,36 +36,45 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set imageTag
+        id: set-image-tag
+        run: |
+          if [[ "${{ github.event_name }}" == "schedule" ]]; then
+            echo "imageTag=latest" >> $GITHUB_ENV
+          elif [[ "${{ github.ref_name }}" == "main" ]]; then
+            echo "imageTag=latest" >> $GITHUB_ENV
+          else
+            echo "imageTag=${{ github.ref_name }}" >> $GITHUB_ENV
+          fi
+
       - name: Pre-build image and run make in dev container
         uses: devcontainers/ci@v0.3
         env:
           AZURE_ENV_NAME: ${{ github.run_id }}
           AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
-          AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
         with:
           imageName: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
           cacheFrom: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
+          imageTag: ${{ env.imageTag }}
           runCmd: make ci && make deploy
-          refFilterForPush: refs/heads/main
+          refFilterForPush: refs/heads/${{ github.event_name == 'schedule' && 'main' || github.ref_name }}
           env: |
             AZURE_CLIENT_ID
             AZURE_CLIENT_SECRET
             AZURE_TENANT_ID
             AZURE_SUBSCRIPTION_ID
             AZURE_ENV_NAME
             AZURE_LOCATION
-            AZURE_RESOURCE_GROUP
-
       - name: Tidy up
         uses: devcontainers/ci@v0.3
         if: always()
         env:
           AZURE_ENV_NAME: ${{ github.run_id }}
           AZURE_LOCATION: ${{ vars.AZURE_LOCATION }}
-          AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
         with:
           push: never
           imageName: ghcr.io/azure-samples/chat-with-your-data-solution-accelerator
+          imageTag: ${{ env.imageTag }}
           runCmd: make destroy
           env: |
             AZURE_CLIENT_ID
@@ -72,13 +83,11 @@ jobs:
             AZURE_SUBSCRIPTION_ID
             AZURE_ENV_NAME
             AZURE_LOCATION
-            AZURE_RESOURCE_GROUP
-
+            
       - name: Send Notification on Failure
         if: failure()
         run: |
           RUN_URL="https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}"
-
           # Construct the email body
           EMAIL_BODY=$(cat <<EOF
           {

.github/workflows/sync-branches.yml

@@ -0,0 +1,44 @@
+name: Sync Main to dependabotchanges
+
+on:
+  # Schedule the sync job to run daily or customize as needed
+  schedule:
+    - cron: '0 1 * * *'  # Runs every day at 1 AM UTC
+  # Trigger the sync job on pushes to the main branch
+  push:
+    branches:
+      - main
+
+jobs:
+  sync:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0  # Fetch all history for accurate branch comparison
+
+      - name: Configure Git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Sync main to dependabotchanges
+        run: |
+          # Ensure we're on the main branch
+          git checkout main
+          # Fetch the latest changes
+          git pull origin main
+
+          # Switch to dependabotchanges branch
+          git checkout dependabotchanges
+          # Merge main branch changes
+          git merge main --no-edit
+
+          # Push changes back to dependabotchanges branch
+          git push origin dependabotchanges
+
+      - name: Notify on Failure
+        if: failure()
+        run: echo "Sync from main to dependabotchanges failed!"

.github/workflows/tests.yml

@@ -2,9 +2,9 @@ name: Tests
 
 on:
   push:
-    branches: [main]
+    branches: [main, dev, demo]
   pull_request:
-    branches: [main]
+    branches: [main, dev, demo]
     types:
       - opened
       - ready_for_review

Makefile

@@ -57,14 +57,10 @@ azd-login: ## 🔑 Login to Azure with azd and a SPN
 	@echo -e "\e[34m$@\e[0m" || true
 	@azd auth login --client-id ${AZURE_CLIENT_ID} --client-secret ${AZURE_CLIENT_SECRET} --tenant-id ${AZURE_TENANT_ID}
 
-az-login: ## 🔑 Login to Azure with az and a SPN
-	az login --service-principal -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} --tenant ${AZURE_TENANT_ID}
-
-deploy: azd-login az-login ## 🚀 Deploy everything to Azure
+deploy: azd-login ## 🚀 Deploy everything to Azure
 	@echo -e "\e[34m$@\e[0m" || true
 	@azd env new ${AZURE_ENV_NAME}
 	@azd env set AZURE_APP_SERVICE_HOSTING_MODEL code --no-prompt
-	@az group create --name ${AZURE_RESOURCE_GROUP} --location ${AZURE_LOCATION}
 	@azd provision --no-prompt
 	@azd deploy web --no-prompt
 	@azd deploy function --no-prompt

README.md

@@ -48,7 +48,7 @@ urlFragment: chat-with-your-data-solution-accelerator
 ## User story
 Welcome to the *Chat with your data* Solution accelerator repository! The *Chat with your data* Solution accelerator is a powerful tool that combines the capabilities of Azure AI Search and Large Language Models (LLMs) to create a conversational search experience. This solution accelerator uses an Azure OpenAI GPT model and an Azure AI Search index generated from your data, which is integrated into a web application to provide a natural language interface, including [speech-to-text](docs/speech_to_text.md) functionality, for search queries. Users can drag and drop files, point to storage, and take care of technical setup to transform documents. Everything can be deployed in your own subscription to accelerate your use of this technology.
 
-![Solution Architecture - Chat with your data](/docs/images/cwyd-solution-architecture.png)
+
 
 ### About this repo
 
@@ -91,12 +91,15 @@ Here is a comparison table with a few features offered by Azure, an available Gi
 - **Single application access to your full data set**: Minimize endpoints required to access internal company knowledgebases. Reuse the same backend with the [Microsoft Teams Extension](docs/teams_extension.md)
 - **Natural language interaction with your unstructured data**: Use natural language to quickly find the answers you need and ask follow-up queries to get the supplemental details, including [Speech-to-text](docs/speech_to_text.md).
 - **Easy access to source documentation when querying**: Review referenced documents in the same chat window for additional context.
+- **Chat history**: Prior conversations and context are maintained and accessible through chat history.
 - **Data upload**: Batch upload documents of [various file types](docs/supported_file_types.md)
 - **Accessible orchestration**: Prompt and document configuration (prompt engineering, document processing, and data retrieval)
+- **Database flexibility**: Dynamic database switching allows users to choose between PostgreSQL and Cosmos DB based on their requirements. If no preference is specified the platform defaults to PostgreSQL.
 
 
 **Note**: The current model allows users to ask questions about unstructured data, such as PDF, text, and docx files. See the [supported file types](docs/supported_file_types.md).
 
+
 ### Target end users
 Company personnel (employees, executives) looking to research against internal unstructured company data would leverage this accelerator using natural language to find what they need quickly.
 
@@ -107,6 +110,11 @@ Tech administrators can use this accelerator to give their colleagues easy acces
 
 ### Use Case scenarios
 
+#### Employee Onboarding Scenario
+The sample data illustrates how this accelerator could be used for an employee onboarding scenario in across industries.
+
+In this scenario, a newly hired employee is in the process of onboarding to their organization. Leveraging the solution accelerator, she navigates through the extensive offerings of her organization’s health and retirement benefits. With the newly integrated chat history capabilities, they can revisit previous conversations, ensuring continuity and context across multiple days of research. This functionality allows the new employee to efficiently gather and consolidate information, streamlining their onboarding experience. [For more details, refer to the README](docs/employee_assistance.md).
+
 #### Financial Advisor Scenario
 The sample data illustrates how this accelerator could be used in the financial services industry (FSI).
 
@@ -120,12 +128,6 @@ Additionally, we have implemented a Legal Review and Summarization Assistant sce
 Note: Some of the sample data included with this accelerator was generated using AI and is for illustrative purposes only.
 
 
-#### Employee Onboarding Scenario
-The sample data illustrates how this accelerator could be used for an employee onboarding scenario in across industries.
-
-In this scenario, a newly hired employee is in the process of onboarding to their organization. Leveraging the solution accelerator, she navigates through the extensive offerings of her organization’s health and retirement benefits. With the newly integrated chat history capabilities, they can revisit previous conversations, ensuring continuity and context across multiple days of research. This functionality allows the new employee to efficiently gather and consolidate information, streamlining their onboarding experience. [For more details, refer to the README](docs/employee_assistance.md).
-
-
 ---
 
 ![One-click Deploy](/docs/images/oneClickDeploy.png)
@@ -146,6 +148,7 @@ In this scenario, a newly hired employee is in the process of onboarding to thei
 - Azure Storage Account
 - Azure Speech Service
 - Azure CosmosDB
+- Azure PostgreSQL
 - Teams (optional: Teams extension only)
 
 ### Required licenses
@@ -163,13 +166,32 @@ The following are links to the pricing details for some of the resources:
 - [Azure AI Document Intelligence pricing](https://azure.microsoft.com/pricing/details/ai-document-intelligence/)
 - [Azure Web App Pricing](https://azure.microsoft.com/pricing/details/app-service/windows/)
 
+### Deployment options: PostgreSQL or Cosmos DB
+With the addition of PostgreSQL, customers can leverage the power of a relationship-based AI solution to enhance historical conversation access, improve data privacy, and open the possibilities for scalability.
+
+Customers have the option to deploy this solution with PostgreSQL or Cosmos DB. Consider the following when deciding which database to use:
+- PostgreSQL enables a relationship-based AI solution and search indexing for Retrieval Augmented Generation (RAG)
+- Cosmos DB enables chat history and is a NoSQL-based solution. With Cosmos DB, Azure AI Search is used for storing extracted documents and embeddings.
+
+
+To review PostgreSQL configuration overview and steps, follow the link [here](docs/postgreSQL.md).
+![Solution Architecture - Chat with your data PostgreSQL](/docs/images/architrecture_pg.png)
+
+To review Cosmos DB configuration overview and steps, follow the link [here](docs/employee_assistance.md).
+![Solution Architecture - Chat with your data CosmosDB](/docs/images/architecture_cdb.png)
+
 ### Deploy instructions
+The "Deploy to Azure" button offers a one-click deployment where you don’t have to clone the code. If you would like a developer experience instead, follow the [local deployment instructions](./docs/LOCAL_DEPLOYMENT.md).
 
-There are two choices; the "Deploy to Azure" offers a one click deployment where you don't have to clone the code, alternatively if you would like a developer experience, follow the [Local deployment instructions](./docs/LOCAL_DEPLOYMENT.md).
+Once you deploy to Azure, you will have the option to select PostgreSQL or Cosmos DB, see screenshot below.
 
-The demo, which uses containers pre-built from the main branch is available by clicking this button:
+[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure-Samples%2Fchat-with-your-data-solution-accelerator%2Frefs%2Fheads%2Fmain%2Finfra%2Fmain.json)
 
-[![Deploy to Azure](https://aka.ms/deploytoazurebutton)](https://portal.azure.com/#create/Microsoft.Template/uri/https%3A%2F%2Fraw.githubusercontent.com%2FAzure-Samples%2Fchat-with-your-data-solution-accelerator%2Fmain%2Finfra%2Fmain.json)
+Select either "PostgreSQL" or "Cosmos DB":
+![Solution Architecture - DB Selection](/docs/images/db_selection.png)
+
+
+When Deployment is complete, follow steps in [Set Up Authentication in Azure App Service](./docs/azure_app_service_auth_setup.md) to add app authentication to your web app running on Azure App Service
 
 **Note**: The default configuration deploys an OpenAI Model "gpt-35-turbo" with version 0613. However, not all
 locations support this version. If you're deploying to a location that doesn't support version 0613, you'll need to
@@ -193,9 +215,11 @@ switch to a lower version. To find out which versions are supported in different
 
     ![A screenshot of the chat app.](./docs/images/web-unstructureddata.png)
 
-\
-\
+
+
+
 ![Supporting documentation](/docs/images/supportingDocuments.png)
+
 ## Supporting documentation
 
 ### Resource links