Add test for backend functions

code/backend/batch/AddURLEmbeddings.py

@@ -1,20 +1,21 @@
 import logging
 import traceback
 import azure.functions as func
-import sys
 
 from utilities.helpers.DocumentProcessorHelper import DocumentProcessor
 from utilities.helpers.ConfigHelper import ConfigHelper
 
-sys.path.append("..")
 
 bp_add_url_embeddings = func.Blueprint()
-
 logger = logging.getLogger(__name__)
 
 
 @bp_add_url_embeddings.route(route="AddURLEmbeddings")
 def add_url_embeddings(req: func.HttpRequest) -> func.HttpResponse:
+    return do_add_url_embeddings(req)
+
+
+def do_add_url_embeddings(req: func.HttpRequest) -> func.HttpResponse:
     logger.info("Python HTTP trigger function processed a request.")
     # Get Url from request
     url = req.params.get("url")

code/backend/batch/BatchPushResults.py

@@ -2,16 +2,15 @@
 import json
 import azure.functions as func
 from urllib.parse import urlparse
-import sys
 
-from utilities.helpers.AzureBlobStorageHelper import AzureBlobStorageClient
+from utilities.helpers.AzureBlobStorageHelper import (
+    AzureBlobStorageClient,
+)
 from utilities.helpers.DocumentProcessorHelper import DocumentProcessor
 from utilities.helpers.ConfigHelper import ConfigHelper
 
-sys.path.append("..")
 
 bp_batch_push_results = func.Blueprint()
-
 logger = logging.getLogger(__name__)
 
 
@@ -29,6 +28,10 @@ def _get_file_name_from_message(msg: func.QueueMessage) -> str:
     arg_name="msg", queue_name="doc-processing", connection="AzureWebJobsStorage"
 )
 def batch_push_results(msg: func.QueueMessage) -> None:
+    do_batch_push_results(msg)
+
+
+def do_batch_push_results(msg: func.QueueMessage) -> None:
     logger.info(
         "Python queue trigger function processed a queue item: %s",
         msg.get_body().decode("utf-8"),
@@ -41,6 +44,7 @@ def batch_push_results(msg: func.QueueMessage) -> None:
     file_sas = blob_client.get_blob_sas(file_name)
     # Get file extension's processors
     file_extension = file_name.split(".")[-1]
+
     processors = list(
         filter(
             lambda x: x.document_type.lower() == file_extension.lower(),

code/backend/batch/BatchStartProcessing.py

@@ -1,22 +1,24 @@
 import logging
 import json
 import azure.functions as func
-import sys
+
 from utilities.helpers.EnvHelper import EnvHelper
 from utilities.helpers.AzureBlobStorageHelper import (
     AzureBlobStorageClient,
     create_queue_client,
 )
 
-sys.path.append("..")
 bp_batch_start_processing = func.Blueprint()
 env_helper: EnvHelper = EnvHelper()
-
 logger = logging.getLogger(__name__)
 
 
 @bp_batch_start_processing.route(route="BatchStartProcessing")
 def batch_start_processing(req: func.HttpRequest) -> func.HttpResponse:
+    return do_batch_start_processing(req)
+
+
+def do_batch_start_processing(req: func.HttpRequest) -> func.HttpResponse:
     logger.info("Requested to start processing all documents received")
     # Set up Blob Storage Client
     azure_blob_storage_client = AzureBlobStorageClient()

code/backend/batch/GetConversationResponse.py

@@ -1,20 +1,23 @@
 import azure.functions as func
 import logging
 import json
-import sys
+
 from utilities.helpers.EnvHelper import EnvHelper
 from utilities.helpers.OrchestratorHelper import Orchestrator
+from utilities.helpers.ConfigHelper import ConfigHelper
 
-sys.path.append("..")
 
 bp_get_conversation_response = func.Blueprint()
 env_helper: EnvHelper = EnvHelper()
-
 logger = logging.getLogger(__name__)
 
 
 @bp_get_conversation_response.route(route="GetConversationResponse")
 def get_conversation_response(req: func.HttpRequest) -> func.HttpResponse:
+    return do_get_conversation_response(req)
+
+
+def do_get_conversation_response(req: func.HttpRequest) -> func.HttpResponse:
     logger.info("Python HTTP trigger function processed a request.")
 
     message_orchestrator = Orchestrator()
@@ -37,7 +40,6 @@ def get_conversation_response(req: func.HttpRequest) -> func.HttpResponse:
                         user_assistant_messages[i + 1]["content"],
                     )
                 )
-        from utilities.helpers.ConfigHelper import ConfigHelper
 
         messages = message_orchestrator.handle_message(
             user_message=user_message,

code/tests/test_AddURLEmbeddings.py

@@ -0,0 +1,58 @@
+import sys
+import os
+from unittest.mock import patch
+import azure.functions as func
+
+
+function_app_path = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../backend/batch")
+)
+sys.path.append(function_app_path)
+
+from backend.batch.AddURLEmbeddings import do_add_url_embeddings  # noqa: E402
+
+
+@patch("backend.batch.BatchPushResults.ConfigHelper")
+@patch("backend.batch.AddURLEmbeddings.DocumentProcessor")
+def test_add_url_embeddings_when_url_set_in_body(_, __):
+    fake_request = func.HttpRequest(
+        method="POST",
+        url="",
+        body=b'{"url": "https://example.com"}',
+        headers={"Content-Type": "application/json"},
+    )
+
+    response = do_add_url_embeddings(fake_request)
+
+    assert response.status_code == 200
+
+
+@patch("backend.batch.BatchPushResults.ConfigHelper")
+@patch("backend.batch.AddURLEmbeddings.DocumentProcessor")
+def test_add_url_embeddings_when_url_set_in_param(_, __):
+    fake_request = func.HttpRequest(
+        method="POST",
+        url="",
+        body=b"",
+        headers={"Content-Type": "application/json"},
+        params={"url": "https://example.com"},
+    )
+
+    response = do_add_url_embeddings(fake_request)
+
+    assert response.status_code == 200
+
+
+@patch("backend.batch.BatchPushResults.ConfigHelper")
+@patch("backend.batch.AddURLEmbeddings.DocumentProcessor")
+def test_add_url_embeddings_returns_400_when_url_not_set(_, __):
+    fake_request = func.HttpRequest(
+        method="POST",
+        url="",
+        body=b"",
+        params={},
+    )
+
+    response = do_add_url_embeddings(fake_request)
+
+    assert response.status_code == 400

code/tests/test_BatchPushResults.py

@@ -0,0 +1,67 @@
+import sys
+import os
+from unittest.mock import patch, Mock
+from azure.functions import QueueMessage
+
+
+function_app_path = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../backend/batch")
+)
+sys.path.append(function_app_path)
+
+from backend.batch.BatchPushResults import do_batch_push_results  # noqa: E402
+from backend.batch.BatchPushResults import _get_file_name_from_message  # noqa: E402
+
+
+def test_get_file_name_from_message():
+    mock_queue_message = QueueMessage(
+        body='{"message": "test message", "filename": "test_filename.md"}'
+    )
+
+    file_name = _get_file_name_from_message(mock_queue_message)
+
+    assert file_name == "test_filename.md"
+
+
+def test_get_file_name_from_message_no_filename():
+    mock_queue_message = QueueMessage(
+        body='{"data": { "url": "test/test/test_filename.md"} }'
+    )
+
+    file_name = _get_file_name_from_message(mock_queue_message)
+
+    assert file_name == "test_filename.md"
+
+
+@patch("backend.batch.BatchPushResults.ConfigHelper")
+@patch("backend.batch.BatchPushResults.AzureBlobStorageClient")
+@patch("backend.batch.BatchPushResults.DocumentProcessor")
+def test_do_batch_push_results(
+    mock_document_processor, mock_azure_blob_storage_client, mock_config_helper
+):
+    mock_queue_message = QueueMessage(
+        body='{"message": "test message", "filename": "test/test/test_filename.md"}'
+    )
+
+    mock_blob_client_instance = mock_azure_blob_storage_client.return_value
+    mock_blob_client_instance.get_blob_sas.return_value = "test_blob_sas"
+
+    mock_document_processor_instance = mock_document_processor.return_value
+
+    md_processor = Mock()
+    md_processor.document_type.lower.return_value = "md"
+    txt_processor = Mock()
+    txt_processor.document_type.lower.return_value = "txt"
+    mock_processors = [md_processor, txt_processor]
+    mock_config_helper.get_active_config_or_default.return_value.document_processors = (
+        mock_processors
+    )
+
+    do_batch_push_results(mock_queue_message)
+
+    mock_document_processor_instance.process.assert_called_once_with(
+        source_url="test_blob_sas", processors=[md_processor]
+    )
+    mock_blob_client_instance.upsert_blob_metadata.assert_called_once_with(
+        "test/test/test_filename.md", {"embeddings_added": "true"}
+    )

code/tests/test_BatchStartProcessing.py

@@ -0,0 +1,63 @@
+import sys
+import os
+from unittest.mock import patch, Mock
+
+function_app_path = os.path.abspath(
+    os.path.join(os.path.dirname(__file__), "../backend/batch")
+)
+sys.path.append(function_app_path)
+
+from backend.batch.BatchStartProcessing import do_batch_start_processing  # noqa: E402
+
+
+@patch("backend.batch.BatchStartProcessing.create_queue_client")
+@patch("backend.batch.BatchStartProcessing.AzureBlobStorageClient")
+def test_batch_start_processing_processes_all(
+    mock_blob_storage_client, mock_create_queue_client
+):
+    mock_http_request = Mock()
+    mock_http_request.params = dict()
+    mock_http_request.params["process_all"] = "true"
+
+    mock_queue_client = Mock()
+    mock_create_queue_client.return_value = mock_queue_client
+
+    mock_blob_storage_client.return_value.get_all_files.return_value = [
+        {"filename": "file_name_one", "embeddings_added": False}
+    ]
+
+    response = do_batch_start_processing(mock_http_request)
+
+    assert response.status_code == 200
+
+    mock_queue_client.send_message.assert_called_once_with(
+        b'{"filename": "file_name_one"}',
+    )
+
+
+@patch("backend.batch.BatchStartProcessing.create_queue_client")
+@patch("backend.batch.BatchStartProcessing.AzureBlobStorageClient")
+def test_batch_start_processing_filters_filter_no_embeddings(
+    mock_blob_storage_client, mock_create_queue_client
+):
+    mock_http_request = Mock()
+    mock_http_request.params = dict()
+    mock_http_request.params["process_all"] = "false"
+
+    mock_queue_client = Mock()
+    mock_create_queue_client.return_value = mock_queue_client
+
+    mock_blob_storage_client.return_value.get_all_files.return_value = [
+        {
+            "filename": "file_name_one",
+            "embeddings_added": True,  # will get filtered out
+        },
+        {"filename": "file_name_two", "embeddings_added": False},
+    ]
+    response = do_batch_start_processing(mock_http_request)
+
+    assert response.status_code == 200
+
+    mock_queue_client.send_message.assert_called_once_with(
+        b'{"filename": "file_name_two"}',
+    )