diff --git a/backends/advanced/init.py b/backends/advanced/init.py index aad7ff0e..e70a1369 100644 --- a/backends/advanced/init.py +++ b/backends/advanced/init.py @@ -309,7 +309,8 @@ def setup_llm(self): choices = { "1": "OpenAI (GPT-4, GPT-3.5 - requires API key)", "2": "Ollama (local models - runs locally)", - "3": "Skip (no memory extraction)" + "3": "OpenAI-Compatible (custom endpoint - Groq, Together AI, LM Studio, etc.)", + "4": "Skip (no memory extraction)" } choice = self.prompt_choice("Which LLM provider will you use?", choices, "1") @@ -347,6 +348,85 @@ def setup_llm(self): self.console.print("[yellow][WARNING][/yellow] Make sure Ollama is running and models are pulled") elif choice == "3": + self.console.print("[blue][INFO][/blue] OpenAI-Compatible custom endpoint selected") + self.console.print("This works with any provider that exposes an OpenAI-compatible API") + self.console.print("(e.g., Groq, Together AI, LM Studio, vLLM, etc.)") + self.console.print() + + # Prompt for base URL (required) + base_url = self.prompt_value( + "API Base URL (e.g., https://api.groq.com/openai/v1)", "" + ) + if not base_url: + self.console.print("[yellow][WARNING][/yellow] No base URL provided - skipping custom LLM setup") + else: + # Prompt for API key + api_key = self.prompt_with_existing_masked( + prompt_text="API Key (leave empty if not required)", + env_key="CUSTOM_LLM_API_KEY", + placeholders=['your_custom_llm_api_key_here'], + is_password=True, + default="" + ) + if api_key: + self.config["CUSTOM_LLM_API_KEY"] = api_key + + # Prompt for model name (required) + model_name = self.prompt_value( + "LLM Model name (e.g., llama-3.1-70b-versatile)", "" + ) + if not model_name: + self.console.print("[yellow][WARNING][/yellow] No model name provided - skipping custom LLM setup") + else: + # Create LLM model entry + llm_model = { + "name": "custom-llm", + "description": "Custom OpenAI-compatible LLM", + "model_type": "llm", + "model_provider": "openai", + "api_family": "openai", + "model_name": model_name, + "model_url": base_url, + "api_key": "${oc.env:CUSTOM_LLM_API_KEY,''}", + "model_params": { + "temperature": 0.2, + "max_tokens": 2000 + }, + "model_output": "json" + } + self.config_manager.add_or_update_model(llm_model) + + # Prompt for optional embedding model + embedding_model_name = self.prompt_value( + "Embedding model name (leave empty to use Ollama local-embed)", "" + ) + + if embedding_model_name: + embed_model = { + "name": "custom-embed", + "description": "Custom OpenAI-compatible embeddings", + "model_type": "embedding", + "model_provider": "openai", + "api_family": "openai", + "model_name": embedding_model_name, + "model_url": base_url, + "api_key": "${oc.env:CUSTOM_LLM_API_KEY,''}", + "embedding_dimensions": 1536, + "model_output": "vector" + } + self.config_manager.add_or_update_model(embed_model) + self.config_manager.update_config_defaults({"llm": "custom-llm", "embedding": "custom-embed"}) + self.console.print("[green][SUCCESS][/green] Custom LLM and embedding configured in config.yml") + self.console.print("[blue][INFO][/blue] Set defaults.llm: custom-llm") + self.console.print("[blue][INFO][/blue] Set defaults.embedding: custom-embed") + else: + self.config_manager.update_config_defaults({"llm": "custom-llm", "embedding": "local-embed"}) + self.console.print("[green][SUCCESS][/green] Custom LLM configured in config.yml") + self.console.print("[blue][INFO][/blue] Set defaults.llm: custom-llm") + self.console.print("[blue][INFO][/blue] Set defaults.embedding: local-embed (Ollama)") + self.console.print("[yellow][WARNING][/yellow] Make sure Ollama is running for embeddings") + + elif choice == "4": self.console.print("[blue][INFO][/blue] Skipping LLM setup - memory extraction disabled") # Disable memory extraction in config.yml self.config_manager.update_memory_config({"extraction": {"enabled": False}}) diff --git a/config_manager.py b/config_manager.py index 6f8a85a6..7919c5bc 100644 --- a/config_manager.py +++ b/config_manager.py @@ -325,6 +325,25 @@ def update_config_defaults(self, updates: Dict[str, str]): self._save_config_yml(config) + def add_or_update_model(self, model_def: Dict[str, Any]): + """ + Add or update a model in the models list by name. + + Args: + model_def: Model definition dict with at least a 'name' key. + """ + config = self._load_config_yml() + if "models" not in config: + config["models"] = [] + # Update existing or append + for i, m in enumerate(config["models"]): + if m.get("name") == model_def["name"]: + config["models"][i] = model_def + break + else: + config["models"].append(model_def) + self._save_config_yml(config) + def get_full_config(self) -> Dict[str, Any]: """ Get complete config.yml as dictionary. diff --git a/extras/asr-services/docker-compose.yml b/extras/asr-services/docker-compose.yml index 7e4b0aa5..d31ea7bf 100644 --- a/extras/asr-services/docker-compose.yml +++ b/extras/asr-services/docker-compose.yml @@ -90,6 +90,8 @@ services: build: context: . dockerfile: providers/vibevoice/Dockerfile + args: + PYTORCH_CUDA_VERSION: ${PYTORCH_CUDA_VERSION:-cu126} image: chronicle-asr-vibevoice:latest ports: - "${ASR_PORT:-8767}:8765" @@ -112,6 +114,9 @@ services: - DEVICE=${DEVICE:-cuda} - TORCH_DTYPE=${TORCH_DTYPE:-bfloat16} - MAX_NEW_TOKENS=${MAX_NEW_TOKENS:-8192} + dns: + - 8.8.8.8 + - 8.8.4.4 restart: unless-stopped # ============================================================================ diff --git a/extras/asr-services/providers/vibevoice/Dockerfile b/extras/asr-services/providers/vibevoice/Dockerfile index 218abb0c..a8d110e9 100644 --- a/extras/asr-services/providers/vibevoice/Dockerfile +++ b/extras/asr-services/providers/vibevoice/Dockerfile @@ -8,6 +8,8 @@ ######################### builder ################################# FROM ghcr.io/astral-sh/uv:python3.12-bookworm-slim AS builder +ARG PYTORCH_CUDA_VERSION=cu126 + WORKDIR /app # Install system dependencies for building @@ -17,7 +19,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ # Dependency manifest first for cache-friendly installs COPY pyproject.toml uv.lock ./ -RUN uv sync --no-install-project --group vibevoice && \ +RUN uv sync --no-install-project --group vibevoice --extra ${PYTORCH_CUDA_VERSION} && \ uv cache clean ######################### runtime ################################# diff --git a/extras/asr-services/tests/test_cuda_version_config.py b/extras/asr-services/tests/test_cuda_version_config.py new file mode 100644 index 00000000..b4ded1c4 --- /dev/null +++ b/extras/asr-services/tests/test_cuda_version_config.py @@ -0,0 +1,220 @@ +""" +Unit tests for CUDA version configuration in ASR service Dockerfiles. + +Tests the configurable PYTORCH_CUDA_VERSION build arg that allows selecting +different CUDA versions (cu121, cu126, cu128) for different GPU architectures. +""" + +import os +import re +import pytest +from pathlib import Path + + +class TestDockerfileCUDASupport: + """Test that Dockerfiles support configurable CUDA versions.""" + + @pytest.fixture + def vibevoice_dockerfile_path(self): + """Path to VibeVoice Dockerfile.""" + return Path(__file__).parent.parent / "providers" / "vibevoice" / "Dockerfile" + + @pytest.fixture + def nemo_dockerfile_path(self): + """Path to NeMo Dockerfile.""" + return Path(__file__).parent.parent / "providers" / "nemo" / "Dockerfile" + + @pytest.fixture + def docker_compose_path(self): + """Path to docker-compose.yml.""" + return Path(__file__).parent.parent / "docker-compose.yml" + + def test_vibevoice_dockerfile_has_cuda_arg(self, vibevoice_dockerfile_path): + """Test that VibeVoice Dockerfile declares PYTORCH_CUDA_VERSION arg.""" + content = vibevoice_dockerfile_path.read_text() + + # Should have ARG declaration + assert re.search(r"ARG\s+PYTORCH_CUDA_VERSION", content), \ + "Dockerfile must declare PYTORCH_CUDA_VERSION build arg" + + # Should have default value + arg_match = re.search(r"ARG\s+PYTORCH_CUDA_VERSION=(\w+)", content) + assert arg_match, "PYTORCH_CUDA_VERSION should have default value" + default_version = arg_match.group(1) + assert default_version in ["cu121", "cu126", "cu128"], \ + f"Default CUDA version {default_version} should be cu121, cu126, or cu128" + + def test_vibevoice_dockerfile_uses_cuda_arg_in_uv_sync(self, vibevoice_dockerfile_path): + """Test that VibeVoice Dockerfile uses CUDA arg in uv sync command.""" + content = vibevoice_dockerfile_path.read_text() + + # Should use --extra ${PYTORCH_CUDA_VERSION} + assert re.search(r"uv\s+sync.*--extra\s+\$\{PYTORCH_CUDA_VERSION\}", content), \ + "uv sync command must include --extra ${PYTORCH_CUDA_VERSION}" + + def test_nemo_dockerfile_has_cuda_support(self, nemo_dockerfile_path): + """Test that NeMo Dockerfile (reference implementation) has CUDA support.""" + content = nemo_dockerfile_path.read_text() + + assert re.search(r"ARG\s+PYTORCH_CUDA_VERSION", content), \ + "NeMo Dockerfile should have PYTORCH_CUDA_VERSION arg" + + assert re.search(r"uv\s+sync.*--extra\s+\$\{PYTORCH_CUDA_VERSION\}", content), \ + "NeMo Dockerfile should use CUDA version in uv sync" + + def test_docker_compose_passes_cuda_arg_to_vibevoice(self, docker_compose_path): + """Test that docker-compose.yml passes PYTORCH_CUDA_VERSION to vibevoice service.""" + content = docker_compose_path.read_text() + + # Find vibevoice-asr service section + vibevoice_section = re.search( + r"vibevoice-asr:.*?(?=^\S|\Z)", + content, + re.MULTILINE | re.DOTALL + ) + assert vibevoice_section, "docker-compose.yml must have vibevoice-asr service" + + section_text = vibevoice_section.group(0) + + # Should have build args section + assert re.search(r"args:", section_text), \ + "vibevoice-asr service should have build args section" + + # Should pass PYTORCH_CUDA_VERSION + assert re.search( + r"PYTORCH_CUDA_VERSION:\s*\$\{PYTORCH_CUDA_VERSION:-cu126\}", + section_text + ), "vibevoice-asr should pass PYTORCH_CUDA_VERSION build arg with cu126 default" + + def test_docker_compose_cuda_arg_consistency(self, docker_compose_path): + """Test that all GPU-enabled services use consistent CUDA version pattern.""" + content = docker_compose_path.read_text() + + # Services that should have CUDA support + gpu_services = ["vibevoice-asr", "nemo-asr", "parakeet-asr"] + + for service_name in gpu_services: + service_match = re.search( + rf"{service_name}:.*?(?=^\S|\Z)", + content, + re.MULTILINE | re.DOTALL + ) + + if service_match: + service_text = service_match.group(0) + + # Check if service has GPU resources + if "devices:" in service_text and "nvidia" in service_text: + # Should have PYTORCH_CUDA_VERSION arg + assert re.search( + r"PYTORCH_CUDA_VERSION:\s*\$\{PYTORCH_CUDA_VERSION:-cu\d+\}", + service_text + ), f"{service_name} with GPU should have PYTORCH_CUDA_VERSION build arg" + + +class TestCUDAVersionEnvironmentVariable: + """Test CUDA version environment variable handling.""" + + def test_cuda_version_env_var_format(self): + """Test that CUDA version environment variables follow correct format.""" + valid_versions = ["cu121", "cu126", "cu128"] + + for version in valid_versions: + assert re.match(r"^cu\d{3}$", version), \ + f"{version} should match pattern cu### (e.g., cu121, cu126)" + + def test_cuda_version_from_env(self): + """Test reading CUDA version from environment.""" + test_version = "cu128" + + with pytest.MonkeyPatch.context() as mp: + mp.setenv("PYTORCH_CUDA_VERSION", test_version) + cuda_version = os.getenv("PYTORCH_CUDA_VERSION") + + assert cuda_version == test_version + assert cuda_version in ["cu121", "cu126", "cu128"] + + def test_cuda_version_default_fallback(self): + """Test that default CUDA version is used when env var not set.""" + with pytest.MonkeyPatch.context() as mp: + mp.delenv("PYTORCH_CUDA_VERSION", raising=False) + + # Simulate docker-compose default: ${PYTORCH_CUDA_VERSION:-cu126} + cuda_version = os.getenv("PYTORCH_CUDA_VERSION", "cu126") + + assert cuda_version == "cu126" + + +class TestGPUArchitectureCUDAMapping: + """Test that GPU architectures map to correct CUDA versions.""" + + def test_rtx_5090_requires_cu128(self): + """ + Test that RTX 5090 (sm_120) requires CUDA 12.8+. + + RTX 5090 has CUDA capability 12.0 (sm_120) which requires + PyTorch built with CUDA 12.8 or higher. + """ + gpu_arch = "sm_120" # RTX 5090 + required_cuda = "cu128" + + # Map GPU architecture to minimum CUDA version + arch_to_cuda = { + "sm_120": "cu128", # RTX 5090, RTX 50 series + "sm_90": "cu126", # RTX 4090, H100 + "sm_89": "cu121", # RTX 4090 + "sm_86": "cu121", # RTX 3090, A6000 + } + + assert arch_to_cuda.get(gpu_arch) == required_cuda, \ + f"GPU architecture {gpu_arch} requires CUDA version {required_cuda}" + + def test_older_gpus_work_with_cu121(self): + """Test that older GPUs (sm_86, sm_80) work with cu121.""" + older_archs = ["sm_86", "sm_80", "sm_75"] # RTX 3090, A100, RTX 2080 + + for arch in older_archs: + # cu121 supports these architectures + assert arch in ["sm_75", "sm_80", "sm_86"], \ + f"{arch} should be supported by CUDA 12.1" + + +class TestPyProjectCUDAExtras: + """Test that pyproject.toml defines CUDA version extras correctly.""" + + @pytest.fixture + def pyproject_path(self): + """Path to pyproject.toml.""" + return Path(__file__).parent.parent / "pyproject.toml" + + def test_pyproject_has_cuda_extras(self, pyproject_path): + """Test that pyproject.toml defines cu121, cu126, cu128 extras.""" + if not pyproject_path.exists(): + pytest.skip("pyproject.toml not found") + + content = pyproject_path.read_text() + + # Should have [project.optional-dependencies] or [tool.uv] with extras + cuda_versions = ["cu121", "cu126", "cu128"] + + for version in cuda_versions: + # Look for the CUDA version as an extra + assert re.search(rf'["\']?{version}["\']?\s*=', content), \ + f"pyproject.toml should define {version} extra" + + def test_pyproject_cuda_extras_have_pytorch(self, pyproject_path): + """Test that CUDA extras include torch/torchaudio dependencies.""" + if not pyproject_path.exists(): + pytest.skip("pyproject.toml not found") + + content = pyproject_path.read_text() + + # Each CUDA extra should reference torch with the appropriate index + # e.g., { extra = "cu128" } or { index = "pytorch-cu128" } + assert re.search(r'extra\s*=\s*["\']cu\d{3}["\']', content) or \ + re.search(r'index\s*=\s*["\']pytorch-cu\d{3}["\']', content), \ + "CUDA extras should reference PyTorch with CUDA version" + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/tests/configuration/test_llm_custom_provider.robot b/tests/configuration/test_llm_custom_provider.robot new file mode 100644 index 00000000..fa9a09c3 --- /dev/null +++ b/tests/configuration/test_llm_custom_provider.robot @@ -0,0 +1,258 @@ +*** Settings *** +Documentation Tests for LLM Custom Provider Setup (ConfigManager) +Library OperatingSystem +Library Collections +Library String +Library ../libs/ConfigTestHelper.py + +*** Keywords *** +Setup Temp Config + [Documentation] Creates a temporary configuration environment + ${random_suffix}= Generate Random String 8 [NUMBERS] + ${temp_path}= Join Path ${OUTPUT DIR} temp_config_${random_suffix} + Create Directory ${temp_path} + + # Create initial default config content + ${defaults}= Create Dictionary llm=openai-llm embedding=openai-embed stt=stt-deepgram + ${model1_params}= Create Dictionary temperature=${0.2} max_tokens=${2000} + ${model1}= Create Dictionary + ... name=openai-llm + ... description=OpenAI GPT-4o-mini + ... model_type=llm + ... model_provider=openai + ... api_family=openai + ... model_name=gpt-4o-mini + ... model_url=https://api.openai.com/v1 + ... api_key=\${oc.env:OPENAI_API_KEY,''} + ... model_params=${model1_params} + ... model_output=json + + ${model2}= Create Dictionary + ... name=local-embed + ... description=Local embeddings via Ollama + ... model_type=embedding + ... model_provider=ollama + ... api_family=openai + ... model_name=nomic-embed-text:latest + ... model_url=http://localhost:11434/v1 + ... api_key=\${oc.env:OPENAI_API_KEY,ollama} + ... embedding_dimensions=${768} + ... model_output=vector + + ${models}= Create List ${model1} ${model2} + ${memory}= Create Dictionary provider=chronicle + ${config}= Create Dictionary defaults=${defaults} models=${models} memory=${memory} + + Create Temp Config Structure ${temp_path} ${config} + Set Test Variable ${TEMP_PATH} ${temp_path} + +Cleanup Temp Config + Remove Directory ${TEMP_PATH} recursive=True + +*** Test Cases *** +Add New Model To Config + [Documentation] add_or_update_model() should append a new model when name doesn't exist. + [Setup] Setup Temp Config + [Teardown] Cleanup Temp Config + + ${params}= Create Dictionary temperature=${0.2} max_tokens=${2000} + ${new_model}= Create Dictionary + ... name=custom-llm + ... description=Custom OpenAI-compatible LLM + ... model_type=llm + ... model_provider=openai + ... api_family=openai + ... model_name=llama-3.1-70b-versatile + ... model_url=https://api.groq.com/openai/v1 + ... api_key=\${oc.env:CUSTOM_LLM_API_KEY,''} + ... model_params=${params} + ... model_output=json + + ${cm}= Get Config Manager Instance ${TEMP_PATH} + Add Model To Config Manager ${cm} ${new_model} + + ${config}= Call Method ${cm} get_full_config + ${models}= Get From Dictionary ${config} models + + ${target_model}= Set Variable ${None} + FOR ${m} IN @{models} + Run Keyword If '${m["name"]}' == 'custom-llm' Set Test Variable ${target_model} ${m} + END + + Should Not Be Equal ${target_model} ${None} + Should Be Equal ${target_model["model_name"]} llama-3.1-70b-versatile + Should Be Equal ${target_model["model_url"]} https://api.groq.com/openai/v1 + Should Be Equal ${target_model["model_type"]} llm + +Update Existing Model + [Documentation] add_or_update_model() should replace an existing model with the same name. + [Setup] Setup Temp Config + [Teardown] Cleanup Temp Config + + ${cm}= Get Config Manager Instance ${TEMP_PATH} + + # First add + ${model_v1}= Create Dictionary name=custom-llm model_type=llm model_name=model-v1 model_url=https://example.com/v1 + Add Model To Config Manager ${cm} ${model_v1} + + # Then update + ${model_v2}= Create Dictionary name=custom-llm model_type=llm model_name=model-v2 model_url=https://example.com/v2 + Add Model To Config Manager ${cm} ${model_v2} + + ${config}= Call Method ${cm} get_full_config + ${models}= Get From Dictionary ${config} models + + ${count}= Set Variable 0 + ${target_model}= Set Variable ${None} + FOR ${m} IN @{models} + IF '${m["name"]}' == 'custom-llm' + Set Test Variable ${target_model} ${m} + ${count}= Evaluate ${count} + 1 + END + END + + Should Be Equal As Integers ${count} 1 + Should Be Equal ${target_model["model_name"]} model-v2 + Should Be Equal ${target_model["model_url"]} https://example.com/v2 + +Add Model To Empty Models List + [Documentation] add_or_update_model() should create models list if it doesn't exist. + [Setup] Setup Temp Config + [Teardown] Cleanup Temp Config + + # Overwrite config with empty models + ${defaults}= Create Dictionary llm=openai-llm + ${empty_config}= Create Dictionary defaults=${defaults} + Create Temp Config Structure ${TEMP_PATH} ${empty_config} + + ${cm}= Get Config Manager Instance ${TEMP_PATH} + ${test_model}= Create Dictionary name=test-model model_type=llm + Add Model To Config Manager ${cm} ${test_model} + + ${config}= Call Method ${cm} get_full_config + Dictionary Should Contain Key ${config} models + ${models}= Get From Dictionary ${config} models + Length Should Be ${models} 1 + Should Be Equal ${models[0]["name"]} test-model + +Custom LLM And Embedding Model Added + [Documentation] Both LLM and embedding models should be created when embedding model is provided. + [Setup] Setup Temp Config + [Teardown] Cleanup Temp Config + + ${cm}= Get Config Manager Instance ${TEMP_PATH} + + ${params}= Create Dictionary temperature=${0.2} max_tokens=${2000} + ${llm_model}= Create Dictionary + ... name=custom-llm + ... model_type=llm + ... model_provider=openai + ... api_family=openai + ... model_name=llama-3.1-70b-versatile + ... model_url=https://api.groq.com/openai/v1 + ... api_key=\${oc.env:CUSTOM_LLM_API_KEY,''} + ... model_params=${params} + ... model_output=json + + ${embed_model}= Create Dictionary + ... name=custom-embed + ... description=Custom OpenAI-compatible embeddings + ... model_type=embedding + ... model_provider=openai + ... api_family=openai + ... model_name=text-embedding-3-small + ... model_url=https://api.groq.com/openai/v1 + ... api_key=\${oc.env:CUSTOM_LLM_API_KEY,''} + ... embedding_dimensions=${1536} + ... model_output=vector + + Add Model To Config Manager ${cm} ${llm_model} + Add Model To Config Manager ${cm} ${embed_model} + + ${config}= Call Method ${cm} get_full_config + ${models}= Get From Dictionary ${config} models + ${model_names}= Create List + FOR ${m} IN @{models} + Append To List ${model_names} ${m["name"]} + END + + List Should Contain Value ${model_names} custom-llm + List Should Contain Value ${model_names} custom-embed + + ${target_embed}= Set Variable ${None} + FOR ${m} IN @{models} + Run Keyword If '${m["name"]}' == 'custom-embed' Set Test Variable ${target_embed} ${m} + END + + Should Be Equal ${target_embed["model_type"]} embedding + Should Be Equal ${target_embed["model_name"]} text-embedding-3-small + Should Be Equal As Integers ${target_embed["embedding_dimensions"]} 1536 + +Custom LLM Without Embedding Falls Back To Local + [Documentation] defaults.embedding should be local-embed when no custom embedding is provided. + [Setup] Setup Temp Config + [Teardown] Cleanup Temp Config + + ${cm}= Get Config Manager Instance ${TEMP_PATH} + + ${llm_model}= Create Dictionary + ... name=custom-llm + ... model_type=llm + ... model_name=some-model + ... model_url=https://api.example.com/v1 + + Add Model To Config Manager ${cm} ${llm_model} + ${defaults_update}= Create Dictionary llm=custom-llm embedding=local-embed + Update Defaults In Config Manager ${cm} ${defaults_update} + + ${defaults}= Call Method ${cm} get_config_defaults + Should Be Equal ${defaults["llm"]} custom-llm + Should Be Equal ${defaults["embedding"]} local-embed + +Custom LLM Updates Defaults With Embedding + [Documentation] defaults.llm and defaults.embedding should be updated correctly with custom embed. + [Setup] Setup Temp Config + [Teardown] Cleanup Temp Config + + ${cm}= Get Config Manager Instance ${TEMP_PATH} + + ${defaults_update}= Create Dictionary llm=custom-llm embedding=custom-embed + Update Defaults In Config Manager ${cm} ${defaults_update} + + ${defaults}= Call Method ${cm} get_config_defaults + Should Be Equal ${defaults["llm"]} custom-llm + Should Be Equal ${defaults["embedding"]} custom-embed + +Existing Models Preserved After Adding Custom + [Documentation] Adding a custom model should not remove existing models. + [Setup] Setup Temp Config + [Teardown] Cleanup Temp Config + + ${cm}= Get Config Manager Instance ${TEMP_PATH} + ${config_before}= Call Method ${cm} get_full_config + ${models_before}= Get From Dictionary ${config_before} models + ${original_count}= Get Length ${models_before} + + ${new_model}= Create Dictionary + ... name=custom-llm + ... model_type=llm + ... model_name=test-model + ... model_url=https://example.com/v1 + + Add Model To Config Manager ${cm} ${new_model} + + ${config_after}= Call Method ${cm} get_full_config + ${models_after}= Get From Dictionary ${config_after} models + ${new_count}= Get Length ${models_after} + ${expected_count}= Evaluate ${original_count} + 1 + + Should Be Equal As Integers ${new_count} ${expected_count} + + ${model_names}= Create List + FOR ${m} IN @{models_after} + Append To List ${model_names} ${m["name"]} + END + + List Should Contain Value ${model_names} openai-llm + List Should Contain Value ${model_names} local-embed + List Should Contain Value ${model_names} custom-llm \ No newline at end of file diff --git a/tests/configuration/test_transcription_url.robot b/tests/configuration/test_transcription_url.robot new file mode 100644 index 00000000..e0ba40e8 --- /dev/null +++ b/tests/configuration/test_transcription_url.robot @@ -0,0 +1,126 @@ +*** Settings *** +Documentation Tests for Transcription Service URL Configuration +Library Collections +Library ../libs/ConfigTestHelper.py + +*** Test Cases *** +Vibevoice Url Without Http Prefix + [Documentation] Test that VIBEVOICE_ASR_URL without http:// prefix works correctly. + ${config_template}= Create Dictionary model_url=http://\${oc.env:VIBEVOICE_ASR_URL,host.docker.internal:8767} + ${env_vars}= Create Dictionary VIBEVOICE_ASR_URL=host.docker.internal:8767 + + ${resolved}= Resolve Omega Config ${config_template} ${env_vars} + Should Be Equal ${resolved["model_url"]} http://host.docker.internal:8767 + Should Not Contain ${resolved["model_url"]} http://http:// + +Vibevoice Url With Http Prefix Causes Double Prefix + [Documentation] Test that VIBEVOICE_ASR_URL WITH http:// causes double prefix (bug scenario). + ${config_template}= Create Dictionary model_url=http://\${oc.env:VIBEVOICE_ASR_URL,host.docker.internal:8767} + ${env_vars}= Create Dictionary VIBEVOICE_ASR_URL=http://host.docker.internal:8767 + + ${resolved}= Resolve Omega Config ${config_template} ${env_vars} + Should Be Equal ${resolved["model_url"]} http://http://host.docker.internal:8767 + Should Contain ${resolved["model_url"]} http://http:// + +Vibevoice Url Default Fallback + [Documentation] Test that default fallback works when VIBEVOICE_ASR_URL is not set. + ${config_template}= Create Dictionary model_url=http://\${oc.env:VIBEVOICE_ASR_URL,host.docker.internal:8767} + ${env_vars}= Create Dictionary + + ${resolved}= Resolve Omega Config ${config_template} ${env_vars} + Should Be Equal ${resolved["model_url"]} http://host.docker.internal:8767 + +Parakeet Url Configuration + [Documentation] Test that PARAKEET_ASR_URL follows same pattern. + ${config_template}= Create Dictionary model_url=http://\${oc.env:PARAKEET_ASR_URL,172.17.0.1:8767} + ${env_vars}= Create Dictionary PARAKEET_ASR_URL=host.docker.internal:8767 + + ${resolved}= Resolve Omega Config ${config_template} ${env_vars} + Should Be Equal ${resolved["model_url"]} http://host.docker.internal:8767 + Should Not Contain ${resolved["model_url"]} http://http:// + +Url Parsing Removes Double Slashes + [Documentation] Test that URL with double http:// causes connection failures (simulated by parsing check). + + # Valid URL + ${valid_url}= Set Variable http://host.docker.internal:8767/transcribe + ${parsed_valid}= Check Url Parsing ${valid_url} + Should Be Equal ${parsed_valid["scheme"]} http + Should Be Equal ${parsed_valid["netloc"]} host.docker.internal:8767 + + # Invalid URL + ${invalid_url}= Set Variable http://http://host.docker.internal:8767/transcribe + ${parsed_invalid}= Check Url Parsing ${invalid_url} + Should Be Equal ${parsed_invalid["scheme"]} http + # In python urlparse, 'http:' becomes the netloc for 'http://http://...' + Should Be Equal ${parsed_invalid["netloc"]} http: + Should Not Be Equal ${parsed_invalid["netloc"]} host.docker.internal:8767 + +Use Provider Segments Default False + [Documentation] Test that use_provider_segments defaults to false. + ${transcription}= Create Dictionary + ${backend}= Create Dictionary transcription=${transcription} + ${config_template}= Create Dictionary backend=${backend} + ${env_vars}= Create Dictionary + + ${resolved}= Resolve Omega Config ${config_template} ${env_vars} + ${val}= Evaluate $resolved.get('backend', {}).get('transcription', {}).get('use_provider_segments', False) + Should Be Equal ${val} ${FALSE} + +Use Provider Segments Explicit True + [Documentation] Test that use_provider_segments can be enabled. + ${transcription}= Create Dictionary use_provider_segments=${TRUE} + ${backend}= Create Dictionary transcription=${transcription} + ${config_template}= Create Dictionary backend=${backend} + ${env_vars}= Create Dictionary + + ${resolved}= Resolve Omega Config ${config_template} ${env_vars} + ${val}= Evaluate $resolved['backend']['transcription']['use_provider_segments'] + Should Be Equal ${val} ${TRUE} + +Vibevoice Should Use Provider Segments + [Documentation] Test that VibeVoice provider should have use_provider_segments=true since it provides diarized segments. + # Logic simulation + ${vibevoice_capabilities}= Create List segments diarization + ${has_diarization}= Evaluate "diarization" in $vibevoice_capabilities + ${has_segments}= Evaluate "segments" in $vibevoice_capabilities + ${should_use_segments}= Evaluate $has_diarization and $has_segments + Should Be Equal ${should_use_segments} ${TRUE} + +Model Registry Url Resolution With Env Var + [Documentation] Test that model URLs resolve correctly from environment. + ${model_def}= Create Dictionary + ... name=stt-vibevoice + ... model_type=stt + ... model_provider=vibevoice + ... model_url=http://\${oc.env:VIBEVOICE_ASR_URL,host.docker.internal:8767} + + ${models}= Create List ${model_def} + ${defaults}= Create Dictionary stt=stt-vibevoice + ${config_template}= Create Dictionary defaults=${defaults} models=${models} + + ${env_vars}= Create Dictionary VIBEVOICE_ASR_URL=host.docker.internal:8767 + + ${resolved}= Resolve Omega Config ${config_template} ${env_vars} + ${resolved_models}= Get From Dictionary ${resolved} models + Should Be Equal ${resolved_models[0]["model_url"]} http://host.docker.internal:8767 + +Multiple Asr Providers Url Resolution + [Documentation] Test that multiple ASR providers can use different URL patterns. + ${m1}= Create Dictionary name=stt-vibevoice model_url=http://\${oc.env:VIBEVOICE_ASR_URL,host.docker.internal:8767} + ${m2}= Create Dictionary name=stt-parakeet model_url=http://\${oc.env:PARAKEET_ASR_URL,172.17.0.1:8767} + ${m3}= Create Dictionary name=stt-deepgram model_url=https://api.deepgram.com/v1 + + ${models}= Create List ${m1} ${m2} ${m3} + ${config_template}= Create Dictionary models=${models} + + ${env_vars}= Create Dictionary + ... VIBEVOICE_ASR_URL=host.docker.internal:8767 + ... PARAKEET_ASR_URL=localhost:8080 + + ${resolved}= Resolve Omega Config ${config_template} ${env_vars} + ${resolved_models}= Get From Dictionary ${resolved} models + + Should Be Equal ${resolved_models[0]["model_url"]} http://host.docker.internal:8767 + Should Be Equal ${resolved_models[1]["model_url"]} http://localhost:8080 + Should Be Equal ${resolved_models[2]["model_url"]} https://api.deepgram.com/v1 diff --git a/tests/libs/ConfigTestHelper.py b/tests/libs/ConfigTestHelper.py new file mode 100644 index 00000000..6fbdcab4 --- /dev/null +++ b/tests/libs/ConfigTestHelper.py @@ -0,0 +1,73 @@ +import os +import sys +import yaml +from pathlib import Path +from typing import Dict, Any, Optional, List +from omegaconf import OmegaConf +from unittest.mock import patch + +# Add repo root to path to import config_manager +sys.path.insert(0, str(Path(__file__).resolve().parent.parent.parent)) +from config_manager import ConfigManager + +class ConfigTestHelper: + """Helper library for testing configuration logic.""" + + def _to_dict(self, obj: Any) -> Any: + """Recursively converts Robot Framework DotDict to standard dict.""" + if isinstance(obj, dict): + return {k: self._to_dict(v) for k, v in obj.items()} + if isinstance(obj, list): + return [self._to_dict(v) for v in obj] + return obj + + def resolve_omega_config(self, config_template: Dict[str, Any], env_vars: Dict[str, str]) -> Dict[str, Any]: + """ + Resolves an OmegaConf configuration template with provided environment variables. + """ + config_template = self._to_dict(config_template) + # We need to ensure values are strings for os.environ + str_env_vars = {k: str(v) for k, v in env_vars.items()} + + with patch.dict(os.environ, str_env_vars): + conf = OmegaConf.create(config_template) + resolved = OmegaConf.to_container(conf, resolve=True) + return resolved + + def check_url_parsing(self, url: str) -> Dict[str, Any]: + """ + Parses a URL and returns its components to verify correct parsing. + """ + from urllib.parse import urlparse + parsed = urlparse(url) + return { + "scheme": parsed.scheme, + "netloc": parsed.netloc, + "path": parsed.path + } + + def create_temp_config_structure(self, base_path: str, content: Dict[str, Any]) -> str: + """ + Creates the config folder structure and config.yml within the given base path. + """ + content = self._to_dict(content) + path = Path(base_path) / "config" + path.mkdir(parents=True, exist_ok=True) + config_file = path / "config.yml" + with open(config_file, "w") as f: + yaml.dump(content, f, default_flow_style=False, sort_keys=False) + return str(base_path) + + def get_config_manager_instance(self, repo_root: str) -> ConfigManager: + """Returns a ConfigManager instance configured with the given repo_root.""" + return ConfigManager(service_path=None, repo_root=Path(repo_root)) + + def add_model_to_config_manager(self, cm: ConfigManager, model_def: Dict[str, Any]): + """Wrapper for add_or_update_model that converts arguments.""" + model_def = self._to_dict(model_def) + cm.add_or_update_model(model_def) + + def update_defaults_in_config_manager(self, cm: ConfigManager, updates: Dict[str, str]): + """Wrapper for update_config_defaults that converts arguments.""" + updates = self._to_dict(updates) + cm.update_config_defaults(updates) \ No newline at end of file diff --git a/tests/test-requirements.txt b/tests/test-requirements.txt index f32614e0..5cd8f020 100644 --- a/tests/test-requirements.txt +++ b/tests/test-requirements.txt @@ -6,4 +6,6 @@ robotframework-databaselibrary python-dotenv websockets pymongo +omegaconf +pyyaml \ No newline at end of file