Enhance configuration management and add new setup scripts

.github/workflows/README.md

@@ -86,6 +86,6 @@ uv sync --dev
 cp .env.template .env.test
 # Add your API keys to .env.test
 
-# Run test (modify CACHED_MODE in test_integration.py if needed)
-uv run pytest test_integration.py::test_full_pipeline_integration -v -s
+# Run Robot Framework integration tests
+uv run robot --outputdir test-results --loglevel INFO tests/integration/integration_test.robot
 ```

.github/workflows/robot-tests.yml

@@ -28,6 +28,7 @@ jobs:
       env:
         DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
       run: |
         echo "Verifying required secrets..."
         if [ -z "$DEEPGRAM_API_KEY" ]; then
@@ -38,8 +39,13 @@ jobs:
           echo "❌ ERROR: OPENAI_API_KEY secret is not set"
           exit 1
         fi
+        if [ -z "$HF_TOKEN" ]; then
+          echo "❌ ERROR: HF_TOKEN secret is not set"
+          exit 1
+        fi
         echo "✓ DEEPGRAM_API_KEY is set (length: ${#DEEPGRAM_API_KEY})"
         echo "✓ OPENAI_API_KEY is set (length: ${#OPENAI_API_KEY})"
+        echo "✓ HF_TOKEN is set (length: ${#HF_TOKEN})"
         echo "✓ All required secrets verified"
 
     - name: Set up Docker Buildx
@@ -61,7 +67,6 @@ jobs:
       uses: actions/setup-python@v5
       with:
         python-version: "3.12"
-        cache: 'pip'
 
     - name: Install uv
       uses: astral-sh/setup-uv@v4
@@ -72,144 +77,25 @@ jobs:
       run: |
         uv pip install --system robotframework robotframework-requests python-dotenv websockets
 
-    - name: Create test environment file
-      working-directory: tests/setup
-      run: |
-        cat > .env.test << EOF
-        # API URLs
-        API_URL=http://localhost:8001
-        BACKEND_URL=http://localhost:8001
-        FRONTEND_URL=http://localhost:3001
-
-        # Test Admin Credentials
-        ADMIN_EMAIL=test-admin@example.com
-        ADMIN_PASSWORD=test-admin-password-123
-
-        # API Keys (from GitHub secrets)
-        OPENAI_API_KEY=${{ secrets.OPENAI_API_KEY }}
-        DEEPGRAM_API_KEY=${{ secrets.DEEPGRAM_API_KEY }}
-
-        # Test Configuration
-        TEST_TIMEOUT=120
-        TEST_DEVICE_NAME=robot-test
-        EOF
-
-    - name: Start test environment
-      working-directory: backends/advanced
-      env:
-        DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        LLM_PROVIDER: openai
-        TRANSCRIPTION_PROVIDER: deepgram
-        MEMORY_PROVIDER: friend_lite
-      run: |
-        # Debug: Check if secrets are available
-        echo "Checking environment variables..."
-        echo "DEEPGRAM_API_KEY is set: $([ -n "$DEEPGRAM_API_KEY" ] && echo 'YES' || echo 'NO')"
-        echo "OPENAI_API_KEY is set: $([ -n "$OPENAI_API_KEY" ] && echo 'YES' || echo 'NO')"
-        echo "LLM_PROVIDER: $LLM_PROVIDER"
-        echo "TRANSCRIPTION_PROVIDER: $TRANSCRIPTION_PROVIDER"
-
-        # Clean any existing test containers for fresh start
-        echo "Cleaning up any existing test containers..."
-        docker compose -f docker-compose-test.yml down -v || true
-
-        # Start ALL services in parallel - Docker Compose handles dependencies via healthchecks
-        echo "Starting all services in parallel (docker-compose-test.yml)..."
-        echo "Note: Using test compose file with source mounts for faster startup"
-
-        # Export API keys so docker-compose can use them
-        export DEEPGRAM_API_KEY
-        export OPENAI_API_KEY
-        export LLM_PROVIDER
-        export TRANSCRIPTION_PROVIDER
-        export MEMORY_PROVIDER
-
-        DOCKER_BUILDKIT=0 docker compose -f docker-compose-test.yml up -d
-
-        # Show container status
-        echo "Container status:"
-        docker compose -f docker-compose-test.yml ps
-
-        # Single wait for backend readiness (backend depends_on ensures infra is ready)
-        echo "Waiting for backend readiness (up to 120s)..."
-        for i in {1..40}; do
-          if curl -s http://localhost:8001/readiness > /dev/null 2>&1; then
-            echo "✓ Backend is ready (all dependencies satisfied)"
-            break
-          fi
-          # Show logs every 10 attempts to help debug
-          if [ $((i % 10)) -eq 0 ]; then
-            echo "Still waiting... showing recent logs:"
-            docker compose -f docker-compose-test.yml logs --tail=20 chronicle-backend-test
-          fi
-          if [ $i -eq 40 ]; then
-            echo "✗ Backend failed to start - showing full logs:"
-            docker compose -f docker-compose-test.yml logs
-            exit 1
-          fi
-          echo "Attempt $i/40..."
-          sleep 3
-        done
-
-        echo "✓ Backend is ready!"
-
-        # Verify workers are registered with Redis (Robot tests need stable workers)
-        echo "Waiting for workers to register with Redis (up to 60s)..."
-        for i in {1..30}; do
-          WORKER_COUNT=$(docker compose -f docker-compose-test.yml exec -T workers-test uv run python -c 'from rq import Worker; from redis import Redis; import os; r = Redis.from_url(os.getenv("REDIS_URL", "redis://redis-test:6379/0")); print(len(Worker.all(connection=r)))' 2>/dev/null || echo "0")
-
-          if [ "$WORKER_COUNT" -ge 6 ]; then
-            echo "✓ Found $WORKER_COUNT workers registered"
-            # Show worker details
-            docker compose -f docker-compose-test.yml exec -T workers-test uv run python -c 'from rq import Worker; from redis import Redis; import os; r = Redis.from_url(os.getenv("REDIS_URL", "redis://redis-test:6379/0")); workers = Worker.all(connection=r); print(f"Total registered workers: {len(workers)}"); [print(f"  - {w.name}: queues={w.queue_names()}, state={w.get_state()}") for w in workers]'
-            break
-          fi
-
-          if [ $i -eq 30 ]; then
-            echo "✗ Workers failed to register after 60s"
-            echo "Showing worker logs:"
-            docker compose -f docker-compose-test.yml logs --tail=50 workers-test
-            exit 1
-          fi
-
-          echo "Attempt $i/30: $WORKER_COUNT workers registered (waiting for 6+)..."
-          sleep 2
-        done
-
-        echo "✓ All services ready!"
-
-    - name: Verify checked out code
-      working-directory: tests
-      run: |
-        echo "Current git commit:"
-        git log -1 --oneline
-        echo ""
-        echo "Test files in current checkout:"
-        find . -name "*.robot" -type f | head -10
-        echo ""
-        echo "Sample of tags in test files:"
-        grep -h "\[Tags\]" endpoints/*.robot infrastructure/*.robot integration/*.robot 2>/dev/null | head -20 || echo "No tag files found"
-
-    - name: Clean previous test results
-      working-directory: tests
+    - name: Create test config.yml
       run: |
-        echo "Cleaning any previous test results..."
-        rm -rf results
-        mkdir -p results
-        echo "✓ Fresh results directory created"
+        echo "Copying test configuration file..."
+        mkdir -p config
+        cp tests/configs/deepgram-openai.yml config/config.yml
+        echo "✓ Test config.yml created from tests/configs/deepgram-openai.yml"
+        ls -lh config/config.yml
 
     - name: Run Robot Framework tests
       working-directory: tests
       env:
-        # Required for backend imports in test libraries
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        OPENAI_BASE_URL: https://api.openai.com/v1
-        OPENAI_MODEL: gpt-4o-mini
+        # Required for test runner script
         DEEPGRAM_API_KEY: ${{ secrets.DEEPGRAM_API_KEY }}
+        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+        HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        CLEANUP_CONTAINERS: "false"  # Don't cleanup in CI - handled by workflow
       run: |
-        # Run all tests (don't fail workflow to allow artifact upload)
-        make all OUTPUTDIR=results
+        # Use the unified test script that mirrors local development
+        ./run-robot-tests.sh
         TEST_EXIT_CODE=$?
         echo "test_exit_code=$TEST_EXIT_CODE" >> $GITHUB_ENV
         exit 0  # Don't fail here, we'll fail at the end after uploading artifacts

.gitignore

@@ -4,6 +4,16 @@
 !**/.env.template
 **/memory_config.yaml
 !**/memory_config.yaml.template
+tests/setup/.env.test
+
+# Main config (user-specific)
+config/config.yml
+!config/config.yml.template
+
+# Config backups
+config/*.backup.*
+config/*.backup*
+
 example/*
 **/node_modules/*
 **/ollama-data/*

CLAUDE.md

@@ -116,11 +116,11 @@ cp .env.template .env  # Configure API keys
 
 # Manual test execution (for debugging)
 source .env && export DEEPGRAM_API_KEY && export OPENAI_API_KEY
-uv run pytest tests/test_integration.py::test_full_pipeline_integration -v -s
+uv run robot --outputdir test-results --loglevel INFO ../../tests/integration/integration_test.robot
 
 # Leave test containers running for debugging (don't auto-cleanup)
 CLEANUP_CONTAINERS=false source .env && export DEEPGRAM_API_KEY && export OPENAI_API_KEY
-uv run pytest tests/test_integration.py::test_full_pipeline_integration -v -s
+uv run robot --outputdir test-results --loglevel INFO ../../tests/integration/integration_test.robot
 
 # Manual cleanup when needed
 docker compose -f docker-compose-test.yml down -v
@@ -390,7 +390,7 @@ docker compose up --build -d
 
 ### Testing Strategy
 - **Local Test Scripts**: Simplified scripts (`./run-test.sh`) mirror CI workflows for local development
-- **End-to-End Integration**: `test_integration.py` validates complete audio processing pipeline
+- **End-to-End Integration**: Robot Framework tests (`tests/integration/integration_test.robot`) validate complete audio processing pipeline
 - **Speaker Recognition Tests**: `test_speaker_service_integration.py` validates speaker identification
 - **Environment Flexibility**: Tests work with both local .env files and CI environment variables
 - **Automated Cleanup**: Test containers are automatically removed after execution

Docs/getting-started.md

@@ -179,9 +179,9 @@ After configuration, verify everything works with the integration test suite:
 
 # Alternative: Manual test with detailed logging
 source .env && export DEEPGRAM_API_KEY OPENAI_API_KEY && \
-  uv run pytest tests/test_integration.py -vv -s --log-cli-level=INFO
+  uv run robot --outputdir ../../test-results --loglevel INFO ../../tests/integration/integration_test.robot
 ```
-This end-to-end test validates the complete audio processing pipeline.
+This end-to-end test validates the complete audio processing pipeline using Robot Framework.
 
 ## Using the System
 
@@ -342,7 +342,7 @@ curl -X POST "http://localhost:8000/api/process-audio-files" \
 
 **Implementation**: 
 - **Memory System**: `src/advanced_omi_backend/memory/memory_service.py` + `src/advanced_omi_backend/controllers/memory_controller.py`
-- **Configuration**: memory settings in `config.yml` (memory section)
+- **Configuration**: memory settings in `config/config.yml` (memory section)
 
 ### Authentication & Security
 - **Email Authentication**: Login with email and password
@@ -541,10 +541,10 @@ OPENMEMORY_MCP_URL=http://host.docker.internal:8765
 
 > 🎯 **New to memory configuration?** Read our [Memory Configuration Guide](./memory-configuration-guide.md) for a step-by-step setup guide with examples.
 
-The system uses **centralized configuration** via `config.yml` for all models (LLM, embeddings, vector store) and memory extraction settings.
+The system uses **centralized configuration** via `config/config.yml` for all models (LLM, embeddings, vector store) and memory extraction settings.
 
 ### Configuration File Location
-- **Path**: repository `config.yml` (override with `CONFIG_FILE` env var)
+- **Path**: repository `config/config.yml` (override with `CONFIG_FILE` env var)
 - **Hot-reload**: Changes are applied on next processing cycle (no restart required)
 - **Fallback**: If file is missing, system uses safe defaults with environment variables
 
@@ -613,7 +613,7 @@ If you experience JSON parsing errors in fact extraction:
 
 2. **Enable fact extraction** with reliable JSON output:
    ```yaml
-   # In config.yml (memory section)
+   # In config/config.yml (memory section)
    fact_extraction:
      enabled: true  # Safe to enable with GPT-4o
    ```
@@ -727,5 +727,5 @@ curl -H "Authorization: Bearer $ADMIN_TOKEN" \
 - **Connect audio clients** using the WebSocket API
 - **Explore the dashboard** to manage conversations and users
 - **Review the user data architecture** for understanding data organization
-- **Customize memory extraction** by editing the `memory` section in `config.yml`
+- **Customize memory extraction** by editing the `memory` section in `config/config.yml`
 - **Monitor processing performance** using debug API endpoints

backends/advanced/.env.template

@@ -45,18 +45,14 @@ OPENAI_MODEL=gpt-4o-mini
 # CHAT_TEMPERATURE=0.7
 
 # ========================================
-# SPEECH-TO-TEXT CONFIGURATION (Choose one)
+# SPEECH-TO-TEXT CONFIGURATION (API Keys Only)
 # ========================================
+# Provider selection is in config.yml (defaults.stt)
 
-# Option 1: Deepgram (recommended for best transcription quality)
+# Deepgram (cloud-based, recommended)
 DEEPGRAM_API_KEY=
 
-# Option 2: Parakeet ASR service from extras/asr-services
-# PARAKEET_ASR_URL=http://host.docker.internal:8767
-
-# Optional: Specify which provider to use ('deepgram' or 'parakeet')
-# If not set, will auto-select based on available configuration (Deepgram preferred)
-# TRANSCRIPTION_PROVIDER=
+# Note: Parakeet ASR URL configured in config.yml
 
 # ========================================
 # SPEECH DETECTION CONFIGURATION