From d594526a6079ab9881507a60e8b8bb28dab343fb Mon Sep 17 00:00:00 2001 From: Adil Hafeez Date: Thu, 25 Dec 2025 17:41:32 -0800 Subject: [PATCH] run e2e tests in parallel --- .github/workflows/e2e_tests_parallel.yml | 82 +++++++++++++++++++ tests/e2e/run_model_alias_tests.sh | 52 ++++++++++++ tests/e2e/run_openai_responses_tests.sh | 52 ++++++++++++ .../run_openai_responses_with_state_tests.sh | 52 ++++++++++++ tests/e2e/run_prompt_gateway_tests.sh | 64 +++++++++++++++ 5 files changed, 302 insertions(+) create mode 100644 .github/workflows/e2e_tests_parallel.yml create mode 100755 tests/e2e/run_model_alias_tests.sh create mode 100755 tests/e2e/run_openai_responses_tests.sh create mode 100755 tests/e2e/run_openai_responses_with_state_tests.sh create mode 100755 tests/e2e/run_prompt_gateway_tests.sh diff --git a/.github/workflows/e2e_tests_parallel.yml b/.github/workflows/e2e_tests_parallel.yml new file mode 100644 index 000000000..0b17e5135 --- /dev/null +++ b/.github/workflows/e2e_tests_parallel.yml @@ -0,0 +1,82 @@ +name: e2e tests (parallel) + +permissions: + contents: read + +on: + push: + branches: + - main # Run tests on pushes to the main branch + pull_request: + +jobs: + e2e_tests: + runs-on: ubuntu-latest + strategy: + fail-fast: false # Continue running other jobs even if one fails + matrix: + test-suite: + - name: "Prompt Gateway" + script: "run_prompt_gateway_tests.sh" + - name: "Model Alias Routing" + script: "run_model_alias_tests.sh" + - name: "OpenAI Responses API" + script: "run_openai_responses_tests.sh" + - name: "OpenAI Responses API with State" + script: "run_openai_responses_with_state_tests.sh" + + name: ${{ matrix.test-suite.name }} + + steps: + - name: Checkout code + uses: actions/checkout@v3 + + # --- Disk inspection & cleanup section (added to free space on GitHub runner) --- + - name: Check disk usage before cleanup + run: | + echo "=== Disk usage before cleanup ===" + df -h + echo "=== Repo size ===" + du -sh . + + - name: Free disk space on runner + run: | + echo "=== Cleaning preinstalled SDKs and toolchains to free space ===" + sudo rm -rf /usr/share/dotnet + sudo rm -rf /usr/local/lib/android + sudo rm -rf /opt/ghc + # If you still hit disk issues, uncomment this to free more space. + # It just removes cached tool versions; setup-python will re-download what it needs. + # sudo rm -rf /opt/hostedtoolcache || true + + echo "=== Docker cleanup (before our builds/compose) ===" + docker system prune -af || true + docker volume prune -f || true + + echo "=== Disk usage after cleanup ===" + df -h + # --- End disk cleanup section --- + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.12" + + - name: Install Poetry + run: | + export POETRY_VERSION=2.2.1 + curl -sSL https://install.python-poetry.org | python3 - + export PATH="$HOME/.local/bin:$PATH" + + - name: Run ${{ matrix.test-suite.name }} tests + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }} + GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + AZURE_API_KEY: ${{ secrets.AZURE_API_KEY }} + AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }} + GROK_API_KEY : ${{ secrets.GROK_API_KEY }} + run: | + python -mvenv venv + source venv/bin/activate && cd tests/e2e && bash ${{ matrix.test-suite.script }} diff --git a/tests/e2e/run_model_alias_tests.sh b/tests/e2e/run_model_alias_tests.sh new file mode 100755 index 000000000..e3c757dbb --- /dev/null +++ b/tests/e2e/run_model_alias_tests.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Run model alias routing e2e tests +# if any of the commands fail, the script will exit +set -e + +. ./common_scripts.sh + +print_disk_usage + +mkdir -p ~/plano_logs +touch ~/plano_logs/modelserver.log + +print_debug() { + log "Received signal to stop" + log "Printing debug logs for docker" + log "====================================" + tail -n 100 ../build.log + planoai logs --debug | tail -n 100 +} + +trap 'print_debug' INT TERM ERR + +log starting > ../build.log + +log building and installing plano cli +log ================================== +cd ../../cli +poetry install +cd - + +log building docker image for arch gateway +log ====================================== +cd ../../ +planoai build +cd - + +# Once we build plano we have to install the dependencies again to a new virtual environment. +poetry install + +log startup arch gateway with model alias routing demo +cd ../../ +planoai down +planoai up demos/use_cases/model_alias_routing/config_with_aliases.yaml +cd - + +log running e2e tests for model alias routing +log ======================================== +poetry run pytest test_model_alias_routing.py + +log shutting down the arch gateway service +log ======================================= +planoai down diff --git a/tests/e2e/run_openai_responses_tests.sh b/tests/e2e/run_openai_responses_tests.sh new file mode 100755 index 000000000..6324c76d6 --- /dev/null +++ b/tests/e2e/run_openai_responses_tests.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Run OpenAI responses API e2e tests +# if any of the commands fail, the script will exit +set -e + +. ./common_scripts.sh + +print_disk_usage + +mkdir -p ~/plano_logs +touch ~/plano_logs/modelserver.log + +print_debug() { + log "Received signal to stop" + log "Printing debug logs for docker" + log "====================================" + tail -n 100 ../build.log + planoai logs --debug | tail -n 100 +} + +trap 'print_debug' INT TERM ERR + +log starting > ../build.log + +log building and installing plano cli +log ================================== +cd ../../cli +poetry install +cd - + +log building docker image for arch gateway +log ====================================== +cd ../../ +planoai build +cd - + +# Once we build plano we have to install the dependencies again to a new virtual environment. +poetry install + +log startup arch gateway with model alias routing demo +cd ../../ +planoai down +planoai up demos/use_cases/model_alias_routing/config_with_aliases.yaml +cd - + +log running e2e tests for openai responses api client +log ================================================ +poetry run pytest test_openai_responses_api_client.py + +log shutting down the arch gateway service +log ======================================= +planoai down diff --git a/tests/e2e/run_openai_responses_with_state_tests.sh b/tests/e2e/run_openai_responses_with_state_tests.sh new file mode 100755 index 000000000..c5562b606 --- /dev/null +++ b/tests/e2e/run_openai_responses_with_state_tests.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Run OpenAI responses API with state storage e2e tests +# if any of the commands fail, the script will exit +set -e + +. ./common_scripts.sh + +print_disk_usage + +mkdir -p ~/plano_logs +touch ~/plano_logs/modelserver.log + +print_debug() { + log "Received signal to stop" + log "Printing debug logs for docker" + log "====================================" + tail -n 100 ../build.log + planoai logs --debug | tail -n 100 +} + +trap 'print_debug' INT TERM ERR + +log starting > ../build.log + +log building and installing plano cli +log ================================== +cd ../../cli +poetry install +cd - + +log building docker image for arch gateway +log ====================================== +cd ../../ +planoai build +cd - + +# Once we build plano we have to install the dependencies again to a new virtual environment. +poetry install + +log startup arch gateway with state storage for openai responses api client demo +cd ../../ +planoai down +planoai up tests/e2e/config_memory_state_v1_responses.yaml +cd - + +log running e2e tests for openai responses api client with state +log ============================================================ +poetry run pytest test_openai_responses_api_client_with_state.py + +log shutting down the arch gateway service +log ======================================= +planoai down diff --git a/tests/e2e/run_prompt_gateway_tests.sh b/tests/e2e/run_prompt_gateway_tests.sh new file mode 100755 index 000000000..b1be05446 --- /dev/null +++ b/tests/e2e/run_prompt_gateway_tests.sh @@ -0,0 +1,64 @@ +#!/bin/bash +# Run prompt gateway e2e tests +# if any of the commands fail, the script will exit +set -e + +. ./common_scripts.sh + +print_disk_usage + +mkdir -p ~/plano_logs +touch ~/plano_logs/modelserver.log + +print_debug() { + log "Received signal to stop" + log "Printing debug logs for docker" + log "====================================" + tail -n 100 ../build.log + planoai logs --debug | tail -n 100 +} + +trap 'print_debug' INT TERM ERR + +log starting > ../build.log + +log building and running function_calling demo +log =========================================== +cd ../../demos/samples_python/weather_forecast/ +docker compose up weather_forecast_service --build -d +cd - + +log building and installing plano cli +log ================================== +cd ../../cli +poetry install +cd - + +log building docker image for arch gateway +log ====================================== +cd ../../ +planoai build +cd - + +# Once we build plano we have to install the dependencies again to a new virtual environment. +poetry install + +log startup arch gateway with function calling demo +cd ../../ +planoai down +planoai up demos/samples_python/weather_forecast/config.yaml +cd - + +log running e2e tests for prompt gateway +log ==================================== +poetry run pytest test_prompt_gateway.py + +log shutting down the arch gateway service for prompt_gateway demo +log =============================================================== +planoai down + +log shutting down the weather_forecast demo +log ======================================= +cd ../../demos/samples_python/weather_forecast +docker compose down +cd -