From 048491cff703d21fe90b08da98e9a1f358e5ee7e Mon Sep 17 00:00:00 2001 From: staverm Date: Sat, 20 Dec 2025 18:28:58 +0100 Subject: [PATCH 1/3] Add a2a tests --- pyproject.toml | 7 ++ tests/conftest.py | 25 ++++++ tests/test_agent.py | 199 ++++++++++++++++++++++++++++++++++++++++++++ uv.lock | 74 ++++++++++++++++ 4 files changed, 305 insertions(+) create mode 100644 tests/conftest.py create mode 100644 tests/test_agent.py diff --git a/pyproject.toml b/pyproject.toml index 275cd2e..a98a765 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,3 +8,10 @@ dependencies = [ "a2a-sdk[http-server]>=0.3.20", "uvicorn>=0.38.0", ] + +[project.optional-dependencies] +test = [ + "pytest>=8.0.0", + "pytest-asyncio>=0.24.0", + "httpx>=0.28.1", +] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..9b4d248 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,25 @@ +import httpx +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--agent-url", + default="http://localhost:9009", + help="Agent URL (default: http://localhost:9009)", + ) + + +@pytest.fixture(scope="session") +def agent(request): + """Agent URL fixture. Agent must be running before tests start.""" + url = request.config.getoption("--agent-url") + + try: + response = httpx.get(f"{url}/.well-known/agent-card.json", timeout=2) + if response.status_code != 200: + pytest.exit(f"Agent at {url} returned status {response.status_code}", returncode=1) + except Exception as e: + pytest.exit(f"Could not connect to agent at {url}: {e}", returncode=1) + + return url diff --git a/tests/test_agent.py b/tests/test_agent.py new file mode 100644 index 0000000..8259536 --- /dev/null +++ b/tests/test_agent.py @@ -0,0 +1,199 @@ +from typing import Any +import pytest +import httpx +from uuid import uuid4 + +from a2a.client import A2ACardResolver, ClientConfig, ClientFactory +from a2a.types import Message, Part, Role, TextPart + + +# A2A validation helpers - adapted from https://github.com/a2aproject/a2a-inspector/blob/main/backend/validators.py + +def validate_agent_card(card_data: dict[str, Any]) -> list[str]: + """Validate the structure and fields of an agent card.""" + errors: list[str] = [] + + # Use a frozenset for efficient checking and to indicate immutability. + required_fields = frozenset( + [ + 'name', + 'description', + 'url', + 'version', + 'capabilities', + 'defaultInputModes', + 'defaultOutputModes', + 'skills', + ] + ) + + # Check for the presence of all required fields + for field in required_fields: + if field not in card_data: + errors.append(f"Required field is missing: '{field}'.") + + # Check if 'url' is an absolute URL (basic check) + if 'url' in card_data and not ( + card_data['url'].startswith('http://') + or card_data['url'].startswith('https://') + ): + errors.append( + "Field 'url' must be an absolute URL starting with http:// or https://." + ) + + # Check if capabilities is a dictionary + if 'capabilities' in card_data and not isinstance( + card_data['capabilities'], dict + ): + errors.append("Field 'capabilities' must be an object.") + + # Check if defaultInputModes and defaultOutputModes are arrays of strings + for field in ['defaultInputModes', 'defaultOutputModes']: + if field in card_data: + if not isinstance(card_data[field], list): + errors.append(f"Field '{field}' must be an array of strings.") + elif not all(isinstance(item, str) for item in card_data[field]): + errors.append(f"All items in '{field}' must be strings.") + + # Check skills array + if 'skills' in card_data: + if not isinstance(card_data['skills'], list): + errors.append( + "Field 'skills' must be an array of AgentSkill objects." + ) + elif not card_data['skills']: + errors.append( + "Field 'skills' array is empty. Agent must have at least one skill if it performs actions." + ) + + return errors + + +def _validate_task(data: dict[str, Any]) -> list[str]: + errors = [] + if 'id' not in data: + errors.append("Task object missing required field: 'id'.") + if 'status' not in data or 'state' not in data.get('status', {}): + errors.append("Task object missing required field: 'status.state'.") + return errors + + +def _validate_status_update(data: dict[str, Any]) -> list[str]: + errors = [] + if 'status' not in data or 'state' not in data.get('status', {}): + errors.append( + "StatusUpdate object missing required field: 'status.state'." + ) + return errors + + +def _validate_artifact_update(data: dict[str, Any]) -> list[str]: + errors = [] + if 'artifact' not in data: + errors.append( + "ArtifactUpdate object missing required field: 'artifact'." + ) + elif ( + 'parts' not in data.get('artifact', {}) + or not isinstance(data.get('artifact', {}).get('parts'), list) + or not data.get('artifact', {}).get('parts') + ): + errors.append("Artifact object must have a non-empty 'parts' array.") + return errors + + +def _validate_message(data: dict[str, Any]) -> list[str]: + errors = [] + if ( + 'parts' not in data + or not isinstance(data.get('parts'), list) + or not data.get('parts') + ): + errors.append("Message object must have a non-empty 'parts' array.") + if 'role' not in data or data.get('role') != 'agent': + errors.append("Message from agent must have 'role' set to 'agent'.") + return errors + + +def validate_event(data: dict[str, Any]) -> list[str]: + """Validate an incoming event from the agent based on its kind.""" + if 'kind' not in data: + return ["Response from agent is missing required 'kind' field."] + + kind = data.get('kind') + validators = { + 'task': _validate_task, + 'status-update': _validate_status_update, + 'artifact-update': _validate_artifact_update, + 'message': _validate_message, + } + + validator = validators.get(str(kind)) + if validator: + return validator(data) + + return [f"Unknown message kind received: '{kind}'."] + + +# A2A messaging helpers + +async def send_text_message(text: str, url: str, context_id: str | None = None, streaming: bool = False): + async with httpx.AsyncClient(timeout=10) as httpx_client: + resolver = A2ACardResolver(httpx_client=httpx_client, base_url=url) + agent_card = await resolver.get_agent_card() + config = ClientConfig(httpx_client=httpx_client, streaming=streaming) + factory = ClientFactory(config) + client = factory.create(agent_card) + + msg = Message( + kind="message", + role=Role.user, + parts=[Part(TextPart(text=text))], + message_id=uuid4().hex, + context_id=context_id, + ) + + events = [event async for event in client.send_message(msg)] + + return events + + +# A2A conformance tests + +def test_agent_card(agent): + """Validate agent card structure and required fields.""" + response = httpx.get(f"{agent}/.well-known/agent-card.json") + assert response.status_code == 200, "Agent card endpoint must return 200" + + card_data = response.json() + errors = validate_agent_card(card_data) + + assert not errors, f"Agent card validation failed:\n" + "\n".join(errors) + +@pytest.mark.asyncio +@pytest.mark.parametrize("streaming", [True, False]) +async def test_message(agent, streaming): + """Test that agent returns valid A2A message format.""" + events = await send_text_message("Hello", agent, streaming=streaming) + + all_errors = [] + for event in events: + match event: + case Message() as msg: + errors = validate_event(msg.model_dump()) + all_errors.extend(errors) + + case (task, update): + errors = validate_event(task.model_dump()) + all_errors.extend(errors) + if update: + errors = validate_event(update.model_dump()) + all_errors.extend(errors) + + case _: + pytest.fail(f"Unexpected event type: {type(event)}") + + assert events, "Agent should respond with at least one event" + assert not all_errors, f"Message validation failed:\n" + "\n".join(all_errors) + +# Add your custom tests here diff --git a/uv.lock b/uv.lock index d159650..204a82a 100644 --- a/uv.lock +++ b/uv.lock @@ -33,9 +33,19 @@ dependencies = [ { name = "uvicorn" }, ] +[package.optional-dependencies] +test = [ + { name = "httpx" }, + { name = "pytest" }, + { name = "pytest-asyncio" }, +] + [package.metadata] requires-dist = [ { name = "a2a-sdk", extras = ["http-server"], specifier = ">=0.3.20" }, + { name = "httpx", marker = "extra == 'test'", specifier = ">=0.28.1" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.0.0" }, + { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.24.0" }, { name = "uvicorn", specifier = ">=0.38.0" }, ] @@ -261,6 +271,33 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008 }, ] +[[package]] +name = "iniconfig" +version = "2.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484 }, +] + +[[package]] +name = "packaging" +version = "25.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469 }, +] + +[[package]] +name = "pluggy" +version = "1.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538 }, +] + [[package]] name = "proto-plus" version = "1.26.1" @@ -385,6 +422,43 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f7/07/34573da085946b6a313d7c42f82f16e8920bfd730665de2d11c0c37a74b5/pydantic_core-2.41.5-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:76d0819de158cd855d1cbb8fcafdf6f5cf1eb8e470abe056d5d161106e38062b", size = 2139017 }, ] +[[package]] +name = "pygments" +version = "2.19.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b0/77/a5b8c569bf593b0140bde72ea885a803b82086995367bf2037de0159d924/pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887", size = 4968631 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/21/705964c7812476f378728bdf590ca4b771ec72385c533964653c68e86bdc/pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b", size = 1225217 }, +] + +[[package]] +name = "pytest" +version = "9.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d1/db/7ef3487e0fb0049ddb5ce41d3a49c235bf9ad299b6a25d5780a89f19230f/pytest-9.0.2.tar.gz", hash = "sha256:75186651a92bd89611d1d9fc20f0b4345fd827c41ccd5c299a868a05d70edf11", size = 1568901 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3b/ab/b3226f0bd7cdcf710fbede2b3548584366da3b19b5021e74f5bde2a8fa3f/pytest-9.0.2-py3-none-any.whl", hash = "sha256:711ffd45bf766d5264d487b917733b453d917afd2b0ad65223959f59089f875b", size = 374801 }, +] + +[[package]] +name = "pytest-asyncio" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/2c/8af215c0f776415f3590cac4f9086ccefd6fd463befeae41cd4d3f193e5a/pytest_asyncio-1.3.0.tar.gz", hash = "sha256:d7f52f36d231b80ee124cd216ffb19369aa168fc10095013c6b014a34d3ee9e5", size = 50087 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/35/f8b19922b6a25bc0880171a2f1a003eaeb93657475193ab516fd87cac9da/pytest_asyncio-1.3.0-py3-none-any.whl", hash = "sha256:611e26147c7f77640e6d0a92a38ed17c3e9848063698d5c93d5aa7aa11cebff5", size = 15075 }, +] + [[package]] name = "requests" version = "2.32.5" From 23199360dcff1931ccbcc200ef14a677e9220fb4 Mon Sep 17 00:00:00 2001 From: staverm Date: Sat, 20 Dec 2025 18:29:14 +0100 Subject: [PATCH 2/3] Run tests in CI --- .../{publish.yml => test-and-publish.yml} | 71 +++++++++++-------- 1 file changed, 41 insertions(+), 30 deletions(-) rename .github/workflows/{publish.yml => test-and-publish.yml} (55%) diff --git a/.github/workflows/publish.yml b/.github/workflows/test-and-publish.yml similarity index 55% rename from .github/workflows/publish.yml rename to .github/workflows/test-and-publish.yml index 0a2e8b5..d0d6113 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/test-and-publish.yml @@ -1,7 +1,7 @@ -name: Publish Agent +name: Test and Publish Agent -# Trigger this workflow when pushing main branch and tags on: + pull_request: push: branches: - main @@ -9,7 +9,7 @@ on: - 'v*' # Trigger on version tags like v1.0.0, v1.1.0 jobs: - publish: + test-and-publish: runs-on: ubuntu-latest # These permissions are required for the workflow to: @@ -23,52 +23,63 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 - - name: Log in to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - # GITHUB_TOKEN is automatically provided by GitHub Actions - # No manual secret configuration needed! - # It has permissions based on the 'permissions' block above - password: ${{ secrets.GITHUB_TOKEN }} - - name: Extract metadata for Docker id: meta uses: docker/metadata-action@v5 with: images: ghcr.io/${{ github.repository }} tags: | - # For tags like v1.0, create tag '1.0' + type=ref,event=pr type=semver,pattern={{version}} - # For tags like v1.0, create tag '1' type=semver,pattern={{major}} - # For main branch, create tag 'latest' type=raw,value=latest,enable={{is_default_branch}} - # For PRs, create tag 'pr-123' - type=ref,event=pr - - name: Build and push Docker image - id: build + - name: Build Docker image uses: docker/build-push-action@v5 with: context: . - file: Dockerfile - # Only push if this is a push event (not a PR) - # PRs will build but not push to avoid polluting the registry - push: ${{ github.event_name != 'pull_request' }} + push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - # Explicitly build for linux/amd64 (GitHub Actions default) + load: true platforms: linux/amd64 + - name: Start agent container + run: | + docker run -d -p 9009:9009 --name agent-container $(echo "${{ steps.meta.outputs.tags }}" | head -n1) --host 0.0.0.0 --port 9009 + timeout 30 bash -c 'until curl -sf http://localhost:9009/.well-known/agent-card.json > /dev/null; do sleep 1; done' + + - name: Set up uv + uses: astral-sh/setup-uv@v4 + + - name: Install test dependencies + run: uv sync --extra test + + - name: Run tests + run: uv run pytest -v --agent-url http://localhost:9009 + + - name: Stop container and show logs + if: always() + run: | + echo "=== Agent Container Logs ===" + docker logs agent-container || true + docker stop agent-container || true + + - name: Log in to GitHub Container Registry + if: success() && github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Push Docker image + if: success() && github.event_name != 'pull_request' + run: docker push --all-tags ghcr.io/${{ github.repository }} + - name: Output image digest - if: github.event_name != 'pull_request' + if: success() && github.event_name != 'pull_request' run: | echo "## Docker Image Published :rocket:" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Tags:** ${{ steps.meta.outputs.tags }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Digest:** \`${{ steps.build.outputs.digest }}\`" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "Use this digest in your MANIFEST.json for reproducibility." >> $GITHUB_STEP_SUMMARY From acb976bad008708ff3dec2bf880e934f3e1bba59 Mon Sep 17 00:00:00 2001 From: staverm Date: Sat, 20 Dec 2025 18:29:26 +0100 Subject: [PATCH 3/3] Update readme --- README.md | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9eaf279..2751535 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ uv.lock # Locked dependencies 1. **Create your repository** - Click "Use this template" to create your own repository from this template -2. **Implement your agent** - Add your agent logic to the `run` method in [`src/agent.py`](src/agent.py) +2. **Implement your agent** - Add your agent logic to [`src/agent.py`](src/agent.py) 3. **Configure your agent card** - Fill in your agent's metadata (name, skills, description) in [`src/server.py`](src/server.py) @@ -43,9 +43,23 @@ docker build -t my-agent . docker run -p 9009:9009 my-agent ``` +## Testing + +Run A2A conformance tests against your agent. + +```bash +# Install test dependencies +uv sync --extra test + +# Start your agent (uv or docker; see above) + +# Run tests against your running agent URL +uv run pytest --agent-url http://localhost:9009 +``` + ## Publishing -The repository includes a GitHub Actions workflow that automatically builds and publishes a Docker image of your agent to GitHub Container Registry: +The repository includes a GitHub Actions workflow that automatically builds, tests, and publishes a Docker image of your agent to GitHub Container Registry: - **Push to `main`** → publishes `latest` tag: ``` @@ -60,4 +74,4 @@ ghcr.io//:1 Once the workflow completes, find your Docker image in the Packages section (right sidebar of your repository). Configure the package visibility in package settings. -> **Note:** Organization repositories may need package write permissions enabled manually (Settings → Actions → General). Version tags must follow [semantic versioning](https://semver.org/) (e.g., `v1.0.0`). \ No newline at end of file +> **Note:** Organization repositories may need package write permissions enabled manually (Settings → Actions → General). Version tags must follow [semantic versioning](https://semver.org/) (e.g., `v1.0.0`).