From 2b0714971a2076873df3671488cadf27cf34f923 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Badyl?= Date: Sat, 28 Feb 2026 13:30:34 +0100 Subject: [PATCH 1/2] fix: fixing expected status bug --- internal/agents/agent.go | 11 ++++++++++- internal/agents/chat.go | 7 ++++++- internal/cli/update_tests.go | 13 +++++++------ 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/internal/agents/agent.go b/internal/agents/agent.go index 552ce13..d7d975b 100644 --- a/internal/agents/agent.go +++ b/internal/agents/agent.go @@ -110,9 +110,18 @@ Return a JSON object with tests array. Each test: "method": "GET", "endpoint": "/users", "body": null, - "requires_auth": true + "requires_auth": true, + "expected_status": 200 } +Always set expected_status to the correct HTTP status code: +- 200 for successful GET, PUT, PATCH +- 201 for successful POST that creates a resource +- 204 for successful DELETE +- 400 for bad request / validation error tests +- 401 for unauthorized tests +- 404 for not found tests + Return pure JSON only - no markdown, no comments.` messages := []ChatMessage{ {Role: "user", Content: prompt}, diff --git a/internal/agents/chat.go b/internal/agents/chat.go index c08fd5f..b9e80ad 100644 --- a/internal/agents/chat.go +++ b/internal/agents/chat.go @@ -126,8 +126,12 @@ func getMainAgentTools() []common.Tool { "type": "boolean", "description": "Whether authentication is required for this test", }, + "expected_status": map[string]any{ + "type": "integer", + "description": "Expected HTTP status code. Defaults to 200 if not set. Set this correctly: 201 for POST that creates resources, 204 for DELETE, 400 for bad requests, 401 for unauthorized, 404 for not found, etc.", + }, }, - "required": []string{"method", "endpoint", "headers", "body", "requires_auth"}, + "required": []string{"method", "endpoint", "headers", "body", "requires_auth", "expected_status"}, }, }, }, @@ -257,6 +261,7 @@ Response includes per test: status_code, response_body, duration_ms, passed, sch - passed=false → status code did not match expected - schema_valid=false → response body does not match the OpenAPI schema (even if passed=true) - Always report schema_errors to the user when schema_valid=false +- expected_status is REQUIRED — always set it based on what the test expects: 200 for GET, 201 for POST that creates, 204 for DELETE, 400 for bad input, 401 for unauthorized, 404 for not found. Never leave it as 200 when testing error cases. ## wait Wait N seconds before proceeding. Use when: diff --git a/internal/cli/update_tests.go b/internal/cli/update_tests.go index d2eb6f2..bea54e4 100644 --- a/internal/cli/update_tests.go +++ b/internal/cli/update_tests.go @@ -25,12 +25,13 @@ func handleGenerateTestPlanResult(m *TestUIModel, msg generateTestPlanResultMsg) }) testCases = append(testCases, map[string]any{ - "method": bt.TestCase.Method, - "endpoint": bt.TestCase.Endpoint, - "headers": bt.TestCase.Headers, - "body": bt.TestCase.Body, - "requires_auth": bt.TestCase.RequiresAuth, - "description": bt.TestCase.Description, + "method": bt.TestCase.Method, + "endpoint": bt.TestCase.Endpoint, + "headers": bt.TestCase.Headers, + "body": bt.TestCase.Body, + "requires_auth": bt.TestCase.RequiresAuth, + "description": bt.TestCase.Description, + "expected_status": bt.TestCase.ExpectedStatus, }) } From b70b5d23862e9285f7e27ae2a9ef76b9526def09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miko=C5=82aj=20Badyl?= Date: Sat, 28 Feb 2026 14:56:40 +0100 Subject: [PATCH 2/2] fix: enforce expected_status in test execution pipeline --- internal/agents/agent.go | 4 +++- internal/agents/chat.go | 3 ++- internal/agents/types.go | 2 +- internal/cli/handlers.go | 31 ++++++++++++++---------- internal/cli/update.go | 31 +++++++++++++++--------- internal/cli/update_tests.go | 8 +++++-- internal/llm/claude/client.go | 45 +++++++++++++++++------------------ 7 files changed, 72 insertions(+), 52 deletions(-) diff --git a/internal/agents/agent.go b/internal/agents/agent.go index d7d975b..467dc5c 100644 --- a/internal/agents/agent.go +++ b/internal/agents/agent.go @@ -3,11 +3,12 @@ package agent import ( "encoding/json" "fmt" + "strings" + "github.com/Octrafic/octrafic-cli/internal/config" "github.com/Octrafic/octrafic-cli/internal/infra/logger" "github.com/Octrafic/octrafic-cli/internal/llm" "github.com/Octrafic/octrafic-cli/internal/llm/common" - "strings" ) const ( @@ -121,6 +122,7 @@ Always set expected_status to the correct HTTP status code: - 400 for bad request / validation error tests - 401 for unauthorized tests - 404 for not found tests +CRITICAL: You MUST include the "expected_status" field in EVERY test case object. Failure to do so will result in an error. Do not default to 200 if you are testing an error condition. Return pure JSON only - no markdown, no comments.` messages := []ChatMessage{ diff --git a/internal/agents/chat.go b/internal/agents/chat.go index b9e80ad..daa0078 100644 --- a/internal/agents/chat.go +++ b/internal/agents/chat.go @@ -2,6 +2,7 @@ package agent import ( "fmt" + "github.com/Octrafic/octrafic-cli/internal/llm/common" ) @@ -261,7 +262,7 @@ Response includes per test: status_code, response_body, duration_ms, passed, sch - passed=false → status code did not match expected - schema_valid=false → response body does not match the OpenAPI schema (even if passed=true) - Always report schema_errors to the user when schema_valid=false -- expected_status is REQUIRED — always set it based on what the test expects: 200 for GET, 201 for POST that creates, 204 for DELETE, 400 for bad input, 401 for unauthorized, 404 for not found. Never leave it as 200 when testing error cases. +- CRITICAL: expected_status is ABSOLUTELY REQUIRED — always set it based on what the test expects: 200 for GET, 201 for POST that creates, 204 for DELETE, 400 for bad input, 401 for unauthorized, 404 for not found. Never omit this field or default to 200 if testing an error condition. ## wait Wait N seconds before proceeding. Use when: diff --git a/internal/agents/types.go b/internal/agents/types.go index 42f5668..30543d5 100644 --- a/internal/agents/types.go +++ b/internal/agents/types.go @@ -74,7 +74,7 @@ Pure JSON, no markdown: "description": "Brief description", "method": "GET", "endpoint": "/exact/path", - "expected_status": 200, + "expected_status": 404, "reasoning": "Why this test matters", "requires_auth": true } diff --git a/internal/cli/handlers.go b/internal/cli/handlers.go index 6baada7..efb1e3c 100644 --- a/internal/cli/handlers.go +++ b/internal/cli/handlers.go @@ -3,13 +3,6 @@ package cli import ( "encoding/json" "fmt" - "github.com/Octrafic/octrafic-cli/internal/agents" - "github.com/Octrafic/octrafic-cli/internal/core/parser" - "github.com/Octrafic/octrafic-cli/internal/core/reporter" - "github.com/Octrafic/octrafic-cli/internal/core/tester" - "github.com/Octrafic/octrafic-cli/internal/exporter" - "github.com/Octrafic/octrafic-cli/internal/infra/logger" - "github.com/Octrafic/octrafic-cli/internal/infra/storage" "maps" "os" "path/filepath" @@ -17,6 +10,14 @@ import ( "strings" "time" + agent "github.com/Octrafic/octrafic-cli/internal/agents" + "github.com/Octrafic/octrafic-cli/internal/core/parser" + "github.com/Octrafic/octrafic-cli/internal/core/reporter" + "github.com/Octrafic/octrafic-cli/internal/core/tester" + "github.com/Octrafic/octrafic-cli/internal/exporter" + "github.com/Octrafic/octrafic-cli/internal/infra/logger" + "github.com/Octrafic/octrafic-cli/internal/infra/storage" + tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "go.uber.org/zap" @@ -419,6 +420,16 @@ func (m *TestUIModel) executeTool(toolCall agent.ToolCall) tea.Cmd { } } + for _, t := range tests { + if _, ok := t["expected_status"]; !ok { + return toolResultMsg{ + toolID: toolCall.ID, + toolName: toolCall.Name, + err: fmt.Errorf("every test MUST include 'expected_status' field (e.g. 200, 201, 204, 400, 401, 404). Re-call ExecuteTestGroup with expected_status set for each test"), + } + } + } + return showTestSelectionMsg{ tests: tests, toolCall: toolCall, @@ -472,9 +483,6 @@ func (m *TestUIModel) handleToolResult(toolName string, toolID string, result an if p, ok := resultMap["passed"].(bool); ok { passed = p } else { - if expectedStatus == 0 { - expectedStatus = 200 - } passed = statusCode == expectedStatus } @@ -558,9 +566,6 @@ func (m *TestUIModel) handleToolResult(toolName string, toolID string, result an if p, ok := testResult["passed"].(bool); ok { passed = p } else { - if expectedStatus == 0 { - expectedStatus = 200 - } passed = statusCode == expectedStatus } diff --git a/internal/cli/update.go b/internal/cli/update.go index 683013b..a177773 100644 --- a/internal/cli/update.go +++ b/internal/cli/update.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/Octrafic/octrafic-cli/internal/agents" + agent "github.com/Octrafic/octrafic-cli/internal/agents" "github.com/Octrafic/octrafic-cli/internal/config" "github.com/Octrafic/octrafic-cli/internal/core/analyzer" "github.com/Octrafic/octrafic-cli/internal/core/auth" @@ -1600,17 +1600,25 @@ func handleShowTestSelection(m *TestUIModel, msg showTestSelectionMsg) (tea.Mode requiresAuth = ra } + expectedStatus := 0 + if es, ok := testMap["expected_status"].(float64); ok { + expectedStatus = int(es) + } else if es, ok := testMap["expected_status"].(int); ok { + expectedStatus = es + } + headers := make(map[string]string) if h, ok := testMap["headers"].(map[string]string); ok { headers = h } testCase := &agent.TestCase{ - Method: method, - Endpoint: endpoint, - Headers: headers, - Body: testMap["body"], - RequiresAuth: requiresAuth, + Method: method, + Endpoint: endpoint, + Headers: headers, + Body: testMap["body"], + RequiresAuth: requiresAuth, + ExpectedStatus: expectedStatus, } m.tests = append(m.tests, Test{ @@ -1640,11 +1648,12 @@ func handleShowTestSelection(m *TestUIModel, msg showTestSelectionMsg) (tea.Mode tests := make([]map[string]any, 0) for _, test := range m.tests { tests = append(tests, map[string]any{ - "method": test.Method, - "endpoint": test.Endpoint, - "headers": test.BackendTest.Headers, - "body": test.BackendTest.Body, - "requires_auth": test.BackendTest.RequiresAuth, + "method": test.Method, + "endpoint": test.Endpoint, + "headers": test.BackendTest.Headers, + "body": test.BackendTest.Body, + "requires_auth": test.BackendTest.RequiresAuth, + "expected_status": test.BackendTest.ExpectedStatus, }) } diff --git a/internal/cli/update_tests.go b/internal/cli/update_tests.go index bea54e4..34e668f 100644 --- a/internal/cli/update_tests.go +++ b/internal/cli/update_tests.go @@ -2,9 +2,9 @@ package cli import ( "fmt" - "github.com/Octrafic/octrafic-cli/internal/agents" "strings" + agent "github.com/Octrafic/octrafic-cli/internal/agents" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" ) @@ -165,13 +165,17 @@ func handleRunNextTest(m *TestUIModel, _ runNextTestMsg) (tea.Model, tea.Cmd) { requiresAuth = ra } - expectedStatus := 200 + expectedStatus := 0 if es, ok := testMap["expected_status"].(float64); ok { expectedStatus = int(es) } else if es, ok := testMap["expected_status"].(int); ok { expectedStatus = es } + if expectedStatus == 0 { + expectedStatus = 200 + } + headers := make(map[string]string) if h, ok := testMap["headers"].(map[string]any); ok { for k, v := range h { diff --git a/internal/llm/claude/client.go b/internal/llm/claude/client.go index cfe6442..3e95719 100644 --- a/internal/llm/claude/client.go +++ b/internal/llm/claude/client.go @@ -10,7 +10,6 @@ import ( "github.com/anthropics/anthropic-sdk-go" "github.com/anthropics/anthropic-sdk-go/option" - "github.com/invopop/jsonschema" "go.uber.org/zap" ) @@ -103,37 +102,37 @@ func NewClientWithConfig(apiKey, model string) (*Client, error) { } // generateToolInputSchema creates a ToolInputSchemaParam from a map +// It preserves the full JSON schema structure including nested objects, +// arrays with items, required fields, enums, etc. func generateToolInputSchema(inputSchema map[string]interface{}) anthropic.ToolInputSchemaParam { - properties := make(map[string]jsonschema.Schema) + result := anthropic.ToolInputSchemaParam{} - // Extract the actual properties from the schema (inputSchema has structure: {type: "object", properties: {...}, required: [...]}) - propertiesRaw, ok := inputSchema["properties"].(map[string]interface{}) - if !ok { - return anthropic.ToolInputSchemaParam{} + if props, ok := inputSchema["properties"]; ok { + result.Properties = props } - for propName, propDef := range propertiesRaw { - propMap, ok := propDef.(map[string]interface{}) - if !ok { - continue - } - - // Create schema for this property - Type is a plain string - schema := jsonschema.Schema{} - if typ, ok := propMap["type"].(string); ok { - schema.Type = typ + if req, ok := inputSchema["required"].([]interface{}); ok { + required := make([]string, 0, len(req)) + for _, r := range req { + if s, ok := r.(string); ok { + required = append(required, s) + } } + result.Required = required + } - if desc, ok := propMap["description"].(string); ok { - schema.Description = desc + extras := make(map[string]any) + for key, val := range inputSchema { + if key == "type" || key == "properties" || key == "required" { + continue } - - properties[propName] = schema + extras[key] = val } - - return anthropic.ToolInputSchemaParam{ - Properties: properties, + if len(extras) > 0 { + result.ExtraFields = extras } + + return result } type StreamCallback func(reasoning string, isThought bool)