diff --git a/internal/agents/agent.go b/internal/agents/agent.go index 552ce13..467dc5c 100644 --- a/internal/agents/agent.go +++ b/internal/agents/agent.go @@ -3,11 +3,12 @@ package agent import ( "encoding/json" "fmt" + "strings" + "github.com/Octrafic/octrafic-cli/internal/config" "github.com/Octrafic/octrafic-cli/internal/infra/logger" "github.com/Octrafic/octrafic-cli/internal/llm" "github.com/Octrafic/octrafic-cli/internal/llm/common" - "strings" ) const ( @@ -110,9 +111,19 @@ Return a JSON object with tests array. Each test: "method": "GET", "endpoint": "/users", "body": null, - "requires_auth": true + "requires_auth": true, + "expected_status": 200 } +Always set expected_status to the correct HTTP status code: +- 200 for successful GET, PUT, PATCH +- 201 for successful POST that creates a resource +- 204 for successful DELETE +- 400 for bad request / validation error tests +- 401 for unauthorized tests +- 404 for not found tests +CRITICAL: You MUST include the "expected_status" field in EVERY test case object. Failure to do so will result in an error. Do not default to 200 if you are testing an error condition. + Return pure JSON only - no markdown, no comments.` messages := []ChatMessage{ {Role: "user", Content: prompt}, diff --git a/internal/agents/chat.go b/internal/agents/chat.go index c08fd5f..daa0078 100644 --- a/internal/agents/chat.go +++ b/internal/agents/chat.go @@ -2,6 +2,7 @@ package agent import ( "fmt" + "github.com/Octrafic/octrafic-cli/internal/llm/common" ) @@ -126,8 +127,12 @@ func getMainAgentTools() []common.Tool { "type": "boolean", "description": "Whether authentication is required for this test", }, + "expected_status": map[string]any{ + "type": "integer", + "description": "Expected HTTP status code. Defaults to 200 if not set. Set this correctly: 201 for POST that creates resources, 204 for DELETE, 400 for bad requests, 401 for unauthorized, 404 for not found, etc.", + }, }, - "required": []string{"method", "endpoint", "headers", "body", "requires_auth"}, + "required": []string{"method", "endpoint", "headers", "body", "requires_auth", "expected_status"}, }, }, }, @@ -257,6 +262,7 @@ Response includes per test: status_code, response_body, duration_ms, passed, sch - passed=false → status code did not match expected - schema_valid=false → response body does not match the OpenAPI schema (even if passed=true) - Always report schema_errors to the user when schema_valid=false +- CRITICAL: expected_status is ABSOLUTELY REQUIRED — always set it based on what the test expects: 200 for GET, 201 for POST that creates, 204 for DELETE, 400 for bad input, 401 for unauthorized, 404 for not found. Never omit this field or default to 200 if testing an error condition. ## wait Wait N seconds before proceeding. Use when: diff --git a/internal/agents/types.go b/internal/agents/types.go index 42f5668..30543d5 100644 --- a/internal/agents/types.go +++ b/internal/agents/types.go @@ -74,7 +74,7 @@ Pure JSON, no markdown: "description": "Brief description", "method": "GET", "endpoint": "/exact/path", - "expected_status": 200, + "expected_status": 404, "reasoning": "Why this test matters", "requires_auth": true } diff --git a/internal/cli/handlers.go b/internal/cli/handlers.go index 6baada7..efb1e3c 100644 --- a/internal/cli/handlers.go +++ b/internal/cli/handlers.go @@ -3,13 +3,6 @@ package cli import ( "encoding/json" "fmt" - "github.com/Octrafic/octrafic-cli/internal/agents" - "github.com/Octrafic/octrafic-cli/internal/core/parser" - "github.com/Octrafic/octrafic-cli/internal/core/reporter" - "github.com/Octrafic/octrafic-cli/internal/core/tester" - "github.com/Octrafic/octrafic-cli/internal/exporter" - "github.com/Octrafic/octrafic-cli/internal/infra/logger" - "github.com/Octrafic/octrafic-cli/internal/infra/storage" "maps" "os" "path/filepath" @@ -17,6 +10,14 @@ import ( "strings" "time" + agent "github.com/Octrafic/octrafic-cli/internal/agents" + "github.com/Octrafic/octrafic-cli/internal/core/parser" + "github.com/Octrafic/octrafic-cli/internal/core/reporter" + "github.com/Octrafic/octrafic-cli/internal/core/tester" + "github.com/Octrafic/octrafic-cli/internal/exporter" + "github.com/Octrafic/octrafic-cli/internal/infra/logger" + "github.com/Octrafic/octrafic-cli/internal/infra/storage" + tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" "go.uber.org/zap" @@ -419,6 +420,16 @@ func (m *TestUIModel) executeTool(toolCall agent.ToolCall) tea.Cmd { } } + for _, t := range tests { + if _, ok := t["expected_status"]; !ok { + return toolResultMsg{ + toolID: toolCall.ID, + toolName: toolCall.Name, + err: fmt.Errorf("every test MUST include 'expected_status' field (e.g. 200, 201, 204, 400, 401, 404). Re-call ExecuteTestGroup with expected_status set for each test"), + } + } + } + return showTestSelectionMsg{ tests: tests, toolCall: toolCall, @@ -472,9 +483,6 @@ func (m *TestUIModel) handleToolResult(toolName string, toolID string, result an if p, ok := resultMap["passed"].(bool); ok { passed = p } else { - if expectedStatus == 0 { - expectedStatus = 200 - } passed = statusCode == expectedStatus } @@ -558,9 +566,6 @@ func (m *TestUIModel) handleToolResult(toolName string, toolID string, result an if p, ok := testResult["passed"].(bool); ok { passed = p } else { - if expectedStatus == 0 { - expectedStatus = 200 - } passed = statusCode == expectedStatus } diff --git a/internal/cli/update.go b/internal/cli/update.go index 683013b..a177773 100644 --- a/internal/cli/update.go +++ b/internal/cli/update.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/Octrafic/octrafic-cli/internal/agents" + agent "github.com/Octrafic/octrafic-cli/internal/agents" "github.com/Octrafic/octrafic-cli/internal/config" "github.com/Octrafic/octrafic-cli/internal/core/analyzer" "github.com/Octrafic/octrafic-cli/internal/core/auth" @@ -1600,17 +1600,25 @@ func handleShowTestSelection(m *TestUIModel, msg showTestSelectionMsg) (tea.Mode requiresAuth = ra } + expectedStatus := 0 + if es, ok := testMap["expected_status"].(float64); ok { + expectedStatus = int(es) + } else if es, ok := testMap["expected_status"].(int); ok { + expectedStatus = es + } + headers := make(map[string]string) if h, ok := testMap["headers"].(map[string]string); ok { headers = h } testCase := &agent.TestCase{ - Method: method, - Endpoint: endpoint, - Headers: headers, - Body: testMap["body"], - RequiresAuth: requiresAuth, + Method: method, + Endpoint: endpoint, + Headers: headers, + Body: testMap["body"], + RequiresAuth: requiresAuth, + ExpectedStatus: expectedStatus, } m.tests = append(m.tests, Test{ @@ -1640,11 +1648,12 @@ func handleShowTestSelection(m *TestUIModel, msg showTestSelectionMsg) (tea.Mode tests := make([]map[string]any, 0) for _, test := range m.tests { tests = append(tests, map[string]any{ - "method": test.Method, - "endpoint": test.Endpoint, - "headers": test.BackendTest.Headers, - "body": test.BackendTest.Body, - "requires_auth": test.BackendTest.RequiresAuth, + "method": test.Method, + "endpoint": test.Endpoint, + "headers": test.BackendTest.Headers, + "body": test.BackendTest.Body, + "requires_auth": test.BackendTest.RequiresAuth, + "expected_status": test.BackendTest.ExpectedStatus, }) } diff --git a/internal/cli/update_tests.go b/internal/cli/update_tests.go index d2eb6f2..34e668f 100644 --- a/internal/cli/update_tests.go +++ b/internal/cli/update_tests.go @@ -2,9 +2,9 @@ package cli import ( "fmt" - "github.com/Octrafic/octrafic-cli/internal/agents" "strings" + agent "github.com/Octrafic/octrafic-cli/internal/agents" tea "github.com/charmbracelet/bubbletea" "github.com/charmbracelet/lipgloss" ) @@ -25,12 +25,13 @@ func handleGenerateTestPlanResult(m *TestUIModel, msg generateTestPlanResultMsg) }) testCases = append(testCases, map[string]any{ - "method": bt.TestCase.Method, - "endpoint": bt.TestCase.Endpoint, - "headers": bt.TestCase.Headers, - "body": bt.TestCase.Body, - "requires_auth": bt.TestCase.RequiresAuth, - "description": bt.TestCase.Description, + "method": bt.TestCase.Method, + "endpoint": bt.TestCase.Endpoint, + "headers": bt.TestCase.Headers, + "body": bt.TestCase.Body, + "requires_auth": bt.TestCase.RequiresAuth, + "description": bt.TestCase.Description, + "expected_status": bt.TestCase.ExpectedStatus, }) } @@ -164,13 +165,17 @@ func handleRunNextTest(m *TestUIModel, _ runNextTestMsg) (tea.Model, tea.Cmd) { requiresAuth = ra } - expectedStatus := 200 + expectedStatus := 0 if es, ok := testMap["expected_status"].(float64); ok { expectedStatus = int(es) } else if es, ok := testMap["expected_status"].(int); ok { expectedStatus = es } + if expectedStatus == 0 { + expectedStatus = 200 + } + headers := make(map[string]string) if h, ok := testMap["headers"].(map[string]any); ok { for k, v := range h { diff --git a/internal/llm/claude/client.go b/internal/llm/claude/client.go index cfe6442..3e95719 100644 --- a/internal/llm/claude/client.go +++ b/internal/llm/claude/client.go @@ -10,7 +10,6 @@ import ( "github.com/anthropics/anthropic-sdk-go" "github.com/anthropics/anthropic-sdk-go/option" - "github.com/invopop/jsonschema" "go.uber.org/zap" ) @@ -103,37 +102,37 @@ func NewClientWithConfig(apiKey, model string) (*Client, error) { } // generateToolInputSchema creates a ToolInputSchemaParam from a map +// It preserves the full JSON schema structure including nested objects, +// arrays with items, required fields, enums, etc. func generateToolInputSchema(inputSchema map[string]interface{}) anthropic.ToolInputSchemaParam { - properties := make(map[string]jsonschema.Schema) + result := anthropic.ToolInputSchemaParam{} - // Extract the actual properties from the schema (inputSchema has structure: {type: "object", properties: {...}, required: [...]}) - propertiesRaw, ok := inputSchema["properties"].(map[string]interface{}) - if !ok { - return anthropic.ToolInputSchemaParam{} + if props, ok := inputSchema["properties"]; ok { + result.Properties = props } - for propName, propDef := range propertiesRaw { - propMap, ok := propDef.(map[string]interface{}) - if !ok { - continue - } - - // Create schema for this property - Type is a plain string - schema := jsonschema.Schema{} - if typ, ok := propMap["type"].(string); ok { - schema.Type = typ + if req, ok := inputSchema["required"].([]interface{}); ok { + required := make([]string, 0, len(req)) + for _, r := range req { + if s, ok := r.(string); ok { + required = append(required, s) + } } + result.Required = required + } - if desc, ok := propMap["description"].(string); ok { - schema.Description = desc + extras := make(map[string]any) + for key, val := range inputSchema { + if key == "type" || key == "properties" || key == "required" { + continue } - - properties[propName] = schema + extras[key] = val } - - return anthropic.ToolInputSchemaParam{ - Properties: properties, + if len(extras) > 0 { + result.ExtraFields = extras } + + return result } type StreamCallback func(reasoning string, isThought bool)