fix: enforce expected_status in test execution pipeline

internal/agents/agent.go

@@ -3,11 +3,12 @@ package agent
 import (
 	"encoding/json"
 	"fmt"
+	"strings"
+
 	"github.com/Octrafic/octrafic-cli/internal/config"
 	"github.com/Octrafic/octrafic-cli/internal/infra/logger"
 	"github.com/Octrafic/octrafic-cli/internal/llm"
 	"github.com/Octrafic/octrafic-cli/internal/llm/common"
-	"strings"
 )
 
 const (
@@ -110,9 +111,19 @@ Return a JSON object with tests array. Each test:
   "method": "GET",
   "endpoint": "/users",
   "body": null,
-  "requires_auth": true
+  "requires_auth": true,
+  "expected_status": 200
 }
 
+Always set expected_status to the correct HTTP status code:
+- 200 for successful GET, PUT, PATCH
+- 201 for successful POST that creates a resource
+- 204 for successful DELETE
+- 400 for bad request / validation error tests
+- 401 for unauthorized tests
+- 404 for not found tests
+CRITICAL: You MUST include the "expected_status" field in EVERY test case object. Failure to do so will result in an error. Do not default to 200 if you are testing an error condition.
+
 Return pure JSON only - no markdown, no comments.`
 	messages := []ChatMessage{
 		{Role: "user", Content: prompt},

internal/agents/chat.go

@@ -2,6 +2,7 @@ package agent
 
 import (
 	"fmt"
+
 	"github.com/Octrafic/octrafic-cli/internal/llm/common"
 )
 
@@ -126,8 +127,12 @@ func getMainAgentTools() []common.Tool {
 									"type":        "boolean",
 									"description": "Whether authentication is required for this test",
 								},
+								"expected_status": map[string]any{
+									"type":        "integer",
+									"description": "Expected HTTP status code. Defaults to 200 if not set. Set this correctly: 201 for POST that creates resources, 204 for DELETE, 400 for bad requests, 401 for unauthorized, 404 for not found, etc.",
+								},
 							},
-							"required": []string{"method", "endpoint", "headers", "body", "requires_auth"},
+							"required": []string{"method", "endpoint", "headers", "body", "requires_auth", "expected_status"},
 						},
 					},
 				},
@@ -257,6 +262,7 @@ Response includes per test: status_code, response_body, duration_ms, passed, sch
 - passed=false → status code did not match expected
 - schema_valid=false → response body does not match the OpenAPI schema (even if passed=true)
 - Always report schema_errors to the user when schema_valid=false
+- CRITICAL: expected_status is ABSOLUTELY REQUIRED — always set it based on what the test expects: 200 for GET, 201 for POST that creates, 204 for DELETE, 400 for bad input, 401 for unauthorized, 404 for not found. Never omit this field or default to 200 if testing an error condition.
 
 ## wait
 Wait N seconds before proceeding. Use when:

internal/agents/types.go

@@ -74,7 +74,7 @@ Pure JSON, no markdown:
       "description": "Brief description",
       "method": "GET",
       "endpoint": "/exact/path",
-      "expected_status": 200,
+      "expected_status": 404,
       "reasoning": "Why this test matters",
       "requires_auth": true
     }

internal/cli/handlers.go

@@ -3,20 +3,21 @@ package cli
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/Octrafic/octrafic-cli/internal/agents"
-	"github.com/Octrafic/octrafic-cli/internal/core/parser"
-	"github.com/Octrafic/octrafic-cli/internal/core/reporter"
-	"github.com/Octrafic/octrafic-cli/internal/core/tester"
-	"github.com/Octrafic/octrafic-cli/internal/exporter"
-	"github.com/Octrafic/octrafic-cli/internal/infra/logger"
-	"github.com/Octrafic/octrafic-cli/internal/infra/storage"
 	"maps"
 	"os"
 	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
+	agent "github.com/Octrafic/octrafic-cli/internal/agents"
+	"github.com/Octrafic/octrafic-cli/internal/core/parser"
+	"github.com/Octrafic/octrafic-cli/internal/core/reporter"
+	"github.com/Octrafic/octrafic-cli/internal/core/tester"
+	"github.com/Octrafic/octrafic-cli/internal/exporter"
+	"github.com/Octrafic/octrafic-cli/internal/infra/logger"
+	"github.com/Octrafic/octrafic-cli/internal/infra/storage"
+
 	tea "github.com/charmbracelet/bubbletea"
 	"github.com/charmbracelet/lipgloss"
 	"go.uber.org/zap"
@@ -419,6 +420,16 @@ func (m *TestUIModel) executeTool(toolCall agent.ToolCall) tea.Cmd {
 				}
 			}
 
+			for _, t := range tests {
+				if _, ok := t["expected_status"]; !ok {
+					return toolResultMsg{
+						toolID:   toolCall.ID,
+						toolName: toolCall.Name,
+						err:      fmt.Errorf("every test MUST include 'expected_status' field (e.g. 200, 201, 204, 400, 401, 404). Re-call ExecuteTestGroup with expected_status set for each test"),
+					}
+				}
+			}
+
 			return showTestSelectionMsg{
 				tests:    tests,
 				toolCall: toolCall,
@@ -472,9 +483,6 @@ func (m *TestUIModel) handleToolResult(toolName string, toolID string, result an
 			if p, ok := resultMap["passed"].(bool); ok {
 				passed = p
 			} else {
-				if expectedStatus == 0 {
-					expectedStatus = 200
-				}
 				passed = statusCode == expectedStatus
 			}
 
@@ -558,9 +566,6 @@ func (m *TestUIModel) handleToolResult(toolName string, toolID string, result an
 				if p, ok := testResult["passed"].(bool); ok {
 					passed = p
 				} else {
-					if expectedStatus == 0 {
-						expectedStatus = 200
-					}
 					passed = statusCode == expectedStatus
 				}
 

internal/cli/update.go

@@ -7,7 +7,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/Octrafic/octrafic-cli/internal/agents"
+	agent "github.com/Octrafic/octrafic-cli/internal/agents"
 	"github.com/Octrafic/octrafic-cli/internal/config"
 	"github.com/Octrafic/octrafic-cli/internal/core/analyzer"
 	"github.com/Octrafic/octrafic-cli/internal/core/auth"
@@ -1600,17 +1600,25 @@ func handleShowTestSelection(m *TestUIModel, msg showTestSelectionMsg) (tea.Mode
 			requiresAuth = ra
 		}
 
+		expectedStatus := 0
+		if es, ok := testMap["expected_status"].(float64); ok {
+			expectedStatus = int(es)
+		} else if es, ok := testMap["expected_status"].(int); ok {
+			expectedStatus = es
+		}
+
 		headers := make(map[string]string)
 		if h, ok := testMap["headers"].(map[string]string); ok {
 			headers = h
 		}
 
 		testCase := &agent.TestCase{
-			Method:       method,
-			Endpoint:     endpoint,
-			Headers:      headers,
-			Body:         testMap["body"],
-			RequiresAuth: requiresAuth,
+			Method:         method,
+			Endpoint:       endpoint,
+			Headers:        headers,
+			Body:           testMap["body"],
+			RequiresAuth:   requiresAuth,
+			ExpectedStatus: expectedStatus,
 		}
 
 		m.tests = append(m.tests, Test{
@@ -1640,11 +1648,12 @@ func handleShowTestSelection(m *TestUIModel, msg showTestSelectionMsg) (tea.Mode
 		tests := make([]map[string]any, 0)
 		for _, test := range m.tests {
 			tests = append(tests, map[string]any{
-				"method":        test.Method,
-				"endpoint":      test.Endpoint,
-				"headers":       test.BackendTest.Headers,
-				"body":          test.BackendTest.Body,
-				"requires_auth": test.BackendTest.RequiresAuth,
+				"method":          test.Method,
+				"endpoint":        test.Endpoint,
+				"headers":         test.BackendTest.Headers,
+				"body":            test.BackendTest.Body,
+				"requires_auth":   test.BackendTest.RequiresAuth,
+				"expected_status": test.BackendTest.ExpectedStatus,
 			})
 		}
 

internal/cli/update_tests.go

@@ -2,9 +2,9 @@ package cli
 
 import (
 	"fmt"
-	"github.com/Octrafic/octrafic-cli/internal/agents"
 	"strings"
 
+	agent "github.com/Octrafic/octrafic-cli/internal/agents"
 	tea "github.com/charmbracelet/bubbletea"
 	"github.com/charmbracelet/lipgloss"
 )
@@ -25,12 +25,13 @@ func handleGenerateTestPlanResult(m *TestUIModel, msg generateTestPlanResultMsg)
 		})
 
 		testCases = append(testCases, map[string]any{
-			"method":        bt.TestCase.Method,
-			"endpoint":      bt.TestCase.Endpoint,
-			"headers":       bt.TestCase.Headers,
-			"body":          bt.TestCase.Body,
-			"requires_auth": bt.TestCase.RequiresAuth,
-			"description":   bt.TestCase.Description,
+			"method":          bt.TestCase.Method,
+			"endpoint":        bt.TestCase.Endpoint,
+			"headers":         bt.TestCase.Headers,
+			"body":            bt.TestCase.Body,
+			"requires_auth":   bt.TestCase.RequiresAuth,
+			"description":     bt.TestCase.Description,
+			"expected_status": bt.TestCase.ExpectedStatus,
 		})
 	}
 
@@ -164,13 +165,17 @@ func handleRunNextTest(m *TestUIModel, _ runNextTestMsg) (tea.Model, tea.Cmd) {
 		requiresAuth = ra
 	}
 
-	expectedStatus := 200
+	expectedStatus := 0
 	if es, ok := testMap["expected_status"].(float64); ok {
 		expectedStatus = int(es)
 	} else if es, ok := testMap["expected_status"].(int); ok {
 		expectedStatus = es
 	}
 
+	if expectedStatus == 0 {
+		expectedStatus = 200
+	}
+
 	headers := make(map[string]string)
 	if h, ok := testMap["headers"].(map[string]any); ok {
 		for k, v := range h {

internal/llm/claude/client.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/anthropics/anthropic-sdk-go"
 	"github.com/anthropics/anthropic-sdk-go/option"
-	"github.com/invopop/jsonschema"
 	"go.uber.org/zap"
 )
 
@@ -103,37 +102,37 @@ func NewClientWithConfig(apiKey, model string) (*Client, error) {
 }
 
 // generateToolInputSchema creates a ToolInputSchemaParam from a map
+// It preserves the full JSON schema structure including nested objects,
+// arrays with items, required fields, enums, etc.
 func generateToolInputSchema(inputSchema map[string]interface{}) anthropic.ToolInputSchemaParam {
-	properties := make(map[string]jsonschema.Schema)
+	result := anthropic.ToolInputSchemaParam{}
 
-	// Extract the actual properties from the schema (inputSchema has structure: {type: "object", properties: {...}, required: [...]})
-	propertiesRaw, ok := inputSchema["properties"].(map[string]interface{})
-	if !ok {
-		return anthropic.ToolInputSchemaParam{}
+	if props, ok := inputSchema["properties"]; ok {
+		result.Properties = props
 	}
 
-	for propName, propDef := range propertiesRaw {
-		propMap, ok := propDef.(map[string]interface{})
-		if !ok {
-			continue
-		}
-
-		// Create schema for this property - Type is a plain string
-		schema := jsonschema.Schema{}
-		if typ, ok := propMap["type"].(string); ok {
-			schema.Type = typ
+	if req, ok := inputSchema["required"].([]interface{}); ok {
+		required := make([]string, 0, len(req))
+		for _, r := range req {
+			if s, ok := r.(string); ok {
+				required = append(required, s)
+			}
 		}
+		result.Required = required
+	}
 
-		if desc, ok := propMap["description"].(string); ok {
-			schema.Description = desc
+	extras := make(map[string]any)
+	for key, val := range inputSchema {
+		if key == "type" || key == "properties" || key == "required" {
+			continue
 		}
-
-		properties[propName] = schema
+		extras[key] = val
 	}
-
-	return anthropic.ToolInputSchemaParam{
-		Properties: properties,
+	if len(extras) > 0 {
+		result.ExtraFields = extras
 	}
+
+	return result
 }
 
 type StreamCallback func(reasoning string, isThought bool)