feat: response schema validation against OpenAPI spec

internal/agents/chat.go

@@ -253,7 +253,10 @@ Parameters:
 
 ## ExecuteTestGroup
 Execute a group of tests against the API. Call AFTER GenerateTestPlan.
-Response includes: status_code, response_body, headers (check for Retry-After), duration_ms, passed.
+Response includes per test: status_code, response_body, duration_ms, passed, schema_valid, schema_errors.
+- passed=false → status code did not match expected
+- schema_valid=false → response body does not match the OpenAPI schema (even if passed=true)
+- Always report schema_errors to the user when schema_valid=false
 
 ## wait
 Wait N seconds before proceeding. Use when:
@@ -262,12 +265,14 @@ Wait N seconds before proceeding. Use when:
 - You want to avoid hitting rate limits between test groups
 
 ## GenerateReport
-Generate PDF report from test results. Call AFTER tests are executed.
+Generate PDF report from test results.
+IMPORTANT: Only call this when the user explicitly asks for a report (e.g. "generate report", "save report", "export PDF"). Never call it automatically after tests.
 
 # Behavior
 - User mentions endpoint (e.g., "users", "auth") → fetch details, show info OR generate tests
 - User says "test X" → fetch details, generate & run tests
 - User says "list endpoints" → show list from available endpoints (no tool call)
+- User says "generate report" / "save PDF" / "export report" → call GenerateReport
 - After 429 response → call wait(seconds=N) where N comes from Retry-After header or default to 5
 - requires_auth=true → CLI adds auth header, requires_auth=false → no auth`, baseURL, endpointsInfo)
 }

internal/cli/handlers.go

@@ -6,12 +6,14 @@ import (
 	"github.com/Octrafic/octrafic-cli/internal/agents"
 	"github.com/Octrafic/octrafic-cli/internal/core/parser"
 	"github.com/Octrafic/octrafic-cli/internal/core/reporter"
+	"github.com/Octrafic/octrafic-cli/internal/core/tester"
 	"github.com/Octrafic/octrafic-cli/internal/exporter"
 	"github.com/Octrafic/octrafic-cli/internal/infra/logger"
 	"github.com/Octrafic/octrafic-cli/internal/infra/storage"
 	"maps"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"time"
 
@@ -327,6 +329,8 @@ func (m *TestUIModel) executeTool(toolCall agent.ToolCall) tea.Cmd {
 
 			passed := result.StatusCode == expectedStatus
 
+			schemaErrors := m.validateResponseSchema(method, endpoint, result.StatusCode, result.ResponseBody)
+
 			return toolResultMsg{
 				toolID:   toolCall.ID,
 				toolName: toolCall.Name,
@@ -339,6 +343,8 @@ func (m *TestUIModel) executeTool(toolCall agent.ToolCall) tea.Cmd {
 					"headers":         result.Headers,
 					"duration_ms":     result.Duration.Milliseconds(),
 					"passed":          passed,
+					"schema_valid":    len(schemaErrors) == 0,
+					"schema_errors":   schemaErrors,
 				},
 				err: nil,
 			}
@@ -781,6 +787,46 @@ func (m *TestUIModel) loadProjectEndpoints() ([]parser.Endpoint, error) {
 	return storage.LoadEndpoints(m.currentProject.ID, m.currentProject.IsTemporary)
 }
 
+// validateResponseSchema checks the response body against the OpenAPI schema for the given endpoint.
+// Returns a list of validation errors, or nil if no schema is available.
+func (m *TestUIModel) validateResponseSchema(method, path string, statusCode int, body string) []string {
+	if m.analysis == nil || m.analysis.Specification == nil {
+		return nil
+	}
+	statusKey := strconv.Itoa(statusCode)
+	for _, ep := range m.analysis.Specification.Endpoints {
+		if strings.EqualFold(ep.Method, method) && matchPath(ep.Path, path) {
+			if schema, ok := ep.ResponseSchemas[statusKey]; ok {
+				return tester.ValidateSchema(body, schema)
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+// matchPath checks if a concrete path matches an OpenAPI path template.
+// Segments wrapped in {} are treated as wildcards.
+func matchPath(template, actual string) bool {
+	if template == actual {
+		return true
+	}
+	tParts := strings.Split(strings.Trim(template, "/"), "/")
+	aParts := strings.Split(strings.Trim(actual, "/"), "/")
+	if len(tParts) != len(aParts) {
+		return false
+	}
+	for i, tp := range tParts {
+		if strings.HasPrefix(tp, "{") && strings.HasSuffix(tp, "}") {
+			continue
+		}
+		if !strings.EqualFold(tp, aParts[i]) {
+			return false
+		}
+	}
+	return true
+}
+
 func (m *TestUIModel) handleExportTests(toolCall agent.ToolCall) tea.Msg {
 	exportsArg, ok := toolCall.Arguments["exports"]
 	if !ok {

internal/cli/handlers_test.go

@@ -0,0 +1,138 @@
+package cli
+
+import (
+	"testing"
+
+	"github.com/Octrafic/octrafic-cli/internal/core/analyzer"
+	"github.com/Octrafic/octrafic-cli/internal/core/parser"
+)
+
+func TestMatchPath_Exact(t *testing.T) {
+	if !matchPath("/users", "/users") {
+		t.Error("expected exact match")
+	}
+}
+
+func TestMatchPath_WithParam(t *testing.T) {
+	if !matchPath("/users/{id}", "/users/42") {
+		t.Error("expected param segment to match any value")
+	}
+}
+
+func TestMatchPath_MultipleParams(t *testing.T) {
+	if !matchPath("/users/{id}/orders/{orderId}", "/users/1/orders/99") {
+		t.Error("expected multiple params to match")
+	}
+}
+
+func TestMatchPath_DifferentSegmentCount(t *testing.T) {
+	if matchPath("/users/{id}", "/users") {
+		t.Error("expected no match for different segment count")
+	}
+}
+
+func TestMatchPath_LiteralMismatch(t *testing.T) {
+	if matchPath("/users/{id}", "/products/42") {
+		t.Error("expected no match when literal segment differs")
+	}
+}
+
+func TestMatchPath_CaseInsensitive(t *testing.T) {
+	if !matchPath("/Users/{id}", "/users/1") {
+		t.Error("expected case-insensitive match on literal segments")
+	}
+}
+
+func TestValidateResponseSchema_NilAnalysis(t *testing.T) {
+	m := &TestUIModel{}
+	errs := m.validateResponseSchema("GET", "/users", 200, `{"id":"1"}`)
+	if errs != nil {
+		t.Errorf("expected nil when analysis is nil, got %v", errs)
+	}
+}
+
+func TestValidateResponseSchema_NoMatchingEndpoint(t *testing.T) {
+	m := &TestUIModel{
+		analysis: &analyzer.Analysis{
+			Specification: &parser.Specification{
+				Endpoints: []parser.Endpoint{
+					{Method: "GET", Path: "/users"},
+				},
+			},
+		},
+	}
+	errs := m.validateResponseSchema("POST", "/other", 200, `{}`)
+	if errs != nil {
+		t.Errorf("expected nil for unmatched endpoint, got %v", errs)
+	}
+}
+
+func TestValidateResponseSchema_NoSchemaForStatus(t *testing.T) {
+	m := &TestUIModel{
+		analysis: &analyzer.Analysis{
+			Specification: &parser.Specification{
+				Endpoints: []parser.Endpoint{
+					{Method: "GET", Path: "/users"},
+				},
+			},
+		},
+	}
+	errs := m.validateResponseSchema("GET", "/users", 200, `{"id":"1"}`)
+	if errs != nil {
+		t.Errorf("expected nil when no schema for status code, got %v", errs)
+	}
+}
+
+func TestValidateResponseSchema_Valid(t *testing.T) {
+	m := &TestUIModel{
+		analysis: &analyzer.Analysis{
+			Specification: &parser.Specification{
+				Endpoints: []parser.Endpoint{
+					{
+						Method: "GET",
+						Path:   "/users/{id}",
+						ResponseSchemas: map[string]map[string]any{
+							"200": {
+								"type": "object",
+								"properties": map[string]any{
+									"id": map[string]any{"type": "string"},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	errs := m.validateResponseSchema("GET", "/users/42", 200, `{"id":"abc"}`)
+	if len(errs) != 0 {
+		t.Errorf("expected no errors, got %v", errs)
+	}
+}
+
+func TestValidateResponseSchema_SchemaError(t *testing.T) {
+	m := &TestUIModel{
+		analysis: &analyzer.Analysis{
+			Specification: &parser.Specification{
+				Endpoints: []parser.Endpoint{
+					{
+						Method: "GET",
+						Path:   "/users/{id}",
+						ResponseSchemas: map[string]map[string]any{
+							"200": {
+								"type": "object",
+								"properties": map[string]any{
+									"id": map[string]any{"type": "string"},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+	errs := m.validateResponseSchema("GET", "/users/42", 200, `{"id":99}`)
+	if len(errs) == 0 {
+		t.Error("expected schema validation error for wrong type")
+	}
+}

internal/cli/update_tests.go

@@ -212,6 +212,9 @@ func handleRunNextTest(m *TestUIModel, _ runNextTestMsg) (tea.Model, tea.Cmd) {
 		})
 	} else {
 		passed := result.StatusCode == expectedStatus
+		schemaErrors := m.validateResponseSchema(method, endpoint, result.StatusCode, result.ResponseBody)
+		schemaValid := len(schemaErrors) == 0
+
 		statusIcon := "✓"
 		statusStyle := m.successStyle
 		if !passed {
@@ -220,6 +223,12 @@ func handleRunNextTest(m *TestUIModel, _ runNextTestMsg) (tea.Model, tea.Cmd) {
 			if m.isHeadless {
 				m.headlessExitCode = 1
 			}
+		} else if !schemaValid {
+			statusIcon = "⚠"
+			statusStyle = lipgloss.NewStyle().Foreground(Theme.Warning)
+			if m.isHeadless {
+				m.headlessExitCode = 1
+			}
 		}
 
 		statusMsg := fmt.Sprintf("    Status: %d", result.StatusCode)
@@ -231,6 +240,14 @@ func handleRunNextTest(m *TestUIModel, _ runNextTestMsg) (tea.Model, tea.Cmd) {
 		m.addMessage(fmt.Sprintf("  %s %s %s%s", statusStyle.Render(statusIcon), methodFormatted, endpoint, authIndicator))
 		m.addMessage(m.subtleStyle.Render(statusMsg))
 
+		if !schemaValid {
+			schemaStyle := lipgloss.NewStyle().Foreground(Theme.Warning)
+			m.addMessage(schemaStyle.Render("    Schema mismatch:"))
+			for _, se := range schemaErrors {
+				m.addMessage(schemaStyle.Render("      · " + se))
+			}
+		}
+
 		m.testGroupResults = append(m.testGroupResults, map[string]any{
 			"method":          method,
 			"endpoint":        endpoint,
@@ -240,6 +257,8 @@ func handleRunNextTest(m *TestUIModel, _ runNextTestMsg) (tea.Model, tea.Cmd) {
 			"duration_ms":     result.Duration.Milliseconds(),
 			"requires_auth":   requiresAuth,
 			"passed":          passed,
+			"schema_valid":    schemaValid,
+			"schema_errors":   schemaErrors,
 		})
 	}
 	m.testGroupCompletedCount++