From c94ee26f1b71e4ecb877bbbefe56561a428af2b1 Mon Sep 17 00:00:00 2001 From: Don Syme Date: Tue, 16 Sep 2025 02:34:26 +0100 Subject: [PATCH 1/5] update docs --- docs/src/content/docs/guides/mcps.md | 4 ++-- docs/src/content/docs/guides/security.md | 4 ++-- docs/src/content/docs/reference/cache-memory.md | 4 ++-- .../content/docs/reference/command-triggers.md | 4 ++-- docs/src/content/docs/reference/frontmatter.md | 12 ++++++------ .../content/docs/reference/include-directives.md | 8 ++++---- docs/src/content/docs/reference/markdown.md | 4 ++-- docs/src/content/docs/reference/network.md | 6 +++--- docs/src/content/docs/reference/safe-outputs.md | 4 ++-- docs/src/content/docs/reference/tools.md | 8 ++++---- .../content/docs/reference/workflow-structure.md | 12 ++++++------ docs/src/content/docs/start-here/concepts.md | 16 ++++++++-------- docs/src/content/docs/tools/cli.md | 8 ++++---- 13 files changed, 47 insertions(+), 47 deletions(-) diff --git a/docs/src/content/docs/guides/mcps.md b/docs/src/content/docs/guides/mcps.md index 55bd28f52f..e40d7a6915 100644 --- a/docs/src/content/docs/guides/mcps.md +++ b/docs/src/content/docs/guides/mcps.md @@ -283,8 +283,8 @@ Error: Tool 'my_tool' not found ## Related Documentation -- [Tools Configuration](../reference/tools.md) - Complete tools reference -- [CLI Commands](../tools/cli.md) - CLI commands including `mcp-inspect` +- [Tools Configuration](../../reference/tools/) - Complete tools reference +- [CLI Commands](../../tools/cli/) - CLI commands including `mcp-inspect` - [Include Directives](../reference/include-directives.md) - Modularizing workflows with includes - [Frontmatter Options](../reference/frontmatter,md) - All configuration options - [Workflow Structure](../reference/workflow-structure.md) - Directory organization diff --git a/docs/src/content/docs/guides/security.md b/docs/src/content/docs/guides/security.md index 6cbb0bcf80..1314fd583b 100644 --- a/docs/src/content/docs/guides/security.md +++ b/docs/src/content/docs/guides/security.md @@ -304,8 +304,8 @@ Claude exposes richer default tools and optional Bash; codex relies more on CLI ## See also -- [Tools Configuration](../reference/tools.md) -- [MCPs](../guides/mcps.md) +- [Tools Configuration](../../reference/tools/) +- [MCPs](../../guides/mcps/) - [Workflow Structure](../reference/workflow-structure.md) ## References diff --git a/docs/src/content/docs/reference/cache-memory.md b/docs/src/content/docs/reference/cache-memory.md index 7ee193807b..c692f4cad2 100644 --- a/docs/src/content/docs/reference/cache-memory.md +++ b/docs/src/content/docs/reference/cache-memory.md @@ -316,7 +316,7 @@ Share memory data across multiple workflows in the same repository. ## Related Documentation -- [Frontmatter Options](../reference/frontmatter.md) - Complete frontmatter configuration guide -- [MCP Tools](../guides/mcps.md) - Model Context Protocol integration +- [Frontmatter Options](../../reference/frontmatter/) - Complete frontmatter configuration guide +- [MCP Tools](../../guides/mcps/) - Model Context Protocol integration - [Safe Outputs](../reference/safe-outputs.md) - Output processing and automation - [GitHub Actions Cache Documentation](https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows) - Official GitHub cache documentation \ No newline at end of file diff --git a/docs/src/content/docs/reference/command-triggers.md b/docs/src/content/docs/reference/command-triggers.md index b68e64cdce..93b695e763 100644 --- a/docs/src/content/docs/reference/command-triggers.md +++ b/docs/src/content/docs/reference/command-triggers.md @@ -94,6 +94,6 @@ See [Reactions](../reference/frontmatter/#reactions) for the complete list of av ## Related Documentation -- [Frontmatter Options](../reference/frontmatter.md) - All configuration options for workflows +- [Frontmatter Options](../../reference/frontmatter/) - All configuration options for workflows - [Workflow Structure](../reference/workflow-structure.md) - Directory layout and organization -- [CLI Commands](../tools/cli.md) - CLI commands for workflow management +- [CLI Commands](../../tools/cli/) - CLI commands for workflow management diff --git a/docs/src/content/docs/reference/frontmatter.md b/docs/src/content/docs/reference/frontmatter.md index eb7f7d8175..290b798d4b 100644 --- a/docs/src/content/docs/reference/frontmatter.md +++ b/docs/src/content/docs/reference/frontmatter.md @@ -186,11 +186,11 @@ tools: bash: ["echo", "ls", "git status"] ``` -For complete tool configuration options, including GitHub tools, Playwright browser automation, custom MCP servers, and security considerations, see [Tools Configuration](../reference/tools.md). +For complete tool configuration options, including GitHub tools, Playwright browser automation, custom MCP servers, and security considerations, see [Tools Configuration](../../reference/tools/). ## Network Permissions (`network:`) -Control network access for AI engines using ecosystem identifiers and domain allowlists. See [Network Permissions](../reference/network.md) for detailed configuration options, security model, and examples. +Control network access for AI engines using ecosystem identifiers and domain allowlists. See [Network Permissions](../../reference/network/) for detailed configuration options, security model, and examples. Quick example: ```yaml @@ -294,10 +294,10 @@ cache: ## Related Documentation -- [CLI Commands](../tools/cli.md) - CLI commands for workflow management +- [CLI Commands](../../tools/cli/) - CLI commands for workflow management - [Workflow Structure](../reference/workflow-structure.md) - Directory layout and organization -- [Network Permissions](../reference/network.md) - Network access control configuration +- [Network Permissions](../../reference/network/) - Network access control configuration - [Command Triggers](../reference/command-triggers.md) - Special @mention triggers and context text -- [MCPs](../guides/mcps.md) - Model Context Protocol setup and configuration -- [Tools Configuration](../reference/tools.md) - GitHub and other tools setup +- [MCPs](../../guides/mcps/) - Model Context Protocol setup and configuration +- [Tools Configuration](../../reference/tools/) - GitHub and other tools setup - [Include Directives](../reference/include-directives.md) - Modularizing workflows with includes diff --git a/docs/src/content/docs/reference/include-directives.md b/docs/src/content/docs/reference/include-directives.md index 6f758dfb70..1e60d0e84f 100644 --- a/docs/src/content/docs/reference/include-directives.md +++ b/docs/src/content/docs/reference/include-directives.md @@ -68,8 +68,8 @@ tools: ## Related Documentation -- [CLI Commands](../tools/cli.md) - CLI commands for workflow management +- [CLI Commands](../../tools/cli/) - CLI commands for workflow management - [Workflow Structure](../reference/workflow-structure.md) - Directory layout and organization -- [Frontmatter Options](../reference/frontmatter.md) - All configuration options -- [Tools Configuration](../reference/tools.md) - GitHub and other tools setup -- [MCPs](../guides/mcps.md) - Model Context Protocol setup and configuration +- [Frontmatter Options](../../reference/frontmatter/) - All configuration options +- [Tools Configuration](../../reference/tools/) - GitHub and other tools setup +- [MCPs](../../guides/mcps/) - Model Context Protocol setup and configuration diff --git a/docs/src/content/docs/reference/markdown.md b/docs/src/content/docs/reference/markdown.md index d3c9c2a078..fe33eec392 100644 --- a/docs/src/content/docs/reference/markdown.md +++ b/docs/src/content/docs/reference/markdown.md @@ -299,5 +299,5 @@ Complex: ${{ toJson(github.workflow) }} ## Related Documentation - [Workflow Structure](./workflow-structure.md) - Overall workflow file organization -- [Frontmatter Options](./frontmatter.md) - YAML configuration options -- [Security Notes](../guides/security.md) - Comprehensive security guidance +- [Frontmatter Options](.././frontmatter/) - YAML configuration options +- [Security Notes](../../guides/security/) - Comprehensive security guidance diff --git a/docs/src/content/docs/reference/network.md b/docs/src/content/docs/reference/network.md index 8a192acdfa..ee6a626a4e 100644 --- a/docs/src/content/docs/reference/network.md +++ b/docs/src/content/docs/reference/network.md @@ -243,6 +243,6 @@ network: ## Related Documentation -- [Frontmatter Options](./frontmatter.md) - Complete frontmatter configuration guide -- [Tools Configuration](./tools.md) - Tool-specific network access configuration -- [Security Notes](../guides/security.md) - Comprehensive security guidance +- [Frontmatter Options](.././frontmatter/) - Complete frontmatter configuration guide +- [Tools Configuration](.././tools/) - Tool-specific network access configuration +- [Security Notes](../../guides/security/) - Comprehensive security guidance diff --git a/docs/src/content/docs/reference/safe-outputs.md b/docs/src/content/docs/reference/safe-outputs.md index 3403598013..a6b667943b 100644 --- a/docs/src/content/docs/reference/safe-outputs.md +++ b/docs/src/content/docs/reference/safe-outputs.md @@ -572,7 +572,7 @@ This is useful when: ## Related Documentation -- [Frontmatter Options](../reference/frontmatter.md) - All configuration options for workflows +- [Frontmatter Options](../../reference/frontmatter/) - All configuration options for workflows - [Workflow Structure](../reference/workflow-structure.md) - Directory layout and organization - [Command Triggers](../reference/command-triggers.md) - Special /my-bot triggers and context text -- [CLI Commands](../tools/cli.md) - CLI commands for workflow management +- [CLI Commands](../../tools/cli/) - CLI commands for workflow management diff --git a/docs/src/content/docs/reference/tools.md b/docs/src/content/docs/reference/tools.md index be20610b06..4be1bd5bea 100644 --- a/docs/src/content/docs/reference/tools.md +++ b/docs/src/content/docs/reference/tools.md @@ -156,9 +156,9 @@ tools: ## Related Documentation -- [Frontmatter Options](../reference/frontmatter.md) - All frontmatter configuration options -- [Network Permissions](../reference/network.md) - Network access control for AI engines -- [MCPs](../guides/mcps.md) - Complete Model Context Protocol setup and usage -- [CLI Commands](../tools/cli.md) - CLI commands for workflow management +- [Frontmatter Options](../../reference/frontmatter/) - All frontmatter configuration options +- [Network Permissions](../../reference/network/) - Network access control for AI engines +- [MCPs](../../guides/mcps/) - Complete Model Context Protocol setup and usage +- [CLI Commands](../../tools/cli/) - CLI commands for workflow management - [Workflow Structure](../reference/workflow-structure.md) - Directory layout and organization - [Include Directives](../reference/include-directives.md) - Modularizing workflows with includes diff --git a/docs/src/content/docs/reference/workflow-structure.md b/docs/src/content/docs/reference/workflow-structure.md index be0e481e11..0345896dd6 100644 --- a/docs/src/content/docs/reference/workflow-structure.md +++ b/docs/src/content/docs/reference/workflow-structure.md @@ -9,8 +9,8 @@ sidebar: Each workflow consists of: -1. **YAML Frontmatter**: Configuration options wrapped in `---`. See [Frontmatter Options](./frontmatter.md) for details. -2. **Markdown Content**: Natural language instructions for the AI. See [Markdown Content](./markdown.md). +1. **YAML Frontmatter**: Configuration options wrapped in `---`. See [Frontmatter Options](.././frontmatter/) for details. +2. **Markdown Content**: Natural language instructions for the AI. See [Markdown Content](.././markdown/). The markdown content is where you write natural language instructions for the AI agent. @@ -65,8 +65,8 @@ gh aw compile ## Related Documentation -- [Frontmatter Options](./frontmatter.md) - Configuration options for workflows -- [Markdown Content](./markdown.md) - The main markdown content of workflows +- [Frontmatter Options](.././frontmatter/) - Configuration options for workflows +- [Markdown Content](.././markdown/) - The main markdown content of workflows - [Include Directives](./include-directives.md) - Modularizing workflows with includes -- [CLI Commands](../tools/cli.md) - CLI commands for workflow management -- [MCPs](../guides/mcps.md) - Model Context Protocol configuration +- [CLI Commands](../../tools/cli/) - CLI commands for workflow management +- [MCPs](../../guides/mcps/) - Model Context Protocol configuration diff --git a/docs/src/content/docs/start-here/concepts.md b/docs/src/content/docs/start-here/concepts.md index 09d972ed9f..e414fe83a1 100644 --- a/docs/src/content/docs/start-here/concepts.md +++ b/docs/src/content/docs/start-here/concepts.md @@ -52,7 +52,7 @@ Analyze this issue and provide helpful triage comments... One crucial difference from traditional agentic prompting is that GitHub Agentic Workflows can contain triggers, permissions and other declarative elements. This works towards more reliable and more secure agentic programming, setting the AI up to contribute to success, in a partially sandboxed way, at the right time in your team's work. -See [Workflow Structure](../reference/workflow-structure.md) and [Frontmatter Options](../reference/frontmatter.md) for details of file layout and configuration options. +See [Workflow Structure](../reference/workflow-structure.md) and [Frontmatter Options](../../reference/frontmatter/) for details of file layout and configuration options. ## Understanding AI Engines @@ -94,7 +94,7 @@ Agentic workflows require careful security consideration: - **Input validation** — All inputs are automatically sanitized - **Human oversight** — Critical actions can require human approval -See [Security Guide](../guides/security.md) for comprehensive guidelines. +See [Security Guide](../../guides/security/) for comprehensive guidelines. ## Tools and MCPs @@ -105,7 +105,7 @@ Workflows can use various tools through the Model Context Protocol (MCP): - **File operations** — Read, write, and analyze repository files - **Custom MCPs** — Build your own tool integrations -Learn more in [Tools Configuration](../reference/tools.md) and [MCPs](../guides/mcps.md). +Learn more in [Tools Configuration](../../reference/tools/) and [MCPs](../../guides/mcps/). ## Best Practices @@ -121,16 +121,16 @@ Learn more in [Tools Configuration](../reference/tools.md) and [MCPs](../guides/ - **Event-driven** — Respond to issues, PRs, pushes, etc. - **Scheduled** — Regular maintenance and reporting tasks - **Alias-triggered** — Activated by @mentions in comments -- **Secure** — User minimal permissions and protect against untrusted content, see [Security Guide](../guides/security.md) +- **Secure** — User minimal permissions and protect against untrusted content, see [Security Guide](../../guides/security/) ## Next Steps Ready to build more sophisticated workflows? Explore: - **[Workflow Structure](../reference/workflow-structure.md)** — Detailed file organization and security -- **[Frontmatter Options](../reference/frontmatter.md)** — Complete configuration reference -- **[Tools Configuration](../reference/tools.md)** — Available tools and permissions -- **[Security Guide](../guides/security.md)** — Important security considerations -- **[VS Code Integration](../tools/vscode.md)** — Enhanced authoring experience +- **[Frontmatter Options](../../reference/frontmatter/)** — Complete configuration reference +- **[Tools Configuration](../../reference/tools/)** — Available tools and permissions +- **[Security Guide](../../guides/security/)** — Important security considerations +- **[VS Code Integration](../../tools/vscode/)** — Enhanced authoring experience The power of agentic workflows lies in their ability to understand context, make intelligent decisions, and take meaningful actions — all while maintaining the reliability you expect from GitHub Actions. diff --git a/docs/src/content/docs/tools/cli.md b/docs/src/content/docs/tools/cli.md index be987e9c6c..5b69772992 100644 --- a/docs/src/content/docs/tools/cli.md +++ b/docs/src/content/docs/tools/cli.md @@ -191,7 +191,7 @@ gh aw logs --format json -o ./exports/ The `mcp-inspect` command allows you to analyze and troubleshoot Model Context Protocol (MCP) servers configured in your workflows. -> **📘 Complete MCP Guide**: For comprehensive MCP setup, configuration examples, and troubleshooting, see the [MCPs](../guides/mcps.md). +> **📘 Complete MCP Guide**: For comprehensive MCP setup, configuration examples, and troubleshooting, see the [MCPs](../../guides/mcps/). ```bash # List all workflows that contain MCP server configurations @@ -224,7 +224,7 @@ gh aw mcp-inspect workflow-name --inspector For detailed MCP debugging and troubleshooting guides, see [MCP Debugging](../guides/mcps/#debugging-and-troubleshooting). ## 👀 Watch Mode for Development -The `--watch` flag provides automatic recompilation during workflow development, monitoring for file changes in real-time. See [Authoring in VS Code](../tools/vscode.md). +The `--watch` flag provides automatic recompilation during workflow development, monitoring for file changes in real-time. See [Authoring in VS Code](../../tools/vscode/). ```bash # Watch all workflow files in .github/workflows/ for changes @@ -296,7 +296,7 @@ gh aw uninstall org/repo --local ## Related Documentation - [Workflow Structure](../reference/workflow-structure.md) - Directory layout and file organization -- [Frontmatter Options](../reference/frontmatter.md) - Configuration options for workflows +- [Frontmatter Options](../../reference/frontmatter/) - Configuration options for workflows - [Safe Outputs](../reference/safe-outputs.md) - Secure output processing including issue updates -- [Tools Configuration](../reference/tools.md) - GitHub and MCP server configuration +- [Tools Configuration](../../reference/tools/) - GitHub and MCP server configuration - [Include Directives](../reference/include-directives.md) - Modularizing workflows with includes From ecb84671ee6d5826838be3baf71ab8f485c4de2c Mon Sep 17 00:00:00 2001 From: Peli de Halleux Date: Tue, 16 Sep 2025 01:35:53 +0000 Subject: [PATCH 2/5] refactor: remove redundant comments and improve log handling in workflow files --- .github/workflows/ci-doctor.lock.yml | 3 --- .github/workflows/dev.lock.yml | 3 --- pkg/cli/logs.go | 8 ++------ .../workflows/test-ai-inference-github-models.lock.yml | 3 --- pkg/cli/workflows/test-claude-add-issue-comment.lock.yml | 3 --- pkg/cli/workflows/test-claude-add-issue-labels.lock.yml | 3 --- pkg/cli/workflows/test-claude-cache-memory.lock.yml | 3 --- pkg/cli/workflows/test-claude-command.lock.yml | 3 --- pkg/cli/workflows/test-claude-create-issue.lock.yml | 3 --- ...est-claude-create-pull-request-review-comment.lock.yml | 3 --- .../workflows/test-claude-create-pull-request.lock.yml | 3 --- ...st-claude-create-repository-security-advisory.lock.yml | 3 --- pkg/cli/workflows/test-claude-mcp.lock.yml | 3 --- pkg/cli/workflows/test-claude-missing-tool.lock.yml | 5 +---- pkg/cli/workflows/test-claude-push-to-pr-branch.lock.yml | 3 --- pkg/cli/workflows/test-claude-update-issue.lock.yml | 3 --- pkg/cli/workflows/test-codex-add-issue-comment.lock.yml | 4 +--- pkg/cli/workflows/test-codex-add-issue-labels.lock.yml | 4 +--- pkg/cli/workflows/test-codex-command.lock.yml | 4 +--- .../test-playwright-accessibility-contrast.lock.yml | 5 +---- pkg/workflow/js/parse_claude_log.cjs | 7 ------- 21 files changed, 7 insertions(+), 72 deletions(-) diff --git a/.github/workflows/ci-doctor.lock.yml b/.github/workflows/ci-doctor.lock.yml index 182ff9226c..6685f0465c 100644 --- a/.github/workflows/ci-doctor.lock.yml +++ b/.github/workflows/ci-doctor.lock.yml @@ -1697,7 +1697,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -1709,9 +1708,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/.github/workflows/dev.lock.yml b/.github/workflows/dev.lock.yml index 1fad30d46f..93b1d8c261 100644 --- a/.github/workflows/dev.lock.yml +++ b/.github/workflows/dev.lock.yml @@ -1693,7 +1693,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -1705,9 +1704,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/logs.go b/pkg/cli/logs.go index fc47aff777..7b06285182 100644 --- a/pkg/cli/logs.go +++ b/pkg/cli/logs.go @@ -1162,6 +1162,7 @@ func displayToolCallReport(processedRuns []ProcessedRun, verbose bool) { // Render compact table without title as requested tableConfig := console.TableConfig{ + Title: "MCP Tool Calls", Headers: headers, Rows: rows, ShowTotal: false, // Keep it simple and compact @@ -1627,7 +1628,6 @@ func extractMCPFailuresFromRun(runDir string, run WorkflowRun, verbose bool) ([] !strings.Contains(fileName, "agent_output") && !strings.Contains(fileName, "access") { - // Parse this log file for MCP server failures failures, parseErr := extractMCPFailuresFromLogFile(path, run, verbose) if parseErr != nil { if verbose { @@ -1670,7 +1670,6 @@ func extractMCPFailuresFromLogFile(logPath string, run WorkflowRun, verbose bool for _, entry := range logEntries { if entryType, ok := entry["type"].(string); ok && entryType == "system" { if subtype, ok := entry["subtype"].(string); ok && subtype == "init" { - // Extract MCP server failures from this init entry if mcpServers, ok := entry["mcp_servers"].([]interface{}); ok { for _, serverInterface := range mcpServers { if server, ok := serverInterface.(map[string]interface{}); ok { @@ -1720,7 +1719,6 @@ func extractMCPFailuresFromLogFile(logPath string, run WorkflowRun, verbose bool // Look for system init entries that contain MCP server information if entryType, ok := entry["type"].(string); ok && entryType == "system" { if subtype, ok := entry["subtype"].(string); ok && subtype == "init" { - // Extract MCP server failures from this init entry if mcpServers, ok := entry["mcp_servers"].([]interface{}); ok { for _, serverInterface := range mcpServers { if server, ok := serverInterface.(map[string]interface{}); ok { @@ -1803,9 +1801,6 @@ func displayMCPFailuresAnalysis(processedRuns []ProcessedRun, verbose bool) { return // No MCP failures to display } - // Display summary header - fmt.Printf("\n%s\n", console.FormatListHeader("🔌 MCP Server Failures")) - // Convert map to slice for sorting var summaries []*MCPFailureSummary for _, summary := range failureSummary { @@ -1845,6 +1840,7 @@ func displayMCPFailuresAnalysis(processedRuns []ProcessedRun, verbose bool) { } tableConfig := console.TableConfig{ + Title: "MCP Server Failures", Headers: headers, Rows: rows, } diff --git a/pkg/cli/workflows/test-ai-inference-github-models.lock.yml b/pkg/cli/workflows/test-ai-inference-github-models.lock.yml index 3dc810e69c..92385c80bf 100644 --- a/pkg/cli/workflows/test-ai-inference-github-models.lock.yml +++ b/pkg/cli/workflows/test-ai-inference-github-models.lock.yml @@ -300,7 +300,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -312,9 +311,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-add-issue-comment.lock.yml b/pkg/cli/workflows/test-claude-add-issue-comment.lock.yml index e64d2e89cb..1c3d875c9a 100644 --- a/pkg/cli/workflows/test-claude-add-issue-comment.lock.yml +++ b/pkg/cli/workflows/test-claude-add-issue-comment.lock.yml @@ -297,7 +297,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -309,9 +308,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-add-issue-labels.lock.yml b/pkg/cli/workflows/test-claude-add-issue-labels.lock.yml index 8ebed77b76..c494021f70 100644 --- a/pkg/cli/workflows/test-claude-add-issue-labels.lock.yml +++ b/pkg/cli/workflows/test-claude-add-issue-labels.lock.yml @@ -297,7 +297,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -309,9 +308,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-cache-memory.lock.yml b/pkg/cli/workflows/test-claude-cache-memory.lock.yml index 0bf05cd803..bcf5c92e59 100644 --- a/pkg/cli/workflows/test-claude-cache-memory.lock.yml +++ b/pkg/cli/workflows/test-claude-cache-memory.lock.yml @@ -362,7 +362,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -374,9 +373,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-command.lock.yml b/pkg/cli/workflows/test-claude-command.lock.yml index 2615bfae2a..c1ff16ef40 100644 --- a/pkg/cli/workflows/test-claude-command.lock.yml +++ b/pkg/cli/workflows/test-claude-command.lock.yml @@ -297,7 +297,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -309,9 +308,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-create-issue.lock.yml b/pkg/cli/workflows/test-claude-create-issue.lock.yml index cc192f0cbd..3f93628263 100644 --- a/pkg/cli/workflows/test-claude-create-issue.lock.yml +++ b/pkg/cli/workflows/test-claude-create-issue.lock.yml @@ -297,7 +297,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -309,9 +308,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-create-pull-request-review-comment.lock.yml b/pkg/cli/workflows/test-claude-create-pull-request-review-comment.lock.yml index d12f4f3105..a73fbff777 100644 --- a/pkg/cli/workflows/test-claude-create-pull-request-review-comment.lock.yml +++ b/pkg/cli/workflows/test-claude-create-pull-request-review-comment.lock.yml @@ -297,7 +297,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -309,9 +308,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-create-pull-request.lock.yml b/pkg/cli/workflows/test-claude-create-pull-request.lock.yml index 416e81571b..85c60fc26f 100644 --- a/pkg/cli/workflows/test-claude-create-pull-request.lock.yml +++ b/pkg/cli/workflows/test-claude-create-pull-request.lock.yml @@ -302,7 +302,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -314,9 +313,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-create-repository-security-advisory.lock.yml b/pkg/cli/workflows/test-claude-create-repository-security-advisory.lock.yml index 996abdd5e4..64c353def0 100644 --- a/pkg/cli/workflows/test-claude-create-repository-security-advisory.lock.yml +++ b/pkg/cli/workflows/test-claude-create-repository-security-advisory.lock.yml @@ -300,7 +300,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -312,9 +311,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-mcp.lock.yml b/pkg/cli/workflows/test-claude-mcp.lock.yml index ff9eddba73..931fe3860e 100644 --- a/pkg/cli/workflows/test-claude-mcp.lock.yml +++ b/pkg/cli/workflows/test-claude-mcp.lock.yml @@ -300,7 +300,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -312,9 +311,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-missing-tool.lock.yml b/pkg/cli/workflows/test-claude-missing-tool.lock.yml index c7fb779e65..92d43c84d1 100644 --- a/pkg/cli/workflows/test-claude-missing-tool.lock.yml +++ b/pkg/cli/workflows/test-claude-missing-tool.lock.yml @@ -263,7 +263,7 @@ jobs: writeMessage(res); } function isToolEnabled(name) { - return safeOutputsConfig[name] && safeOutputsConfig[name].enabled; + return safeOutputsConfig[name]; } function appendSafeOutput(entry) { if (!outputFile) throw new Error("No output file configured"); @@ -1658,7 +1658,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -1670,9 +1669,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-push-to-pr-branch.lock.yml b/pkg/cli/workflows/test-claude-push-to-pr-branch.lock.yml index 83d42cdd59..93a2d97597 100644 --- a/pkg/cli/workflows/test-claude-push-to-pr-branch.lock.yml +++ b/pkg/cli/workflows/test-claude-push-to-pr-branch.lock.yml @@ -302,7 +302,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -314,9 +313,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-claude-update-issue.lock.yml b/pkg/cli/workflows/test-claude-update-issue.lock.yml index 567a12a54b..238f5348dc 100644 --- a/pkg/cli/workflows/test-claude-update-issue.lock.yml +++ b/pkg/cli/workflows/test-claude-update-issue.lock.yml @@ -300,7 +300,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -312,9 +311,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/cli/workflows/test-codex-add-issue-comment.lock.yml b/pkg/cli/workflows/test-codex-add-issue-comment.lock.yml index b3e7a49c3b..41da7e0c1d 100644 --- a/pkg/cli/workflows/test-codex-add-issue-comment.lock.yml +++ b/pkg/cli/workflows/test-codex-add-issue-comment.lock.yml @@ -118,9 +118,7 @@ jobs: mkdir -p /tmp/aw-logs # Run codex with log capture - pipefail ensures codex exit code is preserved - codex exec \ - -c model=o4-mini \ - --full-auto "$INSTRUCTION" 2>&1 | tee /tmp/test-codex-add-issue-comment.log + codex exec --full-auto "$INSTRUCTION" 2>&1 | tee /tmp/test-codex-add-issue-comment.log env: GITHUB_AW_PROMPT: /tmp/aw-prompts/prompt.txt GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} diff --git a/pkg/cli/workflows/test-codex-add-issue-labels.lock.yml b/pkg/cli/workflows/test-codex-add-issue-labels.lock.yml index ebfe19f6d3..2026098281 100644 --- a/pkg/cli/workflows/test-codex-add-issue-labels.lock.yml +++ b/pkg/cli/workflows/test-codex-add-issue-labels.lock.yml @@ -118,9 +118,7 @@ jobs: mkdir -p /tmp/aw-logs # Run codex with log capture - pipefail ensures codex exit code is preserved - codex exec \ - -c model=o4-mini \ - --full-auto "$INSTRUCTION" 2>&1 | tee /tmp/test-codex-add-issue-labels.log + codex exec --full-auto "$INSTRUCTION" 2>&1 | tee /tmp/test-codex-add-issue-labels.log env: GITHUB_AW_PROMPT: /tmp/aw-prompts/prompt.txt GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} diff --git a/pkg/cli/workflows/test-codex-command.lock.yml b/pkg/cli/workflows/test-codex-command.lock.yml index 088f267aef..ee392a2715 100644 --- a/pkg/cli/workflows/test-codex-command.lock.yml +++ b/pkg/cli/workflows/test-codex-command.lock.yml @@ -118,9 +118,7 @@ jobs: mkdir -p /tmp/aw-logs # Run codex with log capture - pipefail ensures codex exit code is preserved - codex exec \ - -c model=o4-mini \ - --full-auto "$INSTRUCTION" 2>&1 | tee /tmp/test-codex-command.log + codex exec --full-auto "$INSTRUCTION" 2>&1 | tee /tmp/test-codex-command.log env: GITHUB_AW_PROMPT: /tmp/aw-prompts/prompt.txt GITHUB_STEP_SUMMARY: ${{ env.GITHUB_STEP_SUMMARY }} diff --git a/pkg/cli/workflows/test-playwright-accessibility-contrast.lock.yml b/pkg/cli/workflows/test-playwright-accessibility-contrast.lock.yml index 029c6b2927..245225fadf 100644 --- a/pkg/cli/workflows/test-playwright-accessibility-contrast.lock.yml +++ b/pkg/cli/workflows/test-playwright-accessibility-contrast.lock.yml @@ -248,7 +248,7 @@ jobs: writeMessage(res); } function isToolEnabled(name) { - return safeOutputsConfig[name] && safeOutputsConfig[name].enabled; + return safeOutputsConfig[name]; } function appendSafeOutput(entry) { if (!outputFile) throw new Error("No output file configured"); @@ -1644,7 +1644,6 @@ jobs: function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); @@ -1656,9 +1655,7 @@ jobs: } const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); diff --git a/pkg/workflow/js/parse_claude_log.cjs b/pkg/workflow/js/parse_claude_log.cjs index 97984a6a5f..934f024fe7 100644 --- a/pkg/workflow/js/parse_claude_log.cjs +++ b/pkg/workflow/js/parse_claude_log.cjs @@ -2,25 +2,18 @@ function main() { const fs = require("fs"); try { - // Get the log file path from environment const logFile = process.env.GITHUB_AW_AGENT_OUTPUT; if (!logFile) { core.info("No agent log file specified"); return; } - if (!fs.existsSync(logFile)) { core.info(`Log file not found: ${logFile}`); return; } - const logContent = fs.readFileSync(logFile, "utf8"); const result = parseClaudeLog(logContent); - - // Append to GitHub step summary core.summary.addRaw(result.markdown).write(); - - // Check for MCP server failures and fail the job if any occurred if (result.mcpFailures && result.mcpFailures.length > 0) { const failedServers = result.mcpFailures.join(", "); core.setFailed(`MCP server(s) failed to launch: ${failedServers}`); From c340453752276fd5aab8e53a57cf5fa401619ed4 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 16 Sep 2025 01:38:19 +0000 Subject: [PATCH 3/5] Initial plan From 5da104cb62071550cddd9730d354f8f397d29992 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 16 Sep 2025 01:52:57 +0000 Subject: [PATCH 4/5] Add comprehensive tests for Claude log parser new format support Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- .../log_parser_new_format_file_test.go | 75 +++++++++++++++++ pkg/workflow/log_parser_new_format_test.go | 83 +++++++++++++++++++ .../sample_claude_log_new_format.txt | 14 ++++ 3 files changed, 172 insertions(+) create mode 100644 pkg/workflow/log_parser_new_format_file_test.go create mode 100644 pkg/workflow/log_parser_new_format_test.go create mode 100644 pkg/workflow/test_data/sample_claude_log_new_format.txt diff --git a/pkg/workflow/log_parser_new_format_file_test.go b/pkg/workflow/log_parser_new_format_file_test.go new file mode 100644 index 0000000000..482d50c6a0 --- /dev/null +++ b/pkg/workflow/log_parser_new_format_file_test.go @@ -0,0 +1,75 @@ +package workflow + +import ( + "os" + "strings" + "testing" +) + +func TestParseClaudeLogNewFormatFile(t *testing.T) { + // Test the new format from file + content, err := os.ReadFile("test_data/sample_claude_log_new_format.txt") + if err != nil { + t.Fatalf("Failed to read test file: %v", err) + } + + engine := NewClaudeEngine() + metrics := engine.ParseLogMetrics(string(content), true) + + // Verify parsing worked correctly + t.Logf("Parsed metrics: Tokens=%d, Cost=%.6f, Turns=%d, Errors=%d", + metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns, metrics.ErrorCount) + + // Should extract the correct final result metrics + if metrics.TokenUsage == 0 { + t.Error("Expected non-zero token usage") + } + if metrics.EstimatedCost == 0 { + t.Error("Expected non-zero cost") + } + if metrics.Turns == 0 { + t.Error("Expected non-zero turns") + } + + // Should count the [ERROR] line in the debug logs + if metrics.ErrorCount == 0 { + t.Error("Expected at least one error from debug logs") + } +} + +func TestParseClaudeLogNewFormatJSScriptFromFile(t *testing.T) { + // Test the JavaScript parser with the new format + script := GetLogParserScript("parse_claude_log") + if script == "" { + t.Skip("parse_claude_log script not available") + } + + content, err := os.ReadFile("test_data/sample_claude_log_new_format.txt") + if err != nil { + t.Fatalf("Failed to read test file: %v", err) + } + + result, err := runJSLogParser(script, string(content)) + if err != nil { + t.Fatalf("Failed to parse new format Claude log: %v", err) + } + + // Verify essential sections are present + if !strings.Contains(result, "🚀 Initialization") { + t.Error("Expected new format Claude log output to contain Initialization section") + } + if !strings.Contains(result, "🤖 Commands and Tools") { + t.Error("Expected new format Claude log output to contain Commands and Tools section") + } + if !strings.Contains(result, "Total Cost") { + t.Error("Expected new format Claude log output to contain cost information") + } + if !strings.Contains(result, "15b818fc-d93c-45e7-b7f2-89bad9ba54f7") { + t.Error("Expected new format Claude log output to contain session ID") + } + if !strings.Contains(result, "safe_outputs::missing-tool") { + t.Error("Expected new format Claude log output to contain MCP tool call") + } + + t.Logf("JavaScript parser output looks correct with proper sections") +} \ No newline at end of file diff --git a/pkg/workflow/log_parser_new_format_test.go b/pkg/workflow/log_parser_new_format_test.go new file mode 100644 index 0000000000..cb08f39a51 --- /dev/null +++ b/pkg/workflow/log_parser_new_format_test.go @@ -0,0 +1,83 @@ +package workflow + +import ( + "strings" + "testing" +) + +func TestParseClaudeLogNewFormat(t *testing.T) { + // Test with the new format that includes debug entries and JSON + newFormatLog := `npm warn exec The following package was not found and will be installed: @anthropic-ai/claude-code@1.0.115 +[DEBUG] Watching for changes in setting files /tmp/.claude/settings.json... +[ERROR] Failed to save config with lock: Error: ENOENT: no such file or directory, lstat '/home/runner/.claude.json' +[DEBUG] Writing to temp file: /home/runner/.claude.json.tmp.2123.1757985980850 +[DEBUG] Temp file written successfully, size: 103 bytes +[DEBUG] Renaming /home/runner/.claude.json.tmp.2123.1757985980850 to /home/runner/.claude.json +[DEBUG] File /home/runner/.claude.json written atomically +{"type":"system","subtype":"init","cwd":"/home/runner/work/gh-aw/gh-aw","session_id":"15b818fc-d93c-45e7-b7f2-89bad9ba54f7","tools":["Task","Bash","Read"],"model":"claude-sonnet-4-20250514"} +[DEBUG] Stream started - received first chunk +{"type":"assistant","message":{"content":[{"type":"text","text":"I'll help you with this task."},{"type":"tool_use","id":"tool_123","name":"Bash","input":{"command":"echo 'Hello World'"}}]}} +[DEBUG] Stream ended +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool_123","content":"Hello World"}]}} +{"type":"result","total_cost_usd":0.0015,"usage":{"input_tokens":150,"output_tokens":50},"num_turns":1}` + + engine := NewClaudeEngine() + + metrics := engine.ParseLogMetrics(newFormatLog, true) + + // Verify that metrics were extracted correctly + if metrics.TokenUsage != 200 { + t.Errorf("Expected token usage 200 (150+50), got %d", metrics.TokenUsage) + } + if metrics.EstimatedCost != 0.0015 { + t.Errorf("Expected cost 0.0015, got %f", metrics.EstimatedCost) + } + if metrics.Turns != 1 { + t.Errorf("Expected 1 turn, got %d", metrics.Turns) + } + + // Check that error count includes the [ERROR] line + if metrics.ErrorCount == 0 { + t.Errorf("Expected at least 1 error (from [ERROR] line), got %d", metrics.ErrorCount) + } +} + +func TestParseClaudeLogNewFormatJSScript(t *testing.T) { + // Test the JavaScript parser with the new format + script := GetLogParserScript("parse_claude_log") + if script == "" { + t.Skip("parse_claude_log script not available") + } + + // Test with new format log + newFormatLog := `[DEBUG] Starting Claude Code CLI +{"type":"system","subtype":"init","session_id":"test-123","tools":["Bash","Read"],"model":"claude-sonnet-4-20250514"} +[DEBUG] Processing user prompt +{"type":"assistant","message":{"content":[{"type":"text","text":"I'll help you with this task."},{"type":"tool_use","id":"tool_123","name":"Bash","input":{"command":"echo 'Hello World'"}}]}} +[DEBUG] Executing bash command +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool_123","content":"Hello World"}]}} +[DEBUG] Workflow completed successfully +{"type":"result","total_cost_usd":0.0015,"usage":{"input_tokens":150,"output_tokens":50},"num_turns":1}` + + result, err := runJSLogParser(script, newFormatLog) + if err != nil { + t.Fatalf("Failed to parse new format Claude log: %v", err) + } + + // Verify essential sections are present + if !strings.Contains(result, "🚀 Initialization") { + t.Error("Expected new format Claude log output to contain Initialization section") + } + if !strings.Contains(result, "🤖 Commands and Tools") { + t.Error("Expected new format Claude log output to contain Commands and Tools section") + } + if !strings.Contains(result, "echo 'Hello World'") { + t.Error("Expected new format Claude log output to contain the bash command") + } + if !strings.Contains(result, "Total Cost") { + t.Error("Expected new format Claude log output to contain cost information") + } + if !strings.Contains(result, "test-123") { + t.Error("Expected new format Claude log output to contain session ID") + } +} \ No newline at end of file diff --git a/pkg/workflow/test_data/sample_claude_log_new_format.txt b/pkg/workflow/test_data/sample_claude_log_new_format.txt new file mode 100644 index 0000000000..9ce9c9a3d5 --- /dev/null +++ b/pkg/workflow/test_data/sample_claude_log_new_format.txt @@ -0,0 +1,14 @@ +npm warn exec The following package was not found and will be installed: @anthropic-ai/claude-code@1.0.115 +[DEBUG] Watching for changes in setting files /tmp/.claude/settings.json... +[ERROR] Failed to save config with lock: Error: ENOENT: no such file or directory, lstat '/home/runner/.claude.json' +[DEBUG] Writing to temp file: /home/runner/.claude.json.tmp.2123.1757985980850 +[DEBUG] Temp file written successfully, size: 103 bytes +[DEBUG] Renaming /home/runner/.claude.json.tmp.2123.1757985980850 to /home/runner/.claude.json +[DEBUG] File /home/runner/.claude.json written atomically +{"type":"system","subtype":"init","cwd":"/home/runner/work/gh-aw/gh-aw","session_id":"15b818fc-d93c-45e7-b7f2-89bad9ba54f7","tools":["Task","Bash","Read","mcp__safe_outputs__missing-tool"],"mcp_servers":[{"name":"safe_outputs","status":"connected"},{"name":"github","status":"connected"}],"model":"claude-sonnet-4-20250514","permissionMode":"bypassPermissions"} +[DEBUG] Stream started - received first chunk +{"type":"assistant","message":{"id":"msg_01G74UPFuoYP6mNpwBdJKs7e","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[{"type":"text","text":"I'll help you with this task."}],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":4,"cache_creation_input_tokens":32823,"cache_read_input_tokens":4802,"cache_creation":{"ephemeral_5m_input_tokens":32823,"ephemeral_1h_input_tokens":0},"output_tokens":159,"service_tier":"standard"}},"parent_tool_use_id":null,"session_id":"15b818fc-d93c-45e7-b7f2-89bad9ba54f7","uuid":"11b5eca5-d045-4e56-8aa8-429b9429fcbd"} +{"type":"assistant","message":{"id":"msg_01G74UPFuoYP6mNpwBdJKs7e","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[{"type":"tool_use","id":"toolu_01CNycNUMWQXFnrBoexCVUUy","name":"mcp__safe_outputs__missing-tool","input":{"tool":"draw_pelican","reason":"User requested to call a tool that draws a pelican, but this tool is not available in the current toolset","alternatives":"Could potentially use ASCII art generation, image creation tools, or text-based drawing if available"}}],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":4,"cache_creation_input_tokens":32823,"cache_read_input_tokens":4802,"cache_creation":{"ephemeral_5m_input_tokens":32823,"ephemeral_1h_input_tokens":0},"output_tokens":159,"service_tier":"standard"}},"parent_tool_use_id":null,"session_id":"15b818fc-d93c-45e7-b7f2-89bad9ba54f7","uuid":"d0990745-c90c-4899-800a-34c8f0bc2b4a"} +{"type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_01CNycNUMWQXFnrBoexCVUUy","type":"tool_result","content":[{"type":"text","text":"success"}]}]},"parent_tool_use_id":null,"session_id":"15b818fc-d93c-45e7-b7f2-89bad9ba54f7","uuid":"bf2776ca-872f-4dad-8d16-0c31d620460b"} +{"type":"assistant","message":{"id":"msg_011wmKmFC5t8MQjkYjcxYfMq","type":"message","role":"assistant","model":"claude-sonnet-4-20250514","content":[{"type":"text","text":"The draw_pelican tool is not available in my current toolset."}],"stop_reason":null,"stop_sequence":null,"usage":{"input_tokens":7,"cache_creation_input_tokens":169,"cache_read_input_tokens":37625,"cache_creation":{"ephemeral_5m_input_tokens":169,"ephemeral_1h_input_tokens":0},"output_tokens":65,"service_tier":"standard"}},"parent_tool_use_id":null,"session_id":"15b818fc-d93c-45e7-b7f2-89bad9ba54f7","uuid":"feb6f3d5-8dec-4c69-8229-0a3640c44827"} +{"type":"result","subtype":"success","is_error":false,"duration_ms":10857,"duration_api_ms":12416,"num_turns":4,"result":"The draw_pelican tool is not available in my current toolset.","session_id":"15b818fc-d93c-45e7-b7f2-89bad9ba54f7","total_cost_usd":0.1401123,"usage":{"input_tokens":11,"cache_creation_input_tokens":32992,"cache_read_input_tokens":42427,"output_tokens":224,"server_tool_use":{"web_search_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":32992}},"modelUsage":{"claude-3-5-haiku-20241022":{"inputTokens":109,"outputTokens":46,"cacheReadInputTokens":0,"cacheCreationInputTokens":0,"webSearchRequests":0,"costUSD":0.0002712},"claude-sonnet-4-20250514":{"inputTokens":11,"outputTokens":224,"cacheReadInputTokens":42427,"cacheCreationInputTokens":32992,"webSearchRequests":0,"costUSD":0.1398411}},"permission_denials":[],"uuid":"9e63a3a0-202b-429d-b9d3-35ae572abfd6"} \ No newline at end of file From f0dbae6c43d899953040b0bd903f1ae54cd31556 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Tue, 16 Sep 2025 01:57:57 +0000 Subject: [PATCH 5/5] Add comprehensive test coverage for Claude log parser edge cases and Docker output Co-authored-by: pelikhan <4175913+pelikhan@users.noreply.github.com> --- pkg/workflow/log_parser_docker_format_test.go | 88 +++++++++++++++++++ .../log_parser_new_format_file_test.go | 4 +- pkg/workflow/log_parser_new_format_test.go | 2 +- 3 files changed, 91 insertions(+), 3 deletions(-) create mode 100644 pkg/workflow/log_parser_docker_format_test.go diff --git a/pkg/workflow/log_parser_docker_format_test.go b/pkg/workflow/log_parser_docker_format_test.go new file mode 100644 index 0000000000..3341ca5379 --- /dev/null +++ b/pkg/workflow/log_parser_docker_format_test.go @@ -0,0 +1,88 @@ +package workflow + +import ( + "strings" + "testing" +) + +func TestParseClaudeLogDockerPullFormat(t *testing.T) { + // Test with the complex format that includes docker pull output + dockerPullLog := `npm warn exec The following package was not found and will be installed: @anthropic-ai/claude-code@1.0.115 +[DEBUG] Watching for changes in setting files /tmp/.claude/settings.json... +[ERROR] Failed to save config with lock: Error: ENOENT: no such file or directory, lstat '/home/runner/.claude.json' +[ERROR] MCP server "github" Server stderr: Unable to find image 'ghcr.io/github/github-mcp-server:sha-09deac4' locally +[DEBUG] Shell snapshot created successfully (242917 bytes) +[ERROR] MCP server "github" Server stderr: sha-09deac4: Pulling from github/github-mcp-server +[ERROR] MCP server "github" Server stderr: 35d697fe2738: Pulling fs layer +[ERROR] MCP server "github" Server stderr: bfb59b82a9b6: Pulling fs layer +4eff9a62d888: Pulling fs layer +62de241dac5f: Pulling fs layer +a62778643d56: Pulling fs layer +[ERROR] MCP server "github" Server stderr: bfb59b82a9b6: Verifying Checksum +bfb59b82a9b6: Download complete +[ERROR] MCP server "github" Server stderr: 4eff9a62d888: Verifying Checksum +{"type":"system","subtype":"init","cwd":"/home/runner/work/gh-aw/gh-aw","session_id":"test-123","tools":["Bash","Read"],"model":"claude-sonnet-4-20250514"} +{"type":"assistant","message":{"content":[{"type":"text","text":"I'll help you with this task."},{"type":"tool_use","id":"tool_123","name":"Bash","input":{"command":"echo 'Hello World'"}}]}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool_123","content":"Hello World"}]}} +{"type":"result","total_cost_usd":0.0015,"usage":{"input_tokens":150,"output_tokens":50},"num_turns":1}` + + engine := NewClaudeEngine() + + // Test parsing with the complex docker format + metrics := engine.ParseLogMetrics(dockerPullLog, true) + + // Should still extract the correct metrics + if metrics.TokenUsage != 200 { + t.Errorf("Expected token usage 200 (150+50), got %d", metrics.TokenUsage) + } + if metrics.EstimatedCost != 0.0015 { + t.Errorf("Expected cost 0.0015, got %f", metrics.EstimatedCost) + } + if metrics.Turns != 1 { + t.Errorf("Expected 1 turn, got %d", metrics.Turns) + } + + // Should count all the error lines including MCP server stderr and the initial [ERROR] + if metrics.ErrorCount < 5 { + t.Errorf("Expected at least 5 errors from various error lines, got %d", metrics.ErrorCount) + } + + t.Logf("Successfully parsed complex docker log with %d errors, %d tokens, cost $%.6f, %d turns", + metrics.ErrorCount, metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns) +} + +func TestParseClaudeLogDockerPullFormatJS(t *testing.T) { + // Test the JavaScript parser with complex docker pull format + script := GetLogParserScript("parse_claude_log") + if script == "" { + t.Skip("parse_claude_log script not available") + } + + dockerPullLog := `[DEBUG] Starting Claude +[ERROR] MCP server "github" Server stderr: Unable to find image 'ghcr.io/github/github-mcp-server:sha-09deac4' locally +[ERROR] MCP server "github" Server stderr: sha-09deac4: Pulling from github/github-mcp-server +4eff9a62d888: Pulling fs layer +62de241dac5f: Pulling fs layer +{"type":"system","subtype":"init","session_id":"test-123","tools":["Bash","Read"],"model":"claude-sonnet-4-20250514"} +{"type":"assistant","message":{"content":[{"type":"text","text":"Working on it."},{"type":"tool_use","id":"tool_456","name":"Bash","input":{"command":"ls -la"}}]}} +{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"tool_456","content":"total 0"}]}} +{"type":"result","total_cost_usd":0.002,"usage":{"input_tokens":100,"output_tokens":40},"num_turns":1}` + + result, err := runJSLogParser(script, dockerPullLog) + if err != nil { + t.Fatalf("Failed to parse docker pull format Claude log: %v", err) + } + + // Verify parsing worked correctly despite docker pull lines + if !strings.Contains(result, "🚀 Initialization") { + t.Error("Expected docker pull format Claude log output to contain Initialization section") + } + if !strings.Contains(result, "ls -la") { + t.Error("Expected docker pull format Claude log output to contain the bash command") + } + if !strings.Contains(result, "test-123") { + t.Error("Expected docker pull format Claude log output to contain session ID") + } + + t.Logf("JavaScript parser correctly handled docker pull format") +} diff --git a/pkg/workflow/log_parser_new_format_file_test.go b/pkg/workflow/log_parser_new_format_file_test.go index 482d50c6a0..06e45ec4b7 100644 --- a/pkg/workflow/log_parser_new_format_file_test.go +++ b/pkg/workflow/log_parser_new_format_file_test.go @@ -17,7 +17,7 @@ func TestParseClaudeLogNewFormatFile(t *testing.T) { metrics := engine.ParseLogMetrics(string(content), true) // Verify parsing worked correctly - t.Logf("Parsed metrics: Tokens=%d, Cost=%.6f, Turns=%d, Errors=%d", + t.Logf("Parsed metrics: Tokens=%d, Cost=%.6f, Turns=%d, Errors=%d", metrics.TokenUsage, metrics.EstimatedCost, metrics.Turns, metrics.ErrorCount) // Should extract the correct final result metrics @@ -72,4 +72,4 @@ func TestParseClaudeLogNewFormatJSScriptFromFile(t *testing.T) { } t.Logf("JavaScript parser output looks correct with proper sections") -} \ No newline at end of file +} diff --git a/pkg/workflow/log_parser_new_format_test.go b/pkg/workflow/log_parser_new_format_test.go index cb08f39a51..1ddc0cbb86 100644 --- a/pkg/workflow/log_parser_new_format_test.go +++ b/pkg/workflow/log_parser_new_format_test.go @@ -80,4 +80,4 @@ func TestParseClaudeLogNewFormatJSScript(t *testing.T) { if !strings.Contains(result, "test-123") { t.Error("Expected new format Claude log output to contain session ID") } -} \ No newline at end of file +}