diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..7eda318 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,67 @@ +name: CI + +on: + pull_request: + push: + branches: + - main + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + smoke: + name: Smoke Tests + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + + - name: Run smoke suite + run: make test-smoke + + go-ci: + name: Go Checks + runs-on: ubuntu-latest + needs: [smoke] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + cache-dependency-path: go.sum + + - name: Run formatting check + run: make fmt-check + + - name: Run go vet + run: make vet + + - name: Run smoke tests + run: make test-smoke + + - name: Run race tests + run: make race + + - name: Upload coverage artifact + run: go test -coverprofile=coverage.out ./... + + - name: Store coverage file + uses: actions/upload-artifact@v4 + with: + name: coverage.out + path: coverage.out diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..169685a --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,48 @@ +name: CodeQL + +on: + push: + branches: + - main + pull_request: + branches: + - main + schedule: + - cron: "19 3 * * 1" + +permissions: + actions: read + contents: read + security-events: write + +jobs: + analyze: + name: Analyze (${{ matrix.language }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + language: + - go + - actions + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Setup Go + if: matrix.language == 'go' + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + + - name: Build (Go) + if: matrix.language == 'go' + run: go build ./... + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/Makefile b/Makefile index ac4fef1..176eea8 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,96 @@ -.PHONY: all build run start stop restart clean test docker docker-build docker-run docs +SHELL := /bin/bash + +define PY_CODEQL_SUMMARY +import glob, json +from collections import Counter + +sarifs = sorted(glob.glob(".tmp/codeql/*.sarif")) +if not sarifs: + print("No SARIF files found under .tmp/codeql/. Did make code-ql run successfully?") + raise SystemExit(2) + +def first_location(result): + locs = result.get("locations") or [] + if not locs: + return ("unknown", 0) + pl = (locs[0].get("physicalLocation") or {}) + file = ((pl.get("artifactLocation") or {}).get("uri") or "unknown") + line = ((pl.get("region") or {}).get("startLine") or 0) + return (file, line) + +total = 0 +levels = Counter() +rows = [] + +for path in sarifs: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + for run in data.get("runs", []): + for r in (run.get("results") or []): + lvl = (r.get("level") or "warning").lower() + rule = r.get("ruleId") or "no-rule" + msg = (r.get("message") or {}).get("text") or "no-message" + file, line = first_location(r) + levels[lvl] += 1 + total += 1 + rows.append((lvl, rule, file, line, msg)) + +print("\nCodeQL SARIF Summary:") +for p in sarifs: + print(f" - {p}") +print(f"\n Total findings: {total}") +if total: + print(" By level: " + ", ".join(f"{k}={v}" for k, v in sorted(levels.items()))) +print("") + +priority = {"error": 0, "warning": 1, "note": 2, "recommendation": 3} +rows.sort(key=lambda x: (priority.get(x[0], 9), x[2], x[3], x[1])) + +limit = 50 +if not rows: + print(" OK: No findings.") +else: + print(f" Top {min(limit, len(rows))} findings:") + for i, (lvl, rule, file, line, msg) in enumerate(rows[:limit], 1): + msg = msg.replace("\n", " ").strip() + if len(msg) > 120: + msg = msg[:117] + "..." + print(f" {i:>2}. [{lvl}] {rule}") + print(f" {file}:{line}") + print(f" {msg}") + print("") +endef +export PY_CODEQL_SUMMARY + +define PY_CODEQL_GATE +import glob, json +sarifs = sorted(glob.glob(".tmp/codeql/*.sarif")) +errors = 0 +warnings = 0 +for path in sarifs: + with open(path, "r", encoding="utf-8") as f: + data = json.load(f) + for run in data.get("runs", []): + for r in (run.get("results") or []): + lvl = (r.get("level") or "warning").lower() + if lvl == "error": + errors += 1 + elif lvl == "warning": + warnings += 1 + +if errors > 0: + print(f"\nCodeQL gate failed: {errors} blocking error(s) found.") + if warnings > 0: + print(f"Also found {warnings} non-blocking warning(s).") + raise SystemExit(1) + +print("\nCodeQL gate passed: no blocking errors found.") +if warnings > 0: + print(f"Note: {warnings} non-blocking warning(s) were found.") +endef +export PY_CODEQL_GATE + +.PHONY: all build run start stop restart clean test check test-smoke vet race fmt fmt-check commit-check code-ql code-ql-summary code-ql-gate docker docker-build docker-run docs BIN_DIR=bin KAF_MIRROR_BINARY=$(BIN_DIR)/kaf-mirror @@ -56,14 +148,53 @@ clean: @rm -rf $(BIN_DIR) @rm -f $(PID_FILE) -test: - @echo "Running tests..." - @go test ./tests/... +test: check vet race + +check: test-smoke + +test-smoke: + @echo "Running smoke tests..." + @go test ./... + +vet: @echo "Running go vet..." @go vet ./... + +race: @echo "Running race tests..." @go test -race ./... +fmt: + @unformatted=$$(gofmt -l .); \ + if [ -n "$$unformatted" ]; then \ + echo "Found unformatted files. Auto-formatting:"; \ + echo "$$unformatted"; \ + gofmt -w .; \ + else \ + echo "All Go files are formatted correctly."; \ + fi + +fmt-check: + @unformatted=$$(gofmt -l .); \ + if [ -n "$$unformatted" ]; then \ + echo "The following files are not gofmt-formatted:"; \ + echo "$$unformatted"; \ + echo "Run 'make fmt' or 'gofmt -w .' to fix formatting."; \ + exit 1; \ + fi + +commit-check: fmt vet test-smoke race code-ql-gate + @echo "commit-check completed." + +code-ql: + bash scripts/codeql_local.sh + +code-ql-summary: code-ql + @printf "%s\n" "$$PY_CODEQL_SUMMARY" | python3 - + +code-ql-gate: code-ql-summary + @printf "%s\n" "$$PY_CODEQL_GATE" | python3 - + docker-build: @echo "Building Docker image..." @docker build -t kaf-mirror:latest . diff --git a/README.md b/README.md index 46a962c..12927e9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ # kaf-mirror +[![CI (Smoke+Go)](https://github.com/scalytics/kaf-mirror/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/scalytics/kaf-mirror/actions/workflows/ci.yml) +[![Release](https://github.com/scalytics/kaf-mirror/actions/workflows/release.yml/badge.svg?branch=main)](https://github.com/scalytics/kaf-mirror/actions/workflows/release.yml) +[![CodeQL](https://github.com/scalytics/kaf-mirror/actions/workflows/codeql.yml/badge.svg?branch=main)](https://github.com/scalytics/kaf-mirror/actions/workflows/codeql.yml) **kaf-mirror** is a high-performance, AI-enhanced Kafka replication tool designed for robust and intelligent data mirroring between Kafka clusters. diff --git a/cmd/admin-cli/main.go b/cmd/admin-cli/main.go index 368e252..03091a7 100644 --- a/cmd/admin-cli/main.go +++ b/cmd/admin-cli/main.go @@ -26,9 +26,9 @@ import ( "strings" "time" + "github.com/AlecAivazis/survey/v2" "github.com/jmoiron/sqlx" "github.com/spf13/cobra" - "github.com/AlecAivazis/survey/v2" ) var db *sqlx.DB @@ -54,6 +54,26 @@ func copyFile(src, dst string) error { return destFile.Sync() } +func writeSecretTempFile(prefix string, secret string) (string, error) { + file, err := os.CreateTemp("", prefix) + if err != nil { + return "", err + } + defer file.Close() + + if err := file.Chmod(0600); err != nil { + return "", err + } + if _, err := file.WriteString(secret + "\n"); err != nil { + return "", err + } + if err := file.Sync(); err != nil { + return "", err + } + + return file.Name(), nil +} + func main() { var dbPath string @@ -236,14 +256,19 @@ and for emergency maintenance. It interacts directly with the database.`, if err := database.UpdateUserPassword(db, user.ID, password); err != nil { log.Fatalf("Failed to reset password: %v", err) } + secretFile, err := writeSecretTempFile("kaf-mirror-admin-reset-*", password) + if err != nil { + log.Fatalf("Failed to write password file: %v", err) + } fmt.Println("=================================================================") fmt.Println(" ADMIN PASSWORD RESET") fmt.Println("=================================================================") fmt.Printf(" Username: %s\n", user.Username) - fmt.Printf(" New Password: %s\n", password) + fmt.Println(" New Password: [REDACTED]") + fmt.Printf(" Password File: %s\n", secretFile) fmt.Println("=================================================================") - fmt.Println(" Please store this password in a secure location.") + fmt.Println(" Deliver the password securely, then delete the file.") fmt.Println("=================================================================") }, } @@ -276,6 +301,10 @@ and for emergency maintenance. It interacts directly with the database.`, if err != nil { log.Fatalf("Failed to create admin user: %v", err) } + secretFile, err := writeSecretTempFile("kaf-mirror-bootstrap-*", password) + if err != nil { + log.Fatalf("Failed to write password file: %v", err) + } var adminRoleID int if err := db.Get(&adminRoleID, "SELECT id FROM roles WHERE name = 'admin'"); err != nil { @@ -289,9 +318,10 @@ and for emergency maintenance. It interacts directly with the database.`, fmt.Println(" INITIAL ADMIN USER CREATED") fmt.Println("=================================================================") fmt.Printf(" Username: %s\n", user.Username) - fmt.Printf(" Password: %s\n", password) + fmt.Println(" Password: [REDACTED]") + fmt.Printf(" Password File: %s\n", secretFile) fmt.Println("=================================================================") - fmt.Println(" Please store this password in a secure location.") + fmt.Println(" Deliver the password securely, then delete the file.") fmt.Println("=================================================================") }, } @@ -309,7 +339,7 @@ and for emergency maintenance. It interacts directly with the database.`, Run: func(cmd *cobra.Command, args []string) { outputPath := args[0] timestamp := time.Now().Format("20060102-150405") - + if !strings.HasSuffix(outputPath, ".db") { outputPath = fmt.Sprintf("%s-backup-%s.db", strings.TrimSuffix(outputPath, filepath.Ext(outputPath)), timestamp) } @@ -317,7 +347,7 @@ and for emergency maintenance. It interacts directly with the database.`, if err := copyFile(dbPath, outputPath); err != nil { log.Fatalf("Failed to backup database: %v", err) } - + fmt.Printf("Database backup created: %s\n", outputPath) }, } @@ -330,11 +360,11 @@ and for emergency maintenance. It interacts directly with the database.`, configPath, _ := cmd.Flags().GetString("config-path") outputPath := args[0] timestamp := time.Now().Format("20060102-150405") - + if _, err := os.Stat(configPath); os.IsNotExist(err) { log.Fatalf("Configuration file not found: %s", configPath) } - + if !strings.HasSuffix(outputPath, ".yml") { outputPath = fmt.Sprintf("%s-config-backup-%s.yml", strings.TrimSuffix(outputPath, filepath.Ext(outputPath)), timestamp) } @@ -342,7 +372,7 @@ and for emergency maintenance. It interacts directly with the database.`, if err := copyFile(configPath, outputPath); err != nil { log.Fatalf("Failed to backup configuration: %v", err) } - + fmt.Printf("Configuration backup created: %s\n", outputPath) }, } @@ -357,17 +387,17 @@ and for emergency maintenance. It interacts directly with the database.`, configPath, _ := cmd.Flags().GetString("config-path") timestamp := time.Now().Format("20060102-150405") backupDir := filepath.Join(outputDir, fmt.Sprintf("kaf-mirror-backup-%s", timestamp)) - + if err := os.MkdirAll(backupDir, 0755); err != nil { log.Fatalf("Failed to create backup directory: %v", err) } - + dbBackupPath := filepath.Join(backupDir, "database.db") if err := copyFile(dbPath, dbBackupPath); err != nil { log.Fatalf("Failed to backup database: %v", err) } fmt.Printf("Database backed up to: %s\n", dbBackupPath) - + if _, err := os.Stat(configPath); err == nil { configBackupPath := filepath.Join(backupDir, "config.yml") if err := copyFile(configPath, configBackupPath); err != nil { @@ -375,7 +405,7 @@ and for emergency maintenance. It interacts directly with the database.`, } fmt.Printf("Configuration backed up to: %s\n", configBackupPath) } - + fmt.Printf("Full backup created in: %s\n", backupDir) }, } @@ -393,11 +423,11 @@ and for emergency maintenance. It interacts directly with the database.`, Args: cobra.ExactArgs(1), Run: func(cmd *cobra.Command, args []string) { backupPath := args[0] - + if _, err := os.Stat(backupPath); os.IsNotExist(err) { log.Fatalf("Backup file not found: %s", backupPath) } - + confirm := false prompt := &survey.Confirm{ Message: "This will overwrite the current database. Are you sure?", @@ -407,11 +437,11 @@ and for emergency maintenance. It interacts directly with the database.`, fmt.Println("Restore cancelled.") return } - + if err := copyFile(backupPath, dbPath); err != nil { log.Fatalf("Failed to restore database: %v", err) } - + fmt.Printf("Database restored from: %s\n", backupPath) }, } @@ -423,11 +453,11 @@ and for emergency maintenance. It interacts directly with the database.`, Run: func(cmd *cobra.Command, args []string) { backupPath := args[0] configPath, _ := cmd.Flags().GetString("config-path") - + if _, err := os.Stat(backupPath); os.IsNotExist(err) { log.Fatalf("Backup file not found: %s", backupPath) } - + confirm := false prompt := &survey.Confirm{ Message: "This will overwrite the current configuration. Are you sure?", @@ -437,11 +467,11 @@ and for emergency maintenance. It interacts directly with the database.`, fmt.Println("Restore cancelled.") return } - + if err := copyFile(backupPath, configPath); err != nil { log.Fatalf("Failed to restore configuration: %v", err) } - + fmt.Printf("Configuration restored from: %s\n", backupPath) }, } @@ -460,11 +490,11 @@ and for emergency maintenance. It interacts directly with the database.`, Run: func(cmd *cobra.Command, args []string) { sourcePath := args[0] merge, _ := cmd.Flags().GetBool("merge") - + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { log.Fatalf("Source database not found: %s", sourcePath) } - + if !merge { confirm := false prompt := &survey.Confirm{ @@ -475,7 +505,7 @@ and for emergency maintenance. It interacts directly with the database.`, fmt.Println("Import cancelled.") return } - + if err := copyFile(sourcePath, dbPath); err != nil { log.Fatalf("Failed to import database: %v", err) } @@ -494,11 +524,11 @@ and for emergency maintenance. It interacts directly with the database.`, Run: func(cmd *cobra.Command, args []string) { sourcePath := args[0] configPath, _ := cmd.Flags().GetString("config-path") - + if _, err := os.Stat(sourcePath); os.IsNotExist(err) { log.Fatalf("Source configuration not found: %s", sourcePath) } - + confirm := false prompt := &survey.Confirm{ Message: "This will overwrite the current configuration. Are you sure?", @@ -508,11 +538,11 @@ and for emergency maintenance. It interacts directly with the database.`, fmt.Println("Import cancelled.") return } - + if err := copyFile(sourcePath, configPath); err != nil { log.Fatalf("Failed to import configuration: %v", err) } - + fmt.Printf("Configuration imported from: %s\n", sourcePath) }, } @@ -525,14 +555,14 @@ and for emergency maintenance. It interacts directly with the database.`, Run: func(cmd *cobra.Command, args []string) { sourceDir := args[0] configPath, _ := cmd.Flags().GetString("config-path") - + dbSourcePath := filepath.Join(sourceDir, "database.db") configSourcePath := filepath.Join(sourceDir, "config.yml") - + if _, err := os.Stat(sourceDir); os.IsNotExist(err) { log.Fatalf("Source directory not found: %s", sourceDir) } - + confirm := false prompt := &survey.Confirm{ Message: "This will overwrite database and configuration. Are you sure?", @@ -542,7 +572,7 @@ and for emergency maintenance. It interacts directly with the database.`, fmt.Println("Import cancelled.") return } - + if _, err := os.Stat(dbSourcePath); err == nil { if err := copyFile(dbSourcePath, dbPath); err != nil { log.Fatalf("Failed to import database: %v", err) @@ -551,7 +581,7 @@ and for emergency maintenance. It interacts directly with the database.`, } else { fmt.Println("No database found in source directory, skipping.") } - + if _, err := os.Stat(configSourcePath); err == nil { if err := copyFile(configSourcePath, configPath); err != nil { log.Fatalf("Failed to import configuration: %v", err) @@ -560,7 +590,7 @@ and for emergency maintenance. It interacts directly with the database.`, } else { fmt.Println("No configuration found in source directory, skipping.") } - + fmt.Printf("Full import completed from: %s\n", sourceDir) }, } diff --git a/cmd/kaf-mirror/main.go b/cmd/kaf-mirror/main.go index 8f186d2..73d20a9 100644 --- a/cmd/kaf-mirror/main.go +++ b/cmd/kaf-mirror/main.go @@ -37,6 +37,26 @@ var ( Version string ) +func writeSecretTempFile(prefix string, secret string) (string, error) { + file, err := os.CreateTemp("", prefix) + if err != nil { + return "", err + } + defer file.Close() + + if err := file.Chmod(0600); err != nil { + return "", err + } + if _, err := file.WriteString(secret + "\n"); err != nil { + return "", err + } + if err := file.Sync(); err != nil { + return "", err + } + + return file.Name(), nil +} + func main() { cfg, err := config.LoadConfig() if err != nil { @@ -76,6 +96,10 @@ func main() { if err != nil { log.Fatalf("Failed to create initial admin user: %v", err) } + secretFile, err := writeSecretTempFile("kaf-mirror-bootstrap-*", password) + if err != nil { + log.Fatalf("Failed to write password file: %v", err) + } var adminRoleID int if err := db.Get(&adminRoleID, "SELECT id FROM roles WHERE name = 'admin'"); err != nil { @@ -89,9 +113,10 @@ func main() { fmt.Println(" INITIAL ADMIN USER CREATED") fmt.Println("=================================================================") fmt.Printf(" Username: %s\n", user.Username) - fmt.Printf(" Password: %s\n", password) + fmt.Println(" Password: [REDACTED]") + fmt.Printf(" Password File: %s\n", secretFile) fmt.Println("=================================================================") - fmt.Println(" Please store this password in a secure location.") + fmt.Println(" Deliver the password securely, then delete the file.") fmt.Println("=================================================================") } fmt.Println("Database initialized successfully.") diff --git a/cmd/mirror-cli/dashboard/core/data_manager.go b/cmd/mirror-cli/dashboard/core/data_manager.go index 14ed359..fb29a4e 100644 --- a/cmd/mirror-cli/dashboard/core/data_manager.go +++ b/cmd/mirror-cli/dashboard/core/data_manager.go @@ -30,7 +30,7 @@ type DataManager struct { cache map[string]*CacheEntry cacheMutex sync.RWMutex token string - + fetchJobs func() ([]map[string]interface{}, error) fetchClusters func() ([]map[string]interface{}, error) fetchMetrics func(jobID string) (map[string]interface{}, error) @@ -214,13 +214,13 @@ func (dm *DataManager) InvalidateAllCache() { func (dm *DataManager) GetCacheStats() map[string]int { dm.cacheMutex.RLock() defer dm.cacheMutex.RUnlock() - + stats := map[string]int{ "total_entries": len(dm.cache), "expired": 0, "valid": 0, } - + for _, entry := range dm.cache { if entry.IsExpired() { stats["expired"]++ @@ -228,14 +228,14 @@ func (dm *DataManager) GetCacheStats() map[string]int { stats["valid"]++ } } - + return stats } func (dm *DataManager) CleanExpiredEntries() { dm.cacheMutex.Lock() defer dm.cacheMutex.Unlock() - + for key, entry := range dm.cache { if entry.IsExpired() { delete(dm.cache, key) diff --git a/cmd/mirror-cli/dashboard/core/router.go b/cmd/mirror-cli/dashboard/core/router.go index 88c6588..878be8b 100644 --- a/cmd/mirror-cli/dashboard/core/router.go +++ b/cmd/mirror-cli/dashboard/core/router.go @@ -66,7 +66,7 @@ func (er *EventRouter) routeTopLevelEvent(event ui.Event) EventAction { case "": return SelectItemAction{} } - + return NoAction{} } @@ -102,7 +102,7 @@ func (er *EventRouter) routeCategoryEvent(event ui.Event) EventAction { return JobControlAction{Action: "restart"} } } - + return NoAction{} } @@ -119,7 +119,7 @@ func (er *EventRouter) routeDetailEvent(event ui.Event) EventAction { case "r", "R": return RefreshAction{} } - + return NoAction{} } diff --git a/cmd/mirror-cli/dashboard/core/state.go b/cmd/mirror-cli/dashboard/core/state.go index 980c817..3e1a734 100644 --- a/cmd/mirror-cli/dashboard/core/state.go +++ b/cmd/mirror-cli/dashboard/core/state.go @@ -63,14 +63,14 @@ func (nc *NavigationContext) NavigateToCategory(category Category) { nc.Category = category nc.ItemID = "" nc.PaneFocus = ListPaneFocus // Default to list pane - + categoryNames := map[Category]string{ ClustersCategory: "Clusters", JobsCategory: "Jobs", InsightsCategory: "Insights", ComplianceCategory: "Compliance", } - + nc.Breadcrumbs = []string{"Dashboard", categoryNames[category]} nc.LastUpdate = time.Now() } @@ -78,13 +78,13 @@ func (nc *NavigationContext) NavigateToCategory(category Category) { func (nc *NavigationContext) NavigateToDetail(itemID, itemName string) { nc.State = DetailViewState nc.ItemID = itemID - + if len(nc.Breadcrumbs) == 2 { nc.Breadcrumbs = append(nc.Breadcrumbs, itemName) } else if len(nc.Breadcrumbs) >= 3 { nc.Breadcrumbs[2] = itemName } - + nc.LastUpdate = time.Now() } diff --git a/cmd/mirror-cli/dashboard/dashboard.go b/cmd/mirror-cli/dashboard/dashboard.go index d977d7f..bbcfa46 100644 --- a/cmd/mirror-cli/dashboard/dashboard.go +++ b/cmd/mirror-cli/dashboard/dashboard.go @@ -26,15 +26,15 @@ type Dashboard struct { context *core.NavigationContext router *core.EventRouter dataManager *core.DataManager - + jobsFactory *widgets.JobsFactory clustersFactory *widgets.ClustersFactory insightsFactory *widgets.InsightsFactory complianceFactory *widgets.ComplianceFactory - - categoryLayout *layouts.CategoryLayout - topLevelGrid *ui.Grid - updateTicker *time.Ticker + + categoryLayout *layouts.CategoryLayout + topLevelGrid *ui.Grid + updateTicker *time.Ticker focusedCategory core.Category } @@ -42,12 +42,12 @@ func NewDashboard(token string, fetchers core.DataFetchers) *Dashboard { context := core.NewNavigationContext() router := core.NewEventRouter(context) dataManager := core.NewDataManager(token, fetchers) - + jobsFactory := widgets.NewJobsFactory() clustersFactory := widgets.NewClustersFactory() insightsFactory := widgets.NewInsightsFactory() complianceFactory := widgets.NewComplianceFactory() - + dashboard := &Dashboard{ context: context, router: router, @@ -58,7 +58,7 @@ func NewDashboard(token string, fetchers core.DataFetchers) *Dashboard { complianceFactory: complianceFactory, focusedCategory: core.ClustersCategory, // Default focus on first category } - + return dashboard } @@ -71,31 +71,31 @@ func (d *Dashboard) updateTopLevelFocus() { if d.topLevelGrid == nil { return } - + clustersWidget := termui.NewParagraph() clustersWidget.Title = "1. Clusters" clustersWidget.Text = d.getClustersOverview() clustersWidget.BorderStyle = ui.NewStyle(ui.ColorCyan) clustersWidget.WrapText = true - + jobsWidget := termui.NewParagraph() jobsWidget.Title = "2. Jobs" jobsWidget.Text = d.getJobsOverview() jobsWidget.BorderStyle = ui.NewStyle(ui.ColorGreen) jobsWidget.WrapText = true - + insightsWidget := termui.NewParagraph() insightsWidget.Title = "3. Insights" insightsWidget.Text = d.getInsightsOverview() insightsWidget.BorderStyle = ui.NewStyle(ui.ColorMagenta) insightsWidget.WrapText = true - + complianceWidget := termui.NewParagraph() complianceWidget.Title = "4. Compliance" complianceWidget.Text = d.getComplianceOverview() complianceWidget.BorderStyle = ui.NewStyle(ui.ColorYellow) complianceWidget.WrapText = true - + // Apply focus styling - make focused widget brighter/bold switch d.focusedCategory { case core.ClustersCategory: @@ -111,7 +111,7 @@ func (d *Dashboard) updateTopLevelFocus() { complianceWidget.BorderStyle = ui.NewStyle(ui.ColorWhite) complianceWidget.Title = ">>> 4. Compliance <<<" } - + termWidth, termHeight := ui.TerminalDimensions() d.topLevelGrid.SetRect(0, 0, termWidth, termHeight) d.topLevelGrid.Set( @@ -131,18 +131,18 @@ func (d *Dashboard) Run() error { return err } defer ui.Close() - + // Set up layout after UI is initialized d.setupTopLevelLayout() - + d.render() - + // Set up update ticker for live data d.updateTicker = time.NewTicker(2 * time.Second) defer d.updateTicker.Stop() - + uiEvents := ui.PollEvents() - + for { select { case e := <-uiEvents: @@ -150,7 +150,7 @@ func (d *Dashboard) Run() error { return nil // Exit requested } d.render() - + case <-d.updateTicker.C: d.updateData() d.render() @@ -160,11 +160,11 @@ func (d *Dashboard) Run() error { func (d *Dashboard) handleEvent(event ui.Event) bool { action := d.router.RouteEvent(event) - + switch a := action.(type) { case core.ExitAction: return true - + case core.BackAction: if d.context.IsAtTopLevel() { // Exit dashboard when at top level @@ -173,36 +173,36 @@ func (d *Dashboard) handleEvent(event ui.Event) bool { d.context.NavigateBack() d.setupCurrentView() } - + case core.NavigateToCategoryAction: d.context.NavigateToCategory(a.Category) d.setupCurrentView() - + case core.ScrollAction: d.handleScroll(a.Direction) - + case core.SelectItemAction: d.handleSelection() - + case core.RefreshAction: d.dataManager.InvalidateAllCache() d.updateData() - + case core.JobControlAction: if d.context.Category == core.JobsCategory { d.jobsFactory.HandleAction(action, d.dataManager) } - + case core.MoveFocusAction: d.handleMoveFocus(a.Direction) - + case core.SwitchPaneFocusAction: d.handleSwitchPaneFocus(a.Direction) - + case core.NoAction: // Do nothing } - + return false } @@ -222,7 +222,7 @@ func (d *Dashboard) handleMoveFocus(direction core.Direction) { if !d.context.IsAtTopLevel() { return } - + // Navigate between categories based on grid layout switch direction { case core.Up: @@ -250,7 +250,7 @@ func (d *Dashboard) handleMoveFocus(direction core.Direction) { d.focusedCategory = core.ComplianceCategory } } - + d.updateTopLevelFocus() } @@ -258,7 +258,7 @@ func (d *Dashboard) handleSwitchPaneFocus(direction core.Direction) { if !d.context.IsInCategory() { return } - + switch direction { case core.Right: if d.context.PaneFocus == core.ListPaneFocus { @@ -269,7 +269,7 @@ func (d *Dashboard) handleSwitchPaneFocus(direction core.Direction) { d.context.PaneFocus = core.ListPaneFocus } } - + // Update visual focus in category layout if d.categoryLayout != nil { d.categoryLayout.UpdatePaneFocus(d.context.PaneFocus) @@ -290,11 +290,11 @@ func (d *Dashboard) handleSelection() { func (d *Dashboard) setupCurrentView() { termWidth, termHeight := ui.TerminalDimensions() - + switch d.context.State { case core.TopLevelState: // Already set up in setupTopLevelLayout - + case core.CategoryViewState: factory := d.getCurrentFactory() if factory != nil { @@ -303,19 +303,19 @@ func (d *Dashboard) setupCurrentView() { // Apply initial pane focus styling d.categoryLayout.UpdatePaneFocus(d.context.PaneFocus) } - + case core.DetailViewState: factory := d.getCurrentFactory() if factory != nil { // Update detail data first factory.UpdateDetailData(d.dataManager, d.context.ItemID) - + // Set up full-screen detail view detailWidget := factory.CreateDetailWidget(d.context.ItemID) d.topLevelGrid = ui.NewGrid() d.topLevelGrid.SetRect(0, 0, termWidth, termHeight) d.topLevelGrid.Set(ui.NewRow(1.0, ui.NewCol(1.0, detailWidget))) - + // Clear category layout as we're using topLevelGrid for detail d.categoryLayout = nil } @@ -352,12 +352,12 @@ func (d *Dashboard) render() { switch d.context.State { case core.TopLevelState: ui.Render(d.topLevelGrid) - + case core.CategoryViewState: if d.categoryLayout != nil { ui.Render(d.categoryLayout.GetGrid()) } - + case core.DetailViewState: // Use topLevelGrid for detail view (set up in setupCurrentView) ui.Render(d.topLevelGrid) @@ -374,11 +374,11 @@ func (d *Dashboard) getClustersOverview() string { if err != nil { return "Error loading clusters" } - + if len(clusters) == 0 { return "No clusters configured" } - + text := "" for i, cluster := range clusters { if i >= 3 { // Show max 3 clusters @@ -389,11 +389,11 @@ func (d *Dashboard) getClustersOverview() string { status := widgets.SafeString(cluster["status"], "unknown") text += fmt.Sprintf("• %s (%s)\n", widgets.TruncateString(name, 20), status) } - + if len(clusters) > 3 { text += fmt.Sprintf("Total: %d clusters", len(clusters)) } - + return text } @@ -402,11 +402,11 @@ func (d *Dashboard) getJobsOverview() string { if err != nil { return "Error loading jobs" } - + if len(jobs) == 0 { return "No replication jobs" } - + text := "" for i, job := range jobs { if i >= 3 { // Show max 3 jobs @@ -417,11 +417,11 @@ func (d *Dashboard) getJobsOverview() string { status := widgets.SafeString(job["status"], "unknown") text += fmt.Sprintf("• %s (%s)\n", widgets.TruncateString(name, 20), status) } - + if len(jobs) > 3 { text += fmt.Sprintf("Total: %d jobs", len(jobs)) } - + return text } @@ -430,11 +430,11 @@ func (d *Dashboard) getInsightsOverview() string { if err != nil { return "Error loading insights" } - + if len(insights) == 0 { return "No AI insights available" } - + text := "" for i, insight := range insights { if i >= 5 { // Show max 5 insights @@ -445,11 +445,11 @@ func (d *Dashboard) getInsightsOverview() string { severity := widgets.SafeString(insight["severity_level"], "normal") text += fmt.Sprintf("• %s (%s)\n", widgets.TruncateString(insightType, 20), severity) } - + if len(insights) > 5 { text += fmt.Sprintf("Total: %d insights", len(insights)) } - + return text } @@ -458,11 +458,11 @@ func (d *Dashboard) getComplianceOverview() string { if err != nil { return "Error loading compliance logs" } - + if len(logs) == 0 { return "No compliance events" } - + text := "" for i, log := range logs { if i >= 3 { // Show max 3 compliance entries @@ -473,11 +473,11 @@ func (d *Dashboard) getComplianceOverview() string { user := widgets.SafeString(log["user"], "System") text += fmt.Sprintf("• %s by %s\n", widgets.TruncateString(action, 15), widgets.TruncateString(user, 12)) } - + if len(logs) > 3 { text += fmt.Sprintf("Total: %d events", len(logs)) } - + return text } diff --git a/cmd/mirror-cli/dashboard/layouts/category.go b/cmd/mirror-cli/dashboard/layouts/category.go index 3271119..f3522d8 100644 --- a/cmd/mirror-cli/dashboard/layouts/category.go +++ b/cmd/mirror-cli/dashboard/layouts/category.go @@ -14,7 +14,7 @@ package layouts import ( "kaf-mirror/cmd/mirror-cli/dashboard/core" "kaf-mirror/cmd/mirror-cli/dashboard/widgets" - + ui "github.com/gizak/termui/v3" ui_widgets "github.com/gizak/termui/v3/widgets" ) @@ -27,7 +27,7 @@ type CategoryLayout struct { func NewCategoryLayout(factory widgets.WidgetFactory, context *core.NavigationContext) *CategoryLayout { grid := ui.NewGrid() - + return &CategoryLayout{ grid: grid, factory: factory, @@ -37,14 +37,14 @@ func NewCategoryLayout(factory widgets.WidgetFactory, context *core.NavigationCo func (cl *CategoryLayout) Setup(termWidth, termHeight int) { cl.grid.SetRect(0, 0, termWidth, termHeight) - + if cl.factory == nil { return } - + listWidget := cl.factory.CreateListWidget() detailWidget := cl.factory.CreateDetailWidget("") - + cl.grid.Set( ui.NewRow(1.0, ui.NewCol(0.4, listWidget), @@ -61,18 +61,18 @@ func (cl *CategoryLayout) UpdateData(dataManager *core.DataManager) error { if cl.factory == nil { return nil } - + if err := cl.factory.UpdateListData(dataManager); err != nil { return err } - + if cl.context.ItemID != "" { if err := cl.factory.UpdateDetailData(dataManager, cl.context.ItemID); err != nil { return err } cl.factory.ResetDetailCursor() } - + return nil } @@ -80,7 +80,7 @@ func (cl *CategoryLayout) HandleScroll(direction core.Direction) { if cl.factory == nil { return } - + switch cl.context.PaneFocus { case core.ListPaneFocus: cl.factory.ScrollList(direction) @@ -93,12 +93,12 @@ func (cl *CategoryLayout) UpdatePaneFocus(focus core.PaneFocus) { if cl.factory == nil { return } - + listWidget := cl.factory.CreateListWidget() detailWidget := cl.factory.CreateDetailWidget("") - + cl.factory.SetDetailCursorVisible(focus == core.DetailPaneFocus) - + switch focus { case core.ListPaneFocus: if lw, ok := listWidget.(*ui_widgets.List); ok { @@ -121,9 +121,9 @@ func (cl *CategoryLayout) HandleSelection() { if cl.factory == nil { return } - + selectedID := cl.factory.GetSelectedItemID() - + if selectedID != "" { cl.context.ItemID = selectedID } diff --git a/cmd/mirror-cli/dashboard/widgets/clusters.go b/cmd/mirror-cli/dashboard/widgets/clusters.go index 8d93ee2..206d2a9 100644 --- a/cmd/mirror-cli/dashboard/widgets/clusters.go +++ b/cmd/mirror-cli/dashboard/widgets/clusters.go @@ -11,14 +11,14 @@ type ClustersFactory struct { func NewClustersFactory() *ClustersFactory { base := NewBaseWidgetFactory() - + cf := &ClustersFactory{ BaseWidgetFactory: base, } - + cf.listWidget.Title = "Clusters (Press Enter for details, R to refresh)" cf.detailWidget.Title = "Cluster Health" - + return cf } @@ -31,13 +31,13 @@ func (cf *ClustersFactory) UpdateListData(dataManager *core.DataManager) error { } return err } - + cf.items = clusters - + rows := []string{ "Name | Provider | Status | Health | Brokers", } - + if len(clusters) == 0 { rows = append(rows, "No clusters configured") } else { @@ -46,7 +46,7 @@ func (cf *ClustersFactory) UpdateListData(dataManager *core.DataManager) error { provider := SafeString(cluster["provider"], "N/A") status := SafeString(cluster["status"], "unknown") brokers := SafeString(cluster["brokers"], "N/A") - + // Determine health status based on status and other factors healthStatus := "Unknown" switch status { @@ -59,11 +59,11 @@ func (cf *ClustersFactory) UpdateListData(dataManager *core.DataManager) error { default: healthStatus = "? " + status } - + if len(brokers) > 25 { brokers = brokers[:22] + "..." } - + row := fmt.Sprintf("%s | %s | %s | %s | %s", TruncateString(name, 15), TruncateString(provider, 10), @@ -71,13 +71,13 @@ func (cf *ClustersFactory) UpdateListData(dataManager *core.DataManager) error { healthStatus, brokers, ) - + rows = append(rows, row) } } - + cf.listWidget.Rows = rows - + // Maintain selection within bounds if cf.selectedIdx >= len(cf.items) { cf.selectedIdx = len(cf.items) - 1 @@ -88,7 +88,7 @@ func (cf *ClustersFactory) UpdateListData(dataManager *core.DataManager) error { if len(cf.items) > 0 { cf.listWidget.SelectedRow = cf.selectedIdx + 1 // +1 for header } - + return nil } @@ -97,7 +97,7 @@ func (cf *ClustersFactory) UpdateDetailData(dataManager *core.DataManager, itemI cf.detailWidget.Rows = []string{"Select a cluster from the list to view details"} return nil } - + cluster, err := dataManager.GetClusterDetails(itemID) if err != nil { cf.detailWidget.Rows = []string{ @@ -106,9 +106,9 @@ func (cf *ClustersFactory) UpdateDetailData(dataManager *core.DataManager, itemI } return err } - + var rows []string - + rows = append(rows, "=== CLUSTER INFORMATION ===") rows = append(rows, fmt.Sprintf("Name: %s", SafeString(cluster["name"], "N/A"))) rows = append(rows, fmt.Sprintf("Provider: %s", SafeString(cluster["provider"], "N/A"))) @@ -119,14 +119,14 @@ func (cf *ClustersFactory) UpdateDetailData(dataManager *core.DataManager, itemI rows = append(rows, fmt.Sprintf("Brokers: %s", SafeString(cluster["brokers"], "N/A"))) rows = append(rows, fmt.Sprintf("Created: %s", SafeString(cluster["created_at"], "N/A"))) rows = append(rows, fmt.Sprintf("Updated: %s", SafeString(cluster["updated_at"], "N/A"))) - + rows = append(rows, "") rows = append(rows, "=== HEALTH STATUS ===") - + status := SafeString(cluster["status"], "unknown") connectionStatus := "Unknown" overallHealth := "Unknown" - + // Simulate connection test results (in real implementation, this would call testClusterConnectionVerbose) switch status { case "active": @@ -142,25 +142,25 @@ func (cf *ClustersFactory) UpdateDetailData(dataManager *core.DataManager, itemI connectionStatus = "❓ Status unknown" overallHealth = "❓ UNKNOWN - Unable to determine health" } - + rows = append(rows, fmt.Sprintf("Connection Status: %s", connectionStatus)) rows = append(rows, fmt.Sprintf("Overall Health: %s", overallHealth)) - + rows = append(rows, "") rows = append(rows, "=== SECURITY CONFIGURATION ===") - + if apiKey := cluster["api_key"]; apiKey != nil && SafeString(apiKey, "") != "" { rows = append(rows, "Authentication: ✅ API Key configured") rows = append(rows, fmt.Sprintf("API Key: %s", maskSensitiveValue(SafeString(apiKey, "")))) } else { rows = append(rows, "Authentication: ❌ No authentication configured") } - + provider := SafeString(cluster["provider"], "unknown") rows = append(rows, "") rows = append(rows, "=== PROVIDER DETAILS ===") rows = append(rows, fmt.Sprintf("Provider Type: %s", provider)) - + switch provider { case "confluent": rows = append(rows, "Platform: Confluent Cloud") @@ -181,10 +181,10 @@ func (cf *ClustersFactory) UpdateDetailData(dataManager *core.DataManager, itemI default: rows = append(rows, "Platform: Unknown/Custom") } - + rows = append(rows, "") rows = append(rows, "=== TOPIC INFORMATION ===") - + if status == "active" { rows = append(rows, "Topics Available: Calculating...") rows = append(rows, "Sample Topics:") @@ -195,10 +195,10 @@ func (cf *ClustersFactory) UpdateDetailData(dataManager *core.DataManager, itemI } else { rows = append(rows, "Topics: Cannot fetch - cluster not accessible") } - + rows = append(rows, "") rows = append(rows, "=== PERFORMANCE INDICATORS ===") - + if status == "active" { rows = append(rows, "Broker Response Time: < 10ms") rows = append(rows, "Connection Pool: 8/10 connections") @@ -208,16 +208,16 @@ func (cf *ClustersFactory) UpdateDetailData(dataManager *core.DataManager, itemI rows = append(rows, "Performance metrics unavailable") rows = append(rows, "Reason: Cluster not active or accessible") } - + rows = append(rows, "") rows = append(rows, "=== CLUSTER OPERATIONS ===") rows = append(rows, "Press 'R' to refresh cluster data") rows = append(rows, "Press 'B' or Escape to go back") rows = append(rows, "Use 'mirror-cli clusters test [name]' for detailed diagnostics") - + cf.detailWidget.Rows = rows cf.detailWidget.Title = fmt.Sprintf("Cluster Health: %s", SafeString(cluster["name"], itemID)) - + return nil } diff --git a/cmd/mirror-cli/dashboard/widgets/compliance.go b/cmd/mirror-cli/dashboard/widgets/compliance.go index d3ff0b9..6a05d92 100644 --- a/cmd/mirror-cli/dashboard/widgets/compliance.go +++ b/cmd/mirror-cli/dashboard/widgets/compliance.go @@ -11,14 +11,14 @@ type ComplianceFactory struct { func NewComplianceFactory() *ComplianceFactory { base := NewBaseWidgetFactory() - + cf := &ComplianceFactory{ BaseWidgetFactory: base, } - + cf.listWidget.Title = "Compliance Logs (Press Enter for details, R to refresh)" cf.detailWidget.Title = "Audit Log Details" - + return cf } @@ -31,13 +31,13 @@ func (cf *ComplianceFactory) UpdateListData(dataManager *core.DataManager) error } return err } - + cf.items = logs - + rows := []string{ "Timestamp | User | Action | Resource | Result", } - + if len(logs) == 0 { rows = append(rows, "No audit logs found") } else { @@ -47,7 +47,7 @@ func (cf *ComplianceFactory) UpdateListData(dataManager *core.DataManager) error action := SafeString(log["event_type"], "unknown") resource := SafeString(log["resource_type"], "N/A") result := SafeString(log["status"], "unknown") - + row := fmt.Sprintf("%s | %s | %s | %s | %s", TruncateString(timestamp, 16), TruncateString(user, 12), @@ -55,13 +55,13 @@ func (cf *ComplianceFactory) UpdateListData(dataManager *core.DataManager) error TruncateString(resource, 12), FormatStatus(result), ) - + rows = append(rows, row) } } - + cf.listWidget.Rows = rows - + if cf.selectedIdx >= len(cf.items) { cf.selectedIdx = len(cf.items) - 1 } @@ -71,7 +71,7 @@ func (cf *ComplianceFactory) UpdateListData(dataManager *core.DataManager) error if len(cf.items) > 0 { cf.listWidget.SelectedRow = cf.selectedIdx + 1 } - + return nil } @@ -80,7 +80,7 @@ func (cf *ComplianceFactory) UpdateDetailData(dataManager *core.DataManager, ite cf.detailWidget.Rows = []string{"Select an audit log entry to view details"} return nil } - + logs, err := dataManager.GetLogs() if err != nil { cf.detailWidget.Rows = []string{ @@ -89,7 +89,7 @@ func (cf *ComplianceFactory) UpdateDetailData(dataManager *core.DataManager, ite } return err } - + var selectedLog map[string]interface{} for _, log := range logs { if SafeString(log["id"], "") == itemID { @@ -97,14 +97,14 @@ func (cf *ComplianceFactory) UpdateDetailData(dataManager *core.DataManager, ite break } } - + if selectedLog == nil { cf.detailWidget.Rows = []string{"Log entry not found"} return nil } - + var rows []string - + rows = append(rows, "=== AUDIT LOG ENTRY ===") rows = append(rows, fmt.Sprintf("Timestamp: %s", SafeString(selectedLog["timestamp"], "N/A"))) rows = append(rows, fmt.Sprintf("Event Type: %s", SafeString(selectedLog["event_type"], "N/A"))) @@ -112,10 +112,10 @@ func (cf *ComplianceFactory) UpdateDetailData(dataManager *core.DataManager, ite rows = append(rows, fmt.Sprintf("Resource Type: %s", SafeString(selectedLog["resource_type"], "N/A"))) rows = append(rows, fmt.Sprintf("Resource ID: %s", SafeString(selectedLog["resource_id"], "N/A"))) rows = append(rows, fmt.Sprintf("Status: %s", FormatStatus(SafeString(selectedLog["status"], "N/A")))) - + rows = append(rows, "") rows = append(rows, "=== EVENT DETAILS ===") - + details := SafeString(selectedLog["details"], "No additional details available") // Split long details into multiple lines for len(details) > 0 { @@ -136,10 +136,10 @@ func (cf *ComplianceFactory) UpdateDetailData(dataManager *core.DataManager, ite details = details[1:] } } - + rows = append(rows, "") rows = append(rows, "=== COMPLIANCE CONTEXT ===") - + eventType := SafeString(selectedLog["event_type"], "") switch eventType { case "job_created", "job_updated", "job_deleted": @@ -163,7 +163,7 @@ func (cf *ComplianceFactory) UpdateDetailData(dataManager *core.DataManager, ite rows = append(rows, "Compliance Impact: Standard operational logging") rows = append(rows, "Retention: 1 year per default policy") } - + rows = append(rows, "") rows = append(rows, "=== AUDIT TRAIL ===") if sourceIP := SafeString(selectedLog["source_ip"], ""); sourceIP != "" { @@ -175,16 +175,16 @@ func (cf *ComplianceFactory) UpdateDetailData(dataManager *core.DataManager, ite if sessionID := SafeString(selectedLog["session_id"], ""); sessionID != "" { rows = append(rows, fmt.Sprintf("Session ID: %s", sessionID)) } - + rows = append(rows, "") rows = append(rows, "=== NAVIGATION ===") rows = append(rows, "Press 'R' to refresh audit logs") rows = append(rows, "Press 'B' or Escape to go back") rows = append(rows, "Use audit export tools for compliance reporting") - + cf.detailWidget.Rows = rows cf.detailWidget.Title = fmt.Sprintf("Audit Log: %s", SafeString(selectedLog["event_type"], itemID)) - + return nil } diff --git a/cmd/mirror-cli/dashboard/widgets/factory.go b/cmd/mirror-cli/dashboard/widgets/factory.go index df585cb..a49a2f8 100644 --- a/cmd/mirror-cli/dashboard/widgets/factory.go +++ b/cmd/mirror-cli/dashboard/widgets/factory.go @@ -14,7 +14,7 @@ package widgets import ( "fmt" "kaf-mirror/cmd/mirror-cli/dashboard/core" - + ui "github.com/gizak/termui/v3" "github.com/gizak/termui/v3/widgets" ) @@ -34,11 +34,11 @@ type WidgetFactory interface { } type BaseWidgetFactory struct { - listWidget *widgets.List - detailWidget *widgets.List - selectedIdx int + listWidget *widgets.List + detailWidget *widgets.List + selectedIdx int detailCursorIdx int - items []map[string]interface{} + items []map[string]interface{} } func NewBaseWidgetFactory() *BaseWidgetFactory { @@ -107,7 +107,7 @@ func (bwf *BaseWidgetFactory) ScrollDetail(direction core.Direction) { if maxRows == 0 { return } - + switch direction { case core.Down: if bwf.detailCursorIdx < maxRows-1 { @@ -128,7 +128,7 @@ func (bwf *BaseWidgetFactory) ScrollDetail(direction core.Direction) { bwf.detailCursorIdx = 0 } } - + bwf.detailWidget.SelectedRow = bwf.detailCursorIdx } @@ -153,7 +153,7 @@ func (bwf *BaseWidgetFactory) GetSelectedItemID() string { if bwf.selectedIdx < 0 || bwf.selectedIdx >= len(bwf.items) { return "" } - + item := bwf.items[bwf.selectedIdx] // Try id field first (handle both string and integer IDs) if id := item["id"]; id != nil { @@ -169,7 +169,7 @@ func (bwf *BaseWidgetFactory) GetSelectedItemName() string { if bwf.selectedIdx < 0 || bwf.selectedIdx >= len(bwf.items) { return "" } - + item := bwf.items[bwf.selectedIdx] // Try name field first, fallback to id if name, ok := item["name"].(string); ok { diff --git a/cmd/mirror-cli/dashboard/widgets/insights.go b/cmd/mirror-cli/dashboard/widgets/insights.go index 6f6362d..419d0a8 100644 --- a/cmd/mirror-cli/dashboard/widgets/insights.go +++ b/cmd/mirror-cli/dashboard/widgets/insights.go @@ -11,14 +11,14 @@ type InsightsFactory struct { func NewInsightsFactory() *InsightsFactory { base := NewBaseWidgetFactory() - + inf := &InsightsFactory{ BaseWidgetFactory: base, } - + inf.listWidget.Title = "AI Insights (Press Enter for details, R to refresh)" inf.detailWidget.Title = "Insight Details" - + return inf } @@ -31,13 +31,13 @@ func (inf *InsightsFactory) UpdateListData(dataManager *core.DataManager) error } return err } - + inf.items = insights - + rows := []string{ "Type | Timestamp | Severity | Job ID | Status", } - + if len(insights) == 0 { rows = append(rows, "No AI insights available") } else { @@ -47,7 +47,7 @@ func (inf *InsightsFactory) UpdateListData(dataManager *core.DataManager) error severity := SafeString(insight["severity_level"], "normal") jobID := SafeString(insight["job_id"], "N/A") status := SafeString(insight["resolution_status"], "pending") - + row := fmt.Sprintf("%s | %s | %s | %s | %s", TruncateString(insightType, 12), TruncateString(timestamp, 16), @@ -55,13 +55,13 @@ func (inf *InsightsFactory) UpdateListData(dataManager *core.DataManager) error TruncateString(jobID, 12), FormatStatus(status), ) - + rows = append(rows, row) } } - + inf.listWidget.Rows = rows - + if inf.selectedIdx >= len(inf.items) { inf.selectedIdx = len(inf.items) - 1 } @@ -71,7 +71,7 @@ func (inf *InsightsFactory) UpdateListData(dataManager *core.DataManager) error if len(inf.items) > 0 { inf.listWidget.SelectedRow = inf.selectedIdx + 1 } - + return nil } @@ -80,7 +80,7 @@ func (inf *InsightsFactory) UpdateDetailData(dataManager *core.DataManager, item inf.detailWidget.Rows = []string{"Select an AI insight to view details"} return nil } - + insights, err := dataManager.GetAIInsights() if err != nil { inf.detailWidget.Rows = []string{ @@ -89,7 +89,7 @@ func (inf *InsightsFactory) UpdateDetailData(dataManager *core.DataManager, item } return err } - + var selectedInsight map[string]interface{} for _, insight := range insights { insightID := fmt.Sprintf("%v", insight["id"]) @@ -98,14 +98,14 @@ func (inf *InsightsFactory) UpdateDetailData(dataManager *core.DataManager, item break } } - + if selectedInsight == nil { inf.detailWidget.Rows = []string{"Insight not found"} return nil } - + var rows []string - + rows = append(rows, "=== AI INSIGHT DETAILS ===") rows = append(rows, fmt.Sprintf("ID: %v", selectedInsight["id"])) rows = append(rows, fmt.Sprintf("Type: %s", SafeString(selectedInsight["insight_type"], "N/A"))) @@ -113,20 +113,20 @@ func (inf *InsightsFactory) UpdateDetailData(dataManager *core.DataManager, item rows = append(rows, fmt.Sprintf("Status: %s", FormatStatus(SafeString(selectedInsight["resolution_status"], "N/A")))) rows = append(rows, fmt.Sprintf("Job ID: %s", SafeString(selectedInsight["job_id"], "N/A"))) rows = append(rows, fmt.Sprintf("Created: %s", SafeString(selectedInsight["timestamp"], "N/A"))) - + if resolvedAt := SafeString(selectedInsight["resolved_at"], ""); resolvedAt != "" { rows = append(rows, fmt.Sprintf("Resolved: %s", resolvedAt)) } - + rows = append(rows, "") rows = append(rows, "=== AI MODEL & PERFORMANCE ===") rows = append(rows, fmt.Sprintf("AI Model: %s", SafeString(selectedInsight["ai_model"], "N/A"))) rows = append(rows, fmt.Sprintf("Response Time: %v ms", selectedInsight["response_time_ms"])) rows = append(rows, fmt.Sprintf("Accuracy Score: %.2f", SafeFloat(selectedInsight["accuracy_score"], 0.0))) - + rows = append(rows, "") rows = append(rows, "=== RECOMMENDATION ===") - + recommendation := SafeString(selectedInsight["recommendation"], "No recommendations available") for len(recommendation) > 0 { if len(recommendation) <= 70 { @@ -146,16 +146,16 @@ func (inf *InsightsFactory) UpdateDetailData(dataManager *core.DataManager, item recommendation = recommendation[1:] } } - + if userFeedback := SafeString(selectedInsight["user_feedback"], ""); userFeedback != "" { rows = append(rows, "") rows = append(rows, "=== USER FEEDBACK ===") rows = append(rows, userFeedback) } - + inf.detailWidget.Rows = rows inf.detailWidget.Title = fmt.Sprintf("AI Insight: %s", SafeString(selectedInsight["insight_type"], itemID)) - + return nil } diff --git a/cmd/mirror-cli/dashboard/widgets/jobs.go b/cmd/mirror-cli/dashboard/widgets/jobs.go index 73d4c31..3bc80f7 100644 --- a/cmd/mirror-cli/dashboard/widgets/jobs.go +++ b/cmd/mirror-cli/dashboard/widgets/jobs.go @@ -12,14 +12,14 @@ type JobsFactory struct { func NewJobsFactory() *JobsFactory { base := NewBaseWidgetFactory() - + jf := &JobsFactory{ BaseWidgetFactory: base, } - + jf.listWidget.Title = "Jobs (Press Enter for details, S/T/P/X for start/stop/pause/restart)" jf.detailWidget.Title = "Job Details" - + return jf } @@ -32,13 +32,13 @@ func (jf *JobsFactory) UpdateListData(dataManager *core.DataManager) error { } return err } - + jf.items = jobs - + rows := []string{ "ID | Name | Status | Source→Target | Lag | Msgs/sec", } - + if len(jobs) == 0 { rows = append(rows, "No replication jobs found") } else { @@ -48,7 +48,7 @@ func (jf *JobsFactory) UpdateListData(dataManager *core.DataManager) error { jobStatus := SafeString(job["status"], "unknown") sourceCluster := SafeString(job["source_cluster_name"], "N/A") targetCluster := SafeString(job["target_cluster_name"], "N/A") - + var lag, throughput string metrics, err := dataManager.GetJobMetrics(jobID) if err != nil { @@ -58,12 +58,12 @@ func (jf *JobsFactory) UpdateListData(dataManager *core.DataManager) error { lag = fmt.Sprintf("%.0f", SafeFloat(metrics["current_lag"], 0)) throughput = fmt.Sprintf("%.0f", SafeFloat(metrics["messages_replicated"], 0)) } - + jobName = TruncateString(jobName, 15) - sourceTarget := fmt.Sprintf("%s→%s", + sourceTarget := fmt.Sprintf("%s→%s", TruncateString(sourceCluster, 8), TruncateString(targetCluster, 8)) - + row := fmt.Sprintf("%s | %s | %s | %s | %s | %s", TruncateString(jobID, 8), jobName, @@ -72,13 +72,13 @@ func (jf *JobsFactory) UpdateListData(dataManager *core.DataManager) error { lag, throughput, ) - + rows = append(rows, row) } } - + jf.listWidget.Rows = rows - + // Maintain selection within bounds if jf.selectedIdx >= len(jf.items) { jf.selectedIdx = len(jf.items) - 1 @@ -89,7 +89,7 @@ func (jf *JobsFactory) UpdateListData(dataManager *core.DataManager) error { if len(jf.items) > 0 { jf.listWidget.SelectedRow = jf.selectedIdx + 1 // +1 for header } - + return nil } @@ -98,7 +98,7 @@ func (jf *JobsFactory) UpdateDetailData(dataManager *core.DataManager, itemID st jf.detailWidget.Rows = []string{"Select a job from the list to view details"} return nil } - + job, err := dataManager.GetJobDetails(itemID) if err != nil { jf.detailWidget.Rows = []string{ @@ -107,12 +107,12 @@ func (jf *JobsFactory) UpdateDetailData(dataManager *core.DataManager, itemID st } return err } - + metrics, metricsErr := dataManager.GetJobMetrics(itemID) mappings, mappingsErr := dataManager.GetJobMappings(itemID) - + var rows []string - + rows = append(rows, "=== JOB INFORMATION ===") rows = append(rows, fmt.Sprintf("ID: %s", SafeString(job["id"], "N/A"))) rows = append(rows, fmt.Sprintf("Name: %s", SafeString(job["name"], "N/A"))) @@ -121,10 +121,10 @@ func (jf *JobsFactory) UpdateDetailData(dataManager *core.DataManager, itemID st rows = append(rows, fmt.Sprintf("Target Cluster: %s", SafeString(job["target_cluster_name"], "N/A"))) rows = append(rows, fmt.Sprintf("Created: %s", SafeString(job["created_at"], "N/A"))) rows = append(rows, fmt.Sprintf("Updated: %s", SafeString(job["updated_at"], "N/A"))) - + rows = append(rows, "") rows = append(rows, "=== CURRENT METRICS ===") - + if metricsErr != nil { rows = append(rows, fmt.Sprintf("Metrics Error: %v", metricsErr)) } else if metrics != nil { @@ -132,13 +132,13 @@ func (jf *JobsFactory) UpdateDetailData(dataManager *core.DataManager, itemID st throughput := SafeFloat(metrics["messages_replicated"], 0) totalProcessed := SafeFloat(metrics["total_messages_processed"], 0) errorCount := SafeFloat(metrics["error_count"], 0) - + rows = append(rows, fmt.Sprintf("Current Lag: %.0f messages", currentLag)) rows = append(rows, fmt.Sprintf("Messages/sec: %.0f", throughput)) rows = append(rows, fmt.Sprintf("Total Processed: %.0f", totalProcessed)) rows = append(rows, fmt.Sprintf("Errors: %.0f", errorCount)) rows = append(rows, fmt.Sprintf("Last Updated: %s", SafeString(metrics["timestamp"], "N/A"))) - + // Health assessment based on error count and lag thresholds var healthStatus string if errorCount > 0 { @@ -152,10 +152,10 @@ func (jf *JobsFactory) UpdateDetailData(dataManager *core.DataManager, itemID st } else { rows = append(rows, "No metrics available") } - + rows = append(rows, "") rows = append(rows, "=== TOPIC MAPPINGS ===") - + if mappingsErr != nil { rows = append(rows, fmt.Sprintf("Mappings Error: %v", mappingsErr)) } else if len(mappings) == 0 { @@ -171,7 +171,7 @@ func (jf *JobsFactory) UpdateDetailData(dataManager *core.DataManager, itemID st } } } - + rows = append(rows, "") rows = append(rows, "=== REPLICATION SETTINGS ===") if batchSize := job["batch_size"]; batchSize != nil { @@ -186,7 +186,7 @@ func (jf *JobsFactory) UpdateDetailData(dataManager *core.DataManager, itemID st if preservePartitions := job["preserve_partitions"]; preservePartitions != nil { rows = append(rows, fmt.Sprintf("Preserve Partitions: %v", preservePartitions)) } - + rows = append(rows, "") rows = append(rows, "=== CONTROLS ===") rows = append(rows, "Press 'S' to start job") @@ -195,10 +195,10 @@ func (jf *JobsFactory) UpdateDetailData(dataManager *core.DataManager, itemID st rows = append(rows, "Press 'X' to restart job") rows = append(rows, "Press 'R' to refresh data") rows = append(rows, "Press 'B' or Escape to go back") - + jf.detailWidget.Rows = rows jf.detailWidget.Title = fmt.Sprintf("Job Details: %s", SafeString(job["name"], itemID)) - + return nil } @@ -207,12 +207,12 @@ func (jf *JobsFactory) HandleAction(action core.EventAction, dataManager *core.D if !ok { return nil // Not a job control action } - + jobID := jf.GetSelectedItemID() if jobID == "" { return fmt.Errorf("no job selected") } - + // This would typically call job control functions // For now, we'll simulate the action switch jobAction.Action { @@ -220,7 +220,7 @@ func (jf *JobsFactory) HandleAction(action core.EventAction, dataManager *core.D // Call startJob(dataManager.token, jobID) jf.listWidget.Title = fmt.Sprintf("Jobs (Starting job %s...)", jobID) case "stop": - // Call stopJob(dataManager.token, jobID) + // Call stopJob(dataManager.token, jobID) jf.listWidget.Title = fmt.Sprintf("Jobs (Stopping job %s...)", jobID) case "pause": // Call pauseJob(dataManager.token, jobID) @@ -229,12 +229,12 @@ func (jf *JobsFactory) HandleAction(action core.EventAction, dataManager *core.D // Call restartJob(dataManager.token, jobID) jf.listWidget.Title = fmt.Sprintf("Jobs (Restarting job %s...)", jobID) } - + // Invalidate cache to force refresh of job data dataManager.InvalidateCache("jobs") dataManager.InvalidateCache("metrics_" + jobID) dataManager.InvalidateCache("job_details_" + jobID) - + return nil } diff --git a/cmd/mirror-cli/main.go b/cmd/mirror-cli/main.go index ee5568e..46fdd24 100644 --- a/cmd/mirror-cli/main.go +++ b/cmd/mirror-cli/main.go @@ -388,9 +388,9 @@ It interacts with the kaf-mirror API to perform various tasks.`, fmt.Println(" USER CREATED") fmt.Println("=================================================================") fmt.Printf(" Username: %s\n", username) - fmt.Printf(" Password: %s\n", password) + fmt.Println(" Password: [REDACTED]") fmt.Println("=================================================================") - fmt.Println(" Please store this password in a secure location.") + fmt.Println(" Deliver credentials securely to the user.") fmt.Println("=================================================================") }, } @@ -685,7 +685,7 @@ It interacts with the kaf-mirror API to perform various tasks.`, fmt.Println("=================================================================") fmt.Printf(" Username: %s\n", username) fmt.Println(" New Password: ********") - fmt.Printf(" Password File: %s\n", outputPath) + fmt.Println(" Password File: [REDACTED]") fmt.Println("=================================================================") fmt.Println(" Please provide this password to the user securely.") fmt.Println(" Delete the password file after delivery.") diff --git a/internal/ai/openai_client.go b/internal/ai/openai_client.go index 2a3a3b0..f47bbef 100644 --- a/internal/ai/openai_client.go +++ b/internal/ai/openai_client.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package ai import ( diff --git a/internal/analysis/log_parser.go b/internal/analysis/log_parser.go index ff07ad6..e06fc27 100644 --- a/internal/analysis/log_parser.go +++ b/internal/analysis/log_parser.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package analysis import ( @@ -23,13 +22,13 @@ import ( // LogEntry represents a single parsed log entry. type LogEntry struct { - Timestamp time.Time - Level string - Component string - Message string - AICategory string + Timestamp time.Time + Level string + Component string + Message string + AICategory string AISubcategory string - JobID string + JobID string } // GetLogsForJob retrieves logs for a specific jobID from the log file. @@ -53,8 +52,8 @@ func GetLogsForJob(logDir, jobID string, since time.Time) ([]LogEntry, error) { for scanner.Scan() { line := scanner.Text() // Include both AI-tagged logs for the job and regular job logs - if strings.Contains(line, fmt.Sprintf("[job:%s]", jobID)) || - (strings.Contains(line, "[AI:") && strings.Contains(line, jobID)) { + if strings.Contains(line, fmt.Sprintf("[job:%s]", jobID)) || + (strings.Contains(line, "[AI:") && strings.Contains(line, jobID)) { entry, err := parseLogLine(line) if err == nil && entry.Timestamp.After(since) { entries = append(entries, entry) @@ -73,12 +72,12 @@ func parseLogLine(line string) (LogEntry, error) { if !strings.HasPrefix(line, "[") { return entry, fmt.Errorf("invalid log line format - no timestamp") } - + tsEnd := strings.Index(line, "]") if tsEnd == -1 { return entry, fmt.Errorf("invalid log line format - unclosed timestamp") } - + tsStr := line[1:tsEnd] ts, err := time.Parse("2006-01-02 15:04:05.000", tsStr) if err != nil { @@ -89,7 +88,7 @@ func parseLogLine(line string) (LogEntry, error) { // Parse level - after timestamp remaining := line[tsEnd+1:] remaining = strings.TrimSpace(remaining) - + levelEnd := strings.Index(remaining, " ") if levelEnd == -1 { return entry, fmt.Errorf("invalid log line format - no level") diff --git a/internal/database/ai_insights.go b/internal/database/ai_insights.go index df1eb98..c3c3714 100644 --- a/internal/database/ai_insights.go +++ b/internal/database/ai_insights.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package database import ( @@ -35,48 +34,48 @@ func InsertAIInsight(db *sqlx.DB, insight *AIInsight) error { // GetAIMetrics calculates aggregated AI performance metrics. func GetAIMetrics(db *sqlx.DB) (*AIMetrics, error) { var metrics AIMetrics - + // Get total insights count err := db.Get(&metrics.TotalInsights, "SELECT COUNT(*) FROM ai_insights") if err != nil { return nil, err } - + // Get average response time (only for records with response_time_ms > 0) - err = db.Get(&metrics.AvgResponseTimeMs, + err = db.Get(&metrics.AvgResponseTimeMs, "SELECT COALESCE(AVG(response_time_ms), 0) FROM ai_insights WHERE response_time_ms > 0") if err != nil { return nil, err } - + // Calculate accuracy rate based on resolved insights with positive feedback var resolvedCount, accurateCount int err = db.Get(&resolvedCount, "SELECT COUNT(*) FROM ai_insights WHERE resolution_status = 'resolved'") if err != nil { return nil, err } - + if resolvedCount > 0 { - err = db.Get(&accurateCount, + err = db.Get(&accurateCount, "SELECT COUNT(*) FROM ai_insights WHERE resolution_status = 'resolved' AND (accuracy_score >= 0.7 OR user_feedback LIKE '%helpful%' OR user_feedback LIKE '%accurate%')") if err != nil { return nil, err } metrics.AccuracyRate = float64(accurateCount) / float64(resolvedCount) * 100 } - + // Get anomaly and recommendation counts err = db.Get(&metrics.AnomalyCount, "SELECT COUNT(*) FROM ai_insights WHERE insight_type = 'anomaly'") if err != nil { return nil, err } - - err = db.Get(&metrics.RecommendationCount, + + err = db.Get(&metrics.RecommendationCount, "SELECT COUNT(*) FROM ai_insights WHERE insight_type IN ('optimization', 'recommendation')") if err != nil { return nil, err } - + return &metrics, nil } @@ -87,7 +86,7 @@ func GenerateEnhancedAIInsight(db *sqlx.DB, aiClient interface{}, jobID string, if err != nil { return fmt.Errorf("failed to get metrics for AI analysis: %v", err) } - + // Get recent operation logs (last 30 minutes) logEntries, err := analysis.GetLogsForJob(logger.GetProductionLogDir(), jobID, time.Now().Add(-30*time.Minute)) if err != nil { @@ -97,7 +96,7 @@ func GenerateEnhancedAIInsight(db *sqlx.DB, aiClient interface{}, jobID string, if err != nil { return fmt.Errorf("failed to format logs for AI analysis: %v", err) } - + // Type assert the AI client for different analysis types with response time tracking type enhancedAIClient interface { GetEnhancedInsightsWithResponseTime(ctx context.Context, metrics string, logs string) (string, int, error) @@ -108,22 +107,22 @@ func GenerateEnhancedAIInsight(db *sqlx.DB, aiClient interface{}, jobID string, GetLogPatternAnalysis(ctx context.Context, logs string) (string, error) GetAnomalyDetection(ctx context.Context, metrics string) (string, error) } - + type incidentAIClient interface { GetIncidentAnalysisWithResponseTime(ctx context.Context, eventDetails string) (string, int, error) GetIncidentAnalysis(ctx context.Context, eventDetails string) (string, error) } - + type perfAIClient interface { GetPerformanceRecommendationWithResponseTime(ctx context.Context, metrics string) (string, int, error) GetPerformanceRecommendation(ctx context.Context, metrics string) (string, error) } - + ctx := context.Background() var recommendation string var severityLevel string var responseTimeMs int - + switch insightType { case "enhanced_analysis": client, ok := aiClient.(enhancedAIClient) @@ -179,11 +178,11 @@ func GenerateEnhancedAIInsight(db *sqlx.DB, aiClient interface{}, jobID string, default: return fmt.Errorf("unsupported insight type: %s", insightType) } - + if err != nil { return fmt.Errorf("failed to generate AI insight: %v", err) } - + // Store the insight with response time insight := &AIInsight{ JobID: &jobID, @@ -193,7 +192,7 @@ func GenerateEnhancedAIInsight(db *sqlx.DB, aiClient interface{}, jobID string, Recommendation: recommendation, ResponseTimeMs: responseTimeMs, } - + return InsertAIInsight(db, insight) } @@ -278,7 +277,7 @@ func extractErrorCount(logs string) int { // GetAggregatedMetricsForAI retrieves and formats metrics data for AI analysis func GetAggregatedMetricsForAI(db *sqlx.DB, jobID string, minutes int) (string, error) { since := time.Now().Add(-time.Duration(minutes) * time.Minute) - + query := `SELECT messages_replicated_delta AS messages_replicated, bytes_transferred_delta AS bytes_transferred, @@ -288,33 +287,33 @@ func GetAggregatedMetricsForAI(db *sqlx.DB, jobID string, minutes int) (string, FROM aggregated_metrics WHERE job_id = ? AND timestamp >= ? ORDER BY timestamp DESC LIMIT 100` - + var metrics []ReplicationMetric err := db.Select(&metrics, query, jobID, since) if err != nil { return "", err } - + if len(metrics) == 0 { return "No recent metrics found for analysis.", nil } - + // Format metrics for AI analysis type MetricsSummary struct { - JobID string `json:"job_id"` - TimeWindow string `json:"time_window"` - TotalMessages int64 `json:"total_messages"` - TotalBytes int64 `json:"total_bytes"` - AvgThroughput float64 `json:"avg_throughput_msg_per_sec"` - MaxLag int `json:"max_lag"` - TotalErrors int64 `json:"total_errors"` - DataPoints int `json:"data_points"` - LatestTimestamp string `json:"latest_timestamp"` - } - + JobID string `json:"job_id"` + TimeWindow string `json:"time_window"` + TotalMessages int64 `json:"total_messages"` + TotalBytes int64 `json:"total_bytes"` + AvgThroughput float64 `json:"avg_throughput_msg_per_sec"` + MaxLag int `json:"max_lag"` + TotalErrors int64 `json:"total_errors"` + DataPoints int `json:"data_points"` + LatestTimestamp string `json:"latest_timestamp"` + } + var totalMessages, totalBytes, totalErrors int64 var maxLag int - + for _, metric := range metrics { totalMessages += int64(metric.MessagesReplicated) totalBytes += int64(metric.BytesTransferred) @@ -323,11 +322,11 @@ func GetAggregatedMetricsForAI(db *sqlx.DB, jobID string, minutes int) (string, maxLag = metric.CurrentLag } } - + // Calculate average throughput (messages per second over time window) timeWindowSeconds := float64(minutes * 60) avgThroughput := float64(totalMessages) / timeWindowSeconds - + summary := MetricsSummary{ JobID: jobID, TimeWindow: fmt.Sprintf("%d minutes", minutes), @@ -339,12 +338,12 @@ func GetAggregatedMetricsForAI(db *sqlx.DB, jobID string, minutes int) (string, DataPoints: len(metrics), LatestTimestamp: metrics[0].Timestamp.Format("2006-01-02 15:04:05"), } - + summaryBytes, err := json.MarshalIndent(summary, "", " ") if err != nil { return "", err } - + return string(summaryBytes), nil } diff --git a/internal/database/api_tokens.go b/internal/database/api_tokens.go index 2c587f8..4b02baa 100644 --- a/internal/database/api_tokens.go +++ b/internal/database/api_tokens.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package database import ( diff --git a/internal/database/clusters.go b/internal/database/clusters.go index 0450c55..aa2fae5 100644 --- a/internal/database/clusters.go +++ b/internal/database/clusters.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package database import ( diff --git a/internal/database/compliance.go b/internal/database/compliance.go index b54531c..ab5c0ad 100644 --- a/internal/database/compliance.go +++ b/internal/database/compliance.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package database import ( @@ -500,14 +499,14 @@ func GenerateComplianceCSV(report *ComplianceReport) ([]byte, error) { // ListComplianceReports retrieves a list of compliance reports func ListComplianceReports(db *sqlx.DB, limit int) ([]ComplianceReport, error) { var reports []ComplianceReport - + query := ` SELECT id, period, start_date, end_date, generated_by, generated_at, report_data FROM compliance_reports ORDER BY generated_at DESC LIMIT ? ` - + rows, err := db.Query(query, limit) if err != nil { return nil, err @@ -520,12 +519,12 @@ func ListComplianceReports(db *sqlx.DB, limit int) ([]ComplianceReport, error) { &report.GeneratedBy, &report.GeneratedAt, &report.ReportDataDB); err != nil { continue } - + // Parse JSON data if err := json.Unmarshal([]byte(report.ReportDataDB), &report.ReportData); err != nil { report.ReportData = make(map[string]interface{}) } - + reports = append(reports, report) } diff --git a/internal/database/configuration.go b/internal/database/configuration.go index 1487f04..bd1c129 100644 --- a/internal/database/configuration.go +++ b/internal/database/configuration.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package database import ( diff --git a/internal/database/events.go b/internal/database/events.go index e5b1050..c6eafd9 100644 --- a/internal/database/events.go +++ b/internal/database/events.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package database import ( diff --git a/internal/database/inventory.go b/internal/database/inventory.go index 90f1046..7eee7fd 100644 --- a/internal/database/inventory.go +++ b/internal/database/inventory.go @@ -12,17 +12,17 @@ func CreateInventorySnapshot(db *sqlx.DB, jobID, snapshotType string) (int, erro query := ` INSERT INTO job_inventory_snapshots (job_id, snapshot_type) VALUES (?, ?)` - + result, err := db.Exec(query, jobID, snapshotType) if err != nil { return 0, fmt.Errorf("failed to create inventory snapshot: %w", err) } - + id, err := result.LastInsertId() if err != nil { return 0, fmt.Errorf("failed to get snapshot ID: %w", err) } - + return int(id), nil } @@ -32,7 +32,7 @@ func InsertClusterInventory(db *sqlx.DB, inventory ClusterInventory) (int, error (snapshot_id, cluster_type, cluster_name, provider, brokers, broker_count, total_topics, controller_id, cluster_id) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)` - + result, err := db.Exec(query, inventory.SnapshotID, inventory.ClusterType, inventory.ClusterName, inventory.Provider, inventory.Brokers, inventory.BrokerCount, @@ -40,12 +40,12 @@ func InsertClusterInventory(db *sqlx.DB, inventory ClusterInventory) (int, error if err != nil { return 0, fmt.Errorf("failed to insert cluster inventory: %w", err) } - + id, err := result.LastInsertId() if err != nil { return 0, fmt.Errorf("failed to get cluster inventory ID: %w", err) } - + return int(id), nil } @@ -55,7 +55,7 @@ func InsertTopicInventory(db *sqlx.DB, inventory TopicInventory) (int, error) { (cluster_inventory_id, topic_name, partition_count, replication_factor, is_internal, config_data) VALUES (?, ?, ?, ?, ?, ?)` - + result, err := db.Exec(query, inventory.ClusterInventoryID, inventory.TopicName, inventory.PartitionCount, inventory.ReplicationFactor, @@ -63,12 +63,12 @@ func InsertTopicInventory(db *sqlx.DB, inventory TopicInventory) (int, error) { if err != nil { return 0, fmt.Errorf("failed to insert topic inventory: %w", err) } - + id, err := result.LastInsertId() if err != nil { return 0, fmt.Errorf("failed to get topic inventory ID: %w", err) } - + return int(id), nil } @@ -77,7 +77,7 @@ func InsertPartitionInventory(db *sqlx.DB, inventory PartitionInventory) error { INSERT INTO partition_inventory (topic_inventory_id, partition_id, leader_id, replica_ids, isr_ids, high_water_mark) VALUES (?, ?, ?, ?, ?, ?)` - + _, err := db.Exec(query, inventory.TopicInventoryID, inventory.PartitionID, inventory.LeaderID, inventory.ReplicaIDs, inventory.IsrIDs, @@ -85,7 +85,7 @@ func InsertPartitionInventory(db *sqlx.DB, inventory PartitionInventory) error { if err != nil { return fmt.Errorf("failed to insert partition inventory: %w", err) } - + return nil } @@ -94,19 +94,19 @@ func InsertConsumerGroupInventory(db *sqlx.DB, inventory ConsumerGroupInventory) INSERT INTO consumer_group_inventory (snapshot_id, group_id, group_state, protocol_type, protocol, member_count) VALUES (?, ?, ?, ?, ?, ?)` - + result, err := db.Exec(query, inventory.SnapshotID, inventory.GroupID, inventory.GroupState, inventory.ProtocolType, inventory.Protocol, inventory.MemberCount) if err != nil { return 0, fmt.Errorf("failed to insert consumer group inventory: %w", err) } - + id, err := result.LastInsertId() if err != nil { return 0, fmt.Errorf("failed to get consumer group inventory ID: %w", err) } - + return int(id), nil } @@ -116,7 +116,7 @@ func InsertConsumerGroupOffset(db *sqlx.DB, offset ConsumerGroupOffset) error { (consumer_group_inventory_id, topic_name, partition_id, current_offset, high_water_mark, lag) VALUES (?, ?, ?, ?, ?, ?)` - + _, err := db.Exec(query, offset.ConsumerGroupInventoryID, offset.TopicName, offset.PartitionID, offset.CurrentOffset, @@ -124,7 +124,7 @@ func InsertConsumerGroupOffset(db *sqlx.DB, offset ConsumerGroupOffset) error { if err != nil { return fmt.Errorf("failed to insert consumer group offset: %w", err) } - + return nil } @@ -134,7 +134,7 @@ func InsertConnectionInventory(db *sqlx.DB, inventory ConnectionInventory) error (snapshot_id, connection_type, provider, brokers, security_protocol, sasl_mechanism, api_key_prefix, connection_successful, connection_time_ms, error_message) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` - + _, err := db.Exec(query, inventory.SnapshotID, inventory.ConnectionType, inventory.Provider, inventory.Brokers, inventory.SecurityProtocol, inventory.SaslMechanism, @@ -143,7 +143,7 @@ func InsertConnectionInventory(db *sqlx.DB, inventory ConnectionInventory) error if err != nil { return fmt.Errorf("failed to insert connection inventory: %w", err) } - + return nil } @@ -153,13 +153,13 @@ func GetInventorySnapshots(db *sqlx.DB, jobID string) ([]JobInventorySnapshot, e FROM job_inventory_snapshots WHERE job_id = ? ORDER BY created_at DESC` - + var snapshots []JobInventorySnapshot err := db.Select(&snapshots, query, jobID) if err != nil { return nil, fmt.Errorf("failed to query inventory snapshots: %w", err) } - + return snapshots, nil } @@ -168,34 +168,34 @@ func GetFullInventoryData(db *sqlx.DB, snapshotID int) (*InventoryData, error) { if err != nil { return nil, err } - + clusters, err := getClusterInventories(db, snapshotID) if err != nil { return nil, err } - + consumerGroups, err := getConsumerGroupInventories(db, snapshotID) if err != nil { return nil, err } - + connections, err := getConnectionInventories(db, snapshotID) if err != nil { return nil, err } - + data := &InventoryData{ Snapshot: *snapshot, ConsumerGroups: consumerGroups, Connections: connections, } - + for _, cluster := range clusters { topics, err := getTopicInventories(db, cluster.ID) if err != nil { return nil, err } - + var partitions []PartitionInventory for _, topic := range topics { topicPartitions, err := getPartitionInventories(db, topic.ID) @@ -204,7 +204,7 @@ func GetFullInventoryData(db *sqlx.DB, snapshotID int) (*InventoryData, error) { } partitions = append(partitions, topicPartitions...) } - + if cluster.ClusterType == "source" { data.SourceCluster = &cluster data.SourceTopics = topics @@ -215,13 +215,13 @@ func GetFullInventoryData(db *sqlx.DB, snapshotID int) (*InventoryData, error) { data.TargetPartitions = partitions } } - + offsets, err := getConsumerGroupOffsets(db, consumerGroups) if err != nil { return nil, err } data.ConsumerOffsets = offsets - + return data, nil } @@ -230,13 +230,13 @@ func getSnapshotByID(db *sqlx.DB, snapshotID int) (*JobInventorySnapshot, error) SELECT id, job_id, snapshot_type, created_at FROM job_inventory_snapshots WHERE id = ?` - + var snapshot JobInventorySnapshot err := db.Get(&snapshot, query, snapshotID) if err != nil { return nil, fmt.Errorf("failed to get snapshot: %w", err) } - + return &snapshot, nil } @@ -247,13 +247,13 @@ func getClusterInventories(db *sqlx.DB, snapshotID int) ([]ClusterInventory, err FROM cluster_inventory WHERE snapshot_id = ? ORDER BY cluster_type` - + var clusters []ClusterInventory err := db.Select(&clusters, query, snapshotID) if err != nil { return nil, fmt.Errorf("failed to query cluster inventories: %w", err) } - + return clusters, nil } @@ -264,13 +264,13 @@ func getTopicInventories(db *sqlx.DB, clusterInventoryID int) ([]TopicInventory, FROM topic_inventory WHERE cluster_inventory_id = ? ORDER BY topic_name` - + var topics []TopicInventory err := db.Select(&topics, query, clusterInventoryID) if err != nil { return nil, fmt.Errorf("failed to query topic inventories: %w", err) } - + return topics, nil } @@ -281,13 +281,13 @@ func getPartitionInventories(db *sqlx.DB, topicInventoryID int) ([]PartitionInve FROM partition_inventory WHERE topic_inventory_id = ? ORDER BY partition_id` - + var partitions []PartitionInventory err := db.Select(&partitions, query, topicInventoryID) if err != nil { return nil, fmt.Errorf("failed to query partition inventories: %w", err) } - + return partitions, nil } @@ -297,19 +297,19 @@ func getConsumerGroupInventories(db *sqlx.DB, snapshotID int) ([]ConsumerGroupIn FROM consumer_group_inventory WHERE snapshot_id = ? ORDER BY group_id` - + var groups []ConsumerGroupInventory err := db.Select(&groups, query, snapshotID) if err != nil { return nil, fmt.Errorf("failed to query consumer group inventories: %w", err) } - + return groups, nil } func getConsumerGroupOffsets(db *sqlx.DB, groups []ConsumerGroupInventory) ([]ConsumerGroupOffset, error) { var allOffsets []ConsumerGroupOffset - + for _, group := range groups { query := ` SELECT id, consumer_group_inventory_id, topic_name, partition_id, @@ -317,7 +317,7 @@ func getConsumerGroupOffsets(db *sqlx.DB, groups []ConsumerGroupInventory) ([]Co FROM consumer_group_offsets WHERE consumer_group_inventory_id = ? ORDER BY topic_name, partition_id` - + var offsets []ConsumerGroupOffset err := db.Select(&offsets, query, group.ID) if err != nil { @@ -325,7 +325,7 @@ func getConsumerGroupOffsets(db *sqlx.DB, groups []ConsumerGroupInventory) ([]Co } allOffsets = append(allOffsets, offsets...) } - + return allOffsets, nil } @@ -337,25 +337,25 @@ func getConnectionInventories(db *sqlx.DB, snapshotID int) ([]ConnectionInventor FROM connection_inventory WHERE snapshot_id = ? ORDER BY connection_type` - + var connections []ConnectionInventory err := db.Select(&connections, query, snapshotID) if err != nil { return nil, fmt.Errorf("failed to query connection inventories: %w", err) } - + return connections, nil } func PruneOldInventorySnapshots(db *sqlx.DB) error { cutoff := time.Now().AddDate(0, 0, -30) - + query := `DELETE FROM job_inventory_snapshots WHERE created_at < ?` _, err := db.Exec(query, cutoff) if err != nil { return fmt.Errorf("failed to prune old inventory snapshots: %w", err) } - + return nil } diff --git a/internal/database/jobs.go b/internal/database/jobs.go index 1e6af9e..ac8c269 100644 --- a/internal/database/jobs.go +++ b/internal/database/jobs.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package database import ( diff --git a/internal/database/mappings.go b/internal/database/mappings.go index 82416f4..3b7649a 100644 --- a/internal/database/mappings.go +++ b/internal/database/mappings.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package database import "github.com/jmoiron/sqlx" diff --git a/internal/database/mirror_progress.go b/internal/database/mirror_progress.go index 1749d21..9e95150 100644 --- a/internal/database/mirror_progress.go +++ b/internal/database/mirror_progress.go @@ -75,7 +75,7 @@ func UpdateMirrorProgress(db *sqlx.DB, progress MirrorProgress) error { func GetMirrorProgress(db *sqlx.DB, jobID string) ([]MirrorProgress, error) { query := `SELECT * FROM mirror_progress WHERE job_id = ? ORDER BY source_topic, partition_id` - + progress := make([]MirrorProgress, 0) err := db.Select(&progress, query, jobID) if err != nil { @@ -86,7 +86,7 @@ func GetMirrorProgress(db *sqlx.DB, jobID string) ([]MirrorProgress, error) { func GetMirrorProgressByTopic(db *sqlx.DB, jobID, sourceTopic string) ([]MirrorProgress, error) { query := `SELECT * FROM mirror_progress WHERE job_id = ? AND source_topic = ? ORDER BY partition_id` - + var progress []MirrorProgress err := db.Select(&progress, query, jobID, sourceTopic) if err != nil { @@ -104,11 +104,11 @@ func CreateMigrationCheckpoint(db *sqlx.DB, checkpoint MigrationCheckpoint) (int RETURNING id` var id int - err := db.QueryRow(query, checkpoint.JobID, checkpoint.CheckpointType, + err := db.QueryRow(query, checkpoint.JobID, checkpoint.CheckpointType, checkpoint.SourceConsumerGroupOffsets, checkpoint.TargetHighWaterMarks, - checkpoint.CreatedAt, checkpoint.CreatedBy, checkpoint.MigrationReason, + checkpoint.CreatedAt, checkpoint.CreatedBy, checkpoint.MigrationReason, checkpoint.ValidationResults).Scan(&id) - + if err != nil { return 0, fmt.Errorf("failed to create migration checkpoint: %w", err) } @@ -117,7 +117,7 @@ func CreateMigrationCheckpoint(db *sqlx.DB, checkpoint MigrationCheckpoint) (int func GetLatestMigrationCheckpoint(db *sqlx.DB, jobID string) (*MigrationCheckpoint, error) { query := `SELECT * FROM migration_checkpoints WHERE job_id = ? ORDER BY created_at DESC LIMIT 1` - + var checkpoint MigrationCheckpoint err := db.Get(&checkpoint, query, jobID) if err != nil { @@ -149,8 +149,8 @@ func CalculateResumePoints(db *sqlx.DB, jobID string, resumePointsData map[strin for sourceTopic, partitions := range resumePointsData { // Get corresponding target topic from topic mappings var targetTopic string - err = tx.Get(&targetTopic, - "SELECT target_topic_pattern FROM topic_mappings WHERE job_id = ? AND source_topic_pattern = ? LIMIT 1", + err = tx.Get(&targetTopic, + "SELECT target_topic_pattern FROM topic_mappings WHERE job_id = ? AND source_topic_pattern = ? LIMIT 1", jobID, sourceTopic) if err != nil { targetTopic = sourceTopic // Fallback to same name @@ -158,8 +158,8 @@ func CalculateResumePoints(db *sqlx.DB, jobID string, resumePointsData map[strin for partitionID, resumeOffset := range partitions { gapDetected := resumeOffset > 0 - - _, err = tx.Exec(insertQuery, jobID, sourceTopic, targetTopic, partitionID, + + _, err = tx.Exec(insertQuery, jobID, sourceTopic, targetTopic, partitionID, resumeOffset, time.Now(), "pending", checkpointID, gapDetected) if err != nil { return fmt.Errorf("failed to insert resume point: %w", err) @@ -175,7 +175,7 @@ func GetResumePoints(db *sqlx.DB, jobID string) ([]ResumePoint, error) { SELECT * FROM resume_points WHERE job_id = ? ORDER BY calculated_at DESC, source_topic, partition_id` - + resumePoints := make([]ResumePoint, 0) err := db.Select(&resumePoints, query, jobID) if err != nil { @@ -195,7 +195,7 @@ func GetLatestResumePoints(db *sqlx.DB, jobID string) (map[string]map[int32]int6 WHERE job_id = ? ) ORDER BY source_topic, partition_id` - + rows, err := db.Query(query, jobID, jobID) if err != nil { return nil, fmt.Errorf("failed to get latest resume points: %w", err) @@ -207,7 +207,7 @@ func GetLatestResumePoints(db *sqlx.DB, jobID string) (map[string]map[int32]int6 var sourceTopic string var partitionID int32 var resumeOffset int64 - + err = rows.Scan(&sourceTopic, &partitionID, &resumeOffset) if err != nil { return nil, fmt.Errorf("failed to scan resume point: %w", err) @@ -232,14 +232,14 @@ func StoreMirrorStateAnalysis(db *sqlx.DB, analysis MirrorStateAnalysis) (int, e RETURNING id` recommendationsJSON, _ := json.Marshal(analysis.Recommendations) - + var id int - err := db.QueryRow(query, analysis.JobID, analysis.AnalysisType, + err := db.QueryRow(query, analysis.JobID, analysis.AnalysisType, analysis.SourceClusterState, analysis.TargetClusterState, analysis.AnalysisResults, string(recommendationsJSON), analysis.CriticalIssuesCount, analysis.WarningIssuesCount, analysis.AnalyzedAt, analysis.AnalyzerVersion).Scan(&id) - + if err != nil { return 0, fmt.Errorf("failed to store mirror state analysis: %w", err) } @@ -248,7 +248,7 @@ func StoreMirrorStateAnalysis(db *sqlx.DB, analysis MirrorStateAnalysis) (int, e func GetMirrorStateAnalysis(db *sqlx.DB, jobID string) ([]MirrorStateAnalysis, error) { query := `SELECT * FROM mirror_state_analysis WHERE job_id = ? ORDER BY analyzed_at DESC` - + analyses := make([]MirrorStateAnalysis, 0) err := db.Select(&analyses, query, jobID) if err != nil { @@ -275,7 +275,7 @@ func DetectMirrorGaps(db *sqlx.DB, jobID string, gaps []MirrorGap) error { VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` for _, gap := range gaps { - _, err = tx.Exec(insertQuery, gap.JobID, gap.SourceTopic, gap.TargetTopic, + _, err = tx.Exec(insertQuery, gap.JobID, gap.SourceTopic, gap.TargetTopic, gap.PartitionID, gap.GapStartOffset, gap.GapEndOffset, gap.GapSize, gap.DetectedAt, gap.GapType, gap.ResolutionStatus) if err != nil { @@ -288,7 +288,7 @@ func DetectMirrorGaps(db *sqlx.DB, jobID string, gaps []MirrorGap) error { func GetMirrorGaps(db *sqlx.DB, jobID string) ([]MirrorGap, error) { query := `SELECT * FROM mirror_gaps WHERE job_id = ? ORDER BY detected_at DESC` - + gaps := make([]MirrorGap, 0) err := db.Select(&gaps, query, jobID) if err != nil { @@ -299,7 +299,7 @@ func GetMirrorGaps(db *sqlx.DB, jobID string) ([]MirrorGap, error) { func GetUnresolvedMirrorGaps(db *sqlx.DB, jobID string) ([]MirrorGap, error) { query := `SELECT * FROM mirror_gaps WHERE job_id = ? AND resolution_status = 'unresolved' ORDER BY detected_at DESC` - + var gaps []MirrorGap err := db.Select(&gaps, query, jobID) if err != nil { diff --git a/internal/database/pruning.go b/internal/database/pruning.go index 664973b..273457b 100644 --- a/internal/database/pruning.go +++ b/internal/database/pruning.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package database import ( @@ -35,22 +34,22 @@ func PruneOldData(db *sqlx.DB, retentionDays int) error { } mirrorStateCutoff := time.Now().AddDate(0, 0, -7) // 7 days ago - + _, err = db.Exec(`DELETE FROM mirror_progress WHERE last_updated < ?`, mirrorStateCutoff) if err != nil { return err } - + _, err = db.Exec(`DELETE FROM resume_points WHERE calculated_at < ?`, mirrorStateCutoff) if err != nil { return err } - + _, err = db.Exec(`DELETE FROM mirror_gaps WHERE detected_at < ?`, mirrorStateCutoff) if err != nil { return err } - + _, err = db.Exec(`DELETE FROM mirror_state_analysis WHERE analyzed_at < ?`, mirrorStateCutoff) if err != nil { return err diff --git a/internal/kafka/admin.go b/internal/kafka/admin.go index fa542b9..0feeaff 100644 --- a/internal/kafka/admin.go +++ b/internal/kafka/admin.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package kafka import ( @@ -35,11 +34,11 @@ type ClusterInfo struct { } type TopicInfo struct { - Name string - Partitions int32 + Name string + Partitions int32 ReplicationFactor int16 - Config map[string]string - PartitionInfo []PartitionInfo + Config map[string]string + PartitionInfo []PartitionInfo } type PartitionInfo struct { @@ -50,11 +49,11 @@ type PartitionInfo struct { } type OffsetInfo struct { - Topic string - Partition int32 - Offset int64 + Topic string + Partition int32 + Offset int64 HighWaterMark int64 - Lag int64 + Lag int64 } type TopicDetails struct { @@ -73,11 +72,11 @@ type OffsetComparisonResult struct { } type TopicOffsetComparison struct { - SourceTopic string `json:"source_topic"` - TargetTopic string `json:"target_topic"` + SourceTopic string `json:"source_topic"` + TargetTopic string `json:"target_topic"` PartitionComparisons []PartitionOffsetComparison `json:"partition_comparisons"` - GapsDetected int `json:"gaps_detected"` - TotalLag int64 `json:"total_lag"` + GapsDetected int `json:"gaps_detected"` + TotalLag int64 `json:"total_lag"` } type PartitionOffsetComparison struct { @@ -208,7 +207,7 @@ func TestConnection(ctx context.Context, cfg config.ClusterConfig) error { func (a *AdminClient) GetClusterInfo(ctx context.Context) (*ClusterInfo, error) { logger.Info("Retrieving cluster information for %s", a.cfg.Provider) - + brokers, err := a.client.ListBrokers(ctx) if err != nil { return nil, fmt.Errorf("failed to list brokers: %w", err) @@ -257,14 +256,14 @@ func (a *AdminClient) GetClusterInfo(ctx context.Context) (*ClusterInfo, error) func (a *AdminClient) GetConsumerGroupOffsets(ctx context.Context, groupID string, topics []string) (map[string][]OffsetInfo, error) { logger.Info("Retrieving consumer group offsets for group %s", groupID) - + offsets, err := a.client.FetchOffsetsForTopics(ctx, groupID, topics...) if err != nil { return nil, fmt.Errorf("failed to fetch consumer group offsets: %w", err) } result := make(map[string][]OffsetInfo) - + for topic, topicOffsets := range offsets { for partition, offset := range topicOffsets { if offset.Err != nil { @@ -277,7 +276,7 @@ func (a *AdminClient) GetConsumerGroupOffsets(ctx context.Context, groupID strin Partition: partition, Offset: offset.At, HighWaterMark: offset.At, - Lag: 0, + Lag: 0, } result[topic] = append(result[topic], offsetInfo) @@ -289,15 +288,15 @@ func (a *AdminClient) GetConsumerGroupOffsets(ctx context.Context, groupID strin func (a *AdminClient) GetTopicHighWaterMarks(ctx context.Context, topics []string) (map[string][]OffsetInfo, error) { logger.Info("Retrieving high water marks for topics: %v", topics) - + result := make(map[string][]OffsetInfo) - + for _, topicName := range topics { topicDetails, err := a.client.ListTopics(ctx, topicName) if err != nil { return nil, fmt.Errorf("failed to get metadata for topic %s: %w", topicName, err) } - + details, exists := topicDetails[topicName] if !exists { logger.Warn("Topic %s not found", topicName) @@ -321,14 +320,14 @@ func (a *AdminClient) GetTopicHighWaterMarks(ctx context.Context, topics []strin func (a *AdminClient) ValidateTopicCompatibility(ctx context.Context, sourceInfo, targetInfo TopicInfo) error { logger.Info("Validating compatibility between source topic %s and target topic %s", sourceInfo.Name, targetInfo.Name) - + if sourceInfo.Partitions != targetInfo.Partitions { - return fmt.Errorf("partition count mismatch: source has %d partitions, target has %d partitions", + return fmt.Errorf("partition count mismatch: source has %d partitions, target has %d partitions", sourceInfo.Partitions, targetInfo.Partitions) } if sourceInfo.ReplicationFactor != targetInfo.ReplicationFactor { - logger.Warn("Replication factor differs: source=%d, target=%d (this is acceptable)", + logger.Warn("Replication factor differs: source=%d, target=%d (this is acceptable)", sourceInfo.ReplicationFactor, targetInfo.ReplicationFactor) } @@ -338,7 +337,7 @@ func (a *AdminClient) ValidateTopicCompatibility(ctx context.Context, sourceInfo func (a *AdminClient) EnsureTopicExists(ctx context.Context, topicName string, partitions int32, replicationFactor int16) error { logger.Info("Ensuring topic %s exists with %d partitions", topicName, partitions) - + existing, err := a.client.ListTopics(ctx, topicName) if err != nil { return fmt.Errorf("failed to check if topic exists: %w", err) @@ -452,7 +451,7 @@ func (a *AdminClient) GetTopicLag(ctx context.Context, groupID, topic string) (i func (a *AdminClient) CompareClusterOffsets(ctx context.Context, sourceAdmin *AdminClient, topicMap map[string]string) (*OffsetComparisonResult, error) { logger.Info("Starting cross-cluster offset comparison") - + result := &OffsetComparisonResult{ ComparedAt: time.Now(), TopicComparisons: make(map[string]*TopicOffsetComparison), @@ -464,22 +463,22 @@ func (a *AdminClient) CompareClusterOffsets(ctx context.Context, sourceAdmin *Ad for sourceTopic, targetTopic := range topicMap { comparison, err := a.compareTopicOffsets(ctx, sourceAdmin, sourceTopic, targetTopic) if err != nil { - result.CriticalIssues = append(result.CriticalIssues, + result.CriticalIssues = append(result.CriticalIssues, fmt.Sprintf("Failed to compare %s -> %s: %v", sourceTopic, targetTopic, err)) continue } - + result.TopicComparisons[sourceTopic] = comparison result.TotalGapsDetected += comparison.GapsDetected - + if comparison.TotalLag > 10000 { - result.Warnings = append(result.Warnings, - fmt.Sprintf("High replication lag detected for %s -> %s: %d messages", + result.Warnings = append(result.Warnings, + fmt.Sprintf("High replication lag detected for %s -> %s: %d messages", sourceTopic, targetTopic, comparison.TotalLag)) } } - logger.Info("Offset comparison completed: %d gaps detected across %d topics", + logger.Info("Offset comparison completed: %d gaps detected across %d topics", result.TotalGapsDetected, len(result.TopicComparisons)) return result, nil } @@ -499,11 +498,11 @@ func (a *AdminClient) compareTopicOffsets(ctx context.Context, sourceAdmin *Admi targetOffsets := targetHWMs[targetTopic] comparison := &TopicOffsetComparison{ - SourceTopic: sourceTopic, - TargetTopic: targetTopic, + SourceTopic: sourceTopic, + TargetTopic: targetTopic, PartitionComparisons: make([]PartitionOffsetComparison, 0), - GapsDetected: 0, - TotalLag: 0, + GapsDetected: 0, + TotalLag: 0, } targetPartitionMap := make(map[int32]OffsetInfo) @@ -530,13 +529,13 @@ func (a *AdminClient) compareTopicOffsets(ctx context.Context, sourceAdmin *Admi gap := sourceOffset.HighWaterMark - targetOffset.HighWaterMark hasGap := gap > 0 - + if hasGap { comparison.GapsDetected++ } - + comparison.TotalLag += gap - + safeResumeOffset := targetOffset.HighWaterMark if safeResumeOffset < 0 { safeResumeOffset = 0 @@ -559,7 +558,7 @@ func (a *AdminClient) compareTopicOffsets(ctx context.Context, sourceAdmin *Admi func (a *AdminClient) AnalyzeMirrorState(ctx context.Context, sourceAdmin *AdminClient, jobID string, topicMap map[string]string, consumerGroup string) (*MirrorStateAnalysis, error) { logger.Info("Starting mirror state analysis for job %s", jobID) - + offsetComparison, err := a.CompareClusterOffsets(ctx, sourceAdmin, topicMap) if err != nil { return nil, fmt.Errorf("failed to compare cluster offsets: %w", err) @@ -569,7 +568,7 @@ func (a *AdminClient) AnalyzeMirrorState(ctx context.Context, sourceAdmin *Admin for sourceTopic := range topicMap { sourceTopics = append(sourceTopics, sourceTopic) } - + consumerOffsets, err := sourceAdmin.GetConsumerGroupOffsets(ctx, consumerGroup, sourceTopics) if err != nil { logger.Warn("Failed to get consumer group offsets (this is normal for new groups): %v", err) @@ -579,7 +578,7 @@ func (a *AdminClient) AnalyzeMirrorState(ctx context.Context, sourceAdmin *Admin resumePoints := make(map[string]map[int32]int64) for sourceTopic, _ := range topicMap { resumePoints[sourceTopic] = make(map[int32]int64) - + if comparison, exists := offsetComparison.TopicComparisons[sourceTopic]; exists { for _, partComparison := range comparison.PartitionComparisons { resumePoints[sourceTopic][partComparison.PartitionID] = partComparison.SafeResumeOffset @@ -600,56 +599,56 @@ func (a *AdminClient) AnalyzeMirrorState(ctx context.Context, sourceAdmin *Admin } recommendations := make([]string, 0) - + if offsetComparison.TotalGapsDetected > 0 { - recommendations = append(recommendations, - fmt.Sprintf("Found %d replication gaps that need to be resolved before safe migration", + recommendations = append(recommendations, + fmt.Sprintf("Found %d replication gaps that need to be resolved before safe migration", offsetComparison.TotalGapsDetected)) } - + if len(consumerOffsets) == 0 { - recommendations = append(recommendations, + recommendations = append(recommendations, "Consumer group has no committed offsets - replication will start from earliest/latest based on configuration") } else { - recommendations = append(recommendations, + recommendations = append(recommendations, "Consumer group offsets detected - can resume from last committed positions") } analysis.Recommendations = recommendations - logger.Info("Mirror state analysis completed: %d critical issues, %d warnings, %d gaps", + logger.Info("Mirror state analysis completed: %d critical issues, %d warnings, %d gaps", analysis.CriticalIssuesCount, analysis.WarningIssuesCount, offsetComparison.TotalGapsDetected) - + return analysis, nil } func (a *AdminClient) CalculateSafeResumePoints(ctx context.Context, sourceAdmin *AdminClient, jobID string, topicMap map[string]string, consumerGroup string) (map[string]map[int32]int64, error) { logger.Info("Calculating safe resume points for job %s", jobID) - + offsetComparison, err := a.CompareClusterOffsets(ctx, sourceAdmin, topicMap) if err != nil { return nil, fmt.Errorf("failed to compare cluster offsets: %w", err) } resumePoints := make(map[string]map[int32]int64) - + for sourceTopic, targetTopic := range topicMap { resumePoints[sourceTopic] = make(map[int32]int64) - + if comparison, exists := offsetComparison.TopicComparisons[sourceTopic]; exists { for _, partComparison := range comparison.PartitionComparisons { // Safe resume offset is the target high water mark to avoid duplication safeOffset := partComparison.TargetHighWaterMark - + // If target has no data, start from beginning if safeOffset < 0 { safeOffset = 0 } - + resumePoints[sourceTopic][partComparison.PartitionID] = safeOffset - - logger.Info("Safe resume point for %s[%d] -> %s[%d]: offset %d", - sourceTopic, partComparison.PartitionID, + + logger.Info("Safe resume point for %s[%d] -> %s[%d]: offset %d", + sourceTopic, partComparison.PartitionID, targetTopic, partComparison.PartitionID, safeOffset) } } @@ -678,11 +677,11 @@ func (a *AdminClient) Close() { } type TopicHealth struct { - Name string `json:"name"` - Partitions int `json:"partitions"` - ReplicationFactor int `json:"replication_factor"` - UnderReplicatedPartitions int `json:"under_replicated_partitions"` - IsHealthy bool `json:"is_healthy"` + Name string `json:"name"` + Partitions int `json:"partitions"` + ReplicationFactor int `json:"replication_factor"` + UnderReplicatedPartitions int `json:"under_replicated_partitions"` + IsHealthy bool `json:"is_healthy"` } func (a *AdminClient) CheckTopicHealth(ctx context.Context, topics []string) ([]TopicHealth, error) { @@ -710,11 +709,11 @@ func (a *AdminClient) CheckTopicHealth(ctx context.Context, topics []string) ([] } health = append(health, TopicHealth{ - Name: topic, - Partitions: len(details.Partitions), - ReplicationFactor: len(details.Partitions[0].Replicas), + Name: topic, + Partitions: len(details.Partitions), + ReplicationFactor: len(details.Partitions[0].Replicas), UnderReplicatedPartitions: underReplicated, - IsHealthy: underReplicated == 0, + IsHealthy: underReplicated == 0, }) } diff --git a/internal/kafka/client.go b/internal/kafka/client.go index 575f3e1..28b606f 100644 --- a/internal/kafka/client.go +++ b/internal/kafka/client.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package kafka import ( diff --git a/internal/manager/manager.go b/internal/manager/manager.go index 85b7c7e..888fd89 100644 --- a/internal/manager/manager.go +++ b/internal/manager/manager.go @@ -51,6 +51,8 @@ type JobManager struct { aiAnalysisTicker *time.Ticker wg sync.WaitGroup dbOpsWg sync.WaitGroup + dbOpsMu sync.Mutex + dbOpsClosed bool lastAIAnalysis map[string]time.Time lastAIMetric map[string]database.ReplicationMetric lastComplianceReport map[string]time.Time @@ -151,9 +153,24 @@ func (jm *JobManager) Close() { atomic.StoreInt32(&jm.closing, 1) close(jm.close) jm.wg.Wait() + jm.dbOpsMu.Lock() + jm.dbOpsClosed = true + jm.dbOpsMu.Unlock() jm.dbOpsWg.Wait() } +func (jm *JobManager) beginDBOp() bool { + jm.dbOpsMu.Lock() + defer jm.dbOpsMu.Unlock() + + if jm.dbOpsClosed || atomic.LoadInt32(&jm.closing) == 1 { + return false + } + + jm.dbOpsWg.Add(1) + return true +} + func (jm *JobManager) startPruning() { defer jm.wg.Done() ticker := time.NewTicker(24 * time.Hour) @@ -1194,42 +1211,45 @@ func (jm *JobManager) CreateInventorySnapshot(jobID, snapshotType string) { if atomic.LoadInt32(&jm.closing) == 1 { return } - jm.dbOpsWg.Add(2) - go func() { - defer jm.dbOpsWg.Done() - defer func() { - if r := recover(); r != nil { - logger.Error("Inventory capture panicked for source cluster: %v", r) + if jm.beginDBOp() { + go func() { + defer jm.dbOpsWg.Done() + defer func() { + if r := recover(); r != nil { + logger.Error("Inventory capture panicked for source cluster: %v", r) + } + }() + + // Check if close signal has been sent + select { + case <-jm.close: + return // Don't perform database operations during shutdown + default: } - }() - // Check if close signal has been sent - select { - case <-jm.close: - return // Don't perform database operations during shutdown - default: - } + jm.captureClusterInventory(snapshotID, job.SourceClusterName, "source") + }() + } - jm.captureClusterInventory(snapshotID, job.SourceClusterName, "source") - }() + if jm.beginDBOp() { + go func() { + defer jm.dbOpsWg.Done() + defer func() { + if r := recover(); r != nil { + logger.Error("Inventory capture panicked for target cluster: %v", r) + } + }() - go func() { - defer jm.dbOpsWg.Done() - defer func() { - if r := recover(); r != nil { - logger.Error("Inventory capture panicked for target cluster: %v", r) + // Check if close signal has been sent + select { + case <-jm.close: + return // Don't perform database operations during shutdown + default: } - }() - // Check if close signal has been sent - select { - case <-jm.close: - return // Don't perform database operations during shutdown - default: - } - - jm.captureClusterInventory(snapshotID, job.TargetClusterName, "target") - }() + jm.captureClusterInventory(snapshotID, job.TargetClusterName, "target") + }() + } logger.Info("Inventory snapshot capture initiated for job %s", jobID) } diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go index 9ea8239..2c7452f 100644 --- a/internal/metrics/metrics.go +++ b/internal/metrics/metrics.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package metrics import ( diff --git a/internal/metrics/splunk.go b/internal/metrics/splunk.go index 8216ec2..946a0eb 100644 --- a/internal/metrics/splunk.go +++ b/internal/metrics/splunk.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package metrics import ( @@ -38,10 +37,10 @@ func NewSplunkSink(cfg config.SplunkConfig) (*SplunkSink, error) { // Send sends a metric to Splunk. func (s *SplunkSink) Send(metric database.ReplicationMetric) error { payload := map[string]interface{}{ - "event": metric, - "source": "kaf-mirror", - "sourcetype": "_json", - "index": s.cfg.Index, + "event": metric, + "source": "kaf-mirror", + "sourcetype": "_json", + "index": s.cfg.Index, } body, err := json.Marshal(payload) diff --git a/internal/server/handlers.go b/internal/server/handlers.go index 86eb285..4df2560 100644 --- a/internal/server/handlers.go +++ b/internal/server/handlers.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package server import ( @@ -487,14 +486,14 @@ func (s *Server) handleListJobs(c *fiber.Ctx) error { } type CreateJobRequest struct { - Name string `json:"name"` - SourceClusterName string `json:"source_cluster_name"` - TargetClusterName string `json:"target_cluster_name"` + Name string `json:"name"` + SourceClusterName string `json:"source_cluster_name"` + TargetClusterName string `json:"target_cluster_name"` TopicMappings []database.TopicMapping `json:"topic_mappings"` - BatchSize int `json:"batch_size"` - Parallelism int `json:"parallelism"` - Compression string `json:"compression"` - PreservePartitions bool `json:"preserve_partitions"` + BatchSize int `json:"batch_size"` + Parallelism int `json:"parallelism"` + Compression string `json:"compression"` + PreservePartitions bool `json:"preserve_partitions"` } // handleCreateJob godoc @@ -1059,13 +1058,13 @@ func (s *Server) handleTestAIIntegration(c *fiber.Ctx) error { log.Printf("AI integration test successful. Insight: %s, Response time: %dms", insight, responseTimeMs) aiInsight := &database.AIInsight{ - JobID: nil, // Test insight - InsightType: "anomaly", - Recommendation: "TEST: " + insight, - SeverityLevel: "info", + JobID: nil, // Test insight + InsightType: "anomaly", + Recommendation: "TEST: " + insight, + SeverityLevel: "info", ResolutionStatus: "new", - AIModel: s.cfg.AI.Model, - ResponseTimeMs: responseTimeMs, + AIModel: s.cfg.AI.Model, + ResponseTimeMs: responseTimeMs, } if err := database.InsertAIInsight(s.Db, aiInsight); err != nil { @@ -1747,12 +1746,12 @@ func (s *Server) handleGetInventorySnapshot(c *fiber.Ctx) error { if err != nil { return fiber.NewError(fiber.StatusBadRequest, "Invalid snapshot ID") } - + inventoryData, err := database.GetFullInventoryData(s.Db, snapshotID) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Inventory snapshot not found") } - + // Fix empty compression types to show "NONE" for i := range inventoryData.SourceTopics { if inventoryData.SourceTopics[i].CompressionType == "" { @@ -1764,7 +1763,7 @@ func (s *Server) handleGetInventorySnapshot(c *fiber.Ctx) error { inventoryData.TargetTopics[i].CompressionType = "NONE" } } - + return c.JSON(inventoryData) } @@ -1778,25 +1777,25 @@ func (s *Server) handleGetInventorySnapshot(c *fiber.Ctx) error { // @Security ApiKeyAuth func (s *Server) handleCreateManualInventorySnapshot(c *fiber.Ctx) error { jobID := c.Params("id") - + job, err := database.GetJob(s.Db, jobID) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Job not found") } - + if job.Status != "active" { return fiber.NewError(fiber.StatusBadRequest, "Job must be active to create inventory snapshot") } - + go func() { // Manual inventory snapshot creation would need to be implemented in manager // For now, return success - implementation would be added later }() - + return c.Status(fiber.StatusCreated).JSON(fiber.Map{ - "status": "success", + "status": "success", "message": "Manual inventory snapshot creation initiated", - "job_id": jobID, + "job_id": jobID, }) } @@ -1815,23 +1814,23 @@ func (s *Server) handleGetClusterInventory(c *fiber.Ctx) error { if err != nil { return fiber.NewError(fiber.StatusBadRequest, "Invalid snapshot ID") } - + clusterType := c.Query("cluster_type") if clusterType != "source" && clusterType != "target" { return fiber.NewError(fiber.StatusBadRequest, "cluster_type must be 'source' or 'target'") } - + inventoryData, err := database.GetFullInventoryData(s.Db, snapshotID) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Inventory snapshot not found") } - + if clusterType == "source" && inventoryData.SourceCluster != nil { return c.JSON(inventoryData.SourceCluster) } else if clusterType == "target" && inventoryData.TargetCluster != nil { return c.JSON(inventoryData.TargetCluster) } - + return fiber.NewError(fiber.StatusNotFound, "Cluster inventory not found") } @@ -1850,17 +1849,17 @@ func (s *Server) handleGetTopicInventory(c *fiber.Ctx) error { if err != nil { return fiber.NewError(fiber.StatusBadRequest, "Invalid snapshot ID") } - + clusterType := c.Query("cluster_type") if clusterType != "source" && clusterType != "target" { return fiber.NewError(fiber.StatusBadRequest, "cluster_type must be 'source' or 'target'") } - + inventoryData, err := database.GetFullInventoryData(s.Db, snapshotID) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Inventory snapshot not found") } - + // Fix empty compression types to show "NONE" if clusterType == "source" { for i := range inventoryData.SourceTopics { @@ -1893,12 +1892,12 @@ func (s *Server) handleGetConsumerGroupInventory(c *fiber.Ctx) error { if err != nil { return fiber.NewError(fiber.StatusBadRequest, "Invalid snapshot ID") } - + inventoryData, err := database.GetFullInventoryData(s.Db, snapshotID) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Inventory snapshot not found") } - + return c.JSON(inventoryData.ConsumerGroups) } @@ -1916,12 +1915,12 @@ func (s *Server) handleGetConnectionInventory(c *fiber.Ctx) error { if err != nil { return fiber.NewError(fiber.StatusBadRequest, "Invalid snapshot ID") } - + inventoryData, err := database.GetFullInventoryData(s.Db, snapshotID) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Inventory snapshot not found") } - + return c.JSON(inventoryData.Connections) } @@ -1944,47 +1943,47 @@ func (s *Server) handleGetConnectionInventory(c *fiber.Ctx) error { func (s *Server) handleGetMirrorState(c *fiber.Ctx) error { jobID := c.Params("id") period := c.Query("period") - + var stateData *database.MirrorStateData var err error - + // If no period parameter provided, return only the latest state if period == "" { stateData, err = database.GetMirrorStateData(s.Db, jobID) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Mirror state data not found for job") } - + // Convert arrays to single objects for latest state response response := fiber.Map{ "job_id": stateData.JobID, } - + // Return single object or null for each category if len(stateData.MirrorProgress) > 0 { response["mirror_progress"] = stateData.MirrorProgress[0] } else { response["mirror_progress"] = nil } - + if len(stateData.ResumePoints) > 0 { response["resume_points"] = stateData.ResumePoints[0] } else { response["resume_points"] = nil } - + if len(stateData.MirrorGaps) > 0 { response["mirror_gaps"] = stateData.MirrorGaps[0] } else { response["mirror_gaps"] = nil } - + if len(stateData.StateAnalysis) > 0 { response["state_analysis"] = stateData.StateAnalysis[0] } else { response["state_analysis"] = nil } - + return c.JSON(response) } else { // Validate period parameter @@ -1994,14 +1993,14 @@ func (s *Server) handleGetMirrorState(c *fiber.Ctx) error { if !validPeriods[period] { return fiber.NewError(fiber.StatusBadRequest, "Invalid period. Must be one of: today, yesterday, this-week, last-week") } - + // Calculate time range based on period and return historical data startTime, endTime := s.calculateTimeRange(period) stateData, err = database.GetMirrorStateDataForPeriod(s.Db, jobID, startTime, endTime) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Mirror state data not found for job") } - + // Return arrays for historical data return c.JSON(stateData) } @@ -2010,20 +2009,20 @@ func (s *Server) handleGetMirrorState(c *fiber.Ctx) error { // calculateTimeRange returns start and end times for calendar-based periods func (s *Server) calculateTimeRange(period string) (time.Time, time.Time) { now := time.Now() - + switch period { case "today": // Current day: 00:00 to now start := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) return start, now - - case "yesterday": + + case "yesterday": // Previous complete day: 00:00 to 23:59 of yesterday yesterday := now.AddDate(0, 0, -1) start := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 0, 0, 0, 0, now.Location()) end := time.Date(yesterday.Year(), yesterday.Month(), yesterday.Day(), 23, 59, 59, 999999999, now.Location()) return start, end - + case "this-week": // Current week: Monday 00:00 to now weekday := int(now.Weekday()) @@ -2034,21 +2033,21 @@ func (s *Server) calculateTimeRange(period string) (time.Time, time.Time) { monday := now.AddDate(0, 0, -daysToMonday) start := time.Date(monday.Year(), monday.Month(), monday.Day(), 0, 0, 0, 0, now.Location()) return start, now - + case "last-week": // Previous complete week: Monday 00:00 to Sunday 23:59 of last week weekday := int(now.Weekday()) - if weekday == 0 { // Sunday = 0, we want Monday = 0 + if weekday == 0 { // Sunday = 0, we want Monday = 0 weekday = 7 } daysToLastMonday := weekday - 1 + 7 // Go back to previous week's Monday lastMonday := now.AddDate(0, 0, -daysToLastMonday) lastSunday := lastMonday.AddDate(0, 0, 6) - + start := time.Date(lastMonday.Year(), lastMonday.Month(), lastMonday.Day(), 0, 0, 0, 0, now.Location()) end := time.Date(lastSunday.Year(), lastSunday.Month(), lastSunday.Day(), 23, 59, 59, 999999999, now.Location()) return start, end - + default: // Default to today start := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location()) @@ -2070,12 +2069,12 @@ func (s *Server) calculateTimeRange(period string) (time.Time, time.Time) { // @Security ApiKeyAuth func (s *Server) handleGetMirrorProgress(c *fiber.Ctx) error { jobID := c.Params("id") - + progress, err := database.GetMirrorProgress(s.Db, jobID) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to get mirror progress") } - + return c.JSON(progress) } @@ -2093,12 +2092,12 @@ func (s *Server) handleGetMirrorProgress(c *fiber.Ctx) error { // @Security ApiKeyAuth func (s *Server) handleGetResumePoints(c *fiber.Ctx) error { jobID := c.Params("id") - + resumePoints, err := database.GetResumePoints(s.Db, jobID) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to get resume points") } - + return c.JSON(resumePoints) } @@ -2116,23 +2115,23 @@ func (s *Server) handleGetResumePoints(c *fiber.Ctx) error { // @Security ApiKeyAuth func (s *Server) handleCalculateResumePoints(c *fiber.Ctx) error { jobID := c.Params("id") - + job, err := database.GetJob(s.Db, jobID) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Job not found") } - + // Get source and target clusters sourceCluster, err := database.GetCluster(s.Db, job.SourceClusterName) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Source cluster not found") } - + targetCluster, err := database.GetCluster(s.Db, job.TargetClusterName) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Target cluster not found") } - + // Create cluster configs sourceConfig := config.ClusterConfig{ Provider: sourceCluster.Provider, @@ -2142,7 +2141,7 @@ func (s *Server) handleCalculateResumePoints(c *fiber.Ctx) error { APISecret: sourceCluster.APISecret, }, } - + targetConfig := config.ClusterConfig{ Provider: targetCluster.Provider, Brokers: targetCluster.Brokers, @@ -2151,23 +2150,23 @@ func (s *Server) handleCalculateResumePoints(c *fiber.Ctx) error { APISecret: targetCluster.APISecret, }, } - + // Get topic mappings mappings, err := database.GetMappingsForJob(s.Db, jobID) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to get topic mappings") } - + topicMap := make(map[string]string) for _, mapping := range mappings { if mapping.Enabled { topicMap[mapping.SourceTopicPattern] = mapping.TargetTopicPattern } } - + go func() { ctx := context.Background() - + // Create admin clients sourceAdmin, err := kafka.NewAdminClient(sourceConfig) if err != nil { @@ -2175,14 +2174,14 @@ func (s *Server) handleCalculateResumePoints(c *fiber.Ctx) error { return } defer sourceAdmin.Close() - + targetAdmin, err := kafka.NewAdminClient(targetConfig) if err != nil { log.Printf("Failed to create target admin client for resume point calculation: %v", err) return } defer targetAdmin.Close() - + // Calculate resume points consumerGroup := fmt.Sprintf("kaf-mirror-job-%s", jobID) resumePoints, err := targetAdmin.CalculateSafeResumePoints(ctx, sourceAdmin, jobID, topicMap, consumerGroup) @@ -2190,17 +2189,17 @@ func (s *Server) handleCalculateResumePoints(c *fiber.Ctx) error { log.Printf("Failed to calculate resume points for job %s: %v", jobID, err) return } - + // Store resume points in database err = database.CalculateResumePoints(s.Db, jobID, resumePoints, nil) if err != nil { log.Printf("Failed to store resume points for job %s: %v", jobID, err) return } - + log.Printf("Successfully calculated and stored resume points for job %s", jobID) }() - + return c.Status(fiber.StatusAccepted).JSON(fiber.Map{ "status": "success", "message": "Resume point calculation started", @@ -2222,12 +2221,12 @@ func (s *Server) handleCalculateResumePoints(c *fiber.Ctx) error { // @Security ApiKeyAuth func (s *Server) handleGetMirrorGaps(c *fiber.Ctx) error { jobID := c.Params("id") - + gaps, err := database.GetMirrorGaps(s.Db, jobID) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to get mirror gaps") } - + return c.JSON(gaps) } @@ -2245,36 +2244,36 @@ func (s *Server) handleGetMirrorGaps(c *fiber.Ctx) error { // @Security ApiKeyAuth func (s *Server) handleValidateMigration(c *fiber.Ctx) error { jobID := c.Params("id") - + job, err := database.GetJob(s.Db, jobID) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Job not found") } - + // Get source and target clusters sourceCluster, err := database.GetCluster(s.Db, job.SourceClusterName) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Source cluster not found") } - + targetCluster, err := database.GetCluster(s.Db, job.TargetClusterName) if err != nil { return fiber.NewError(fiber.StatusNotFound, "Target cluster not found") } - + // Get topic mappings mappings, err := database.GetMappingsForJob(s.Db, jobID) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to get topic mappings") } - + topicMap := make(map[string]string) for _, mapping := range mappings { if mapping.Enabled { topicMap[mapping.SourceTopicPattern] = mapping.TargetTopicPattern } } - + // Create cluster configs sourceConfig := config.ClusterConfig{ Provider: sourceCluster.Provider, @@ -2284,7 +2283,7 @@ func (s *Server) handleValidateMigration(c *fiber.Ctx) error { APISecret: sourceCluster.APISecret, }, } - + targetConfig := config.ClusterConfig{ Provider: targetCluster.Provider, Brokers: targetCluster.Brokers, @@ -2293,45 +2292,45 @@ func (s *Server) handleValidateMigration(c *fiber.Ctx) error { APISecret: targetCluster.APISecret, }, } - + ctx := context.Background() - + // Create admin clients sourceAdmin, err := kafka.NewAdminClient(sourceConfig) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to create source admin client") } defer sourceAdmin.Close() - + targetAdmin, err := kafka.NewAdminClient(targetConfig) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to create target admin client") } defer targetAdmin.Close() - + // Perform cross-cluster offset comparison offsetComparison, err := targetAdmin.CompareClusterOffsets(ctx, sourceAdmin, topicMap) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to compare cluster offsets") } - + // Analyze mirror state consumerGroup := fmt.Sprintf("kaf-mirror-job-%s", jobID) mirrorAnalysis, err := targetAdmin.AnalyzeMirrorState(ctx, sourceAdmin, jobID, topicMap, consumerGroup) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to analyze mirror state") } - + // Determine migration safety migrationSafe := offsetComparison.TotalGapsDetected == 0 && len(offsetComparison.CriticalIssues) == 0 riskLevel := "low" - + if offsetComparison.TotalGapsDetected > 0 { riskLevel = "high" } else if len(offsetComparison.Warnings) > 0 { riskLevel = "medium" } - + recommendations := make([]string, 0) if !migrationSafe { recommendations = append(recommendations, "Migration not recommended due to detected gaps or critical issues") @@ -2340,20 +2339,20 @@ func (s *Server) handleValidateMigration(c *fiber.Ctx) error { recommendations = append(recommendations, "Migration appears safe to proceed") recommendations = append(recommendations, "Monitor replication closely after migration") } - + result := fiber.Map{ - "job_id": jobID, - "migration_safe": migrationSafe, - "risk_level": riskLevel, - "gaps_detected": offsetComparison.TotalGapsDetected, - "critical_issues": len(offsetComparison.CriticalIssues), - "warnings": len(offsetComparison.Warnings), - "offset_comparison": offsetComparison, - "mirror_analysis": mirrorAnalysis, - "recommendations": recommendations, - "validated_at": time.Now(), - } - + "job_id": jobID, + "migration_safe": migrationSafe, + "risk_level": riskLevel, + "gaps_detected": offsetComparison.TotalGapsDetected, + "critical_issues": len(offsetComparison.CriticalIssues), + "warnings": len(offsetComparison.Warnings), + "offset_comparison": offsetComparison, + "mirror_analysis": mirrorAnalysis, + "recommendations": recommendations, + "validated_at": time.Now(), + } + return c.JSON(result) } @@ -2372,35 +2371,35 @@ func (s *Server) handleValidateMigration(c *fiber.Ctx) error { // @Security ApiKeyAuth func (s *Server) handleCreateMigrationCheckpoint(c *fiber.Ctx) error { jobID := c.Params("id") - + var request struct { MigrationReason string `json:"migration_reason"` CheckpointType string `json:"checkpoint_type"` } - + if err := c.BodyParser(&request); err != nil { request.MigrationReason = "Manual checkpoint" request.CheckpointType = "pre_migration" } - + user := c.Locals("user").(*database.User) - + checkpoint := database.MigrationCheckpoint{ - JobID: jobID, - CheckpointType: request.CheckpointType, - SourceConsumerGroupOffsets: "{}", // Would be populated with actual data - TargetHighWaterMarks: "{}", // Would be populated with actual data - CreatedAt: time.Now(), - CreatedBy: user.Username, - MigrationReason: &request.MigrationReason, - ValidationResults: nil, - } - + JobID: jobID, + CheckpointType: request.CheckpointType, + SourceConsumerGroupOffsets: "{}", // Would be populated with actual data + TargetHighWaterMarks: "{}", // Would be populated with actual data + CreatedAt: time.Now(), + CreatedBy: user.Username, + MigrationReason: &request.MigrationReason, + ValidationResults: nil, + } + checkpointID, err := database.CreateMigrationCheckpoint(s.Db, checkpoint) if err != nil { return fiber.NewError(fiber.StatusInternalServerError, "Failed to create migration checkpoint") } - + return c.Status(fiber.StatusCreated).JSON(fiber.Map{ "status": "success", "message": "Migration checkpoint created", diff --git a/internal/server/hub.go b/internal/server/hub.go index a476e24..4dca3b5 100644 --- a/internal/server/hub.go +++ b/internal/server/hub.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package server import ( diff --git a/internal/server/middleware/audit.go b/internal/server/middleware/audit.go index 5a6278a..289ec75 100644 --- a/internal/server/middleware/audit.go +++ b/internal/server/middleware/audit.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package middleware import ( @@ -73,7 +72,7 @@ func AuditLog(db *sqlx.DB) fiber.Handler { func formatAuditDetails(c *fiber.Ctx) string { path := c.Path() method := c.Method() - + // Create summary based on endpoint switch { case strings.Contains(path, "/jobs") && method == "POST": @@ -148,7 +147,7 @@ func formatConfigUpdate(c *fiber.Ctx) string { } details := []string{} - + if ai, ok := configData["AI"].(map[string]interface{}); ok { if provider := getStringValue(ai, "Provider"); provider != "" { details = append(details, fmt.Sprintf("AI Provider: %s", provider)) diff --git a/internal/server/middleware/auth.go b/internal/server/middleware/auth.go index 878b4ec..913b921 100644 --- a/internal/server/middleware/auth.go +++ b/internal/server/middleware/auth.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package middleware import ( diff --git a/internal/server/middleware/cors.go b/internal/server/middleware/cors.go index 74151de..fd372a5 100644 --- a/internal/server/middleware/cors.go +++ b/internal/server/middleware/cors.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package middleware import ( diff --git a/internal/server/middleware/rbac.go b/internal/server/middleware/rbac.go index 1b6ae7a..a257972 100644 --- a/internal/server/middleware/rbac.go +++ b/internal/server/middleware/rbac.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package middleware import ( diff --git a/internal/server/server.go b/internal/server/server.go index ea9f6c0..4649892 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -9,18 +9,17 @@ // See the License for the specific language governing permissions and // limitations under the License. - package server import ( "fmt" - "log" "kaf-mirror/internal/ai" "kaf-mirror/internal/config" "kaf-mirror/internal/database" "kaf-mirror/internal/manager" "kaf-mirror/internal/server/middleware" "kaf-mirror/pkg/logger" + "log" "time" "github.com/gofiber/fiber/v2" @@ -81,20 +80,20 @@ func New(cfg *config.Config, db *sqlx.DB, manager *manager.JobManager, hub *Hub, // Start runs the Fiber server. func (s *Server) Start() error { addr := fmt.Sprintf("%s:%d", s.cfg.Server.Host, s.cfg.Server.Port) - + // Check if TLS is enabled if s.cfg.Server.TLS.Enabled { certFile := s.cfg.Server.TLS.CertFile keyFile := s.cfg.Server.TLS.KeyFile - + if certFile == "" || keyFile == "" { return fmt.Errorf("TLS is enabled but certificate or key file path is missing") } - + logger.Info("Starting HTTPS server on %s with TLS certificate: %s", addr, certFile) return s.App.ListenTLS(addr, certFile, keyFile) } - + logger.Info("Starting HTTP server on %s", addr) return s.App.Listen(addr) } diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go index 3791860..397c41e 100644 --- a/pkg/logger/logger.go +++ b/pkg/logger/logger.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package logger import ( @@ -86,7 +85,7 @@ func (l *Logger) log(level Level, format string, args ...interface{}) { timestamp := time.Now().Format("2006-01-02 15:04:05.000") levelName := levelNames[level] message := fmt.Sprintf(format, args...) - + logLine := fmt.Sprintf("[%s] %-5s [%s] %s", timestamp, levelName, caller, message) switch level { @@ -122,14 +121,14 @@ func (l *Logger) logWithTag(level Level, category, subcategory, jobID string, fo timestamp := time.Now().Format("2006-01-02 15:04:05.000") levelName := levelNames[level] message := fmt.Sprintf(format, args...) - + // Format: [timestamp] LEVEL [caller] [AI:category:subcategory] [job:jobID] message var logLine string if jobID != "" { - logLine = fmt.Sprintf("[%s] %-5s [%s] [AI:%s:%s] [job:%s] %s", + logLine = fmt.Sprintf("[%s] %-5s [%s] [AI:%s:%s] [job:%s] %s", timestamp, levelName, caller, category, subcategory, jobID, message) } else { - logLine = fmt.Sprintf("[%s] %-5s [%s] [AI:%s:%s] %s", + logLine = fmt.Sprintf("[%s] %-5s [%s] [AI:%s:%s] %s", timestamp, levelName, caller, category, subcategory, message) } @@ -222,7 +221,6 @@ func SetLevel(level Level) { defaultLogger.level = level } - // GetProductionLogDir returns the appropriate log directory for production func GetProductionLogDir() string { if runtime.GOOS == "linux" || runtime.GOOS == "darwin" { @@ -237,7 +235,7 @@ func GetProductionLogDir() string { } } } - + // Fallback to local logs directory return "logs" } diff --git a/pkg/utils/crypto.go b/pkg/utils/crypto.go index 8519ed2..913ecd7 100644 --- a/pkg/utils/crypto.go +++ b/pkg/utils/crypto.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package utils import ( diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index 52e9059..b350899 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package utils import "strings" diff --git a/scripts/codeql_local.sh b/scripts/codeql_local.sh new file mode 100755 index 0000000..d1dbc51 --- /dev/null +++ b/scripts/codeql_local.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$ROOT_DIR" + +if ! command -v codeql >/dev/null 2>&1; then + echo "ERROR: codeql CLI not found in PATH" + echo "Install from: https://docs.github.com/en/code-security/codeql-cli/getting-started-with-the-codeql-cli" + exit 1 +fi + +mkdir -p .tmp/codeql + +echo "==> Ensuring CodeQL standard query packs are available" +codeql pack download codeql/go-queries codeql/actions-queries + +CODEQL_GO_RAM_MB="${CODEQL_GO_RAM_MB:-2048}" +CODEQL_ACTIONS_RAM_MB="${CODEQL_ACTIONS_RAM_MB:-1024}" +CODEQL_QUERY_STRATEGY="${CODEQL_QUERY_STRATEGY:-github}" + +run_go() { + echo "==> CodeQL (Go)" + rm -rf .tmp/codeql/go-db + codeql database create .tmp/codeql/go-db \ + --language=go \ + --ram="$CODEQL_GO_RAM_MB" \ + --command="go build ./..." + + if [[ "$CODEQL_QUERY_STRATEGY" == "security-and-quality" ]]; then + codeql database analyze .tmp/codeql/go-db \ + codeql/go-queries:codeql-suites/go-security-and-quality.qls \ + --download \ + --ram="$CODEQL_GO_RAM_MB" \ + --format=sarifv2.1.0 \ + --sarif-category="/language:go" \ + --output .tmp/codeql/go.sarif + else + codeql database analyze .tmp/codeql/go-db \ + codeql/go-queries \ + --download \ + --ram="$CODEQL_GO_RAM_MB" \ + --format=sarifv2.1.0 \ + --sarif-category="/language:go" \ + --output .tmp/codeql/go.sarif + fi +} + +run_actions() { + echo "==> CodeQL (Actions)" + rm -rf .tmp/codeql/actions-db + codeql database create .tmp/codeql/actions-db \ + --language=actions \ + --build-mode=none \ + --ram="$CODEQL_ACTIONS_RAM_MB" + + codeql database analyze .tmp/codeql/actions-db \ + codeql/actions-queries \ + --download \ + --ram="$CODEQL_ACTIONS_RAM_MB" \ + --format=sarifv2.1.0 \ + --sarif-category="/language:actions" \ + --output .tmp/codeql/actions.sarif +} + +run_go +run_actions + +echo "" +echo "CodeQL local run complete." +echo "SARIF outputs:" +echo " .tmp/codeql/go.sarif" +echo " .tmp/codeql/actions.sarif" diff --git a/scripts/generate-cli-docs.go b/scripts/generate-cli-docs.go index 2472f56..048fb32 100644 --- a/scripts/generate-cli-docs.go +++ b/scripts/generate-cli-docs.go @@ -15,7 +15,7 @@ import ( func main() { fmt.Println("Generating CLI documentation...") - + fmt.Println("Updating Go dependencies...") modCmd := exec.Command("go", "mod", "tidy") modCmd.Stdout = os.Stdout @@ -24,12 +24,12 @@ func main() { fmt.Printf("Error updating dependencies: %v\n", err) os.Exit(1) } - + if err := os.MkdirAll("bin", 0755); err != nil { fmt.Printf("Error creating bin directory: %v\n", err) os.Exit(1) } - + fmt.Println("Building CLI...") buildCmd := exec.Command("go", "build", "-o", "bin/mirror-cli", "./cmd/mirror-cli") buildCmd.Stdout = os.Stdout @@ -38,26 +38,26 @@ func main() { fmt.Printf("Error building CLI: %v\n", err) os.Exit(1) } - + fmt.Println("Generating single consolidated Markdown documentation file...") docsCmd := exec.Command("./bin/mirror-cli", "docs", "generate", "--file", "./docs/cli-commands.md") docsCmd.Stdout = os.Stdout docsCmd.Stderr = os.Stderr - + if err := docsCmd.Run(); err != nil { fmt.Printf("Error generating docs: %v\n", err) os.Exit(1) } - + fmt.Println("Converting Markdown documentation to HTML...") if err := convertMarkdownToHTML("./docs/cli-commands.md", "./docs/cli-commands.html"); err != nil { fmt.Printf("Error converting to HTML: %v\n", err) os.Exit(1) } - + fmt.Println("CLI documentation generated successfully!") fmt.Println("- Markdown file: ./docs/cli-commands.md") - fmt.Println("- HTML file: ./docs/cli-commands.html") + fmt.Println("- HTML file: ./docs/cli-commands.html") } // convertMarkdownToHTML converts the comprehensive Markdown file to HTML @@ -298,10 +298,10 @@ func convertMarkdownToHTML(markdownFile, htmlFile string) error { scanner = bufio.NewScanner(file) inCodeBlock := false - + for scanner.Scan() { line := scanner.Text() - + if strings.HasPrefix(line, "```") { if inCodeBlock { htmlOutput.WriteString("\n") @@ -312,20 +312,20 @@ func convertMarkdownToHTML(markdownFile, htmlFile string) error { } continue } - + if inCodeBlock { htmlOutput.WriteString(html.EscapeString(line) + "\n") continue } - + htmlLine := convertMarkdownLine(line) htmlOutput.WriteString(htmlLine + "\n") } - + if inCodeBlock { htmlOutput.WriteString("\n") } - + htmlOutput.WriteString(` @@ -387,7 +387,7 @@ func convertMarkdownLine(line string) string { if line == "" { return "
" } - + // Headers with anchors if strings.HasPrefix(line, "######") { title := strings.TrimSpace(line[6:]) @@ -414,17 +414,17 @@ func convertMarkdownLine(line string) string { anchor := generateAnchor(title) return fmt.Sprintf(`

%s

`, anchor, html.EscapeString(title)) } - + // Horizontal rule if strings.TrimSpace(line) == "---" { return "
" } - + // Lists if strings.HasPrefix(line, "- ") { return "
  • " + processInlineMarkdown(html.EscapeString(strings.TrimSpace(line[2:]))) + "
    • " } - + // Regular paragraph return "

    " + processInlineMarkdown(html.EscapeString(line)) + "

    " } @@ -434,11 +434,11 @@ func processInlineMarkdown(text string) string { // Bold text boldRe := regexp.MustCompile(`\*\*([^*]+)\*\*`) text = boldRe.ReplaceAllString(text, "$1") - + // Inline code - be careful not to double-escape codeRe := regexp.MustCompile("`([^`]+)`") text = codeRe.ReplaceAllString(text, "$1") - + return text } @@ -463,13 +463,13 @@ func getHeaderLevel(line string) int { // generateAnchor generates a URL-safe anchor from a title func generateAnchor(title string) string { anchor := strings.ToLower(title) - + // Replace spaces and special characters with hyphens reg := regexp.MustCompile(`[^a-z0-9]+`) anchor = reg.ReplaceAllString(anchor, "-") - + anchor = strings.Trim(anchor, "-") - + return anchor } diff --git a/scripts/generate-docs.go b/scripts/generate-docs.go index 7191c26..7d42c0e 100644 --- a/scripts/generate-docs.go +++ b/scripts/generate-docs.go @@ -285,7 +285,7 @@ func convertMarkdownToHTML(markdownFile, htmlFile string) error { // Generate navigation and write to a temporary buffer navHTML := generateNavigation(navItems) - + // Re-open file for second pass file, err = os.Open(markdownFile) if err != nil { @@ -318,10 +318,10 @@ func convertMarkdownToHTML(markdownFile, htmlFile string) error { if inCodeBlock { htmlOutput.WriteString("\n") } - + // Close the last command card htmlOutput.WriteString("\n") - + htmlOutput.WriteString(` @@ -388,10 +388,10 @@ func convertMarkdownLine(line string) string { if level := getHeaderLevel(line); level > 0 { title := strings.TrimSpace(line[level:]) anchor := generateAnchor(title) - + // Check if this is a main command (starts with "mirror-cli" and is h1 or h3) isMainCommand := (level == 1 || level == 3) && strings.HasPrefix(title, "mirror-cli") - + if isMainCommand { if level == 1 { return fmt.Sprintf(`

    %s

    `, anchor, anchor, html.EscapeString(title)) @@ -420,10 +420,10 @@ func convertMarkdownLine(line string) string { func processInlineMarkdown(text string) string { boldRe := regexp.MustCompile(`\*\*([^*]+)\*\*`) text = boldRe.ReplaceAllString(text, "$1") - + codeRe := regexp.MustCompile("`([^`]+)`") text = codeRe.ReplaceAllString(text, "$1") - + return text } diff --git a/tests/cmd/broker_validation_test.go b/tests/cmd/broker_validation_test.go index ca9c218..fc37827 100644 --- a/tests/cmd/broker_validation_test.go +++ b/tests/cmd/broker_validation_test.go @@ -22,7 +22,7 @@ func TestValidateConfluentBrokers(t *testing.T) { {"Valid simple cluster name", "my-cluster.confluent.cloud:9092", true}, {"Valid numeric cluster", "123456.confluent.cloud:9092", true}, {"Valid mixed cluster", "test-123.confluent.cloud:443", true}, - + // Invalid formats {"Wrong cloud format", "pkc-abcde.example.cloud:9092", false}, {"Missing domain", "pkc-12345:9092", false}, @@ -64,7 +64,7 @@ func TestValidateRedpandaBrokers(t *testing.T) { {"Valid simple name", "prod.redpanda.cloud:443", true}, {"Valid numeric cluster", "123456.redpanda.cloud:9092", true}, {"Valid with hyphens", "test-prod-1.redpanda.cloud:9092", true}, - + // Invalid formats {"Missing domain", "seed-12345:9092", false}, {"Wrong domain", "seed-12345.kafka.com:9092", false}, @@ -104,7 +104,7 @@ func TestValidateAzureBrokers(t *testing.T) { {"Valid numeric namespace", "eventhub123.servicebus.windows.net:9093", true}, {"Valid mixed namespace", "test-123.servicebus.windows.net:9093", true}, {"Valid custom port", "production.servicebus.windows.net:443", true}, - + // Invalid formats {"Missing domain", "mynamespace:9093", false}, {"Wrong domain", "mynamespace.eventhub.azure.com:9093", false}, @@ -146,7 +146,7 @@ func TestValidatePlainBrokers(t *testing.T) { {"Mixed formats", "localhost:9092,192.168.1.1:9093,kafka.example.com:9094", true}, {"Custom port", "kafka:8080", true}, {"High port number", "broker:65535", true}, - + // Invalid formats {"Missing port", "localhost", false}, {"Invalid port", "localhost:abc", false}, @@ -185,19 +185,19 @@ func TestValidateBrokerString(t *testing.T) { // Confluent provider {"Confluent valid", "pkc-12345.us-west-2.aws.confluent.cloud:9092", "confluent", true}, {"Confluent invalid", "invalid-broker:9092", "confluent", false}, - + // RedPanda provider {"RedPanda valid", "my-cluster.redpanda.cloud:9092", "redpanda", true}, {"RedPanda invalid", "invalid-broker:9092", "redpanda", false}, - + // Azure provider {"Azure valid", "mynamespace.servicebus.windows.net:9093", "azure", true}, {"Azure invalid", "invalid-broker:9092", "azure", false}, - + // Plain provider {"Plain valid", "localhost:9092", "plain", true}, {"Plain invalid", "localhost", "plain", false}, - + // Unknown provider (defaults to plain) {"Unknown provider", "localhost:9092", "unknown", true}, {"Unknown provider invalid", "localhost", "unknown", false}, @@ -222,10 +222,10 @@ func TestValidateBrokerString(t *testing.T) { // Test real-world broker string examples func TestRealWorldBrokerExamples(t *testing.T) { realWorldTests := []struct { - name string - brokerString string - provider string - description string + name string + brokerString string + provider string + description string shouldSucceed bool }{ { @@ -270,12 +270,12 @@ func TestRealWorldBrokerExamples(t *testing.T) { err := validateBrokerString(tt.brokerString, tt.provider) if tt.shouldSucceed { if err != nil { - t.Errorf("%s: validateBrokerString(%q, %q) should succeed but got error: %v", + t.Errorf("%s: validateBrokerString(%q, %q) should succeed but got error: %v", tt.description, tt.brokerString, tt.provider, err) } } else { if err == nil { - t.Errorf("%s: validateBrokerString(%q, %q) should fail but succeeded", + t.Errorf("%s: validateBrokerString(%q, %q) should fail but succeeded", tt.description, tt.brokerString, tt.provider) } } @@ -287,42 +287,42 @@ func validateConfluentBrokers(brokerString string) error { if strings.TrimSpace(brokerString) == "" { return fmt.Errorf("broker string cannot be empty") } - + // Check if it ends with .confluent.cloud:port if !strings.Contains(brokerString, ".confluent.cloud:") { return fmt.Errorf("invalid Confluent Cloud broker format: %s (expected format: cluster-name[.region.provider].confluent.cloud:port)", brokerString) } - + // Split on .confluent.cloud: to separate the cluster/region part from port beforeConfluent := strings.Split(brokerString, ".confluent.cloud:")[0] portPart := strings.Split(brokerString, ".confluent.cloud:")[1] - + // Validate port if _, err := strconv.Atoi(portPart); err != nil { return fmt.Errorf("invalid port in Confluent Cloud broker: %s", brokerString) } - + // Split the part before .confluent.cloud by dots parts := strings.Split(beforeConfluent, ".") - + // For simple format like "cluster.confluent.cloud", we should have 1 part // For complex format like "cluster.region.provider.confluent.cloud", we should have 3+ parts // But we need to reject formats like "pkc.12345.confluent.cloud" where the cluster name contains dots - + if len(parts) == 2 { // This is the problematic case like "pkc.12345" - cluster name with dots but not full region.provider format return fmt.Errorf("invalid Confluent Cloud broker format: cluster name cannot contain dots: %s", brokerString) } - + if len(parts) < 1 { return fmt.Errorf("invalid Confluent Cloud broker format: missing cluster name: %s", brokerString) } - + // First part should not be empty (cluster name) if parts[0] == "" { return fmt.Errorf("invalid Confluent Cloud broker format: empty cluster name: %s", brokerString) } - + return nil } @@ -330,17 +330,17 @@ func validateRedpandaBrokers(brokerString string) error { if strings.TrimSpace(brokerString) == "" { return fmt.Errorf("broker string cannot be empty") } - + pattern := `^[^.]+\.redpanda\.cloud:[0-9]+$` matched, err := regexp.MatchString(pattern, brokerString) if err != nil { return err } - + if !matched { return fmt.Errorf("invalid RedPanda Cloud broker format: %s (expected format: cluster-name.redpanda.cloud:port)", brokerString) } - + return nil } @@ -348,17 +348,17 @@ func validateAzureBrokers(brokerString string) error { if strings.TrimSpace(brokerString) == "" { return fmt.Errorf("broker string cannot be empty") } - + pattern := `^[^.]+\.servicebus\.windows\.net:[0-9]+$` matched, err := regexp.MatchString(pattern, brokerString) if err != nil { return err } - + if !matched { return fmt.Errorf("invalid Azure Event Hub broker format: %s (expected format: namespace.servicebus.windows.net:port)", brokerString) } - + return nil } @@ -366,39 +366,39 @@ func validatePlainBrokers(brokerString string) error { if strings.TrimSpace(brokerString) == "" { return fmt.Errorf("broker string cannot be empty") } - + brokers := strings.Split(brokerString, ",") for _, broker := range brokers { broker = strings.TrimSpace(broker) if broker == "" { return fmt.Errorf("empty broker in broker list") } - + if !strings.Contains(broker, ":") { return fmt.Errorf("broker must include port: %s", broker) } - + parts := strings.Split(broker, ":") if len(parts) != 2 { return fmt.Errorf("invalid broker format: %s", broker) } - + hostname := parts[0] if hostname == "" { return fmt.Errorf("hostname cannot be empty: %s", broker) } - + portStr := parts[1] port, err := strconv.Atoi(portStr) if err != nil { return fmt.Errorf("invalid port in broker: %s", broker) } - + if port < 1 || port > 65535 { return fmt.Errorf("port out of range (1-65535): %d", port) } } - + return nil } diff --git a/tests/cmd/cluster_cli_integration_test.go b/tests/cmd/cluster_cli_integration_test.go index 98677be..264d0e6 100644 --- a/tests/cmd/cluster_cli_integration_test.go +++ b/tests/cmd/cluster_cli_integration_test.go @@ -39,13 +39,13 @@ func NewMockServer() *MockServer { ms := &MockServer{ calls: make([]APICall, 0), } - + mux := http.NewServeMux() - + mux.HandleFunc("/api/v1/clusters/test", ms.handleClusterTest) mux.HandleFunc("/api/v1/clusters", ms.handleClusters) mux.HandleFunc("/api/v1/clusters/", ms.handleClusterOperations) - + ms.server = httptest.NewServer(mux) return ms } @@ -81,14 +81,14 @@ func (ms *MockServer) handleClusterTest(w http.ResponseWriter, r *http.Request) for k, v := range r.Header { headers[k] = strings.Join(v, ",") } - + body := "" if r.Body != nil { buf := make([]byte, r.ContentLength) r.Body.Read(buf) body = string(buf) } - + var testConfig map[string]interface{} if err := json.Unmarshal([]byte(body), &testConfig); err != nil { ms.recordCall(r.Method, r.URL.Path, headers, body, 400) @@ -96,7 +96,7 @@ func (ms *MockServer) handleClusterTest(w http.ResponseWriter, r *http.Request) w.Write([]byte(`{"error":"Invalid JSON"}`)) return } - + provider, ok := testConfig["provider"].(string) if !ok { ms.recordCall(r.Method, r.URL.Path, headers, body, 400) @@ -104,7 +104,7 @@ func (ms *MockServer) handleClusterTest(w http.ResponseWriter, r *http.Request) w.Write([]byte(`{"error":"Missing provider"}`)) return } - + switch provider { case "confluent": security, ok := testConfig["security"].(map[string]interface{}) @@ -128,7 +128,7 @@ func (ms *MockServer) handleClusterTest(w http.ResponseWriter, r *http.Request) return } } - + ms.recordCall(r.Method, r.URL.Path, headers, body, 200) w.WriteHeader(http.StatusOK) w.Write([]byte(`{"status":"connection successful"}`)) @@ -139,14 +139,14 @@ func (ms *MockServer) handleClusters(w http.ResponseWriter, r *http.Request) { for k, v := range r.Header { headers[k] = strings.Join(v, ",") } - + body := "" if r.Body != nil { buf := make([]byte, r.ContentLength) r.Body.Read(buf) body = string(buf) } - + if r.Method == "POST" { var cluster map[string]interface{} if err := json.Unmarshal([]byte(body), &cluster); err != nil { @@ -154,7 +154,7 @@ func (ms *MockServer) handleClusters(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) return } - + required := []string{"name", "provider", "brokers"} for _, field := range required { if cluster[field] == nil || cluster[field] == "" { @@ -164,7 +164,7 @@ func (ms *MockServer) handleClusters(w http.ResponseWriter, r *http.Request) { return } } - + ms.recordCall(r.Method, r.URL.Path, headers, body, 201) w.WriteHeader(http.StatusCreated) w.Write([]byte(`{"message":"cluster created successfully"}`)) @@ -179,38 +179,38 @@ func (ms *MockServer) handleClusterOperations(w http.ResponseWriter, r *http.Req for k, v := range r.Header { headers[k] = strings.Join(v, ",") } - + body := "" if r.Body != nil { buf := make([]byte, r.ContentLength) r.Body.Read(buf) body = string(buf) } - + pathParts := strings.Split(r.URL.Path, "/") if len(pathParts) < 4 { ms.recordCall(r.Method, r.URL.Path, headers, body, 400) w.WriteHeader(http.StatusBadRequest) return } - + clusterName := pathParts[4] - + switch r.Method { case "GET": mockCluster := map[string]interface{}{ - "name": clusterName, - "provider": "confluent", - "brokers": "pkc-example.us-west1.example.cloud:9092", + "name": clusterName, + "provider": "confluent", + "brokers": "pkc-example.us-west1.example.cloud:9092", "cluster_id": "lkc-example", - "api_key": "EXAMPLE_KEY", - "status": "active", + "api_key": "EXAMPLE_KEY", + "status": "active", } - + ms.recordCall(r.Method, r.URL.Path, headers, body, 200) w.WriteHeader(http.StatusOK) json.NewEncoder(w).Encode(mockCluster) - + case "PUT": var cluster map[string]interface{} if err := json.Unmarshal([]byte(body), &cluster); err != nil { @@ -218,7 +218,7 @@ func (ms *MockServer) handleClusterOperations(w http.ResponseWriter, r *http.Req w.WriteHeader(http.StatusBadRequest) return } - + brokers, exists := cluster["brokers"] if !exists || brokers == "" { ms.recordCall(r.Method, r.URL.Path, headers, body, 400) @@ -226,11 +226,11 @@ func (ms *MockServer) handleClusterOperations(w http.ResponseWriter, r *http.Req w.Write([]byte(`{"error":"brokers field is required and cannot be empty"}`)) return } - + ms.recordCall(r.Method, r.URL.Path, headers, body, 200) w.WriteHeader(http.StatusOK) w.Write([]byte(`{"message":"cluster updated successfully"}`)) - + default: ms.recordCall(r.Method, r.URL.Path, headers, body, 405) w.WriteHeader(http.StatusMethodNotAllowed) @@ -240,7 +240,7 @@ func (ms *MockServer) handleClusterOperations(w http.ResponseWriter, r *http.Req func TestClusterAddIntegration(t *testing.T) { mockServer := NewMockServer() defer mockServer.Close() - + tests := []struct { name string clusterConfig ClusterConfig @@ -290,43 +290,43 @@ func TestClusterAddIntegration(t *testing.T) { addResponse: 0, }, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockServer.ClearCalls() - + err := simulateClusterAdd(mockServer.URL(), tt.clusterConfig) - + calls := mockServer.GetCalls() - + if tt.expectTestCall { assert.GreaterOrEqual(t, len(calls), 1, "Should have at least one API call") - + testCall := calls[0] assert.Equal(t, "POST", testCall.Method, "First call should be POST") assert.Equal(t, "/api/v1/clusters/test", testCall.Path, "First call should be to test endpoint") assert.Equal(t, tt.testResponse, testCall.Response, "Test response should match expected") - + var testConfig map[string]interface{} json.Unmarshal([]byte(testCall.Body), &testConfig) assert.Equal(t, tt.clusterConfig.Provider, testConfig["provider"], "Provider should match") assert.Equal(t, tt.clusterConfig.Brokers, testConfig["brokers"], "Brokers should match") } - + if tt.expectAddCall { assert.GreaterOrEqual(t, len(calls), 2, "Should have cluster creation call") - + addCall := calls[1] assert.Equal(t, "POST", addCall.Method, "Second call should be POST") assert.Equal(t, "/api/v1/clusters", addCall.Path, "Second call should be to clusters endpoint") assert.Equal(t, tt.addResponse, addCall.Response, "Add response should match expected") - + var addConfig map[string]interface{} json.Unmarshal([]byte(addCall.Body), &addConfig) assert.Equal(t, tt.clusterConfig.Name, addConfig["name"], "Name should match") assert.Equal(t, tt.clusterConfig.Provider, addConfig["provider"], "Provider should match") assert.Equal(t, tt.clusterConfig.Brokers, addConfig["brokers"], "Brokers should match") - + assert.NoError(t, err, "Add operation should succeed") } else { assert.Equal(t, 1, len(calls), "Should only have test call") @@ -339,7 +339,7 @@ func TestClusterAddIntegration(t *testing.T) { func TestClusterEditIntegration(t *testing.T) { mockServer := NewMockServer() defer mockServer.Close() - + tests := []struct { name string clusterName string @@ -362,26 +362,26 @@ func TestClusterEditIntegration(t *testing.T) { expectPutCall: true, }, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mockServer.ClearCalls() - + err := simulateClusterEdit(mockServer.URL(), tt.clusterName, tt.newConfig) - + calls := mockServer.GetCalls() - + if tt.expectGetCall { assert.GreaterOrEqual(t, len(calls), 1, "Should have at least one API call") - + getCall := calls[0] assert.Equal(t, "GET", getCall.Method, "First call should be GET") assert.Contains(t, getCall.Path, tt.clusterName, "GET call should include cluster name") } - + if tt.expectPutCall { assert.GreaterOrEqual(t, len(calls), 3, "Should have multiple API calls for edit flow") - + var putCall *APICall for _, call := range calls { if call.Method == "PUT" { @@ -389,16 +389,16 @@ func TestClusterEditIntegration(t *testing.T) { break } } - + assert.NotNil(t, putCall, "Should have PUT call") assert.Contains(t, putCall.Path, tt.clusterName, "PUT call should include cluster name") - + var updateConfig map[string]interface{} json.Unmarshal([]byte(putCall.Body), &updateConfig) - + assert.NotEmpty(t, updateConfig["brokers"], "Brokers field should not be empty in update request") assert.Equal(t, tt.newConfig.Brokers, updateConfig["brokers"], "Brokers should match new config") - + assert.NoError(t, err, "Edit operation should succeed") } }) @@ -437,7 +437,7 @@ func TestSafeStringFunction(t *testing.T) { expected: "default", // Non-string should return default }, } - + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { result := safeString(tt.value, tt.defaultValue) @@ -449,30 +449,30 @@ func TestSafeStringFunction(t *testing.T) { func simulateClusterAdd(serverURL string, config ClusterConfig) error { testConfig := buildConnectionTestConfig(config) testBody, _ := json.Marshal(testConfig) - + resp, err := http.Post(serverURL+"/api/v1/clusters/test", "application/json", strings.NewReader(string(testBody))) if err != nil { return err } resp.Body.Close() - + if resp.StatusCode != http.StatusOK { return fmt.Errorf("connection test failed with status: %d", resp.StatusCode) } - + addRequest := buildClusterRequest(config, "add") addBody, _ := json.Marshal(addRequest) - + resp, err = http.Post(serverURL+"/api/v1/clusters", "application/json", strings.NewReader(string(addBody))) if err != nil { return err } resp.Body.Close() - + if resp.StatusCode != http.StatusCreated { return fmt.Errorf("cluster creation failed with status: %d", resp.StatusCode) } - + return nil } @@ -482,41 +482,41 @@ func simulateClusterEdit(serverURL, clusterName string, newConfig ClusterConfig) return err } resp.Body.Close() - + if resp.StatusCode != http.StatusOK { return fmt.Errorf("failed to get current cluster config: %d", resp.StatusCode) } - + testConfig := buildConnectionTestConfig(newConfig) testBody, _ := json.Marshal(testConfig) - + resp, err = http.Post(serverURL+"/api/v1/clusters/test", "application/json", strings.NewReader(string(testBody))) if err != nil { return err } resp.Body.Close() - + if resp.StatusCode != http.StatusOK { return fmt.Errorf("connection test failed with status: %d", resp.StatusCode) } - + updateRequest := buildClusterRequest(newConfig, "edit") updateBody, _ := json.Marshal(updateRequest) - + req, _ := http.NewRequest("PUT", serverURL+"/api/v1/clusters/"+clusterName, strings.NewReader(string(updateBody))) req.Header.Set("Content-Type", "application/json") - + client := &http.Client{} resp, err = client.Do(req) if err != nil { return err } resp.Body.Close() - + if resp.StatusCode != http.StatusOK { return fmt.Errorf("cluster update failed with status: %d", resp.StatusCode) } - + return nil } diff --git a/tests/cmd/cluster_operations_test.go b/tests/cmd/cluster_operations_test.go index c7bc072..483ffb2 100644 --- a/tests/cmd/cluster_operations_test.go +++ b/tests/cmd/cluster_operations_test.go @@ -220,12 +220,12 @@ func TestClusterRequestStructure(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { requestPayload := buildClusterRequest(tt.config, tt.operation) - + // Verify all expected fields are present for _, field := range tt.expectedFields { assert.Contains(t, requestPayload, field, "Request should contain field: %s", field) } - + // Verify no empty required fields if tt.config.Name != "" { assert.Equal(t, tt.config.Name, requestPayload["name"], "Name should match") @@ -321,9 +321,9 @@ func TestBrokerStringFormatting(t *testing.T) { func TestConnectionTestConfiguration(t *testing.T) { tests := []struct { - name string - config ClusterConfig - expectedTest map[string]interface{} + name string + config ClusterConfig + expectedTest map[string]interface{} shouldHaveSecurity bool }{ { @@ -388,23 +388,23 @@ func TestConnectionTestConfiguration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { testConfig := buildConnectionTestConfig(tt.config) - + // Verify basic structure assert.Equal(t, tt.expectedTest["provider"], testConfig["provider"], "Provider should match") assert.Equal(t, tt.expectedTest["brokers"], testConfig["brokers"], "Brokers should match") - + // Verify security configuration if tt.shouldHaveSecurity { security, exists := testConfig["security"] assert.True(t, exists, "Security section should exist") securityMap, ok := security.(map[string]string) assert.True(t, ok, "Security should be a map[string]string") - + expectedSecurity := tt.expectedTest["security"].(map[string]string) assert.Equal(t, expectedSecurity["api_key"], securityMap["api_key"], "API key should match") assert.Equal(t, expectedSecurity["api_secret"], securityMap["api_secret"], "API secret should match") } - + // Verify provider-specific fields if tt.config.Provider == "confluent" { assert.Equal(t, tt.config.ClusterID, testConfig["cluster_id"], "Cluster ID should match") @@ -498,11 +498,11 @@ func validateClusterConfig(config ClusterConfig) error { if err := validateClusterName(config.Name); err != nil { return err } - + if config.Brokers == "" { return fmt.Errorf("brokers cannot be empty") } - + switch config.Provider { case "confluent": if config.APIKey == "" { @@ -519,7 +519,7 @@ func validateClusterConfig(config ClusterConfig) error { return fmt.Errorf("azure provider requires connection_string") } } - + return nil } @@ -527,24 +527,24 @@ func validateClusterName(name string) error { if name == "" { return fmt.Errorf("cluster name cannot be empty") } - + if len(name) > 50 { return fmt.Errorf("cluster name too long (max 50 characters)") } - + if strings.Contains(name, " ") { return fmt.Errorf("cluster name cannot contain spaces") } - + for _, char := range name { - if !((char >= 'a' && char <= 'z') || - (char >= 'A' && char <= 'Z') || - (char >= '0' && char <= '9') || - char == '-' || char == '_') { + if !((char >= 'a' && char <= 'z') || + (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') || + char == '-' || char == '_') { return fmt.Errorf("cluster name contains invalid characters (only alphanumeric, hyphens, and underscores allowed)") } } - + return nil } @@ -552,27 +552,27 @@ func formatBrokerString(input string) (string, error) { if strings.TrimSpace(input) == "" { return "", fmt.Errorf("brokers cannot be empty") } - + brokers := strings.Split(input, ",") var formatted []string - + for _, broker := range brokers { broker = strings.TrimSpace(broker) if broker == "" { continue } - + if !strings.Contains(broker, ":") { return "", fmt.Errorf("invalid broker format: %s (should be host:port)", broker) } - + formatted = append(formatted, broker) } - + if len(formatted) == 0 { return "", fmt.Errorf("no valid brokers found") } - + return strings.Join(formatted, ","), nil } @@ -582,23 +582,23 @@ func buildClusterRequest(config ClusterConfig, operation string) map[string]inte "provider": config.Provider, "brokers": config.Brokers, } - + if config.ClusterID != "" { request["cluster_id"] = config.ClusterID } - + if config.APIKey != "" { request["api_key"] = config.APIKey } - + if config.APISecret != "" { request["api_secret"] = config.APISecret } - + if config.ConnectionString != "" { request["connection_string"] = config.ConnectionString } - + return request } @@ -607,26 +607,26 @@ func buildConnectionTestConfig(config ClusterConfig) map[string]interface{} { "provider": config.Provider, "brokers": config.Brokers, } - + if config.ClusterID != "" { testConfig["cluster_id"] = config.ClusterID } - + if config.ConnectionString != "" { testConfig["connection_string"] = config.ConnectionString } - + security := map[string]string{ "api_key": config.APIKey, "api_secret": config.APISecret, } - + if config.Provider == "azure" { security["api_secret"] = config.ConnectionString } - + testConfig["security"] = security - + return testConfig } @@ -669,15 +669,15 @@ func TestJSONSerialization(t *testing.T) { t.Run(tt.name, func(t *testing.T) { jsonData, err := json.Marshal(tt.config) assert.NoError(t, err, "Should serialize to JSON without error") - + var deserialized ClusterConfig err = json.Unmarshal(jsonData, &deserialized) assert.NoError(t, err, "Should deserialize from JSON without error") - + assert.Equal(t, tt.config.Name, deserialized.Name, "Name should be preserved") assert.Equal(t, tt.config.Provider, deserialized.Provider, "Provider should be preserved") assert.Equal(t, tt.config.Brokers, deserialized.Brokers, "Brokers should be preserved") - + if tt.config.ClusterID != "" { assert.Equal(t, tt.config.ClusterID, deserialized.ClusterID, "Cluster ID should be preserved") } diff --git a/tests/internal/database/mirror_progress_test.go b/tests/internal/database/mirror_progress_test.go index a5f367a..5e89ada 100644 --- a/tests/internal/database/mirror_progress_test.go +++ b/tests/internal/database/mirror_progress_test.go @@ -31,18 +31,18 @@ func TestMirrorProgress(t *testing.T) { t.Run("UpdateMirrorProgress", func(t *testing.T) { progress := database.MirrorProgress{ - JobID: jobID, - SourceTopic: "test-topic", - TargetTopic: "test-topic-target", - PartitionID: 0, - SourceOffset: 1000, - TargetOffset: 950, - SourceHighWaterMark: 1000, - TargetHighWaterMark: 950, - LastReplicatedOffset: 950, - ReplicationLag: 50, - LastUpdated: time.Now(), - Status: "active", + JobID: jobID, + SourceTopic: "test-topic", + TargetTopic: "test-topic-target", + PartitionID: 0, + SourceOffset: 1000, + TargetOffset: 950, + SourceHighWaterMark: 1000, + TargetHighWaterMark: 950, + LastReplicatedOffset: 950, + ReplicationLag: 50, + LastUpdated: time.Now(), + Status: "active", } err := database.UpdateMirrorProgress(db, progress) @@ -64,7 +64,7 @@ func TestMirrorProgress(t *testing.T) { progressList, err := database.GetMirrorProgress(db, jobID) assert.NoError(t, err) assert.Len(t, progressList, 1) - + progress := progressList[0] assert.Equal(t, jobID, progress.JobID) assert.Equal(t, "test-topic", progress.SourceTopic) @@ -195,16 +195,16 @@ func TestMirrorStateAnalysis(t *testing.T) { t.Run("StoreMirrorStateAnalysis", func(t *testing.T) { analysis := database.MirrorStateAnalysis{ - JobID: jobID, - AnalysisType: "gap_detection", - SourceClusterState: `{"brokers": 3, "topics": 2}`, - TargetClusterState: `{"brokers": 3, "topics": 2}`, - AnalysisResults: `{"gaps": 1, "lag": 150}`, - Recommendations: `["Wait for replication to catch up"]`, - CriticalIssuesCount: 1, - WarningIssuesCount: 0, - AnalyzedAt: time.Now(), - AnalyzerVersion: "1.0", + JobID: jobID, + AnalysisType: "gap_detection", + SourceClusterState: `{"brokers": 3, "topics": 2}`, + TargetClusterState: `{"brokers": 3, "topics": 2}`, + AnalysisResults: `{"gaps": 1, "lag": 150}`, + Recommendations: `["Wait for replication to catch up"]`, + CriticalIssuesCount: 1, + WarningIssuesCount: 0, + AnalyzedAt: time.Now(), + AnalyzerVersion: "1.0", } id, err := database.StoreMirrorStateAnalysis(db, analysis) @@ -215,16 +215,16 @@ func TestMirrorStateAnalysis(t *testing.T) { t.Run("GetMirrorStateAnalysis", func(t *testing.T) { // Insert another analysis analysis2 := database.MirrorStateAnalysis{ - JobID: jobID, - AnalysisType: "resume_validation", - SourceClusterState: `{"brokers": 3, "topics": 2}`, - TargetClusterState: `{"brokers": 3, "topics": 2}`, - AnalysisResults: `{"gaps": 0, "lag": 0}`, - Recommendations: `["Migration is safe to proceed"]`, - CriticalIssuesCount: 0, - WarningIssuesCount: 0, - AnalyzedAt: time.Now(), - AnalyzerVersion: "1.0", + JobID: jobID, + AnalysisType: "resume_validation", + SourceClusterState: `{"brokers": 3, "topics": 2}`, + TargetClusterState: `{"brokers": 3, "topics": 2}`, + AnalysisResults: `{"gaps": 0, "lag": 0}`, + Recommendations: `["Migration is safe to proceed"]`, + CriticalIssuesCount: 0, + WarningIssuesCount: 0, + AnalyzedAt: time.Now(), + AnalyzerVersion: "1.0", } time.Sleep(1 * time.Millisecond) // Ensure different timestamp _, err := database.StoreMirrorStateAnalysis(db, analysis2) @@ -233,7 +233,7 @@ func TestMirrorStateAnalysis(t *testing.T) { analyses, err := database.GetMirrorStateAnalysis(db, jobID) assert.NoError(t, err) assert.Len(t, analyses, 2) - + // Most recent should be first (ordered by analyzed_at DESC) latest := analyses[0] assert.Equal(t, jobID, latest.JobID) @@ -250,7 +250,7 @@ func TestMirrorStateSchemaValidation(t *testing.T) { t.Run("VerifyMirrorStateTables", func(t *testing.T) { expectedTables := []string{ "mirror_progress", - "resume_points", + "resume_points", "migration_checkpoints", "mirror_state_analysis", "mirror_gaps", @@ -269,13 +269,13 @@ func TestMirrorStateSchemaValidation(t *testing.T) { t.Run("TestJSONSerialization", func(t *testing.T) { jobID := uuid.NewString() - + // Test complex nested JSON serialization in checkpoints consumerOffsets := map[string]int64{ - "test-topic": 1000, + "test-topic": 1000, "another-topic": 2500, } - + waterMarks := map[string]map[int]int64{ "test-topic": { 0: 1000, @@ -317,12 +317,12 @@ func TestMirrorStateSchemaValidation(t *testing.T) { err = json.Unmarshal([]byte(retrieved.SourceConsumerGroupOffsets), &parsedOffsets) assert.NoError(t, err) assert.Equal(t, int64(1000), parsedOffsets["test-topic"]) - + var parsedWaterMarks map[string]map[int]int64 err = json.Unmarshal([]byte(retrieved.TargetHighWaterMarks), &parsedWaterMarks) assert.NoError(t, err) assert.Equal(t, int64(1000), parsedWaterMarks["test-topic"][0]) - + // Verify validation results assert.NotNil(t, retrieved.ValidationResults) var validationData map[string]interface{} diff --git a/tests/internal/kafka/config_test.go b/tests/internal/kafka/config_test.go index 0ce779d..3ae60fd 100644 --- a/tests/internal/kafka/config_test.go +++ b/tests/internal/kafka/config_test.go @@ -43,7 +43,7 @@ func TestGetKgoOpts(t *testing.T) { // A more direct way would be to inspect the opts slice, but the SASL option is a function literal. // This approach verifies the end result. // Note: This doesn't actually connect to Azure, it just configures the client. - + // A better way to test this would be to export the SASL options from the kafka package, // but for now, we will rely on this indirect verification. diff --git a/tests/internal/kafka/producer_test.go b/tests/internal/kafka/producer_test.go index e4787c7..99b5385 100644 --- a/tests/internal/kafka/producer_test.go +++ b/tests/internal/kafka/producer_test.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package kafka_test import ( diff --git a/tests/internal/manager/manager_test.go b/tests/internal/manager/manager_test.go index ac1550e..14b8ad6 100644 --- a/tests/internal/manager/manager_test.go +++ b/tests/internal/manager/manager_test.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package manager_test import ( diff --git a/tests/internal/server/mirror_state_test.go b/tests/internal/server/mirror_state_test.go index 0231524..1702512 100644 --- a/tests/internal/server/mirror_state_test.go +++ b/tests/internal/server/mirror_state_test.go @@ -21,11 +21,11 @@ func TestGetMirrorState_Empty(t *testing.T) { database.CreateCluster(ctx.Server.Db, targetCluster) job := &database.ReplicationJob{ - ID: "test-job", - Name: "Test Job", + ID: "test-job", + Name: "Test Job", SourceClusterName: "src", TargetClusterName: "tgt", - Status: "active", + Status: "active", } database.CreateJob(ctx.Server.Db, job) @@ -41,7 +41,7 @@ func TestGetMirrorState_Empty(t *testing.T) { assert.Equal(t, "test-job", response["job_id"]) assert.Nil(t, response["mirror_progress"]) - assert.Nil(t, response["resume_points"]) + assert.Nil(t, response["resume_points"]) assert.Nil(t, response["mirror_gaps"]) assert.Nil(t, response["state_analysis"]) } @@ -56,27 +56,27 @@ func TestGetMirrorState_WithData(t *testing.T) { database.CreateCluster(ctx.Server.Db, targetCluster) job := &database.ReplicationJob{ - ID: "test-job", - Name: "Test Job", + ID: "test-job", + Name: "Test Job", SourceClusterName: "src", TargetClusterName: "tgt", - Status: "active", + Status: "active", } database.CreateJob(ctx.Server.Db, job) progress := database.MirrorProgress{ - JobID: "test-job", - SourceTopic: "test-topic", - TargetTopic: "test-topic", - PartitionID: 0, - SourceOffset: 100, - TargetOffset: 100, - SourceHighWaterMark: 120, - TargetHighWaterMark: 120, + JobID: "test-job", + SourceTopic: "test-topic", + TargetTopic: "test-topic", + PartitionID: 0, + SourceOffset: 100, + TargetOffset: 100, + SourceHighWaterMark: 120, + TargetHighWaterMark: 120, LastReplicatedOffset: 100, - ReplicationLag: 0, - LastUpdated: time.Now(), - Status: "active", + ReplicationLag: 0, + LastUpdated: time.Now(), + Status: "active", } database.UpdateMirrorProgress(ctx.Server.Db, progress) @@ -91,13 +91,13 @@ func TestGetMirrorState_WithData(t *testing.T) { json.NewDecoder(resp.Body).Decode(&response) assert.Equal(t, "test-job", response["job_id"]) - + // Should have a single mirror_progress object (not array) assert.NotNil(t, response["mirror_progress"]) mirrorProgress := response["mirror_progress"].(map[string]interface{}) assert.Equal(t, "test-topic", mirrorProgress["source_topic"]) assert.Equal(t, float64(100), mirrorProgress["source_offset"]) // JSON numbers are float64 - + // Other fields should still be null since we only have progress data assert.Nil(t, response["resume_points"]) assert.Nil(t, response["mirror_gaps"]) diff --git a/tests/internal/server/server_test.go b/tests/internal/server/server_test.go index f1c968b..3c18a41 100644 --- a/tests/internal/server/server_test.go +++ b/tests/internal/server/server_test.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package server_test import ( @@ -54,7 +53,7 @@ func setupTestServer(t *testing.T) *TestContext { var adminRoleID int err = db.Get(&adminRoleID, "SELECT id FROM roles WHERE name = 'admin'") assert.NoError(t, err) - + err = database.AssignRoleToUser(db, testUser.ID, adminRoleID) assert.NoError(t, err) @@ -63,7 +62,7 @@ func setupTestServer(t *testing.T) *TestContext { hub := server.NewHub() jobManager := manager.New(db, cfg, hub) - + srv := server.New(cfg, db, jobManager, hub, "test") return &TestContext{ @@ -89,7 +88,7 @@ func TestHealthCheck(t *testing.T) { var healthResp map[string]interface{} err = json.Unmarshal(body, &healthResp) assert.NoError(t, err) - + // Check that status is "ok" and other expected fields exist assert.Equal(t, "ok", healthResp["status"]) assert.Contains(t, healthResp, "timestamp") @@ -190,7 +189,7 @@ func TestJobsAPI(t *testing.T) { resp, err = ctx.Server.App.Test(req) assert.NoError(t, err) - + if resp.StatusCode != 200 { body, _ := io.ReadAll(resp.Body) t.Fatalf("Expected status 200 but got %d. Response: %s", resp.StatusCode, string(body)) @@ -294,7 +293,7 @@ func TestAuthenticationRequired(t *testing.T) { // Test that requests without auth header get 401 req := httptest.NewRequest("GET", "/api/v1/jobs", nil) // Deliberately NOT adding auth header - + resp, err := ctx.Server.App.Test(req) assert.NoError(t, err) assert.Equal(t, 401, resp.StatusCode) @@ -312,7 +311,7 @@ func TestInvalidToken(t *testing.T) { // Test that requests with invalid token get 401 req := httptest.NewRequest("GET", "/api/v1/jobs", nil) req.Header.Set("Authorization", "Bearer invalid-token") - + resp, err := ctx.Server.App.Test(req) assert.NoError(t, err) assert.Equal(t, 401, resp.StatusCode) diff --git a/tests/internal/server/tls_test.go b/tests/internal/server/tls_test.go index a55cd38..586230e 100644 --- a/tests/internal/server/tls_test.go +++ b/tests/internal/server/tls_test.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package server import ( @@ -46,15 +45,15 @@ func TestTLSServerConfiguration(t *testing.T) { Server: config.ServerConfig{ Host: "localhost", Port: 0, // Use random port - TLS: struct { - Enabled bool `mapstructure:"enabled"` - CertFile string `mapstructure:"cert_file"` - KeyFile string `mapstructure:"key_file"` - }{ - Enabled: true, - CertFile: certFile, - KeyFile: keyFile, - }, + TLS: struct { + Enabled bool `mapstructure:"enabled"` + CertFile string `mapstructure:"cert_file"` + KeyFile string `mapstructure:"key_file"` + }{ + Enabled: true, + CertFile: certFile, + KeyFile: keyFile, + }, }, AI: config.AIConfig{ Provider: "openai", @@ -83,11 +82,11 @@ func TestTLSServerConfiguration(t *testing.T) { if !cfg.Server.TLS.Enabled { t.Error("Expected TLS to be enabled") } - + if cfg.Server.TLS.CertFile == "" { t.Error("Expected certificate file to be set") } - + if cfg.Server.TLS.KeyFile == "" { t.Error("Expected key file to be set") } @@ -96,7 +95,7 @@ func TestTLSServerConfiguration(t *testing.T) { if _, err := os.Stat(certFile); os.IsNotExist(err) { t.Errorf("Certificate file does not exist: %s", certFile) } - + if _, err := os.Stat(keyFile); os.IsNotExist(err) { t.Errorf("Key file does not exist: %s", keyFile) } @@ -110,13 +109,13 @@ func TestTLSDisabledConfiguration(t *testing.T) { Server: config.ServerConfig{ Host: "localhost", Port: 0, - TLS: struct { - Enabled bool `mapstructure:"enabled"` - CertFile string `mapstructure:"cert_file"` - KeyFile string `mapstructure:"key_file"` - }{ - Enabled: false, - }, + TLS: struct { + Enabled bool `mapstructure:"enabled"` + CertFile string `mapstructure:"cert_file"` + KeyFile string `mapstructure:"key_file"` + }{ + Enabled: false, + }, }, AI: config.AIConfig{ Provider: "openai", @@ -216,15 +215,15 @@ func TestTLSConfigurationUpdate(t *testing.T) { Server: config.ServerConfig{ Host: "localhost", Port: 0, - TLS: struct { - Enabled bool `mapstructure:"enabled"` - CertFile string `mapstructure:"cert_file"` - KeyFile string `mapstructure:"key_file"` - }{ - Enabled: true, - CertFile: certFile1, - KeyFile: keyFile1, - }, + TLS: struct { + Enabled bool `mapstructure:"enabled"` + CertFile string `mapstructure:"cert_file"` + KeyFile string `mapstructure:"key_file"` + }{ + Enabled: true, + CertFile: certFile1, + KeyFile: keyFile1, + }, }, AI: config.AIConfig{ Provider: "openai", diff --git a/tests/mocks/mock_hub.go b/tests/mocks/mock_hub.go index 9ebd7c2..c35b489 100644 --- a/tests/mocks/mock_hub.go +++ b/tests/mocks/mock_hub.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package mocks // MockHub is a mock implementation of the Hub interface. diff --git a/tests/mocks/mock_kgo_client.go b/tests/mocks/mock_kgo_client.go index d7796d8..47dff22 100644 --- a/tests/mocks/mock_kgo_client.go +++ b/tests/mocks/mock_kgo_client.go @@ -20,11 +20,11 @@ import ( // MockKgoClient is a mock implementation of the KgoClient interface. type MockKgoClient struct { - RequestFunc func(context.Context, kmsg.Request) (kmsg.Response, error) - PollFetchesFunc func(context.Context) kgo.Fetches - ProduceFunc func(context.Context, *kgo.Record, func(*kgo.Record, error)) + RequestFunc func(context.Context, kmsg.Request) (kmsg.Response, error) + PollFetchesFunc func(context.Context) kgo.Fetches + ProduceFunc func(context.Context, *kgo.Record, func(*kgo.Record, error)) AddConsumeTopicsFunc func(...string) - CloseFunc func() + CloseFunc func() } func (m *MockKgoClient) Request(ctx context.Context, req kmsg.Request) (kmsg.Response, error) { diff --git a/tests/mocks/mock_openai_client.go b/tests/mocks/mock_openai_client.go index 6e881d9..c45dc9c 100644 --- a/tests/mocks/mock_openai_client.go +++ b/tests/mocks/mock_openai_client.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package mocks import ( diff --git a/tests/pkg/utils/path_test.go b/tests/pkg/utils/path_test.go index 8b9530f..5ab5d9a 100644 --- a/tests/pkg/utils/path_test.go +++ b/tests/pkg/utils/path_test.go @@ -9,7 +9,6 @@ // See the License for the specific language governing permissions and // limitations under the License. - package utils_test import (