Add commit-check + CodeQL workflow, fix manager race, and redact secrets

.github/workflows/ci.yml

@@ -0,0 +1,67 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ci-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  smoke:
+    name: Smoke Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Run smoke suite
+        run: make test-smoke
+
+  go-ci:
+    name: Go Checks
+    runs-on: ubuntu-latest
+    needs: [smoke]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+          cache-dependency-path: go.sum
+
+      - name: Run formatting check
+        run: make fmt-check
+
+      - name: Run go vet
+        run: make vet
+
+      - name: Run smoke tests
+        run: make test-smoke
+
+      - name: Run race tests
+        run: make race
+
+      - name: Upload coverage artifact
+        run: go test -coverprofile=coverage.out ./...
+
+      - name: Store coverage file
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage.out
+          path: coverage.out

.github/workflows/codeql.yml

@@ -0,0 +1,48 @@
+name: CodeQL
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  schedule:
+    - cron: "19 3 * * 1"
+
+permissions:
+  actions: read
+  contents: read
+  security-events: write
+
+jobs:
+  analyze:
+    name: Analyze (${{ matrix.language }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        language:
+          - go
+          - actions
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Go
+        if: matrix.language == 'go'
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+
+      - name: Build (Go)
+        if: matrix.language == 'go'
+        run: go build ./...
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

Makefile

@@ -1,4 +1,96 @@
-.PHONY: all build run start stop restart clean test docker docker-build docker-run docs
+SHELL := /bin/bash
+
+define PY_CODEQL_SUMMARY
+import glob, json
+from collections import Counter
+
+sarifs = sorted(glob.glob(".tmp/codeql/*.sarif"))
+if not sarifs:
+    print("No SARIF files found under .tmp/codeql/. Did make code-ql run successfully?")
+    raise SystemExit(2)
+
+def first_location(result):
+    locs = result.get("locations") or []
+    if not locs:
+        return ("unknown", 0)
+    pl = (locs[0].get("physicalLocation") or {})
+    file = ((pl.get("artifactLocation") or {}).get("uri") or "unknown")
+    line = ((pl.get("region") or {}).get("startLine") or 0)
+    return (file, line)
+
+total = 0
+levels = Counter()
+rows = []
+
+for path in sarifs:
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    for run in data.get("runs", []):
+        for r in (run.get("results") or []):
+            lvl = (r.get("level") or "warning").lower()
+            rule = r.get("ruleId") or "no-rule"
+            msg = (r.get("message") or {}).get("text") or "no-message"
+            file, line = first_location(r)
+            levels[lvl] += 1
+            total += 1
+            rows.append((lvl, rule, file, line, msg))
+
+print("\nCodeQL SARIF Summary:")
+for p in sarifs:
+    print(f"  - {p}")
+print(f"\n  Total findings: {total}")
+if total:
+    print("  By level: " + ", ".join(f"{k}={v}" for k, v in sorted(levels.items())))
+print("")
+
+priority = {"error": 0, "warning": 1, "note": 2, "recommendation": 3}
+rows.sort(key=lambda x: (priority.get(x[0], 9), x[2], x[3], x[1]))
+
+limit = 50
+if not rows:
+    print("  OK: No findings.")
+else:
+    print(f"  Top {min(limit, len(rows))} findings:")
+    for i, (lvl, rule, file, line, msg) in enumerate(rows[:limit], 1):
+        msg = msg.replace("\n", " ").strip()
+        if len(msg) > 120:
+            msg = msg[:117] + "..."
+        print(f"  {i:>2}. [{lvl}] {rule}")
+        print(f"      {file}:{line}")
+        print(f"      {msg}")
+        print("")
+endef
+export PY_CODEQL_SUMMARY
+
+define PY_CODEQL_GATE
+import glob, json
+sarifs = sorted(glob.glob(".tmp/codeql/*.sarif"))
+errors = 0
+warnings = 0
+for path in sarifs:
+    with open(path, "r", encoding="utf-8") as f:
+        data = json.load(f)
+    for run in data.get("runs", []):
+        for r in (run.get("results") or []):
+            lvl = (r.get("level") or "warning").lower()
+            if lvl == "error":
+                errors += 1
+            elif lvl == "warning":
+                warnings += 1
+
+if errors > 0:
+    print(f"\nCodeQL gate failed: {errors} blocking error(s) found.")
+    if warnings > 0:
+        print(f"Also found {warnings} non-blocking warning(s).")
+    raise SystemExit(1)
+
+print("\nCodeQL gate passed: no blocking errors found.")
+if warnings > 0:
+    print(f"Note: {warnings} non-blocking warning(s) were found.")
+endef
+export PY_CODEQL_GATE
+
+.PHONY: all build run start stop restart clean test check test-smoke vet race fmt fmt-check commit-check code-ql code-ql-summary code-ql-gate docker docker-build docker-run docs
 
 BIN_DIR=bin
 KAF_MIRROR_BINARY=$(BIN_DIR)/kaf-mirror
@@ -56,14 +148,53 @@ clean:
 	@rm -rf $(BIN_DIR)
 	@rm -f $(PID_FILE)
 
-test:
-	@echo "Running tests..."
-	@go test ./tests/...
+test: check vet race
+
+check: test-smoke
+
+test-smoke:
+	@echo "Running smoke tests..."
+	@go test ./...
+
+vet:
 	@echo "Running go vet..."
 	@go vet ./...
+
+race:
 	@echo "Running race tests..."
 	@go test -race ./...
 
+fmt:
+	@unformatted=$$(gofmt -l .); \
+	if [ -n "$$unformatted" ]; then \
+		echo "Found unformatted files. Auto-formatting:"; \
+		echo "$$unformatted"; \
+		gofmt -w .; \
+	else \
+		echo "All Go files are formatted correctly."; \
+	fi
+
+fmt-check:
+	@unformatted=$$(gofmt -l .); \
+	if [ -n "$$unformatted" ]; then \
+		echo "The following files are not gofmt-formatted:"; \
+		echo "$$unformatted"; \
+		echo "Run 'make fmt' or 'gofmt -w .' to fix formatting."; \
+		exit 1; \
+	fi
+
+commit-check: fmt vet test-smoke race code-ql-gate
+	@echo "commit-check completed."
+
+code-ql:
+	bash scripts/codeql_local.sh
+
+code-ql-summary: code-ql
+	@printf "%s\n" "$$PY_CODEQL_SUMMARY" | python3 -
+
+code-ql-gate: code-ql-summary
+	@printf "%s\n" "$$PY_CODEQL_GATE" | python3 -
+
 docker-build:
 	@echo "Building Docker image..."
 	@docker build -t kaf-mirror:latest .

README.md

@@ -1,4 +1,7 @@
 # kaf-mirror
+[![CI (Smoke+Go)](https://github.com/scalytics/kaf-mirror/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/scalytics/kaf-mirror/actions/workflows/ci.yml)
+[![Release](https://github.com/scalytics/kaf-mirror/actions/workflows/release.yml/badge.svg?branch=main)](https://github.com/scalytics/kaf-mirror/actions/workflows/release.yml)
+[![CodeQL](https://github.com/scalytics/kaf-mirror/actions/workflows/codeql.yml/badge.svg?branch=main)](https://github.com/scalytics/kaf-mirror/actions/workflows/codeql.yml)
 
 **kaf-mirror** is a high-performance, AI-enhanced Kafka replication tool designed for robust and intelligent data mirroring between Kafka clusters.