(migrate) kvs to sqlite3 for race conditions

.gitignore

@@ -23,3 +23,4 @@ tmp/
 *.buntdb
 .DS_Store
 go-watch-logs
+*.sqlite

go.mod

@@ -5,29 +5,18 @@ go 1.21
 toolchain go1.22.3
 
 require (
-	github.com/gookit/color v1.5.4
 	github.com/jasonlvhit/gocron v0.0.1
-	github.com/k0kubun/pp v3.0.1+incompatible
 	github.com/kevincobain2000/go-msteams v0.0.0-20231124044510-4369c04dd224
+	github.com/lmittmann/tint v1.0.4
+	github.com/mattn/go-isatty v0.0.16
+	github.com/mattn/go-sqlite3 v1.14.22
 	github.com/stretchr/testify v1.9.0
-	github.com/tidwall/buntdb v1.3.1
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
-	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-isatty v0.0.16 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/tidwall/btree v1.4.2 // indirect
-	github.com/tidwall/gjson v1.14.3 // indirect
-	github.com/tidwall/grect v0.1.4 // indirect
-	github.com/tidwall/match v1.1.1 // indirect
-	github.com/tidwall/pretty v1.2.0 // indirect
-	github.com/tidwall/rtred v0.1.2 // indirect
-	github.com/tidwall/tinyqueue v0.1.1 // indirect
-	github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 // indirect
 	golang.org/x/sys v0.10.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect

go.sum

@@ -5,15 +5,9 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/go-redis/redis v6.15.5+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/gookit/color v1.5.4 h1:FZmqs7XOyGgCAxmWyPslpiok1k05wmY3SJTytgvYFs0=
-github.com/gookit/color v1.5.4/go.mod h1:pZJOeOS8DM43rXbp4AZo1n9zCU2qjpcRko0b6/QJi9w=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5YU=
 github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
-github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
-github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
-github.com/k0kubun/pp v3.0.1+incompatible h1:3tqvf7QgUnZ5tXO6pNAZlrvHgl6DvifjDrd9g2S9Z40=
-github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg=
 github.com/kevincobain2000/go-msteams v0.0.0-20231124044510-4369c04dd224 h1:YLj2tqX+dD34tRTW7WUuOW00G+RPBQAKGRYGNmkYuBs=
 github.com/kevincobain2000/go-msteams v0.0.0-20231124044510-4369c04dd224/go.mod h1:+HowoQQHg9HLfx3CYQGImGGYw20+kN9rFmUXgxrqBzo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -23,10 +17,12 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
-github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/lmittmann/tint v1.0.4 h1:LeYihpJ9hyGvE0w+K2okPTGUdVLfng1+nDNVR4vWISc=
+github.com/lmittmann/tint v1.0.4/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
 github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
@@ -39,29 +35,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/tidwall/assert v0.1.0 h1:aWcKyRBUAdLoVebxo95N7+YZVTFF/ASTr7BN4sLP6XI=
-github.com/tidwall/assert v0.1.0/go.mod h1:QLYtGyeqse53vuELQheYl9dngGCJQ+mTtlxcktb+Kj8=
-github.com/tidwall/btree v1.4.2 h1:PpkaieETJMUxYNADsjgtNRcERX7mGc/GP2zp/r5FM3g=
-github.com/tidwall/btree v1.4.2/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE=
-github.com/tidwall/buntdb v1.3.1 h1:HKoDF01/aBhl9RjYtbaLnvX9/OuenwvQiC3OP1CcL4o=
-github.com/tidwall/buntdb v1.3.1/go.mod h1:lZZrZUWzlyDJKlLQ6DKAy53LnG7m5kHyrEHvvcDmBpU=
-github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/gjson v1.14.3 h1:9jvXn7olKEHU1S9vwoMGliaT8jq1vJ7IH/n9zD9Dnlw=
-github.com/tidwall/gjson v1.14.3/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
-github.com/tidwall/grect v0.1.4 h1:dA3oIgNgWdSspFzn1kS4S/RDpZFLrIxAZOdJKjYapOg=
-github.com/tidwall/grect v0.1.4/go.mod h1:9FBsaYRaR0Tcy4UwefBX/UDcDcDy9V5jUcxHzv2jd5Q=
-github.com/tidwall/lotsa v1.0.2 h1:dNVBH5MErdaQ/xd9s769R31/n2dXavsQ0Yf4TMEHHw8=
-github.com/tidwall/lotsa v1.0.2/go.mod h1:X6NiU+4yHA3fE3Puvpnn1XMDrFZrE9JO2/w+UMuqgR8=
-github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
-github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM=
-github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs=
-github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
-github.com/tidwall/rtred v0.1.2 h1:exmoQtOLvDoO8ud++6LwVsAMTu0KPzLTUrMln8u1yu8=
-github.com/tidwall/rtred v0.1.2/go.mod h1:hd69WNXQ5RP9vHd7dqekAz+RIdtfBogmglkZSRxCHFQ=
-github.com/tidwall/tinyqueue v0.1.1 h1:SpNEvEggbpyN5DIReaJ2/1ndroY8iyEGxPYxoSaymYE=
-github.com/tidwall/tinyqueue v0.1.1/go.mod h1:O/QNHwrnjqr6IHItYrzoHAKYhBkLI67Q096fQP5zMYw=
-github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778 h1:QldyIu/L63oPpyvQmHgvgickp1Yw510KJOqX7H24mg8=
-github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=

main.go

@@ -3,11 +3,10 @@ package main
 import (
 	"flag"
 	"fmt"
+	"log/slog"
 	"os"
 
-	"github.com/gookit/color"
 	"github.com/jasonlvhit/gocron"
-	"github.com/k0kubun/pp"
 	gmt "github.com/kevincobain2000/go-msteams/src"
 
 	"github.com/rakutentech/go-watch-logs/pkg"
@@ -32,29 +31,30 @@ var f Flags
 var version = "dev"
 
 func main() {
+	pkg.SetupLoggingStdout()
 	flags()
 	parseProxy()
 	wantsVersion()
 	validate()
-	pp.Println(f)
+	slog.Info("Flags", "filePath", f.filePath, "match", f.match, "ignore", f.ignore, "dbPath", f.dbPath, "min", f.min, "every", f.every, "noCache", f.noCache, "version", f.version, "proxy", f.proxy, "msTeamsHook", f.msTeamsHook)
 
 	filePaths, err := pkg.FilesByPattern(f.filePath)
 	if err != nil {
-		color.Danger.Println(err)
+		slog.Error("Error finding files", "error", err.Error())
 		return
 	}
 	if len(filePaths) == 0 {
-		color.Danger.Println("no files found", f.filePath)
+		slog.Error("No files found", "filePath", f.filePath)
 		return
 	}
 	for _, filePath := range filePaths {
 		isText, err := pkg.IsTextFile(filePath)
 		if err != nil {
-			color.Danger.Println(err)
+			slog.Error("Error checking if file is text", "error", err.Error(), "filePath", filePath)
 			return
 		}
 		if !isText {
-			color.Danger.Println("file is not a text file", filePath)
+			slog.Error("File is not a text file", "filePath", filePath)
 			return
 		}
 	}
@@ -70,47 +70,46 @@ func main() {
 func cron(filePaths []string) {
 	for _, filePath := range filePaths {
 		if err := gocron.Every(f.every).Second().Do(watch, filePath); err != nil {
-			color.Danger.Println(err)
+			slog.Error("Error scheduling watch", "error", err.Error(), "filePath", filePath)
 			return
 		}
 	}
+	if err := gocron.Every(f.every).Second().Do(pkg.PrintMemUsage); err != nil {
+		slog.Error("Error scheduling memory usage", "error", err.Error())
+		return
+	}
 	<-gocron.Start()
 }
 
 func validate() {
 	if f.filePath == "" {
-		color.Danger.Println("file-path is required")
+		slog.Error("file-path is required")
 		os.Exit(1)
 	}
 }
 
 func watch(filePath string) {
 	watcher, err := pkg.NewWatcher(f.dbPath, filePath, f.match, f.ignore, f.noCache)
 	if err != nil {
-		color.Danger.Println(err)
+		slog.Error("Error creating watcher", "error", err.Error(), "filePath", filePath)
 		return
 	}
 	defer watcher.Close()
 
-	color.Secondary.Print("scanning..................... ")
-	fmt.Println(filePath)
+	slog.Info("Scanning file", "filePath", filePath)
 
 	result, err := watcher.Scan()
 	if err != nil {
-		color.Danger.Println(err)
+		slog.Error("Error scanning file", "error", err.Error(), "filePath", filePath)
 		return
 	}
-	color.Secondary.Print("error count.................. ")
-	color.Danger.Println(result.ErrorCount)
+	slog.Info("Error count", "count", result.ErrorCount)
 
 	// first line
-	color.Secondary.Print("1st line..................... ")
-	fmt.Println(pkg.Truncate(result.FirstLine, 50))
-
-	color.Secondary.Print("last line.................... ")
-	fmt.Println(pkg.Truncate(result.LastLine, 50))
+	slog.Info("1st line", "line", pkg.Truncate(result.FirstLine, 50))
 
-	fmt.Println()
+	// last line
+	slog.Info("Last line", "line", pkg.Truncate(result.LastLine, 50))
 
 	if result.ErrorCount < 0 {
 		return
@@ -125,8 +124,7 @@ func notify(errorCount int, firstLine, lastLine string) {
 	if f.msTeamsHook != "" {
 		teamsMsg := fmt.Sprintf("total errors: %d\n\n", errorCount)
 		teamsMsg += fmt.Sprintf("1st error<pre>\n\n%s</pre>\n\nlast error<pre>\n\n%s</pre>", firstLine, lastLine)
-		color.Secondary.Print("Sending to Teams.............")
-		color.Warn.Println("Work in Progress")
+		slog.Info("Sending to Teams")
 
 		hostname, _ := os.Hostname()
 		subject := fmt.Sprintf("match: <code>%s</code>", f.match)
@@ -136,10 +134,11 @@ func notify(errorCount int, firstLine, lastLine string) {
 		subject += fmt.Sprintf("min error: <code>%d</code>", f.min)
 		err := gmt.Send(hostname, f.filePath, subject, "red", teamsMsg, f.msTeamsHook, f.proxy)
 		if err != nil {
-			color.Danger.Println(err)
+			slog.Error("Error sending to Teams", "error", err.Error())
+		} else {
+			slog.Info("Sent to Teams")
+			slog.Info("Done")
 		}
-		color.Secondary.Print("Sent to Teams................ ")
-		color.Success.Println("Done")
 	}
 }
 
@@ -168,7 +167,7 @@ func parseProxy() string {
 }
 func wantsVersion() {
 	if f.version {
-		color.Secondary.Println(version)
+		slog.Info("Version", "version", version)
 		os.Exit(0)
 	}
 }

pkg/log.go

@@ -0,0 +1,35 @@
+package pkg
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/lmittmann/tint"
+	"github.com/mattn/go-isatty"
+)
+
+func SetupLoggingStdout() {
+	w := os.Stderr
+	handler := tint.NewHandler(w, &tint.Options{
+		NoColor:   !isatty.IsTerminal(w.Fd()),
+		AddSource: true,
+		Level:     slog.LevelInfo,
+		ReplaceAttr: func(_ []string, a slog.Attr) slog.Attr {
+			if a.Key == slog.TimeKey {
+				t := a.Value.Time()
+				a.Value = slog.StringValue(t.Format(time.DateTime))
+			}
+			// change file to short file with line number
+			if a.Key == slog.SourceKey {
+				source := a.Value.Any().(*slog.Source)
+				a.Value = slog.StringValue(filepath.Base(source.File) + ":" + fmt.Sprint(source.Line))
+			}
+			return a
+		},
+	})
+
+	slog.SetDefault(slog.New(handler))
+}

pkg/system.go

@@ -1,7 +1,9 @@
 package pkg
 
 import (
+	"log/slog"
 	"os"
+	"runtime"
 )
 
 func SystemProxy() string {
@@ -14,3 +16,16 @@ func SystemProxy() string {
 	}
 	return ""
 }
+
+func PrintMemUsage() {
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	slog.Info("Memory Usage",
+		"Alloc (MB)", bToMb(m.Alloc),
+		"Sys", bToMb(m.Sys),
+	)
+}
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}