feat: anomaly detection

.github/workflows/test.yml

@@ -20,10 +20,7 @@ jobs:
 
       - name: Install Tools
         run: |
-          go install github.com/axw/gocov/gocov@latest
-          go install github.com/AlekSi/gocov-xml@latest
           go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest
-          curl -sLk https://raw.githubusercontent.com/kevincobain2000/cover-totalizer/master/install.sh | sh
 
       - run: go mod tidy
       - run: golangci-lint run ./...

.gitignore

@@ -24,4 +24,6 @@ tmp/
 .DS_Store
 go-watch-logs
 *.sqlite
-*.sqlite3
+*.sqlite3
+logs/
+testdata/

README.md

@@ -36,6 +36,8 @@ curl -sL https://raw.githubusercontent.com/rakutentech/go-watch-logs/master/inst
 
 ## Examples
 
+### Watching a log file for errors
+
 ```sh
 # match error patterns and notify on MS Teams
 go-watch-logs --file-path=my.log --match="error:pattern1|error:pattern2" --ms-teams-hook="https://outlook.office.com/webhook/xxxxx"
@@ -50,7 +52,6 @@ go-watch-logs --file-path=my.log --match='HTTP/1.1" 50|HTTP/1.1" 40' --ignore='H
 go-watch-logs --file-path=my.log --match='HTTP/1.1" 50' --every=60
 ```
 
-
 **All done!**
 
 ## Help

go.mod

@@ -8,21 +8,25 @@ require (
 	github.com/gravwell/gravwell/v3 v3.8.34
 	github.com/jasonlvhit/gocron v0.0.1
 	github.com/kevincobain2000/go-msteams v1.1.1
-	github.com/lmittmann/tint v1.0.5
 	github.com/mattn/go-isatty v0.0.20
-	github.com/stretchr/testify v1.9.0
+	github.com/natefinch/lumberjack v2.0.0+incompatible
+	github.com/stretchr/testify v1.10.0
 )
 
 require (
+	github.com/BurntSushi/toml v1.4.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fatih/color v1.16.0 // indirect
 	github.com/google/uuid v1.6.0 // indirect
+	github.com/kr/pretty v0.3.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/ncruces/go-strftime v0.1.9 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	golang.org/x/sys v0.23.0 // indirect
+	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
+	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	modernc.org/libc v1.57.0 // indirect
 	modernc.org/mathutil v1.6.0 // indirect

go.sum

@@ -1,5 +1,8 @@
+github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
+github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/MatusOllah/slogcolor v1.4.0 h1:NW4xI8BdBOY6Lt14004OInbbr0p+NURAMg15jzOjKds=
 github.com/MatusOllah/slogcolor v1.4.0/go.mod h1:5y1H50XuQIBvuYTJlmokWi+4FuPiJN5L7Z0jM4K4bYA=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -23,26 +26,36 @@ github.com/jasonlvhit/gocron v0.0.1 h1:qTt5qF3b3srDjeOIR4Le1LfeyvoYzJlYpqvG7tJX5
 github.com/jasonlvhit/gocron v0.0.1/go.mod h1:k9a3TV8VcU73XZxfVHCHWMWF9SOqgoku0/QlY2yvlA4=
 github.com/kevincobain2000/go-msteams v1.1.1 h1:vZ8AYvVmiCdC+VZwsw7RFhb89RG/GasX9kvbdKheFN4=
 github.com/kevincobain2000/go-msteams v1.1.1/go.mod h1:+HowoQQHg9HLfx3CYQGImGGYw20+kN9rFmUXgxrqBzo=
-github.com/lmittmann/tint v1.0.5 h1:NQclAutOfYsqs2F1Lenue6OoWCajs5wJcP3DfWVpePw=
-github.com/lmittmann/tint v1.0.5/go.mod h1:HIS3gSy7qNwGCj+5oRjAutErFBl4BzdQP6cJZ0NfMwE=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
 github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM=
+github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk=
 github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
 github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
 github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
-github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
-github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/mod v0.19.0 h1:fEdghXQSo20giMthA7cd28ZC+jts4amQ3YMXiP5oMQ8=
 golang.org/x/mod v0.19.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
@@ -62,11 +75,15 @@ golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.23.0 h1:SGsXPZ+2l4JsgaCKkx+FQ9YZ5XEtA1GZYuoDjenLjvg=
 golang.org/x/tools v0.23.0/go.mod h1:pnu6ufv6vQkll6szChhK3C3L/ruaIv5eBeztNG8wtsI=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc=
+gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

install.sh

@@ -8,6 +8,9 @@ THIS_PROJECT_NAME='go-watch-logs'
 THISOS=$(uname -s)
 ARCH=$(uname -m)
 
+INSTALL_VERSION=${INSTALL_VERSION:-latest}
+echo "Installing $THIS_PROJECT_NAME version: $INSTALL_VERSION"
+
 case $THISOS in
    Linux*)
       case $ARCH in
@@ -49,9 +52,19 @@ if [ -z "$THE_ARCH_BIN" ]; then
    exit 1
 fi
 
-curl -kL --progress-bar https://github.com/rakutentech/$THIS_PROJECT_NAME/releases/latest/download/$THE_ARCH_BIN -o "$BIN_DIR"/$THIS_PROJECT_NAME
+DOWNLOAD_URL="https://github.com/rakutentech/$THIS_PROJECT_NAME/releases/download/$INSTALL_VERSION/$THE_ARCH_BIN"
+if [ "$INSTALL_VERSION" = "latest" ]; then
+  DOWNLOAD_URL="https://github.com/rakutentech/$THIS_PROJECT_NAME/releases/$INSTALL_VERSION/download/$THE_ARCH_BIN"
+fi
 
-chmod +x "$BIN_DIR"/$THIS_PROJECT_NAME
+echo "Downloading from $DOWNLOAD_URL..."
+HTTP_STATUS=$(curl -kL --progress-bar -w "%{http_code}" -o "$BIN_DIR/$THIS_PROJECT_NAME" "$DOWNLOAD_URL")
 
-echo "Installed successfully to: $BIN_DIR/$THIS_PROJECT_NAME"
+if [ "$HTTP_STATUS" -ne 200 ]; then
+  echo "Error: Failed to download $THIS_PROJECT_NAME. HTTP status code: $HTTP_STATUS"
+  exit 1
+fi
+
+chmod +x "$BIN_DIR/$THIS_PROJECT_NAME"
 
+echo "Installed successfully to: $BIN_DIR/$THIS_PROJECT_NAME"

main.go

@@ -21,8 +21,11 @@ var filePaths []string
 var filePathsMutex sync.Mutex
 
 func main() {
-	flags()
-	pkg.SetupLoggingStdout(f.LogLevel)
+	pkg.Parseflags(&f)
+	pkg.SetupLoggingStdout(f.LogLevel, f.LogFile) // nolint: errcheck
+	flag.VisitAll(func(f *flag.Flag) {
+		slog.Info(f.Name, slog.String("value", f.Value.String()))
+	})
 	parseProxy()
 	wantsVersion()
 	validate()
@@ -65,26 +68,31 @@ func main() {
 		watch(filePath)
 	}
 	if f.Every > 0 {
-		if err := gocron.Every(1).Second().Do(pkg.PrintMemUsage, &f); err != nil {
-			slog.Error("Error scheduling memory usage", "error", err.Error())
-			return
-		}
-		if err := gocron.Every(f.Every).Second().Do(cron); err != nil {
-			slog.Error("Error scheduling cron", "error", err.Error())
-			return
-		}
-		if err := gocron.Every(f.Every).Second().Do(syncFilePaths); err != nil {
-			slog.Error("Error scheduling syncFilePaths", "error", err.Error())
+		startCron()
+	}
+}
+
+func startCron() {
+	if err := gocron.Every(1).Second().Do(pkg.PrintMemUsage, &f); err != nil {
+		slog.Error("Error scheduling memory usage", "error", err.Error())
+		return
+	}
+	if err := gocron.Every(f.Every).Second().Do(syncFilePaths); err != nil {
+		slog.Error("Error scheduling syncFilePaths", "error", err.Error())
+		return
+	}
+	if f.HealthCheckEvery > 0 {
+		if err := gocron.Every(f.HealthCheckEvery).Second().Do(sendHealthCheck); err != nil {
+			slog.Error("Error scheduling health check", "error", err.Error())
 			return
 		}
-		if f.HealthCheckEvery > 0 {
-			if err := gocron.Every(f.HealthCheckEvery).Second().Do(sendHealthCheck); err != nil {
-				slog.Error("Error scheduling health check", "error", err.Error())
-				return
-			}
-		}
-		<-gocron.Start()
 	}
+
+	if err := gocron.Every(f.Every).Second().Do(cron); err != nil {
+		slog.Error("Error scheduling cron", "error", err.Error())
+		return
+	}
+	<-gocron.Start()
 }
 
 func cron() {
@@ -157,7 +165,8 @@ func validate() {
 }
 
 func watch(filePath string) {
-	watcher, err := pkg.NewWatcher(f.DBPath, filePath, f.Match, f.Ignore)
+	watcher, err := pkg.NewWatcher(filePath, f)
+
 	if err != nil {
 		slog.Error("Error creating watcher", "error", err.Error(), "filePath", filePath)
 		return
@@ -171,82 +180,33 @@ func watch(filePath string) {
 		slog.Error("Error scanning file", "error", err.Error(), "filePath", filePath)
 		return
 	}
-	slog.Info("1st line", "date", result.FirstDate, "line", pkg.Truncate(result.FirstLine, pkg.TruncateMax))
-	slog.Info("Preview line", "line", pkg.Truncate(result.PreviewLine, pkg.TruncateMax))
-	slog.Info("Last line", "date", result.LastDate, "line", pkg.Truncate(result.LastLine, pkg.TruncateMax))
-	slog.Info("Error count", "percent", fmt.Sprintf("%d (%.2f)", result.ErrorCount, result.ErrorPercent)+"%")
-
 	slog.Info("Lines read", "count", result.LinesRead)
-
 	slog.Info("Scanning complete", "filePath", result.FilePath)
+	slog.Info("1st line (truncated to 200 chars)", "date", result.FirstDate, "line", pkg.Truncate(result.FirstLine, pkg.TruncateMax))
+	slog.Info("Preview line (truncated to 200 chars)", "line", pkg.Truncate(result.PreviewLine, pkg.TruncateMax))
+	slog.Info("Last line (truncated to 200 chars)", "date", result.LastDate, "line", pkg.Truncate(result.LastLine, pkg.TruncateMax))
+	slog.Info("Error count", "percent", fmt.Sprintf("%d (%.2f)", result.ErrorCount, result.ErrorPercent)+"%")
 
 	if result.ErrorCount < 0 {
 		return
 	}
 	if result.ErrorCount < f.Min {
 		return
 	}
-	notify(result)
-	if f.PostMin != "" {
-		if _, err := pkg.ExecShell(f.PostMin); err != nil {
-			slog.Error("Error running post command", "error", err.Error())
-		}
-	}
-}
-
-func notify(result *pkg.ScanResult) {
-	slog.Info("Sending to MS Teams")
-	details := pkg.GetAlertDetails(&f, version, result)
-
-	var logDetails []interface{} // nolint: prealloc
-	for _, detail := range details {
-		logDetails = append(logDetails, detail.Label, detail.Message)
+	if !f.NotifyOnlyRecent {
+		pkg.Notify(result, f, version)
 	}
 
-	if f.MSTeamsHook == "" {
-		slog.Warn("MS Teams hook not set")
-		return
+	if f.NotifyOnlyRecent && pkg.IsRecentlyModified(result.FileInfo, f.Every) {
+		pkg.Notify(result, f, version)
 	}
-	slog.Info("Sending Alert Notify", logDetails...)
-
-	hostname, _ := os.Hostname()
-
-	err := gmt.Send(hostname, details, f.MSTeamsHook, f.Proxy)
-	if err != nil {
-		slog.Error("Error sending to Teams", "error", err.Error())
-	} else {
-		slog.Info("Successfully sent to MS Teams")
+	if f.PostCommand != "" {
+		if _, err := pkg.ExecShell(f.PostCommand); err != nil {
+			slog.Error("Error running post command", "error", err.Error())
+		}
 	}
 }
 
-func flags() {
-	flag.StringVar(&f.FilePath, "file-path", "", "full path to the log file")
-	flag.StringVar(&f.FilePath, "f", "", "(short for --file-path) full path to the log file")
-	flag.StringVar(&f.DBPath, "db-path", pkg.GetHomedir()+"/.go-watch-logs.db", "path to store db file")
-	flag.StringVar(&f.Match, "match", "", "regex for matching errors (empty to match all lines)")
-	flag.StringVar(&f.Ignore, "ignore", "", "regex for ignoring errors (empty to ignore none)")
-	flag.StringVar(&f.PostAlways, "post-always", "", "run this shell command after every scan")
-	flag.StringVar(&f.PostMin, "post-min", "", "run this shell command after every scan when min errors are found")
-	flag.Uint64Var(&f.Every, "every", 0, "run every n seconds (0 to run once)")
-	flag.Uint64Var(&f.HealthCheckEvery, "health-check-every", 0, "run health check every n seconds (0 to disable)")
-	flag.IntVar(&f.LogLevel, "log-level", 0, "log level (0=info, -4=debug, 4=warn, 8=error)")
-	flag.IntVar(&f.MemLimit, "mem-limit", 100, "memory limit in MB (0 to disable)")
-	flag.IntVar(&f.FilePathsCap, "file-paths-cap", 100, "max number of file paths to watch")
-	flag.IntVar(&f.Min, "min", 1, "on minimum num of matches, it should notify")
-	flag.BoolVar(&f.Version, "version", false, "")
-	flag.BoolVar(&f.Test, "test", false, `Quickly test paths or regex
-# will test if the input matches the regex
-echo test123 | go-watch-logs --match=123 --test
-# will test if the file paths are found and list them
-go-watch-logs --file-path=./ssl_access.*log --test
-	`)
-
-	flag.StringVar(&f.Proxy, "proxy", "", "http proxy for webhooks")
-	flag.StringVar(&f.MSTeamsHook, "ms-teams-hook", "", "ms teams webhook")
-
-	flag.Parse()
-}
-
 func parseProxy() string {
 	systemProxy := pkg.SystemProxy()
 	if systemProxy != "" && f.Proxy == "" {