Move timez to logmatch.

cmd/wasm/wasm.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/prequel-dev/preq/internal/pkg/config"
+	"github.com/prequel-dev/preq/internal/pkg/eval"
 	"github.com/prequel-dev/preq/internal/pkg/ux"
 	"github.com/prequel-dev/preq/internal/pkg/verz"
-	"github.com/prequel-dev/preq/pkg/eval"
 	"github.com/rs/zerolog/log"
 )
 
@@ -66,10 +66,10 @@ func detectWrapper(ctx context.Context) js.Func {
 	detectFunc := js.FuncOf(func(this js.Value, args []js.Value) any {
 
 		var (
-			cfg, inputData, ruleData string
-			reportDoc                ux.ReportDocT
-			stats                    ux.StatsT
-			err                      error
+			inputData, ruleData string
+			reportDoc           ux.ReportDocT
+			stats               ux.StatsT
+			err                 error
 		)
 
 		log.Info().
@@ -82,7 +82,7 @@ func detectWrapper(ctx context.Context) js.Func {
 		ruleData = args[1].String()
 
 		// Permit events to arrive out of order within a 1 hour window by default
-		cfg = config.Marshal(config.WithWindow(time.Hour))
+		cfg := config.DefaultConfig(config.WithWindow(time.Hour))
 
 		if reportDoc, stats, err = eval.Detect(ctx, cfg, inputData, ruleData); err != nil {
 			return errJson(err)

go.mod

@@ -9,7 +9,7 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible
 	github.com/jedib0t/go-pretty/v6 v6.7.5
 	github.com/posener/complete v1.2.3
-	github.com/prequel-dev/prequel-logmatch v0.0.16
+	github.com/prequel-dev/prequel-logmatch v0.0.19
 	github.com/rs/zerolog v1.34.0
 	github.com/willabides/kongplete v0.4.0
 	gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -215,8 +215,8 @@ github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXq
 github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
 github.com/prequel-dev/prequel-compiler v0.0.20 h1:NTDfvUgmT83on6RDA+GihAgNM4ZvGLo9r34DAdc2tFQ=
 github.com/prequel-dev/prequel-compiler v0.0.20/go.mod h1:sqZKM2senjfoqjBqnq6u/r3qoWjTSCxI8rh4sW9zQkQ=
-github.com/prequel-dev/prequel-logmatch v0.0.16 h1:7e/ATexC8MmRGDyxpNdy8taDdFWUNRp1y3JQUOwVPfQ=
-github.com/prequel-dev/prequel-logmatch v0.0.16/go.mod h1:hRiK9FGVqbiQWLbJReXg5Fz759BYM3xCw4hxiLF3/Jg=
+github.com/prequel-dev/prequel-logmatch v0.0.19 h1:0YslB6BqcEPGv0LUampO6rnfvVhpYGWIVseWgcJlsek=
+github.com/prequel-dev/prequel-logmatch v0.0.19/go.mod h1:hRiK9FGVqbiQWLbJReXg5Fz759BYM3xCw4hxiLF3/Jg=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=

internal/pkg/cli/cli.go

@@ -13,10 +13,10 @@ import (
 	"github.com/prequel-dev/preq/internal/pkg/resolve"
 	"github.com/prequel-dev/preq/internal/pkg/rules"
 	"github.com/prequel-dev/preq/internal/pkg/runbook"
-	"github.com/prequel-dev/preq/internal/pkg/timez"
 	"github.com/prequel-dev/preq/internal/pkg/utils"
 	"github.com/prequel-dev/preq/internal/pkg/ux"
 	"github.com/prequel-dev/prequel-compiler/pkg/datasrc"
+	"github.com/prequel-dev/prequel-logmatch/pkg/timez"
 	"github.com/rs/zerolog/log"
 )
 

internal/pkg/config/config.go

@@ -1,180 +1,18 @@
 package config
 
 import (
-	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/prequel-dev/preq/internal/pkg/resolve"
+	"github.com/prequel-dev/prequel-logmatch/pkg/timez"
 	"github.com/rs/zerolog/log"
 	"gopkg.in/yaml.v3"
 )
 
-var (
-	defaultConfig = `timestamps:
-
-  # Example: {"level":"error","error":"context deadline exceeded","time":1744570895480541,"caller":"server.go:462"}
-  - format: epochany
-    pattern: |
-      "time":(\d{16,19})
-
-  # Example: 2006-01-02T15:04:05Z07:00 <log message>
-  - format: rfc3339
-    pattern: |
-      ^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+\-]\d{2}:\d{2}))
-
-  # Example: 2006/01/02 03:04:05 <log message>
-  - format: "2006/01/02 03:04:05"
-    pattern: |
-      ^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2})
-
-  # Example: 2006-01-02 15:04:05.000 <log message>
-  # Source: ISO 8601
-  - format: "2006-01-02 15:04:05.000"
-    pattern: |
-      ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})
-
-  # Example: Apr 30 23:36:47.715984 WRN <log message>
-  # Source: RFC 3164 extended
-  - format: "Jan 2 15:04:05.000000"
-    pattern: |
-      ^([A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}\.\d{6})
-
-  # Example: Jan 2 15:04:05 <log message>
-  # Source: RFC 3164
-  - format: "Jan 2 15:04:05"
-    pattern: |
-      ^([A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2})
-
-  # Example: 2006-01-02 15:04:05 <log message>
-  # Source: w3c, Postgres
-  - format: "2006-01-02 15:04:05"
-    pattern: |
-      ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})
-
-  # Example: I0102 15:04:05.000000 <log message>
-  # Source: go/klog
-  - format: "0102 15:04:05.000000"
-    pattern: |
-      ^[IWEF](\d{4} \d{2}:\d{2}:\d{2}\.\d{6})
-
-  # Example: [2006-01-02 15:04:05,000] <log message>
-  - format: "2006-01-02 15:04:05,000"
-    pattern: |
-      ^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]
-
-  # Example: 2006-01-02 15:04:05.000000-0700 <log message>
-  - format: "2006-01-02 15:04:05.000000-0700"
-    pattern: |
-      ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}[+-]\d{4})
-
-  # Example: 2006/01/02 15:04:05 <log message>
-  - format: "2006/01/02 15:04:05"
-    pattern: |
-      ^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2})
-
-  # Example: 01/02/2006, 15:04:05 <log message>
-  # Source: IIS format
-  - format: "01/02/2006, 15:04:05"
-    pattern: |
-      ^(\d{2}/\d{2}/\d{4}, \d{2}:\d{2}:\d{2})
-
-  # Example: 02 Jan 2006 15:04:05.000 <log message>
-  - format: "02 Jan 2006 15:04:05.000" 
-    pattern: |
-      ^(\d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2}\.\d{3})
-
-  # Example: 2006 Jan 02 15:04:05.000 <log message>
-  - format: "2006 Jan 02 15:04:05.000"
-    pattern: |
-      ^(\d{4} [A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2}\.\d{3})
-
-  # Example: 02/Jan/2006:15:04:05.000 <log message>
-  - format: "02/Jan/2006:15:04:05.000"
-    pattern: |
-      ^(\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2}\.\d{3})
-
-  # Example: 01/02/2006 03:04:05 PM <log message>
-  - format: "01/02/2006 03:04:05 PM"
-    pattern: |
-      ^(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2} (AM|PM))
-
-  # Example: 2006 Jan 02 15:04:05 <log message>
-  - format: "2006 Jan 02 15:04:05" 
-    pattern: |
-      ^(\d{4} [A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2})
-
-  # Example: 2006-01-02 15:04:05.000 <log message>
-  - format: "2006-01-02 15:04:05.000"
-    pattern: |
-      ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3})
-
-  # Example: {"timestamp":"2025-03-26T14:01:02Z","level":"info", "message":"..."}
-  # Source: Postgres JSON output
-  - format: rfc3339
-    pattern: |
-      "timestamp"\s*:\s*"([^"]+)"
-
-  # Example: {"ts":"2025-03-26T14:01:02Z","level":"info", "message":"..."}
-  # Source: metallb
-  - format: rfc3339
-    pattern: |
-      "ts"\s*:\s*"([^"]+)"
-
-  # Example: [7] 2025/04/25 02:01:04.339092 [ERR] 10.0.6.53:27827 - cid:10110160 - TLS handshake error: EOF
-  # Source: NATS
-  - format: "2006/01/02 15:04:05.000000"
-    pattern: |
-      ^\[\d+\]\s+(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{6})
-
-  # Example: {"creationTimestamp":"2025-04-23T20:50:35Z","name":"insecure-nginx-conf","namespace":"default","resourceVersion":"825013"}
-  # Source: Kubernetes events, configmaps
-  - format: rfc3339
-    pattern: |
-      "creationTimestamp":"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)"
-
-  # Example: 2025-04-24T21:55:08.535-0500	INFO	example-log-entry
-  # Source: ZAP production
-  - format: "2006-01-02T15:04:05.000-0700"
-    pattern: |
-      ^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{4})
-
-  # Example: {"level":"info","ts":1745549708.5355184,"msg":"example-log-entry"}
-  # Source: ZAP development
-  - format: epochany
-    pattern: |
-      "ts"\s*:\s*([0-9]+)(?:\.[0-9]+)?
-
-  # Example: ts=2025-03-10T13:52:40.623431174Z level=info msg="tail routine: tail channel closed...
-  # Source: Loki
-  - format: "2006-01-02T15:04:05.000000000Z"
-    pattern: |
-      ts=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9}Z)
-
-  # Example: {"event": "DD_API_KEY undefined. Metrics, logs and events will not be reported to DataDog", "timestamp": "2025-02-12T18:12:58.715528Z", "level": "warn...
-  # Source: DataDog
-  - format: "2006-01-02T15:04:05.000000Z"
-    pattern: |
-      "timestamp"\s*:\s*"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z)"
-
-  # Example: {"Id":19,"Version":1,"Opcode":13,"RecordId":1493,"LogName":"System","ProcessId":4324,"ThreadId":10456,"MachineName":"windows","TimeCreated":"\/Date(1743448267142)\/"}
-  # Source; Windows events via Get-Events w/ JSON output
-  - format: epochany
-    pattern: |
-      /Date\((\d+)\)
-
-  # Example: time="2025-02-12T18:12:58.715528Z"
-  # Source: argocd
-  - format: rfc3339
-    pattern: |
-      time="([^"]+)"
-`
-
-	windowConfig = `window: %s`
-)
-
 type Config struct {
 	TimestampRegexes []Regex        `yaml:"timestamps"`
 	Rules            Rules          `yaml:"rules"`
@@ -216,57 +54,60 @@ func parseOpts(opts ...OptT) *optsT {
 	return o
 }
 
-func Marshal(opts ...OptT) string {
-	o := parseOpts(opts...)
-
-	if o.window > 0 {
-		wcfg := fmt.Sprintf(windowConfig, o.window)
-		return fmt.Sprintf("%s\n%s\n", defaultConfig, wcfg)
-	}
-
-	return defaultConfig
-}
+// LoadConfig loads the configuration from the specified directory and file.
+// If the file does not exist, it returns a default configuration.
 
 func LoadConfig(dir, file string, opts ...OptT) (*Config, error) {
 
-	var config Config
-
-	if _, err := os.Stat(dir); os.IsNotExist(err) {
-		if err := os.MkdirAll(dir, 0755); err != nil {
-			return nil, err
-		}
+	spec := filepath.Join(dir, file)
+	_, err := os.Stat(spec)
+
+	switch {
+	case err == nil: // NOOP
+	case os.IsNotExist(err):
+		log.Info().
+			Str("file", spec).
+			Msg("Configuration file does not exist, using default configuration")
+		return DefaultConfig(opts...), nil
+	default:
+		return nil, err
 	}
 
-	if _, err := os.Stat(filepath.Join(dir, file)); os.IsNotExist(err) {
-		if err := WriteDefaultConfig(filepath.Join(dir, file), opts...); err != nil {
-			log.Error().Err(err).Msg("Failed to write default config")
-			return nil, err
-		}
+	log.Info().Str("file", spec).Msg("Loading configuration file")
+	fh, err := os.OpenFile(spec, os.O_RDONLY, 0644)
+	if err != nil {
+		return nil, err
 	}
+	defer fh.Close()
+
+	return ReadConfig(fh)
+}
 
-	data, err := os.ReadFile(filepath.Join(dir, file))
+func ReadConfig(rd io.Reader) (*Config, error) {
+	data, err := io.ReadAll(rd)
 	if err != nil {
 		return nil, err
 	}
-
+	var config Config
 	if err := yaml.Unmarshal(data, &config); err != nil {
 		return nil, err
 	}
-
 	return &config, nil
 }
 
-func WriteDefaultConfig(path string, opts ...OptT) error {
-	cfg := Marshal(opts...)
-	return os.WriteFile(path, []byte(cfg), 0644)
-}
+func DefaultConfig(opts ...OptT) *Config {
+	o := parseOpts(opts...)
 
-func LoadConfigFromBytes(data string) (*Config, error) {
-	var config Config
-	if err := yaml.Unmarshal([]byte(data), &config); err != nil {
-		return nil, err
+	c := &Config{
+		Window: o.window,
 	}
-	return &config, nil
+	for _, r := range timez.Defaults {
+		c.TimestampRegexes = append(c.TimestampRegexes, Regex{
+			Pattern: r.Pattern,
+			Format:  string(r.Format),
+		})
+	}
+	return c
 }
 
 func (c *Config) ResolveOpts() (opts []resolve.OptT) {