From fa23501c76a2e362962b42ca31efa1f8847bb0d4 Mon Sep 17 00:00:00 2001 From: Sean Cunningham Date: Fri, 9 Jan 2026 11:45:56 -0500 Subject: [PATCH 1/2] Move timez to logmatch. Remove side effect of config file creation if non-existent; this should be the responsibility of the caller if desired. Additionally, the user will not pick up new default timestamp formats once the config file is created, so this avoids that confusion. --- go.mod | 2 +- go.sum | 4 +- internal/pkg/cli/cli.go | 2 +- internal/pkg/config/config.go | 235 +++++----------------------- internal/pkg/config/config_test.go | 238 ++++++++++++++++++++++++----- internal/pkg/resolve/logfactory.go | 2 +- internal/pkg/timez/timez.go | 154 ------------------- internal/pkg/timez/timez_test.go | 52 ------- internal/pkg/ux/ux.go | 2 +- pkg/eval/eval.go | 18 ++- 10 files changed, 257 insertions(+), 452 deletions(-) delete mode 100644 internal/pkg/timez/timez.go delete mode 100644 internal/pkg/timez/timez_test.go diff --git a/go.mod b/go.mod index fa4710b..23707f4 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/jedib0t/go-pretty/v6 v6.7.5 github.com/posener/complete v1.2.3 - github.com/prequel-dev/prequel-logmatch v0.0.16 + github.com/prequel-dev/prequel-logmatch v0.0.19 github.com/rs/zerolog v1.34.0 github.com/willabides/kongplete v0.4.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index cf3ff2d..47290c5 100644 --- a/go.sum +++ b/go.sum @@ -215,8 +215,8 @@ github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXq github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prequel-dev/prequel-compiler v0.0.20 h1:NTDfvUgmT83on6RDA+GihAgNM4ZvGLo9r34DAdc2tFQ= github.com/prequel-dev/prequel-compiler v0.0.20/go.mod h1:sqZKM2senjfoqjBqnq6u/r3qoWjTSCxI8rh4sW9zQkQ= -github.com/prequel-dev/prequel-logmatch v0.0.16 h1:7e/ATexC8MmRGDyxpNdy8taDdFWUNRp1y3JQUOwVPfQ= -github.com/prequel-dev/prequel-logmatch v0.0.16/go.mod h1:hRiK9FGVqbiQWLbJReXg5Fz759BYM3xCw4hxiLF3/Jg= +github.com/prequel-dev/prequel-logmatch v0.0.19 h1:0YslB6BqcEPGv0LUampO6rnfvVhpYGWIVseWgcJlsek= +github.com/prequel-dev/prequel-logmatch v0.0.19/go.mod h1:hRiK9FGVqbiQWLbJReXg5Fz759BYM3xCw4hxiLF3/Jg= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= diff --git a/internal/pkg/cli/cli.go b/internal/pkg/cli/cli.go index dce7122..b91a230 100644 --- a/internal/pkg/cli/cli.go +++ b/internal/pkg/cli/cli.go @@ -13,10 +13,10 @@ import ( "github.com/prequel-dev/preq/internal/pkg/resolve" "github.com/prequel-dev/preq/internal/pkg/rules" "github.com/prequel-dev/preq/internal/pkg/runbook" - "github.com/prequel-dev/preq/internal/pkg/timez" "github.com/prequel-dev/preq/internal/pkg/utils" "github.com/prequel-dev/preq/internal/pkg/ux" "github.com/prequel-dev/prequel-compiler/pkg/datasrc" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/rs/zerolog/log" ) diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 9cb7c9c..f41fb75 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -1,180 +1,18 @@ package config import ( - "fmt" + "io" "os" "path/filepath" "strings" "time" "github.com/prequel-dev/preq/internal/pkg/resolve" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" ) -var ( - defaultConfig = `timestamps: - - # Example: {"level":"error","error":"context deadline exceeded","time":1744570895480541,"caller":"server.go:462"} - - format: epochany - pattern: | - "time":(\d{16,19}) - - # Example: 2006-01-02T15:04:05Z07:00 - - format: rfc3339 - pattern: | - ^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+\-]\d{2}:\d{2})) - - # Example: 2006/01/02 03:04:05 - - format: "2006/01/02 03:04:05" - pattern: | - ^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) - - # Example: 2006-01-02 15:04:05.000 - # Source: ISO 8601 - - format: "2006-01-02 15:04:05.000" - pattern: | - ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: Apr 30 23:36:47.715984 WRN - # Source: RFC 3164 extended - - format: "Jan 2 15:04:05.000000" - pattern: | - ^([A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}\.\d{6}) - - # Example: Jan 2 15:04:05 - # Source: RFC 3164 - - format: "Jan 2 15:04:05" - pattern: | - ^([A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}) - - # Example: 2006-01-02 15:04:05 - # Source: w3c, Postgres - - format: "2006-01-02 15:04:05" - pattern: | - ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) - - # Example: I0102 15:04:05.000000 - # Source: go/klog - - format: "0102 15:04:05.000000" - pattern: | - ^[IWEF](\d{4} \d{2}:\d{2}:\d{2}\.\d{6}) - - # Example: [2006-01-02 15:04:05,000] - - format: "2006-01-02 15:04:05,000" - pattern: | - ^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\] - - # Example: 2006-01-02 15:04:05.000000-0700 - - format: "2006-01-02 15:04:05.000000-0700" - pattern: | - ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}[+-]\d{4}) - - # Example: 2006/01/02 15:04:05 - - format: "2006/01/02 15:04:05" - pattern: | - ^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) - - # Example: 01/02/2006, 15:04:05 - # Source: IIS format - - format: "01/02/2006, 15:04:05" - pattern: | - ^(\d{2}/\d{2}/\d{4}, \d{2}:\d{2}:\d{2}) - - # Example: 02 Jan 2006 15:04:05.000 - - format: "02 Jan 2006 15:04:05.000" - pattern: | - ^(\d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: 2006 Jan 02 15:04:05.000 - - format: "2006 Jan 02 15:04:05.000" - pattern: | - ^(\d{4} [A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: 02/Jan/2006:15:04:05.000 - - format: "02/Jan/2006:15:04:05.000" - pattern: | - ^(\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: 01/02/2006 03:04:05 PM - - format: "01/02/2006 03:04:05 PM" - pattern: | - ^(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2} (AM|PM)) - - # Example: 2006 Jan 02 15:04:05 - - format: "2006 Jan 02 15:04:05" - pattern: | - ^(\d{4} [A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2}) - - # Example: 2006-01-02 15:04:05.000 - - format: "2006-01-02 15:04:05.000" - pattern: | - ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: {"timestamp":"2025-03-26T14:01:02Z","level":"info", "message":"..."} - # Source: Postgres JSON output - - format: rfc3339 - pattern: | - "timestamp"\s*:\s*"([^"]+)" - - # Example: {"ts":"2025-03-26T14:01:02Z","level":"info", "message":"..."} - # Source: metallb - - format: rfc3339 - pattern: | - "ts"\s*:\s*"([^"]+)" - - # Example: [7] 2025/04/25 02:01:04.339092 [ERR] 10.0.6.53:27827 - cid:10110160 - TLS handshake error: EOF - # Source: NATS - - format: "2006/01/02 15:04:05.000000" - pattern: | - ^\[\d+\]\s+(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{6}) - - # Example: {"creationTimestamp":"2025-04-23T20:50:35Z","name":"insecure-nginx-conf","namespace":"default","resourceVersion":"825013"} - # Source: Kubernetes events, configmaps - - format: rfc3339 - pattern: | - "creationTimestamp":"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)" - - # Example: 2025-04-24T21:55:08.535-0500 INFO example-log-entry - # Source: ZAP production - - format: "2006-01-02T15:04:05.000-0700" - pattern: | - ^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{4}) - - # Example: {"level":"info","ts":1745549708.5355184,"msg":"example-log-entry"} - # Source: ZAP development - - format: epochany - pattern: | - "ts"\s*:\s*([0-9]+)(?:\.[0-9]+)? - - # Example: ts=2025-03-10T13:52:40.623431174Z level=info msg="tail routine: tail channel closed... - # Source: Loki - - format: "2006-01-02T15:04:05.000000000Z" - pattern: | - ts=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9}Z) - - # Example: {"event": "DD_API_KEY undefined. Metrics, logs and events will not be reported to DataDog", "timestamp": "2025-02-12T18:12:58.715528Z", "level": "warn... - # Source: DataDog - - format: "2006-01-02T15:04:05.000000Z" - pattern: | - "timestamp"\s*:\s*"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z)" - - # Example: {"Id":19,"Version":1,"Opcode":13,"RecordId":1493,"LogName":"System","ProcessId":4324,"ThreadId":10456,"MachineName":"windows","TimeCreated":"\/Date(1743448267142)\/"} - # Source; Windows events via Get-Events w/ JSON output - - format: epochany - pattern: | - /Date\((\d+)\) - - # Example: time="2025-02-12T18:12:58.715528Z" - # Source: argocd - - format: rfc3339 - pattern: | - time="([^"]+)" -` - - windowConfig = `window: %s` -) - type Config struct { TimestampRegexes []Regex `yaml:"timestamps"` Rules Rules `yaml:"rules"` @@ -216,57 +54,60 @@ func parseOpts(opts ...OptT) *optsT { return o } -func Marshal(opts ...OptT) string { - o := parseOpts(opts...) - - if o.window > 0 { - wcfg := fmt.Sprintf(windowConfig, o.window) - return fmt.Sprintf("%s\n%s\n", defaultConfig, wcfg) - } - - return defaultConfig -} +// LoadConfig loads the configuration from the specified directory and file. +// If the file does not exist, it creates a return a default configuration. func LoadConfig(dir, file string, opts ...OptT) (*Config, error) { - var config Config - - if _, err := os.Stat(dir); os.IsNotExist(err) { - if err := os.MkdirAll(dir, 0755); err != nil { - return nil, err - } + spec := filepath.Join(dir, file) + _, err := os.Stat(spec) + + switch { + case err == nil: // NOOP + case os.IsNotExist(err): + log.Info(). + Str("file", spec). + Msg("Configuration file does not exist, using default configuration") + return DefaultConfig(opts...), nil + default: + return nil, err } - if _, err := os.Stat(filepath.Join(dir, file)); os.IsNotExist(err) { - if err := WriteDefaultConfig(filepath.Join(dir, file), opts...); err != nil { - log.Error().Err(err).Msg("Failed to write default config") - return nil, err - } + log.Info().Str("file", spec).Msg("Loading configuration file") + fh, err := os.OpenFile(spec, os.O_RDONLY, 0644) + if err != nil { + return nil, err } + defer fh.Close() + + return ReadConfig(fh) +} - data, err := os.ReadFile(filepath.Join(dir, file)) +func ReadConfig(rd io.Reader) (*Config, error) { + data, err := io.ReadAll(rd) if err != nil { return nil, err } - + var config Config if err := yaml.Unmarshal(data, &config); err != nil { return nil, err } - return &config, nil } -func WriteDefaultConfig(path string, opts ...OptT) error { - cfg := Marshal(opts...) - return os.WriteFile(path, []byte(cfg), 0644) -} +func DefaultConfig(opts ...OptT) *Config { + o := parseOpts(opts...) -func LoadConfigFromBytes(data string) (*Config, error) { - var config Config - if err := yaml.Unmarshal([]byte(data), &config); err != nil { - return nil, err + c := &Config{ + Window: o.window, } - return &config, nil + for _, r := range timez.Defaults { + c.TimestampRegexes = append(c.TimestampRegexes, Regex{ + Pattern: r.Pattern, + Format: string(r.Format), + }) + } + return c } func (c *Config) ResolveOpts() (opts []resolve.OptT) { diff --git a/internal/pkg/config/config_test.go b/internal/pkg/config/config_test.go index 26e6ccc..f264e38 100644 --- a/internal/pkg/config/config_test.go +++ b/internal/pkg/config/config_test.go @@ -10,19 +10,7 @@ import ( "github.com/prequel-dev/preq/internal/pkg/config" ) -func TestMarshal(t *testing.T) { - out := config.Marshal() - if !strings.Contains(out, "timestamps:") { - t.Fatalf("expected timestamps in output") - } - - out = config.Marshal(config.WithWindow(2 * time.Second)) - if !strings.Contains(out, "window: 2s") { - t.Fatalf("expected window option in output") - } -} - -func TestLoadConfig(t *testing.T) { +func TestLoadConfig_FileDoesNotExist(t *testing.T) { dir := t.TempDir() cfg, err := config.LoadConfig(dir, "cfg.yaml", config.WithWindow(3*time.Second)) if err != nil { @@ -34,41 +22,221 @@ func TestLoadConfig(t *testing.T) { if len(cfg.TimestampRegexes) == 0 { t.Fatalf("expected default timestamp regexes") } - if _, err := os.Stat(filepath.Join(dir, "cfg.yaml")); err != nil { - t.Fatalf("expected config file written: %v", err) - } } -func TestLoadConfigFromBytes(t *testing.T) { - data := "timestamps: []\nwindow: 1s\n" - cfg, err := config.LoadConfigFromBytes(data) +func TestLoadConfig_FileExists(t *testing.T) { + dir := t.TempDir() + + // Create a config file + configPath := filepath.Join(dir, "cfg.yaml") + configContent := `window: 5s +skip: 10 +dataSources: "test-source" +acceptUpdates: true +rulesVersion: "v1.0" +timestamps: + - pattern: "test-pattern" + format: "test-format" +rules: + paths: + - "/path/to/rules" + disableCommunityRules: true +` + if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { + t.Fatalf("Failed to write config file: %v", err) + } + + cfg, err := config.LoadConfig(dir, "cfg.yaml") if err != nil { - t.Fatalf("LoadConfigFromBytes: %v", err) + t.Fatalf("LoadConfig error: %v", err) + } + if cfg.Window != 5*time.Second { + t.Fatalf("expected window 5s got %v", cfg.Window) + } + if cfg.Skip != 10 { + t.Fatalf("expected skip 10 got %v", cfg.Skip) + } + if cfg.DataSources != "test-source" { + t.Fatalf("expected dataSources 'test-source' got %v", cfg.DataSources) + } + if !cfg.AcceptUpdates { + t.Fatalf("expected acceptUpdates true") + } + if cfg.RulesVersion != "v1.0" { + t.Fatalf("expected rulesVersion 'v1.0' got %v", cfg.RulesVersion) + } + if len(cfg.TimestampRegexes) == 0 { + t.Fatalf("expected timestamp regexes") + } + if cfg.TimestampRegexes[0].Pattern != "test-pattern" { + t.Fatalf("expected pattern 'test-pattern' got %v", cfg.TimestampRegexes[0].Pattern) } - if cfg.Window != time.Second { - t.Fatalf("expected 1s window, got %v", cfg.Window) + if len(cfg.Rules.Paths) == 0 { + t.Fatalf("expected rules paths") + } + if !cfg.Rules.Disabled { + t.Fatalf("expected rules disabled true") } } -func TestWriteDefaultConfigAndResolveOpts(t *testing.T) { +func TestLoadConfig_StatError(t *testing.T) { dir := t.TempDir() - path := filepath.Join(dir, "cfg.yaml") - if err := config.WriteDefaultConfig(path, config.WithWindow(2*time.Second)); err != nil { - t.Fatalf("WriteDefaultConfig: %v", err) + + // Create a directory and then make it unreadable + subdir := filepath.Join(dir, "unreadable") + if err := os.Mkdir(subdir, 0755); err != nil { + t.Fatalf("Failed to create directory: %v", err) } - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read file: %v", err) + + // Create a file in the directory first + configPath := filepath.Join(subdir, "cfg.yaml") + if err := os.WriteFile(configPath, []byte("test"), 0644); err != nil { + t.Fatalf("Failed to write config file: %v", err) + } + + // Make the directory unreadable (no execute permission means can't access files in it) + if err := os.Chmod(subdir, 0000); err != nil { + t.Fatalf("Failed to chmod directory: %v", err) } - if !strings.Contains(string(data), "window: 2s") { - t.Fatalf("window option missing") + defer os.Chmod(subdir, 0755) // Restore for cleanup + + _, err := config.LoadConfig(subdir, "cfg.yaml") + if err == nil { + t.Fatalf("expected error for inaccessible directory") } +} - cfg, err := config.LoadConfig(dir, "cfg.yaml") +func TestLoadConfig_OpenFileError(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "cfg.yaml") + + // Create a file with restricted permissions + if err := os.WriteFile(configPath, []byte("test"), 0000); err != nil { + t.Fatalf("Failed to write config file: %v", err) + } + + _, err := config.LoadConfig(dir, "cfg.yaml") + if err == nil { + t.Fatalf("expected error when opening file with no permissions") + } +} + +func TestReadConfig(t *testing.T) { + configContent := `window: 2s +skip: 5 +dataSources: "my-source" +acceptUpdates: false +rulesVersion: "v2.0" +timestamps: + - pattern: "\\d{4}-\\d{2}-\\d{2}" + format: "2006-01-02" +rules: + paths: + - "/rules/path1" + - "/rules/path2" + disableCommunityRules: false +` + reader := strings.NewReader(configContent) + cfg, err := config.ReadConfig(reader) if err != nil { - t.Fatalf("LoadConfig: %v", err) + t.Fatalf("ReadConfig error: %v", err) + } + + if cfg.Window != 2*time.Second { + t.Fatalf("expected window 2s got %v", cfg.Window) + } + if cfg.Skip != 5 { + t.Fatalf("expected skip 5 got %v", cfg.Skip) + } + if cfg.DataSources != "my-source" { + t.Fatalf("expected dataSources 'my-source' got %v", cfg.DataSources) } - if len(cfg.ResolveOpts()) == 0 { - t.Fatalf("expected resolve options") + if cfg.AcceptUpdates { + t.Fatalf("expected acceptUpdates false") + } + if len(cfg.TimestampRegexes) != 1 { + t.Fatalf("expected 1 timestamp regex got %v", len(cfg.TimestampRegexes)) + } + if len(cfg.Rules.Paths) != 2 { + t.Fatalf("expected 2 rules paths got %v", len(cfg.Rules.Paths)) + } +} + +func TestReadConfig_InvalidYAML(t *testing.T) { + invalidYAML := `invalid: yaml: content: [[[` + reader := strings.NewReader(invalidYAML) + _, err := config.ReadConfig(reader) + if err == nil { + t.Fatalf("expected error for invalid YAML") + } +} + +type errorReader struct{} + +func (e *errorReader) Read(p []byte) (n int, err error) { + return 0, os.ErrInvalid +} + +func TestReadConfig_ReadError(t *testing.T) { + _, err := config.ReadConfig(&errorReader{}) + if err == nil { + t.Fatalf("expected error when reading fails") + } +} + +func TestDefaultConfig(t *testing.T) { + cfg := config.DefaultConfig() + + if cfg.Window != 0 { + t.Fatalf("expected default window 0 got %v", cfg.Window) + } + if len(cfg.TimestampRegexes) == 0 { + t.Fatalf("expected default timestamp regexes") + } +} + +func TestDefaultConfig_WithWindow(t *testing.T) { + cfg := config.DefaultConfig(config.WithWindow(10 * time.Second)) + + if cfg.Window != 10*time.Second { + t.Fatalf("expected window 10s got %v", cfg.Window) + } + if len(cfg.TimestampRegexes) == 0 { + t.Fatalf("expected default timestamp regexes") + } +} + +func TestResolveOpts(t *testing.T) { + cfg := &config.Config{ + TimestampRegexes: []config.Regex{ + {Pattern: " test-pattern ", Format: " test-format "}, + {Pattern: "pattern2", Format: "format2"}, + }, + } + + opts := cfg.ResolveOpts() + if len(opts) == 0 { + t.Fatalf("expected resolve opts") + } +} + +func TestResolveOpts_NoTimestamps(t *testing.T) { + cfg := &config.Config{ + TimestampRegexes: []config.Regex{}, + } + + opts := cfg.ResolveOpts() + if len(opts) != 0 { + t.Fatalf("expected no resolve opts got %v", len(opts)) + } +} + +func TestWithWindow(t *testing.T) { + duration := 5 * time.Second + optFunc := config.WithWindow(duration) + + cfg := config.DefaultConfig(optFunc) + if cfg.Window != duration { + t.Fatalf("expected window %v got %v", duration, cfg.Window) } } diff --git a/internal/pkg/resolve/logfactory.go b/internal/pkg/resolve/logfactory.go index 697d909..95ae030 100644 --- a/internal/pkg/resolve/logfactory.go +++ b/internal/pkg/resolve/logfactory.go @@ -3,8 +3,8 @@ package resolve import ( "bytes" - "github.com/prequel-dev/preq/internal/pkg/timez" "github.com/prequel-dev/prequel-logmatch/pkg/format" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/rs/zerolog/log" ) diff --git a/internal/pkg/timez/timez.go b/internal/pkg/timez/timez.go deleted file mode 100644 index 06080eb..0000000 --- a/internal/pkg/timez/timez.go +++ /dev/null @@ -1,154 +0,0 @@ -package timez - -import ( - "bytes" - "errors" - "strconv" - "time" - - "github.com/prequel-dev/prequel-logmatch/pkg/format" - "github.com/rs/zerolog/log" -) - -const ( - DefaultSkip = 50 -) - -var ( - ErrInvalidTimestampFormat = errors.New("invalid timestamp format") -) - -type TimestampFmt string - -func (f TimestampFmt) String() string { - return string(f) -} - -const ( - FmtRfc3339 TimestampFmt = "rfc3339" - FmtRfc3339Nano TimestampFmt = "rfc3339nano" - FmtUnix TimestampFmt = "unix" - FmtEpochAny TimestampFmt = "epochany" - FmtEpochSeconds TimestampFmt = "epochseconds" - FmtEpochMillis TimestampFmt = "epochmillis" - FmtEpochMicros TimestampFmt = "epochmicros" - FmtEpochNanos TimestampFmt = "epochnanos" -) - -func GetTimestampFormat(f TimestampFmt) (format.TimeFormatCbT, error) { - - switch f { - case FmtRfc3339: - return format.WithTimeFormat(time.RFC3339), nil - case FmtRfc3339Nano: - return format.WithTimeFormat(time.RFC3339Nano), nil - case FmtUnix: - return format.WithTimeFormat(time.UnixDate), nil - case FmtEpochAny: - return epochAny, nil - case FmtEpochSeconds: - return epochSeconds, nil - case FmtEpochMillis: - return epochMillis, nil - case FmtEpochMicros: - return epochMicros, nil - case FmtEpochNanos: - return epochNanos, nil - default: - return format.WithTimeFormat(string(f)), nil - } -} - -var ( - epochSeconds = epochParser(time.Second) - epochMillis = epochParser(time.Millisecond) - epochMicros = epochParser(time.Microsecond) - epochNanos = epochParser(time.Nanosecond) -) - -func epochParser(unit time.Duration) format.TimeFormatCbT { - return func(m []byte) (int64, error) { - v, err := strconv.ParseInt(string(m), 10, 64) - if err != nil { - return 0, ErrInvalidTimestampFormat - } - return v * int64(unit), nil - } -} - -func epochAny(m []byte) (int64, error) { - v, err := strconv.ParseInt(string(m), 10, 64) - if err != nil { - return 0, ErrInvalidTimestampFormat - } - - sz := len(m) - switch { - case sz > 16: - // NOOP: v *= int64(time.Nanosecond) - case sz > 13: - v *= int64(time.Microsecond) - case sz > 10: - v *= int64(time.Millisecond) - default: - v *= int64(time.Second) - } - return v, nil - -} - -func TryTimestampFormat(exp string, fmtStr TimestampFmt, buf []byte, maxTries int) (format.FactoryI, int64, error) { - - var ( - ts int64 - factory format.FactoryI - cb format.TimeFormatCbT - err error - ) - - log.Debug(). - Str("exp", exp). - Str("fmt", fmtStr.String()). - Msg("Trying timestamp format") - - if cb, err = GetTimestampFormat(fmtStr); err != nil { - log.Warn().Err(err).Msg("Failed to get timestamp format") - return nil, 0, err - } - - if factory, err = format.NewRegexFactory(exp, cb); err != nil { - log.Warn().Err(err).Msg("Failed to create regex factory") - return nil, 0, err - } - - f := factory.New() - ts, err = f.ReadTimestamp(bytes.NewReader(buf)) - - tries := 0 - for (err != nil || ts == 0) && tries < maxTries { - // First line may contain a header; try up to N lines - tries += 1 - if index := bytes.IndexByte(buf, '\n'); index != -1 { - buf = buf[index+1:] - ts, err = f.ReadTimestamp(bytes.NewReader(buf)) - } else { - break - } - } - - if err != nil { - log.Warn().Err(err).Msg("Failed to read timestamp") - return nil, 0, err - } - - if ts == 0 { - return nil, 0, ErrInvalidTimestampFormat - } - - log.Debug(). - Str("exp", exp). - Str("fmt", fmtStr.String()). - Msg("Selected timestamp format") - - return factory, ts, nil -} diff --git a/internal/pkg/timez/timez_test.go b/internal/pkg/timez/timez_test.go deleted file mode 100644 index 8305160..0000000 --- a/internal/pkg/timez/timez_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package timez_test - -import ( - "testing" - "time" - - "github.com/prequel-dev/preq/internal/pkg/timez" -) - -func TestGetTimestampFormat(t *testing.T) { - cb, err := timez.GetTimestampFormat(timez.FmtRfc3339) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - ts, err := cb([]byte("2025-01-02T03:04:05Z")) - if err != nil { - t.Fatalf("parse failed: %v", err) - } - want := time.Date(2025, 1, 2, 3, 4, 5, 0, time.UTC).UnixNano() - if ts != want { - t.Fatalf("expected %d got %d", want, ts) - } -} - -func TestEpochParserAndAny(t *testing.T) { - cb, _ := timez.GetTimestampFormat(timez.FmtEpochMillis) - ts, err := cb([]byte("42")) - if err != nil || ts != 42*int64(time.Millisecond) { - t.Fatalf("epoch millis failed") - } - - cb, _ = timez.GetTimestampFormat(timez.FmtEpochAny) - ts, err = cb([]byte("1000")) - if err != nil || ts != 1000*int64(time.Second) { - t.Fatalf("epoch any failed") - } -} - -func TestTryTimestampFormat(t *testing.T) { - line := "2025-06-06T12:00:00Z first line\nsecond" // newline ensures only first line used - factory, ts, err := timez.TryTimestampFormat(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)`, timez.FmtRfc3339, []byte(line), 1) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if factory == nil { - t.Fatal("expected factory") - } - want := time.Date(2025, 6, 6, 12, 0, 0, 0, time.UTC).UnixNano() - if ts != want { - t.Fatalf("timestamp mismatch") - } -} diff --git a/internal/pkg/ux/ux.go b/internal/pkg/ux/ux.go index 203b082..6e0e73e 100644 --- a/internal/pkg/ux/ux.go +++ b/internal/pkg/ux/ux.go @@ -9,9 +9,9 @@ import ( "sync/atomic" "time" - "github.com/prequel-dev/preq/internal/pkg/timez" "github.com/prequel-dev/preq/internal/pkg/verz" "github.com/prequel-dev/prequel-logmatch/pkg/format" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/Masterminds/semver" "github.com/fatih/color" diff --git a/pkg/eval/eval.go b/pkg/eval/eval.go index b7341de..607a247 100644 --- a/pkg/eval/eval.go +++ b/pkg/eval/eval.go @@ -2,13 +2,14 @@ package eval import ( "context" + "strings" "github.com/prequel-dev/preq/internal/pkg/config" "github.com/prequel-dev/preq/internal/pkg/engine" "github.com/prequel-dev/preq/internal/pkg/resolve" - "github.com/prequel-dev/preq/internal/pkg/timez" "github.com/prequel-dev/preq/internal/pkg/utils" "github.com/prequel-dev/preq/internal/pkg/ux" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/rs/zerolog/log" ) @@ -25,13 +26,14 @@ func Detect(ctx context.Context, cfg, data, rule string) (ux.ReportDocT, ux.Stat err error ) - if len(cfg) == 0 { - cfg = config.Marshal() - } - - if c, err = config.LoadConfigFromBytes(cfg); err != nil { - log.Error().Err(err).Msg("Failed to load config") - return nil, nil, err + switch len(cfg) { + case 0: + c = config.DefaultConfig() + default: + if c, err = config.ReadConfig(strings.NewReader(cfg)); err != nil { + log.Error().Err(err).Msg("Failed to load config") + return nil, nil, err + } } opts := c.ResolveOpts() From 1688a5279bf63e87570efb269c93c1ad8399f606 Mon Sep 17 00:00:00 2001 From: Sean Cunningham Date: Fri, 9 Jan 2026 12:37:53 -0500 Subject: [PATCH 2/2] Make eval an internal package. Change interface to explicitly require a Config object. --- cmd/wasm/wasm.go | 12 ++++++------ internal/pkg/config/config.go | 2 +- {pkg => internal/pkg}/eval/eval.go | 14 +------------- test/preq_test.go | 9 +++++---- 4 files changed, 13 insertions(+), 24 deletions(-) rename {pkg => internal/pkg}/eval/eval.go (81%) diff --git a/cmd/wasm/wasm.go b/cmd/wasm/wasm.go index 0259ee4..99eb371 100644 --- a/cmd/wasm/wasm.go +++ b/cmd/wasm/wasm.go @@ -10,9 +10,9 @@ import ( "time" "github.com/prequel-dev/preq/internal/pkg/config" + "github.com/prequel-dev/preq/internal/pkg/eval" "github.com/prequel-dev/preq/internal/pkg/ux" "github.com/prequel-dev/preq/internal/pkg/verz" - "github.com/prequel-dev/preq/pkg/eval" "github.com/rs/zerolog/log" ) @@ -66,10 +66,10 @@ func detectWrapper(ctx context.Context) js.Func { detectFunc := js.FuncOf(func(this js.Value, args []js.Value) any { var ( - cfg, inputData, ruleData string - reportDoc ux.ReportDocT - stats ux.StatsT - err error + inputData, ruleData string + reportDoc ux.ReportDocT + stats ux.StatsT + err error ) log.Info(). @@ -82,7 +82,7 @@ func detectWrapper(ctx context.Context) js.Func { ruleData = args[1].String() // Permit events to arrive out of order within a 1 hour window by default - cfg = config.Marshal(config.WithWindow(time.Hour)) + cfg := config.DefaultConfig(config.WithWindow(time.Hour)) if reportDoc, stats, err = eval.Detect(ctx, cfg, inputData, ruleData); err != nil { return errJson(err) diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index f41fb75..f4d0e82 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -55,7 +55,7 @@ func parseOpts(opts ...OptT) *optsT { } // LoadConfig loads the configuration from the specified directory and file. -// If the file does not exist, it creates a return a default configuration. +// If the file does not exist, it returns a default configuration. func LoadConfig(dir, file string, opts ...OptT) (*Config, error) { diff --git a/pkg/eval/eval.go b/internal/pkg/eval/eval.go similarity index 81% rename from pkg/eval/eval.go rename to internal/pkg/eval/eval.go index 607a247..40eea8e 100644 --- a/pkg/eval/eval.go +++ b/internal/pkg/eval/eval.go @@ -2,7 +2,6 @@ package eval import ( "context" - "strings" "github.com/prequel-dev/preq/internal/pkg/config" "github.com/prequel-dev/preq/internal/pkg/engine" @@ -13,10 +12,9 @@ import ( "github.com/rs/zerolog/log" ) -func Detect(ctx context.Context, cfg, data, rule string) (ux.ReportDocT, ux.StatsT, error) { +func Detect(ctx context.Context, c *config.Config, data, rule string) (ux.ReportDocT, ux.StatsT, error) { var ( - c *config.Config run *engine.RuntimeT report *ux.ReportT ruleMatchers *engine.RuleMatchersT @@ -26,16 +24,6 @@ func Detect(ctx context.Context, cfg, data, rule string) (ux.ReportDocT, ux.Stat err error ) - switch len(cfg) { - case 0: - c = config.DefaultConfig() - default: - if c, err = config.ReadConfig(strings.NewReader(cfg)); err != nil { - log.Error().Err(err).Msg("Failed to load config") - return nil, nil, err - } - } - opts := c.ResolveOpts() opts = append(opts, resolve.WithTimestampTries(timez.DefaultSkip)) diff --git a/test/preq_test.go b/test/preq_test.go index ef951a5..e070ad5 100644 --- a/test/preq_test.go +++ b/test/preq_test.go @@ -5,8 +5,9 @@ import ( "os" "testing" + "github.com/prequel-dev/preq/internal/pkg/config" + "github.com/prequel-dev/preq/internal/pkg/eval" "github.com/prequel-dev/preq/internal/pkg/logs" - "github.com/prequel-dev/preq/pkg/eval" "github.com/rs/zerolog/log" ) @@ -138,7 +139,7 @@ func TestSuccessExamples(t *testing.T) { t.Fatalf("Error reading data file %s: %v", test.dataPath, err) } - _, stats, err := eval.Detect(ctx, "", string(data), string(ruleData)) + _, stats, err := eval.Detect(ctx, config.DefaultConfig(), string(data), string(ruleData)) if err != nil { t.Fatalf("Error running detection: %v", err) } @@ -226,7 +227,7 @@ func TestMissExamples(t *testing.T) { t.Fatalf("Error reading data file %s: %v", test.dataPath, err) } - _, stats, err := eval.Detect(ctx, "", string(data), string(ruleData)) + _, stats, err := eval.Detect(ctx, config.DefaultConfig(), string(data), string(ruleData)) if err != nil { t.Fatalf("Error running detection: %v", err) } @@ -294,7 +295,7 @@ func TestFailureExamples(t *testing.T) { t.Fatalf("Error reading data file %s: %v", test.dataPath, err) } - _, _, err = eval.Detect(ctx, "", string(data), string(ruleData)) + _, _, err = eval.Detect(ctx, config.DefaultConfig(), string(data), string(ruleData)) if err == nil { t.Fatalf("Expected error running detection: %v", err) }