diff --git a/cmd/wasm/wasm.go b/cmd/wasm/wasm.go index 0259ee4..99eb371 100644 --- a/cmd/wasm/wasm.go +++ b/cmd/wasm/wasm.go @@ -10,9 +10,9 @@ import ( "time" "github.com/prequel-dev/preq/internal/pkg/config" + "github.com/prequel-dev/preq/internal/pkg/eval" "github.com/prequel-dev/preq/internal/pkg/ux" "github.com/prequel-dev/preq/internal/pkg/verz" - "github.com/prequel-dev/preq/pkg/eval" "github.com/rs/zerolog/log" ) @@ -66,10 +66,10 @@ func detectWrapper(ctx context.Context) js.Func { detectFunc := js.FuncOf(func(this js.Value, args []js.Value) any { var ( - cfg, inputData, ruleData string - reportDoc ux.ReportDocT - stats ux.StatsT - err error + inputData, ruleData string + reportDoc ux.ReportDocT + stats ux.StatsT + err error ) log.Info(). @@ -82,7 +82,7 @@ func detectWrapper(ctx context.Context) js.Func { ruleData = args[1].String() // Permit events to arrive out of order within a 1 hour window by default - cfg = config.Marshal(config.WithWindow(time.Hour)) + cfg := config.DefaultConfig(config.WithWindow(time.Hour)) if reportDoc, stats, err = eval.Detect(ctx, cfg, inputData, ruleData); err != nil { return errJson(err) diff --git a/go.mod b/go.mod index fa4710b..23707f4 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/jedib0t/go-pretty/v6 v6.7.5 github.com/posener/complete v1.2.3 - github.com/prequel-dev/prequel-logmatch v0.0.16 + github.com/prequel-dev/prequel-logmatch v0.0.19 github.com/rs/zerolog v1.34.0 github.com/willabides/kongplete v0.4.0 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index cf3ff2d..47290c5 100644 --- a/go.sum +++ b/go.sum @@ -215,8 +215,8 @@ github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXq github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= github.com/prequel-dev/prequel-compiler v0.0.20 h1:NTDfvUgmT83on6RDA+GihAgNM4ZvGLo9r34DAdc2tFQ= github.com/prequel-dev/prequel-compiler v0.0.20/go.mod h1:sqZKM2senjfoqjBqnq6u/r3qoWjTSCxI8rh4sW9zQkQ= -github.com/prequel-dev/prequel-logmatch v0.0.16 h1:7e/ATexC8MmRGDyxpNdy8taDdFWUNRp1y3JQUOwVPfQ= -github.com/prequel-dev/prequel-logmatch v0.0.16/go.mod h1:hRiK9FGVqbiQWLbJReXg5Fz759BYM3xCw4hxiLF3/Jg= +github.com/prequel-dev/prequel-logmatch v0.0.19 h1:0YslB6BqcEPGv0LUampO6rnfvVhpYGWIVseWgcJlsek= +github.com/prequel-dev/prequel-logmatch v0.0.19/go.mod h1:hRiK9FGVqbiQWLbJReXg5Fz759BYM3xCw4hxiLF3/Jg= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= diff --git a/internal/pkg/cli/cli.go b/internal/pkg/cli/cli.go index dce7122..b91a230 100644 --- a/internal/pkg/cli/cli.go +++ b/internal/pkg/cli/cli.go @@ -13,10 +13,10 @@ import ( "github.com/prequel-dev/preq/internal/pkg/resolve" "github.com/prequel-dev/preq/internal/pkg/rules" "github.com/prequel-dev/preq/internal/pkg/runbook" - "github.com/prequel-dev/preq/internal/pkg/timez" "github.com/prequel-dev/preq/internal/pkg/utils" "github.com/prequel-dev/preq/internal/pkg/ux" "github.com/prequel-dev/prequel-compiler/pkg/datasrc" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/rs/zerolog/log" ) diff --git a/internal/pkg/config/config.go b/internal/pkg/config/config.go index 9cb7c9c..f4d0e82 100644 --- a/internal/pkg/config/config.go +++ b/internal/pkg/config/config.go @@ -1,180 +1,18 @@ package config import ( - "fmt" + "io" "os" "path/filepath" "strings" "time" "github.com/prequel-dev/preq/internal/pkg/resolve" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/rs/zerolog/log" "gopkg.in/yaml.v3" ) -var ( - defaultConfig = `timestamps: - - # Example: {"level":"error","error":"context deadline exceeded","time":1744570895480541,"caller":"server.go:462"} - - format: epochany - pattern: | - "time":(\d{16,19}) - - # Example: 2006-01-02T15:04:05Z07:00 - - format: rfc3339 - pattern: | - ^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+\-]\d{2}:\d{2})) - - # Example: 2006/01/02 03:04:05 - - format: "2006/01/02 03:04:05" - pattern: | - ^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) - - # Example: 2006-01-02 15:04:05.000 - # Source: ISO 8601 - - format: "2006-01-02 15:04:05.000" - pattern: | - ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: Apr 30 23:36:47.715984 WRN - # Source: RFC 3164 extended - - format: "Jan 2 15:04:05.000000" - pattern: | - ^([A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}\.\d{6}) - - # Example: Jan 2 15:04:05 - # Source: RFC 3164 - - format: "Jan 2 15:04:05" - pattern: | - ^([A-Z][a-z]{2}\s{1,2}\d{1,2}\s\d{2}:\d{2}:\d{2}) - - # Example: 2006-01-02 15:04:05 - # Source: w3c, Postgres - - format: "2006-01-02 15:04:05" - pattern: | - ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) - - # Example: I0102 15:04:05.000000 - # Source: go/klog - - format: "0102 15:04:05.000000" - pattern: | - ^[IWEF](\d{4} \d{2}:\d{2}:\d{2}\.\d{6}) - - # Example: [2006-01-02 15:04:05,000] - - format: "2006-01-02 15:04:05,000" - pattern: | - ^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\] - - # Example: 2006-01-02 15:04:05.000000-0700 - - format: "2006-01-02 15:04:05.000000-0700" - pattern: | - ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{6}[+-]\d{4}) - - # Example: 2006/01/02 15:04:05 - - format: "2006/01/02 15:04:05" - pattern: | - ^(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}) - - # Example: 01/02/2006, 15:04:05 - # Source: IIS format - - format: "01/02/2006, 15:04:05" - pattern: | - ^(\d{2}/\d{2}/\d{4}, \d{2}:\d{2}:\d{2}) - - # Example: 02 Jan 2006 15:04:05.000 - - format: "02 Jan 2006 15:04:05.000" - pattern: | - ^(\d{2} [A-Z][a-z]{2} \d{4} \d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: 2006 Jan 02 15:04:05.000 - - format: "2006 Jan 02 15:04:05.000" - pattern: | - ^(\d{4} [A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: 02/Jan/2006:15:04:05.000 - - format: "02/Jan/2006:15:04:05.000" - pattern: | - ^(\d{2}/[A-Z][a-z]{2}/\d{4}:\d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: 01/02/2006 03:04:05 PM - - format: "01/02/2006 03:04:05 PM" - pattern: | - ^(\d{2}/\d{2}/\d{4} \d{2}:\d{2}:\d{2} (AM|PM)) - - # Example: 2006 Jan 02 15:04:05 - - format: "2006 Jan 02 15:04:05" - pattern: | - ^(\d{4} [A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2}) - - # Example: 2006-01-02 15:04:05.000 - - format: "2006-01-02 15:04:05.000" - pattern: | - ^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) - - # Example: {"timestamp":"2025-03-26T14:01:02Z","level":"info", "message":"..."} - # Source: Postgres JSON output - - format: rfc3339 - pattern: | - "timestamp"\s*:\s*"([^"]+)" - - # Example: {"ts":"2025-03-26T14:01:02Z","level":"info", "message":"..."} - # Source: metallb - - format: rfc3339 - pattern: | - "ts"\s*:\s*"([^"]+)" - - # Example: [7] 2025/04/25 02:01:04.339092 [ERR] 10.0.6.53:27827 - cid:10110160 - TLS handshake error: EOF - # Source: NATS - - format: "2006/01/02 15:04:05.000000" - pattern: | - ^\[\d+\]\s+(\d{4}/\d{2}/\d{2} \d{2}:\d{2}:\d{2}\.\d{6}) - - # Example: {"creationTimestamp":"2025-04-23T20:50:35Z","name":"insecure-nginx-conf","namespace":"default","resourceVersion":"825013"} - # Source: Kubernetes events, configmaps - - format: rfc3339 - pattern: | - "creationTimestamp":"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)" - - # Example: 2025-04-24T21:55:08.535-0500 INFO example-log-entry - # Source: ZAP production - - format: "2006-01-02T15:04:05.000-0700" - pattern: | - ^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}[+-]\d{4}) - - # Example: {"level":"info","ts":1745549708.5355184,"msg":"example-log-entry"} - # Source: ZAP development - - format: epochany - pattern: | - "ts"\s*:\s*([0-9]+)(?:\.[0-9]+)? - - # Example: ts=2025-03-10T13:52:40.623431174Z level=info msg="tail routine: tail channel closed... - # Source: Loki - - format: "2006-01-02T15:04:05.000000000Z" - pattern: | - ts=([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9}Z) - - # Example: {"event": "DD_API_KEY undefined. Metrics, logs and events will not be reported to DataDog", "timestamp": "2025-02-12T18:12:58.715528Z", "level": "warn... - # Source: DataDog - - format: "2006-01-02T15:04:05.000000Z" - pattern: | - "timestamp"\s*:\s*"([0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{6}Z)" - - # Example: {"Id":19,"Version":1,"Opcode":13,"RecordId":1493,"LogName":"System","ProcessId":4324,"ThreadId":10456,"MachineName":"windows","TimeCreated":"\/Date(1743448267142)\/"} - # Source; Windows events via Get-Events w/ JSON output - - format: epochany - pattern: | - /Date\((\d+)\) - - # Example: time="2025-02-12T18:12:58.715528Z" - # Source: argocd - - format: rfc3339 - pattern: | - time="([^"]+)" -` - - windowConfig = `window: %s` -) - type Config struct { TimestampRegexes []Regex `yaml:"timestamps"` Rules Rules `yaml:"rules"` @@ -216,57 +54,60 @@ func parseOpts(opts ...OptT) *optsT { return o } -func Marshal(opts ...OptT) string { - o := parseOpts(opts...) - - if o.window > 0 { - wcfg := fmt.Sprintf(windowConfig, o.window) - return fmt.Sprintf("%s\n%s\n", defaultConfig, wcfg) - } - - return defaultConfig -} +// LoadConfig loads the configuration from the specified directory and file. +// If the file does not exist, it returns a default configuration. func LoadConfig(dir, file string, opts ...OptT) (*Config, error) { - var config Config - - if _, err := os.Stat(dir); os.IsNotExist(err) { - if err := os.MkdirAll(dir, 0755); err != nil { - return nil, err - } + spec := filepath.Join(dir, file) + _, err := os.Stat(spec) + + switch { + case err == nil: // NOOP + case os.IsNotExist(err): + log.Info(). + Str("file", spec). + Msg("Configuration file does not exist, using default configuration") + return DefaultConfig(opts...), nil + default: + return nil, err } - if _, err := os.Stat(filepath.Join(dir, file)); os.IsNotExist(err) { - if err := WriteDefaultConfig(filepath.Join(dir, file), opts...); err != nil { - log.Error().Err(err).Msg("Failed to write default config") - return nil, err - } + log.Info().Str("file", spec).Msg("Loading configuration file") + fh, err := os.OpenFile(spec, os.O_RDONLY, 0644) + if err != nil { + return nil, err } + defer fh.Close() + + return ReadConfig(fh) +} - data, err := os.ReadFile(filepath.Join(dir, file)) +func ReadConfig(rd io.Reader) (*Config, error) { + data, err := io.ReadAll(rd) if err != nil { return nil, err } - + var config Config if err := yaml.Unmarshal(data, &config); err != nil { return nil, err } - return &config, nil } -func WriteDefaultConfig(path string, opts ...OptT) error { - cfg := Marshal(opts...) - return os.WriteFile(path, []byte(cfg), 0644) -} +func DefaultConfig(opts ...OptT) *Config { + o := parseOpts(opts...) -func LoadConfigFromBytes(data string) (*Config, error) { - var config Config - if err := yaml.Unmarshal([]byte(data), &config); err != nil { - return nil, err + c := &Config{ + Window: o.window, } - return &config, nil + for _, r := range timez.Defaults { + c.TimestampRegexes = append(c.TimestampRegexes, Regex{ + Pattern: r.Pattern, + Format: string(r.Format), + }) + } + return c } func (c *Config) ResolveOpts() (opts []resolve.OptT) { diff --git a/internal/pkg/config/config_test.go b/internal/pkg/config/config_test.go index 26e6ccc..f264e38 100644 --- a/internal/pkg/config/config_test.go +++ b/internal/pkg/config/config_test.go @@ -10,19 +10,7 @@ import ( "github.com/prequel-dev/preq/internal/pkg/config" ) -func TestMarshal(t *testing.T) { - out := config.Marshal() - if !strings.Contains(out, "timestamps:") { - t.Fatalf("expected timestamps in output") - } - - out = config.Marshal(config.WithWindow(2 * time.Second)) - if !strings.Contains(out, "window: 2s") { - t.Fatalf("expected window option in output") - } -} - -func TestLoadConfig(t *testing.T) { +func TestLoadConfig_FileDoesNotExist(t *testing.T) { dir := t.TempDir() cfg, err := config.LoadConfig(dir, "cfg.yaml", config.WithWindow(3*time.Second)) if err != nil { @@ -34,41 +22,221 @@ func TestLoadConfig(t *testing.T) { if len(cfg.TimestampRegexes) == 0 { t.Fatalf("expected default timestamp regexes") } - if _, err := os.Stat(filepath.Join(dir, "cfg.yaml")); err != nil { - t.Fatalf("expected config file written: %v", err) - } } -func TestLoadConfigFromBytes(t *testing.T) { - data := "timestamps: []\nwindow: 1s\n" - cfg, err := config.LoadConfigFromBytes(data) +func TestLoadConfig_FileExists(t *testing.T) { + dir := t.TempDir() + + // Create a config file + configPath := filepath.Join(dir, "cfg.yaml") + configContent := `window: 5s +skip: 10 +dataSources: "test-source" +acceptUpdates: true +rulesVersion: "v1.0" +timestamps: + - pattern: "test-pattern" + format: "test-format" +rules: + paths: + - "/path/to/rules" + disableCommunityRules: true +` + if err := os.WriteFile(configPath, []byte(configContent), 0644); err != nil { + t.Fatalf("Failed to write config file: %v", err) + } + + cfg, err := config.LoadConfig(dir, "cfg.yaml") if err != nil { - t.Fatalf("LoadConfigFromBytes: %v", err) + t.Fatalf("LoadConfig error: %v", err) + } + if cfg.Window != 5*time.Second { + t.Fatalf("expected window 5s got %v", cfg.Window) + } + if cfg.Skip != 10 { + t.Fatalf("expected skip 10 got %v", cfg.Skip) + } + if cfg.DataSources != "test-source" { + t.Fatalf("expected dataSources 'test-source' got %v", cfg.DataSources) + } + if !cfg.AcceptUpdates { + t.Fatalf("expected acceptUpdates true") + } + if cfg.RulesVersion != "v1.0" { + t.Fatalf("expected rulesVersion 'v1.0' got %v", cfg.RulesVersion) + } + if len(cfg.TimestampRegexes) == 0 { + t.Fatalf("expected timestamp regexes") + } + if cfg.TimestampRegexes[0].Pattern != "test-pattern" { + t.Fatalf("expected pattern 'test-pattern' got %v", cfg.TimestampRegexes[0].Pattern) } - if cfg.Window != time.Second { - t.Fatalf("expected 1s window, got %v", cfg.Window) + if len(cfg.Rules.Paths) == 0 { + t.Fatalf("expected rules paths") + } + if !cfg.Rules.Disabled { + t.Fatalf("expected rules disabled true") } } -func TestWriteDefaultConfigAndResolveOpts(t *testing.T) { +func TestLoadConfig_StatError(t *testing.T) { dir := t.TempDir() - path := filepath.Join(dir, "cfg.yaml") - if err := config.WriteDefaultConfig(path, config.WithWindow(2*time.Second)); err != nil { - t.Fatalf("WriteDefaultConfig: %v", err) + + // Create a directory and then make it unreadable + subdir := filepath.Join(dir, "unreadable") + if err := os.Mkdir(subdir, 0755); err != nil { + t.Fatalf("Failed to create directory: %v", err) } - data, err := os.ReadFile(path) - if err != nil { - t.Fatalf("read file: %v", err) + + // Create a file in the directory first + configPath := filepath.Join(subdir, "cfg.yaml") + if err := os.WriteFile(configPath, []byte("test"), 0644); err != nil { + t.Fatalf("Failed to write config file: %v", err) + } + + // Make the directory unreadable (no execute permission means can't access files in it) + if err := os.Chmod(subdir, 0000); err != nil { + t.Fatalf("Failed to chmod directory: %v", err) } - if !strings.Contains(string(data), "window: 2s") { - t.Fatalf("window option missing") + defer os.Chmod(subdir, 0755) // Restore for cleanup + + _, err := config.LoadConfig(subdir, "cfg.yaml") + if err == nil { + t.Fatalf("expected error for inaccessible directory") } +} - cfg, err := config.LoadConfig(dir, "cfg.yaml") +func TestLoadConfig_OpenFileError(t *testing.T) { + dir := t.TempDir() + configPath := filepath.Join(dir, "cfg.yaml") + + // Create a file with restricted permissions + if err := os.WriteFile(configPath, []byte("test"), 0000); err != nil { + t.Fatalf("Failed to write config file: %v", err) + } + + _, err := config.LoadConfig(dir, "cfg.yaml") + if err == nil { + t.Fatalf("expected error when opening file with no permissions") + } +} + +func TestReadConfig(t *testing.T) { + configContent := `window: 2s +skip: 5 +dataSources: "my-source" +acceptUpdates: false +rulesVersion: "v2.0" +timestamps: + - pattern: "\\d{4}-\\d{2}-\\d{2}" + format: "2006-01-02" +rules: + paths: + - "/rules/path1" + - "/rules/path2" + disableCommunityRules: false +` + reader := strings.NewReader(configContent) + cfg, err := config.ReadConfig(reader) if err != nil { - t.Fatalf("LoadConfig: %v", err) + t.Fatalf("ReadConfig error: %v", err) + } + + if cfg.Window != 2*time.Second { + t.Fatalf("expected window 2s got %v", cfg.Window) + } + if cfg.Skip != 5 { + t.Fatalf("expected skip 5 got %v", cfg.Skip) + } + if cfg.DataSources != "my-source" { + t.Fatalf("expected dataSources 'my-source' got %v", cfg.DataSources) } - if len(cfg.ResolveOpts()) == 0 { - t.Fatalf("expected resolve options") + if cfg.AcceptUpdates { + t.Fatalf("expected acceptUpdates false") + } + if len(cfg.TimestampRegexes) != 1 { + t.Fatalf("expected 1 timestamp regex got %v", len(cfg.TimestampRegexes)) + } + if len(cfg.Rules.Paths) != 2 { + t.Fatalf("expected 2 rules paths got %v", len(cfg.Rules.Paths)) + } +} + +func TestReadConfig_InvalidYAML(t *testing.T) { + invalidYAML := `invalid: yaml: content: [[[` + reader := strings.NewReader(invalidYAML) + _, err := config.ReadConfig(reader) + if err == nil { + t.Fatalf("expected error for invalid YAML") + } +} + +type errorReader struct{} + +func (e *errorReader) Read(p []byte) (n int, err error) { + return 0, os.ErrInvalid +} + +func TestReadConfig_ReadError(t *testing.T) { + _, err := config.ReadConfig(&errorReader{}) + if err == nil { + t.Fatalf("expected error when reading fails") + } +} + +func TestDefaultConfig(t *testing.T) { + cfg := config.DefaultConfig() + + if cfg.Window != 0 { + t.Fatalf("expected default window 0 got %v", cfg.Window) + } + if len(cfg.TimestampRegexes) == 0 { + t.Fatalf("expected default timestamp regexes") + } +} + +func TestDefaultConfig_WithWindow(t *testing.T) { + cfg := config.DefaultConfig(config.WithWindow(10 * time.Second)) + + if cfg.Window != 10*time.Second { + t.Fatalf("expected window 10s got %v", cfg.Window) + } + if len(cfg.TimestampRegexes) == 0 { + t.Fatalf("expected default timestamp regexes") + } +} + +func TestResolveOpts(t *testing.T) { + cfg := &config.Config{ + TimestampRegexes: []config.Regex{ + {Pattern: " test-pattern ", Format: " test-format "}, + {Pattern: "pattern2", Format: "format2"}, + }, + } + + opts := cfg.ResolveOpts() + if len(opts) == 0 { + t.Fatalf("expected resolve opts") + } +} + +func TestResolveOpts_NoTimestamps(t *testing.T) { + cfg := &config.Config{ + TimestampRegexes: []config.Regex{}, + } + + opts := cfg.ResolveOpts() + if len(opts) != 0 { + t.Fatalf("expected no resolve opts got %v", len(opts)) + } +} + +func TestWithWindow(t *testing.T) { + duration := 5 * time.Second + optFunc := config.WithWindow(duration) + + cfg := config.DefaultConfig(optFunc) + if cfg.Window != duration { + t.Fatalf("expected window %v got %v", duration, cfg.Window) } } diff --git a/pkg/eval/eval.go b/internal/pkg/eval/eval.go similarity index 80% rename from pkg/eval/eval.go rename to internal/pkg/eval/eval.go index b7341de..40eea8e 100644 --- a/pkg/eval/eval.go +++ b/internal/pkg/eval/eval.go @@ -6,16 +6,15 @@ import ( "github.com/prequel-dev/preq/internal/pkg/config" "github.com/prequel-dev/preq/internal/pkg/engine" "github.com/prequel-dev/preq/internal/pkg/resolve" - "github.com/prequel-dev/preq/internal/pkg/timez" "github.com/prequel-dev/preq/internal/pkg/utils" "github.com/prequel-dev/preq/internal/pkg/ux" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/rs/zerolog/log" ) -func Detect(ctx context.Context, cfg, data, rule string) (ux.ReportDocT, ux.StatsT, error) { +func Detect(ctx context.Context, c *config.Config, data, rule string) (ux.ReportDocT, ux.StatsT, error) { var ( - c *config.Config run *engine.RuntimeT report *ux.ReportT ruleMatchers *engine.RuleMatchersT @@ -25,15 +24,6 @@ func Detect(ctx context.Context, cfg, data, rule string) (ux.ReportDocT, ux.Stat err error ) - if len(cfg) == 0 { - cfg = config.Marshal() - } - - if c, err = config.LoadConfigFromBytes(cfg); err != nil { - log.Error().Err(err).Msg("Failed to load config") - return nil, nil, err - } - opts := c.ResolveOpts() opts = append(opts, resolve.WithTimestampTries(timez.DefaultSkip)) diff --git a/internal/pkg/resolve/logfactory.go b/internal/pkg/resolve/logfactory.go index 697d909..95ae030 100644 --- a/internal/pkg/resolve/logfactory.go +++ b/internal/pkg/resolve/logfactory.go @@ -3,8 +3,8 @@ package resolve import ( "bytes" - "github.com/prequel-dev/preq/internal/pkg/timez" "github.com/prequel-dev/prequel-logmatch/pkg/format" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/rs/zerolog/log" ) diff --git a/internal/pkg/timez/timez.go b/internal/pkg/timez/timez.go deleted file mode 100644 index 06080eb..0000000 --- a/internal/pkg/timez/timez.go +++ /dev/null @@ -1,154 +0,0 @@ -package timez - -import ( - "bytes" - "errors" - "strconv" - "time" - - "github.com/prequel-dev/prequel-logmatch/pkg/format" - "github.com/rs/zerolog/log" -) - -const ( - DefaultSkip = 50 -) - -var ( - ErrInvalidTimestampFormat = errors.New("invalid timestamp format") -) - -type TimestampFmt string - -func (f TimestampFmt) String() string { - return string(f) -} - -const ( - FmtRfc3339 TimestampFmt = "rfc3339" - FmtRfc3339Nano TimestampFmt = "rfc3339nano" - FmtUnix TimestampFmt = "unix" - FmtEpochAny TimestampFmt = "epochany" - FmtEpochSeconds TimestampFmt = "epochseconds" - FmtEpochMillis TimestampFmt = "epochmillis" - FmtEpochMicros TimestampFmt = "epochmicros" - FmtEpochNanos TimestampFmt = "epochnanos" -) - -func GetTimestampFormat(f TimestampFmt) (format.TimeFormatCbT, error) { - - switch f { - case FmtRfc3339: - return format.WithTimeFormat(time.RFC3339), nil - case FmtRfc3339Nano: - return format.WithTimeFormat(time.RFC3339Nano), nil - case FmtUnix: - return format.WithTimeFormat(time.UnixDate), nil - case FmtEpochAny: - return epochAny, nil - case FmtEpochSeconds: - return epochSeconds, nil - case FmtEpochMillis: - return epochMillis, nil - case FmtEpochMicros: - return epochMicros, nil - case FmtEpochNanos: - return epochNanos, nil - default: - return format.WithTimeFormat(string(f)), nil - } -} - -var ( - epochSeconds = epochParser(time.Second) - epochMillis = epochParser(time.Millisecond) - epochMicros = epochParser(time.Microsecond) - epochNanos = epochParser(time.Nanosecond) -) - -func epochParser(unit time.Duration) format.TimeFormatCbT { - return func(m []byte) (int64, error) { - v, err := strconv.ParseInt(string(m), 10, 64) - if err != nil { - return 0, ErrInvalidTimestampFormat - } - return v * int64(unit), nil - } -} - -func epochAny(m []byte) (int64, error) { - v, err := strconv.ParseInt(string(m), 10, 64) - if err != nil { - return 0, ErrInvalidTimestampFormat - } - - sz := len(m) - switch { - case sz > 16: - // NOOP: v *= int64(time.Nanosecond) - case sz > 13: - v *= int64(time.Microsecond) - case sz > 10: - v *= int64(time.Millisecond) - default: - v *= int64(time.Second) - } - return v, nil - -} - -func TryTimestampFormat(exp string, fmtStr TimestampFmt, buf []byte, maxTries int) (format.FactoryI, int64, error) { - - var ( - ts int64 - factory format.FactoryI - cb format.TimeFormatCbT - err error - ) - - log.Debug(). - Str("exp", exp). - Str("fmt", fmtStr.String()). - Msg("Trying timestamp format") - - if cb, err = GetTimestampFormat(fmtStr); err != nil { - log.Warn().Err(err).Msg("Failed to get timestamp format") - return nil, 0, err - } - - if factory, err = format.NewRegexFactory(exp, cb); err != nil { - log.Warn().Err(err).Msg("Failed to create regex factory") - return nil, 0, err - } - - f := factory.New() - ts, err = f.ReadTimestamp(bytes.NewReader(buf)) - - tries := 0 - for (err != nil || ts == 0) && tries < maxTries { - // First line may contain a header; try up to N lines - tries += 1 - if index := bytes.IndexByte(buf, '\n'); index != -1 { - buf = buf[index+1:] - ts, err = f.ReadTimestamp(bytes.NewReader(buf)) - } else { - break - } - } - - if err != nil { - log.Warn().Err(err).Msg("Failed to read timestamp") - return nil, 0, err - } - - if ts == 0 { - return nil, 0, ErrInvalidTimestampFormat - } - - log.Debug(). - Str("exp", exp). - Str("fmt", fmtStr.String()). - Msg("Selected timestamp format") - - return factory, ts, nil -} diff --git a/internal/pkg/timez/timez_test.go b/internal/pkg/timez/timez_test.go deleted file mode 100644 index 8305160..0000000 --- a/internal/pkg/timez/timez_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package timez_test - -import ( - "testing" - "time" - - "github.com/prequel-dev/preq/internal/pkg/timez" -) - -func TestGetTimestampFormat(t *testing.T) { - cb, err := timez.GetTimestampFormat(timez.FmtRfc3339) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - ts, err := cb([]byte("2025-01-02T03:04:05Z")) - if err != nil { - t.Fatalf("parse failed: %v", err) - } - want := time.Date(2025, 1, 2, 3, 4, 5, 0, time.UTC).UnixNano() - if ts != want { - t.Fatalf("expected %d got %d", want, ts) - } -} - -func TestEpochParserAndAny(t *testing.T) { - cb, _ := timez.GetTimestampFormat(timez.FmtEpochMillis) - ts, err := cb([]byte("42")) - if err != nil || ts != 42*int64(time.Millisecond) { - t.Fatalf("epoch millis failed") - } - - cb, _ = timez.GetTimestampFormat(timez.FmtEpochAny) - ts, err = cb([]byte("1000")) - if err != nil || ts != 1000*int64(time.Second) { - t.Fatalf("epoch any failed") - } -} - -func TestTryTimestampFormat(t *testing.T) { - line := "2025-06-06T12:00:00Z first line\nsecond" // newline ensures only first line used - factory, ts, err := timez.TryTimestampFormat(`^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z)`, timez.FmtRfc3339, []byte(line), 1) - if err != nil { - t.Fatalf("unexpected error: %v", err) - } - if factory == nil { - t.Fatal("expected factory") - } - want := time.Date(2025, 6, 6, 12, 0, 0, 0, time.UTC).UnixNano() - if ts != want { - t.Fatalf("timestamp mismatch") - } -} diff --git a/internal/pkg/ux/ux.go b/internal/pkg/ux/ux.go index 203b082..6e0e73e 100644 --- a/internal/pkg/ux/ux.go +++ b/internal/pkg/ux/ux.go @@ -9,9 +9,9 @@ import ( "sync/atomic" "time" - "github.com/prequel-dev/preq/internal/pkg/timez" "github.com/prequel-dev/preq/internal/pkg/verz" "github.com/prequel-dev/prequel-logmatch/pkg/format" + "github.com/prequel-dev/prequel-logmatch/pkg/timez" "github.com/Masterminds/semver" "github.com/fatih/color" diff --git a/test/preq_test.go b/test/preq_test.go index ef951a5..e070ad5 100644 --- a/test/preq_test.go +++ b/test/preq_test.go @@ -5,8 +5,9 @@ import ( "os" "testing" + "github.com/prequel-dev/preq/internal/pkg/config" + "github.com/prequel-dev/preq/internal/pkg/eval" "github.com/prequel-dev/preq/internal/pkg/logs" - "github.com/prequel-dev/preq/pkg/eval" "github.com/rs/zerolog/log" ) @@ -138,7 +139,7 @@ func TestSuccessExamples(t *testing.T) { t.Fatalf("Error reading data file %s: %v", test.dataPath, err) } - _, stats, err := eval.Detect(ctx, "", string(data), string(ruleData)) + _, stats, err := eval.Detect(ctx, config.DefaultConfig(), string(data), string(ruleData)) if err != nil { t.Fatalf("Error running detection: %v", err) } @@ -226,7 +227,7 @@ func TestMissExamples(t *testing.T) { t.Fatalf("Error reading data file %s: %v", test.dataPath, err) } - _, stats, err := eval.Detect(ctx, "", string(data), string(ruleData)) + _, stats, err := eval.Detect(ctx, config.DefaultConfig(), string(data), string(ruleData)) if err != nil { t.Fatalf("Error running detection: %v", err) } @@ -294,7 +295,7 @@ func TestFailureExamples(t *testing.T) { t.Fatalf("Error reading data file %s: %v", test.dataPath, err) } - _, _, err = eval.Detect(ctx, "", string(data), string(ruleData)) + _, _, err = eval.Detect(ctx, config.DefaultConfig(), string(data), string(ruleData)) if err == nil { t.Fatalf("Expected error running detection: %v", err) }