Add promql support for the prequel product.

go.mod

@@ -6,7 +6,6 @@ require (
 	github.com/btcsuite/btcutil v1.0.2
 	github.com/prequel-dev/prequel-logmatch v0.0.13
 	github.com/rs/zerolog v1.34.0
-	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 

go.sum

@@ -63,7 +63,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

pkg/ast/ast.go

@@ -161,11 +161,17 @@ func (b *builderT) buildTree(parserNode *parser.NodeT, parentMachineAddress *Ast
 	)
 
 	// Build children (either matcher children or nested machines)
-	if isMatcherNode(parserNode) {
+	if parserNode.IsMatcherNode() {
 		if matchNode, err = b.buildMatcherChildren(parserNode, machineAddress, termIdx); err != nil {
 			return nil, err
 		}
 		children = append(children, matchNode)
+	} else if parserNode.IsPromNode() {
+		if matchNode, err = b.buildPromQLNode(parserNode, machineAddress, termIdx); err != nil {
+			return nil, err
+		}
+		children = append(children, matchNode)
+
 	} else {
 		if children, err = b.buildMachineChildren(parserNode, machineAddress); err != nil {
 			return nil, err
@@ -210,20 +216,6 @@ func newAstNode(parserNode *parser.NodeT, typ schema.NodeTypeT, scope string, pa
 	}
 }
 
-func isMatcherNode(node *parser.NodeT) bool {
-	var (
-		hasMatcher = true
-	)
-
-	for _, child := range node.Children {
-		if _, ok := child.(*parser.MatcherT); !ok {
-			hasMatcher = false
-		}
-	}
-
-	return hasMatcher
-}
-
 func (b *builderT) buildMatcherChildren(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
 
 	var (
@@ -265,6 +257,8 @@ func (b *builderT) buildMatcherNodes(parserNode *parser.NodeT, machineAddress *A
 	switch parserNode.Metadata.Type {
 	case schema.NodeTypeLogSeq:
 	case schema.NodeTypeLogSet:
+	case schema.NodeTypePromQL:
+		return b.buildPromQLNode(parserNode, machineAddress, termIdx)
 	default:
 		return nil, parserNode.WrapError(ErrInvalidNodeType)
 	}
@@ -372,7 +366,7 @@ func (b *builderT) buildStateMachine(parserNode *parser.NodeT, parentMachineAddr
 				Msg("Window is required for sequences")
 			return nil, parserNode.WrapError(ErrInvalidWindow)
 		}
-	case schema.NodeTypeSet, schema.NodeTypeLogSet:
+	case schema.NodeTypeSet, schema.NodeTypeLogSet, schema.NodeTypePromQL:
 	default:
 		log.Error().
 			Any("address", machineAddress).

pkg/ast/ast_log.go

@@ -202,6 +202,7 @@ func newMatchTerm(field parser.FieldT) (AstFieldT, error) {
 	}
 
 	return t, nil
+
 }
 
 func newNegateTerm(field parser.FieldT, anchors uint32) (AstFieldT, error) {

pkg/ast/ast_machine.go

@@ -24,25 +24,31 @@ type AstSetMatcherT struct {
 
 func (b *builderT) buildMachineNode(parserNode *parser.NodeT, parentMachineAddress, machineAddress *AstNodeAddressT, children []*AstNodeT) (*AstNodeT, error) {
 	var (
-		seqMatcher *AstSeqMatcherT
-		setMatcher *AstSetMatcherT
-		matchNode  = newAstNode(parserNode, parserNode.Metadata.Type, schema.ScopeCluster, parentMachineAddress, machineAddress)
-		err        error
+		matchNode = newAstNode(parserNode, parserNode.Metadata.Type, schema.ScopeCluster, parentMachineAddress, machineAddress)
 	)
 
 	switch parserNode.Metadata.Type {
 	case schema.NodeTypeSeq, schema.NodeTypeLogSeq:
 		matchNode.Metadata.Type = schema.NodeTypeSeq
-		if seqMatcher, err = buildSeqMatcher(parserNode, children); err != nil {
+		if seqMatcher, err := buildSeqMatcher(parserNode, children); err != nil {
 			return nil, err
+		} else {
+			matchNode.Object = seqMatcher
 		}
-		matchNode.Object = seqMatcher
 	case schema.NodeTypeSet, schema.NodeTypeLogSet:
 		matchNode.Metadata.Type = schema.NodeTypeSet
-		if setMatcher, err = buildSetMatcher(parserNode, children); err != nil {
+		if setMatcher, err := buildSetMatcher(parserNode, children); err != nil {
 			return nil, err
+		} else {
+			matchNode.Object = setMatcher
+		}
+	case schema.NodeTypePromQL:
+		matchNode.Metadata.Type = schema.NodeTypePromQL
+		if promMatcher, err := b.buildPromQLNode(parserNode, machineAddress, nil); err != nil {
+			return nil, err
+		} else {
+			matchNode.Object = promMatcher
 		}
-		matchNode.Object = setMatcher
 	default:
 		log.Error().
 			Str("type", parserNode.Metadata.Type.String()).

pkg/ast/ast_metrics.go

@@ -0,0 +1,69 @@
+package ast
+
+import (
+	"time"
+
+	"github.com/prequel-dev/prequel-compiler/pkg/parser"
+	"github.com/prequel-dev/prequel-compiler/pkg/schema"
+	"github.com/rs/zerolog/log"
+)
+
+type AstPromQL struct {
+	Expr     string
+	For      time.Duration
+	Interval time.Duration
+	Event    *AstEventT
+}
+
+func (b *builderT) buildPromQLNode(parserNode *parser.NodeT, machineAddress *AstNodeAddressT, termIdx *uint32) (*AstNodeT, error) {
+
+	// Expects one child of type ParsePromQL
+
+	if len(parserNode.Children) != 1 {
+		log.Error().Int("child_count", len(parserNode.Children)).Msg("PromQL node must have exactly one child")
+		return nil, parserNode.WrapError(ErrInvalidNodeType)
+	}
+
+	promNode, ok := parserNode.Children[0].(*parser.PromQLT)
+
+	if !ok {
+		log.Error().Any("promql", parserNode.Children[0]).Msg("Failed to build PromQL node")
+		return nil, parserNode.WrapError(ErrMissingScalar)
+	}
+
+	if promNode.Expr == "" {
+		log.Error().Msg("PromQL Expr string is empty")
+		return nil, parserNode.WrapError(ErrMissingScalar)
+	}
+
+	pn := &AstPromQL{
+		Expr: promNode.Expr,
+	}
+
+	if parserNode.Metadata.Event != nil {
+		if parserNode.Metadata.Event.Origin {
+			b.HasOrigin = true
+		}
+		pn.Event = &AstEventT{
+			Source: parserNode.Metadata.Event.Source,
+			Origin: parserNode.Metadata.Event.Origin,
+		}
+	}
+
+	if promNode.Interval != nil {
+		pn.Interval = *promNode.Interval
+	}
+
+	if promNode.For != nil {
+		pn.For = *promNode.For
+	}
+
+	var (
+		address = b.newAstNodeAddress(parserNode.Metadata.RuleHash, parserNode.Metadata.Type.String(), termIdx)
+		node    = newAstNode(parserNode, parserNode.Metadata.Type, schema.ScopeCluster, machineAddress, address)
+	)
+
+	node.Object = pn
+	return node, nil
+
+}

pkg/ast/ast_test.go

@@ -69,6 +69,10 @@ func TestAstSuccess(t *testing.T) {
 			rule:              testdata.TestSuccessSimpleExtraction,
 			expectedNodeTypes: []string{"machine_seq", "log_seq"},
 		},
+		"Success_PromQLMetric": {
+			rule:              testdata.TestSuccessSimplePromQL,
+			expectedNodeTypes: []string{"machine_set", "promql", "log_set"},
+		},
 	}
 
 	for name, test := range tests {
@@ -172,7 +176,7 @@ func TestAstFail(t *testing.T) {
 		"Fail_TermsSemanticError4": {
 			rule: testdata.TestFailTermsSemanticError4,
 			err:  ErrInvalidEventType,
-			line: 16,
+			line: 14,
 			col:  11,
 		},
 		"Fail_TermsSemanticError5": {

pkg/datasrc/parse.go

@@ -4,7 +4,7 @@ import (
 	"os"
 	"time"
 
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
 )
 
 // version: 0.0.1

pkg/parser/parse.go

@@ -101,6 +101,7 @@ type ParseTermT struct {
 	Set        *ParseSetT        `yaml:"set,omitempty"`
 	Sequence   *ParseSequenceT   `yaml:"sequence,omitempty"`
 	NegateOpts *ParseNegateOptsT `yaml:",inline,omitempty"`
+	PromQL     *ParsePromQL      `yaml:"promql,omitempty"`
 	Extract    []ParseExtractT   `yaml:"extract,omitempty"`
 }
 
@@ -118,22 +119,30 @@ type ParseExtractT struct {
 	RegexValue string `yaml:"regex,omitempty"`
 }
 
+type ParsePromQL struct {
+	Expr     string       `yaml:"expr"`
+	Interval string       `yaml:"interval,omitempty"`
+	For      string       `yaml:"for,omitempty"`
+	Event    *ParseEventT `yaml:"event,omitempty"`
+}
+
 func (o *ParseTermT) UnmarshalYAML(unmarshal func(any) error) error {
 	var str string
 	if err := unmarshal(&str); err == nil {
 		o.StrValue = str
 		return nil
 	}
 	var temp struct {
-		Field      string            `yaml:"field,omitempty"`
-		StrValue   string            `yaml:"value,omitempty"`
-		JqValue    string            `yaml:"jq,omitempty"`
-		RegexValue string            `yaml:"regex,omitempty"`
-		Count      int               `yaml:"count,omitempty"`
-		Set        *ParseSetT        `yaml:"set,omitempty"`
-		Sequence   *ParseSequenceT   `yaml:"sequence,omitempty"`
-		NegateOpts *ParseNegateOptsT `yaml:",inline,omitempty"`
-		Extract    []ParseExtractT   `yaml:"extract,omitempty"`
+		Field       string            `yaml:"field,omitempty"`
+		StrValue    string            `yaml:"value,omitempty"`
+		JqValue     string            `yaml:"jq,omitempty"`
+		RegexValue  string            `yaml:"regex,omitempty"`
+		Count       int               `yaml:"count,omitempty"`
+		Set         *ParseSetT        `yaml:"set,omitempty"`
+		Sequence    *ParseSequenceT   `yaml:"sequence,omitempty"`
+		NegateOpts  *ParseNegateOptsT `yaml:",inline,omitempty"`
+		ParsePromQL *ParsePromQL      `yaml:"promql,omitempty"`
+		Extract     []ParseExtractT   `yaml:"extract,omitempty"`
 	}
 	if err := unmarshal(&temp); err != nil {
 		return err
@@ -146,6 +155,7 @@ func (o *ParseTermT) UnmarshalYAML(unmarshal func(any) error) error {
 	o.Set = temp.Set
 	o.Sequence = temp.Sequence
 	o.NegateOpts = temp.NegateOpts
+	o.PromQL = temp.ParsePromQL
 	o.Extract = temp.Extract
 	return nil
 }

pkg/parser/parse_test.go

@@ -61,15 +61,20 @@ func TestParseSuccess(t *testing.T) {
 			expectedNegIndexes: []int{-1, 2, 2, -1, -1, -1, -1},
 		},
 		"Success_MissingRuleId": {
-                        rule: testdata.TestFailMissingRuleIdRule,
+			rule:               testdata.TestFailMissingRuleIdRule,
 			expectedNodeTypes:  []string{"log_set"},
 			expectedNegIndexes: []int{-1},
-                },
-                "Success_MissingRuleHash": {
-                        rule: testdata.TestFailMissingRuleHashRule,
+		},
+		"Success_MissingRuleHash": {
+			rule:               testdata.TestFailMissingRuleHashRule,
 			expectedNodeTypes:  []string{"log_set"},
 			expectedNegIndexes: []int{-1},
-                },
+		},
+		"Success_PromQL": {
+			rule:               testdata.TestSuccessSimplePromQL,
+			expectedNodeTypes:  []string{"machine_set", "promql", "log_set"},
+			expectedNegIndexes: []int{-1, -1, -1},
+		},
 	}
 
 	for name, test := range tests {