From 97049219d76a4822a7ee163cca39b11d4834c6c4 Mon Sep 17 00:00:00 2001 From: Panos Stergiotis Date: Sun, 5 Jan 2025 22:25:24 +0100 Subject: [PATCH] wip: ch dsl --- public/db/clickhouse/dsl/dsl.go | 4 ++ public/db/clickhouse/dsl/params.go | 51 +++++++++++++------ public/db/clickhouse/dsl/parseddqlquery.go | 43 ++++++++++++++-- .../db/clickhouse/dsl/tableidtransformer.go | 16 +++--- public/imzero/main.go | 2 + public/observability/logging/flags.go | 15 ++++-- 6 files changed, 100 insertions(+), 31 deletions(-) diff --git a/public/db/clickhouse/dsl/dsl.go b/public/db/clickhouse/dsl/dsl.go index 6fb6aea..2b7a6b5 100644 --- a/public/db/clickhouse/dsl/dsl.go +++ b/public/db/clickhouse/dsl/dsl.go @@ -29,6 +29,10 @@ func (inst *Dsl) Parse(sql string) (err error) { } return } +func (inst *Dsl) LoadDql(dql *ParsedDqlQuery) (err error) { + inst.Exprs = []chparser.Expr{dql.GetAst()} + return +} func (inst *Dsl) Transform() (err error) { err = inst.checkParsed() if err != nil { diff --git a/public/db/clickhouse/dsl/params.go b/public/db/clickhouse/dsl/params.go index 91885b9..8350062 100644 --- a/public/db/clickhouse/dsl/params.go +++ b/public/db/clickhouse/dsl/params.go @@ -2,24 +2,28 @@ package dsl import ( chparser "github.com/AfterShip/clickhouse-sql-parser/parser" + "github.com/stergiotis/boxer/public/containers" "github.com/stergiotis/boxer/public/observability/eh" "github.com/stergiotis/boxer/public/observability/eh/eb" "golang.org/x/exp/maps" "iter" ) -type ParamSet struct { +type ParamSlotSet struct { typesLu map[string][]*chparser.QueryParam paramOccurrences int } -func NewParamSet() *ParamSet { - return &ParamSet{typesLu: nil} +func NewParamSlotsSet() *ParamSlotSet { + return &ParamSlotSet{ + typesLu: nil, + paramOccurrences: 0, + } } var ErrIncompatibleParam = eh.Errorf("a param with an incompatible type is already contained in param set") -func (inst *ParamSet) Add(param *chparser.QueryParam) (err error) { +func (inst *ParamSlotSet) Add(param *chparser.QueryParam) (err error) { if param == nil { return } @@ -42,7 +46,7 @@ func (inst *ParamSet) Add(param *chparser.QueryParam) (err error) { inst.paramOccurrences++ return } -func (inst *ParamSet) UnionMod(other *ParamSet) (err error) { +func (inst *ParamSlotSet) UnionMod(other *ParamSlotSet) (err error) { for _, ps := range other.All() { for _, p := range ps { err = inst.Add(p) @@ -54,16 +58,16 @@ func (inst *ParamSet) UnionMod(other *ParamSet) (err error) { } return } -func (inst *ParamSet) TotalParamOccurrences() int { +func (inst *ParamSlotSet) TotalParamOccurrences() int { return inst.paramOccurrences } -func (inst *ParamSet) TotalDistinctParams() int { +func (inst *ParamSlotSet) TotalDistinctParams() int { return len(inst.typesLu) } -func (inst *ParamSet) IsEmpty() bool { +func (inst *ParamSlotSet) IsEmpty() bool { return len(inst.typesLu) == 0 } -func (inst *ParamSet) All() iter.Seq2[string, []*chparser.QueryParam] { +func (inst *ParamSlotSet) All() iter.Seq2[string, []*chparser.QueryParam] { return func(yield func(string, []*chparser.QueryParam) bool) { for k, vs := range inst.typesLu { if !yield(k, vs) { @@ -72,7 +76,22 @@ func (inst *ParamSet) All() iter.Seq2[string, []*chparser.QueryParam] { } } } -func (inst *ParamSet) Clear() { +func (inst *ParamSlotSet) NamesAndTypes() iter.Seq2[string, *containers.HashSet[string]] { + return func(yield func(string, *containers.HashSet[string]) bool) { + types := containers.NewHashSet[string](32) + defer types.Clear() + for k, vs := range inst.typesLu { + for _, v := range vs { + types.Add(v.Type.String()) + } + if !yield(k, types) { + return + } + types.Clear() + } + } +} +func (inst *ParamSlotSet) Clear() { if len(inst.typesLu) > 0 { maps.Clear(inst.typesLu) } @@ -82,21 +101,21 @@ func isParamTypeCompatible(t1 string, t2 string) (compatible bool) { return t1 == t2 } -type paramsDiscoverer struct { +type paramSlotsDiscoverer struct { chparser.DefaultASTVisitor - params *ParamSet + params *ParamSlotSet } -func newParamsDiscoverer() *paramsDiscoverer { - return ¶msDiscoverer{ +func newParamSlotsDiscoverer() *paramSlotsDiscoverer { + return ¶mSlotsDiscoverer{ DefaultASTVisitor: chparser.DefaultASTVisitor{}, params: nil, } } -func (inst *paramsDiscoverer) VisitQueryParam(expr *chparser.QueryParam) error { +func (inst *paramSlotsDiscoverer) VisitQueryParam(expr *chparser.QueryParam) error { return inst.params.Add(expr) } -func (inst *paramsDiscoverer) discover(ast chparser.Expr, params *ParamSet) (err error) { +func (inst *paramSlotsDiscoverer) discover(ast chparser.Expr, params *ParamSlotSet) (err error) { if params == nil { return eh.Errorf("paramset is nil") } diff --git a/public/db/clickhouse/dsl/parseddqlquery.go b/public/db/clickhouse/dsl/parseddqlquery.go index b69a492..7792c02 100644 --- a/public/db/clickhouse/dsl/parseddqlquery.go +++ b/public/db/clickhouse/dsl/parseddqlquery.go @@ -4,15 +4,18 @@ import ( "fmt" chparser "github.com/AfterShip/clickhouse-sql-parser/parser" "github.com/fxamacker/cbor/v2" + "github.com/rs/zerolog/log" "github.com/stergiotis/boxer/public/observability/eh" "github.com/stergiotis/boxer/public/observability/eh/eb" + "slices" + "strings" ) type ParsedDqlQuery struct { inputSql string ast *chparser.SelectQuery - paramSet *ParamSet + paramSet *ParamSlotSet paramSetErr error noParams bool } @@ -20,13 +23,16 @@ type ParsedDqlQuery struct { func (inst *ParsedDqlQuery) String() string { return inst.ast.String() } -func (inst *ParsedDqlQuery) GetParamSet() (paramSet *ParamSet, err error) { +func (inst *ParsedDqlQuery) GetAst() *chparser.SelectQuery { + return inst.ast +} +func (inst *ParsedDqlQuery) GetParamSlotSet() (paramSet *ParamSlotSet, err error) { if inst.noParams { return } - if inst.paramSet == nil && inst.paramSetErr != nil { - ps := NewParamSet() - d := newParamsDiscoverer() + if inst.paramSet == nil && inst.paramSetErr == nil { + ps := NewParamSlotsSet() + d := newParamSlotsDiscoverer() err = d.discover(inst.ast, ps) if err != nil { err = eh.Errorf("error while discovering paramset: %w", err) @@ -56,6 +62,31 @@ func NewParsedDqlQuery(sql string) (inst *ParsedDqlQuery, err error) { } return } +func removeParamSettingsFromExprs(exprs []chparser.Expr) (exprsOut []chparser.Expr) { + for _, expr := range exprs { + switch exprt := expr.(type) { + case *chparser.SetStmt: + items := slices.DeleteFunc(exprt.Settings.Items, func(list *chparser.SettingExprList) bool { + name := list.Name.Name + if strings.HasPrefix(list.Name.Name, "param_") { + log.Info().Str("name", name).Msg("removing set param value expression") + return true + } + return false + }) + exprt.Settings.Items = items + break + } + } + exprsOut = slices.DeleteFunc(exprs, func(expr chparser.Expr) bool { + switch exprt := expr.(type) { + case *chparser.SetStmt: + return len(exprt.Settings.Items) == 0 + } + return false + }) + return +} func (inst *ParsedDqlQuery) parse() (err error) { p := chparser.NewParser(inst.inputSql) var exprs []chparser.Expr @@ -64,6 +95,8 @@ func (inst *ParsedDqlQuery) parse() (err error) { err = eh.Errorf("unable to parse sql: %w", err) return } + + exprs = removeParamSettingsFromExprs(exprs) if len(exprs) != 1 { err = eb.Build().Int("nExprs", len(exprs)).Errorf("sql must contain exactly on expression") return diff --git a/public/db/clickhouse/dsl/tableidtransformer.go b/public/db/clickhouse/dsl/tableidtransformer.go index 6cfbd97..8eea9be 100644 --- a/public/db/clickhouse/dsl/tableidtransformer.go +++ b/public/db/clickhouse/dsl/tableidtransformer.go @@ -79,14 +79,16 @@ func (inst *TableIdTransformer) Apply(ast chparser.Expr) (err error) { } switch astt := ast.(type) { case *chparser.SelectQuery: - if astt.With == nil { - astt.With = &chparser.WithClause{ - WithPos: 0, - EndPos: 0, - CTEs: inst.ctes, + if len(inst.ctes) > 0 { + if astt.With == nil { + astt.With = &chparser.WithClause{ + WithPos: 0, + EndPos: 0, + CTEs: inst.ctes, + } + } else { + astt.With.CTEs = append(inst.ctes, astt.With.CTEs...) } - } else { - astt.With.CTEs = append(inst.ctes, astt.With.CTEs...) } break default: diff --git a/public/imzero/main.go b/public/imzero/main.go index bedb47c..f5fdfe6 100644 --- a/public/imzero/main.go +++ b/public/imzero/main.go @@ -2,6 +2,7 @@ package main import ( "github.com/rs/zerolog/log" + "github.com/stergiotis/boxer/public/db/clickhouse/dsl" "github.com/stergiotis/boxer/public/dev" "github.com/stergiotis/boxer/public/fffi/compiletime" "github.com/stergiotis/boxer/public/imzero/demo" @@ -47,6 +48,7 @@ func main() { dev.DebuggerFlags...), dev.IoOverrideFlags...), Commands: []*cli.Command{ + dsl.NewCommand(), compiletime.NewCommand(nil, nil), demo.NewCommand(), { diff --git a/public/observability/logging/flags.go b/public/observability/logging/flags.go index bbff8f1..e1ad7d2 100644 --- a/public/observability/logging/flags.go +++ b/public/observability/logging/flags.go @@ -1,6 +1,7 @@ package logging import ( + "encoding/json" "github.com/fxamacker/cbor/v2" "github.com/stergiotis/boxer/public/observability/eh" "github.com/yassinebenaid/godump" @@ -92,7 +93,7 @@ var LoggingFlags = []cli.Flag{ var cbordiagmode cbor.DiagMode var cborencmode cbor.EncMode var err error - if false { + if true { cborencmode, err = cbor.CanonicalEncOptions().EncMode() if err != nil { log.Warn().Err(err).Msg("unable to create cbor encoder, skipping") @@ -109,6 +110,10 @@ var LoggingFlags = []cli.Flag{ MaxArrayElements: 0, MaxMapPairs: 0, }.DiagMode() + if err != nil { + log.Warn().Err(err).Msg("unable to create cbor diagmode, skipping") + err = nil + } } zerolog.InterfaceMarshalFunc = func(v any) ([]byte, error) { if cborencmode != nil && cbordiagmode != nil { @@ -120,8 +125,12 @@ var LoggingFlags = []cli.Flag{ } } } - return []byte(dumper.Sprintln(v)), nil - //return json.MarshalIndent(v, "", " ") + var js []byte + js, err = json.MarshalIndent(v, "", " ") + if err != nil { + return []byte(dumper.Sprintln(v)), nil + } + return js, nil } break case "diag":