Skip to content

Commit e7560e0

Browse files
timvaillancourtharshit-gangaldeepthi
committed
Add sql text counts stats to vtcombo,vtgate+vttablet (vitessio#15897)
Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com> Signed-off-by: Harshit Gangal <harshit@planetscale.com> Co-authored-by: Harshit Gangal <harshit@planetscale.com> Co-authored-by: Deepthi Sigireddi <deepthi.sigireddi@gmail.com>
1 parent 19844b2 commit e7560e0

File tree

5 files changed

+67
-32
lines changed

5 files changed

+67
-32
lines changed

‎go/vt/vtgate/plugin_mysql_server_test.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -349,7 +349,7 @@ func TestKillMethods(t *testing.T) {
349349
func TestGracefulShutdown(t *testing.T) {
350350
executor, _, _, _, _ := createExecutorEnv(t)
351351

352-
vh := newVtgateHandler(&VTGate{executor: executor, timings: timings, rowsReturned: rowsReturned, rowsAffected: rowsAffected})
352+
vh := newVtgateHandler(&VTGate{executor: executor, timings: timings, rowsReturned: rowsReturned, rowsAffected: rowsAffected, queryTextCharsProcessed: queryTextCharsProcessed})
353353
th := &testHandler{}
354354
listener, err := mysql.NewListener("tcp", "127.0.0.1:", mysql.NewAuthServerNone(), th, 0, 0, false, false, 0, 0)
355355
require.NoError(t, err)
@@ -379,7 +379,7 @@ func TestGracefulShutdown(t *testing.T) {
379379
func TestGracefulShutdownWithTransaction(t *testing.T) {
380380
executor, _, _, _, _ := createExecutorEnv(t)
381381

382-
vh := newVtgateHandler(&VTGate{executor: executor, timings: timings, rowsReturned: rowsReturned, rowsAffected: rowsAffected})
382+
vh := newVtgateHandler(&VTGate{executor: executor, timings: timings, rowsReturned: rowsReturned, rowsAffected: rowsAffected, queryTextCharsProcessed: queryTextCharsProcessed})
383383
th := &testHandler{}
384384
listener, err := mysql.NewListener("tcp", "127.0.0.1:", mysql.NewAuthServerNone(), th, 0, 0, false, false, 0, 0)
385385
require.NoError(t, err)

‎go/vt/vtgate/vtgate.go

Lines changed: 19 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -208,6 +208,11 @@ var (
208208
"VtgateApiRowsAffected",
209209
"Rows affected by a write (DML) operation through the VTgate API",
210210
[]string{"Operation", "Keyspace", "DbType"})
211+
212+
queryTextCharsProcessed = stats.NewCountersWithMultiLabels(
213+
"VtgateQueryTextCharactersProcessed",
214+
"Query text characters processed through the VTGate API",
215+
[]string{"Operation", "Keyspace", "DbType"})
211216
)
212217

213218
// VTGate is the rpc interface to vtgate. Only one instance
@@ -224,9 +229,10 @@ type VTGate struct {
224229
// stats objects.
225230
// TODO(sougou): This needs to be cleaned up. There
226231
// are global vars that depend on this member var.
227-
timings *stats.MultiTimings
228-
rowsReturned *stats.CountersWithMultiLabels
229-
rowsAffected *stats.CountersWithMultiLabels
232+
timings *stats.MultiTimings
233+
rowsReturned *stats.CountersWithMultiLabels
234+
rowsAffected *stats.CountersWithMultiLabels
235+
queryTextCharsProcessed *stats.CountersWithMultiLabels
230236

231237
// the throttled loggers for all errors, one per API entry
232238
logExecute *logutil.ThrottledLogger
@@ -458,6 +464,7 @@ func (vtg *VTGate) Execute(ctx context.Context, mysqlCtx vtgateservice.MySQLConn
458464
if err == nil {
459465
vtg.rowsReturned.Add(statsKey, int64(len(qr.Rows)))
460466
vtg.rowsAffected.Add(statsKey, int64(qr.RowsAffected))
467+
vtg.queryTextCharsProcessed.Add(statsKey, int64(len(sql)))
461468
return session, qr, nil
462469
}
463470

@@ -668,14 +675,15 @@ func (vtg *VTGate) HandlePanic(err *error) {
668675

669676
func newVTGate(executor *Executor, resolver *Resolver, vsm *vstreamManager, tc *TxConn, gw *TabletGateway) *VTGate {
670677
return &VTGate{
671-
executor: executor,
672-
resolver: resolver,
673-
vsm: vsm,
674-
txConn: tc,
675-
gw: gw,
676-
timings: timings,
677-
rowsReturned: rowsReturned,
678-
rowsAffected: rowsAffected,
678+
executor: executor,
679+
resolver: resolver,
680+
vsm: vsm,
681+
txConn: tc,
682+
gw: gw,
683+
timings: timings,
684+
rowsReturned: rowsReturned,
685+
rowsAffected: rowsAffected,
686+
queryTextCharsProcessed: queryTextCharsProcessed,
679687

680688
logExecute: logutil.NewThrottledLogger("Execute", 5*time.Second),
681689
logPrepare: logutil.NewThrottledLogger("Prepare", 5*time.Second),

‎go/vt/vttablet/tabletserver/query_engine.go

Lines changed: 10 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ type QueryEngine struct {
187187

188188
// stats
189189
// Note: queryErrorCountsWithCode is similar to queryErrorCounts except it contains error code as an additional dimension
190-
queryCounts, queryCountsWithTabletType, queryTimes, queryErrorCounts, queryErrorCountsWithCode, queryRowsAffected, queryRowsReturned *stats.CountersWithMultiLabels
190+
queryCounts, queryCountsWithTabletType, queryTimes, queryErrorCounts, queryErrorCountsWithCode, queryRowsAffected, queryRowsReturned, queryTextCharsProcessed *stats.CountersWithMultiLabels
191191

192192
// stats flags
193193
enablePerWorkloadTableMetrics bool
@@ -291,6 +291,7 @@ func NewQueryEngine(env tabletenv.Env, se *schema.Engine) *QueryEngine {
291291
qe.queryTimes = env.Exporter().NewCountersWithMultiLabels("QueryTimesNs", "query times in ns", labels)
292292
qe.queryRowsAffected = env.Exporter().NewCountersWithMultiLabels("QueryRowsAffected", "query rows affected", labels)
293293
qe.queryRowsReturned = env.Exporter().NewCountersWithMultiLabels("QueryRowsReturned", "query rows returned", labels)
294+
qe.queryTextCharsProcessed = env.Exporter().NewCountersWithMultiLabels("QueryTextCharactersProcessed", "query text characters processed", labels)
294295
qe.queryErrorCounts = env.Exporter().NewCountersWithMultiLabels("QueryErrorCounts", "query error counts", labels)
295296
qe.queryErrorCountsWithCode = env.Exporter().NewCountersWithMultiLabels("QueryErrorCountsWithCode", "query error counts with error code", []string{"Table", "Plan", "Code"})
296297

@@ -552,30 +553,33 @@ func (qe *QueryEngine) QueryPlanCacheLen() (count int) {
552553
}
553554

554555
// AddStats adds the given stats for the planName.tableName
555-
func (qe *QueryEngine) AddStats(planType planbuilder.PlanType, tableName, workload string, tabletType topodata.TabletType, queryCount int64, duration, mysqlTime time.Duration, rowsAffected, rowsReturned, errorCount int64, errorCode string) {
556+
func (qe *QueryEngine) AddStats(plan *TabletPlan, tableName, workload string, tabletType topodata.TabletType, queryCount int64, duration, mysqlTime time.Duration, rowsAffected, rowsReturned, errorCount int64, errorCode string) {
556557
// table names can contain "." characters, replace them!
557-
keys := []string{tableName, planType.String()}
558+
keys := []string{tableName, plan.PlanID.String()}
558559
// Only use the workload as a label if that's enabled in the configuration.
559560
if qe.enablePerWorkloadTableMetrics {
560561
keys = append(keys, workload)
561562
}
562563
qe.queryCounts.Add(keys, queryCount)
563564
qe.queryTimes.Add(keys, int64(duration))
564565
qe.queryErrorCounts.Add(keys, errorCount)
566+
if plan.FullQuery != nil {
567+
qe.queryTextCharsProcessed.Add(keys, int64(len(plan.FullQuery.Query)))
568+
}
565569

566-
qe.queryCountsWithTabletType.Add([]string{tableName, planType.String(), tabletType.String()}, queryCount)
570+
qe.queryCountsWithTabletType.Add([]string{tableName, plan.PlanID.String(), tabletType.String()}, queryCount)
567571

568572
// queryErrorCountsWithCode is similar to queryErrorCounts except we have an additional dimension
569573
// of error code.
570574
if errorCount > 0 {
571-
errorKeys := []string{tableName, planType.String(), errorCode}
575+
errorKeys := []string{tableName, plan.PlanID.String(), errorCode}
572576
qe.queryErrorCountsWithCode.Add(errorKeys, errorCount)
573577
}
574578

575579
// For certain plan types like select, we only want to add their metrics to rows returned
576580
// But there are special cases like `SELECT ... INTO OUTFILE ''` which return positive rows affected
577581
// So we check if it is positive and add that too.
578-
switch planType {
582+
switch plan.PlanID {
579583
case planbuilder.PlanSelect, planbuilder.PlanSelectStream, planbuilder.PlanSelectImpossible, planbuilder.PlanShow, planbuilder.PlanOtherRead:
580584
qe.queryRowsReturned.Add(keys, rowsReturned)
581585
if rowsAffected > 0 {

‎go/vt/vttablet/tabletserver/query_engine_test.go

Lines changed: 34 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -619,9 +619,21 @@ func TestPlanCachePollution(t *testing.T) {
619619
}
620620

621621
func TestAddQueryStats(t *testing.T) {
622+
fakeSelectPlan := &TabletPlan{
623+
Plan: &planbuilder.Plan{
624+
PlanID: planbuilder.PlanSelect,
625+
FullQuery: &sqlparser.ParsedQuery{Query: `select * from something where something=123`}, // 43 length
626+
},
627+
}
628+
fakeInsertPlan := &TabletPlan{
629+
Plan: &planbuilder.Plan{
630+
PlanID: planbuilder.PlanInsert,
631+
FullQuery: &sqlparser.ParsedQuery{Query: `insert into something (id, msg) values(123, 'hello world!')`}, // 59 length
632+
},
633+
}
622634
testcases := []struct {
623635
name string
624-
planType planbuilder.PlanType
636+
plan *TabletPlan
625637
tableName string
626638
tabletType topodata.TabletType
627639
queryCount int64
@@ -638,12 +650,13 @@ func TestAddQueryStats(t *testing.T) {
638650
expectedQueryTimes string
639651
expectedQueryRowsAffected string
640652
expectedQueryRowsReturned string
653+
expectedQueryTextCharsProcessed string
641654
expectedQueryErrorCounts string
642655
expectedQueryErrorCountsWithCode string
643656
}{
644657
{
645658
name: "select query",
646-
planType: planbuilder.PlanSelect,
659+
plan: fakeSelectPlan,
647660
tableName: "A",
648661
tabletType: topodata.TabletType_PRIMARY,
649662
queryCount: 1,
@@ -658,12 +671,13 @@ func TestAddQueryStats(t *testing.T) {
658671
expectedQueryTimes: `{"A.Select": 10}`,
659672
expectedQueryRowsAffected: `{}`,
660673
expectedQueryRowsReturned: `{"A.Select": 15}`,
674+
expectedQueryTextCharsProcessed: `{"A.Select": 43}`,
661675
expectedQueryErrorCounts: `{"A.Select": 0}`,
662676
expectedQueryErrorCountsWithCode: `{}`,
663677
expectedQueryCountsWithTableType: `{"A.Select.PRIMARY": 1}`,
664678
}, {
665679
name: "select query against a replica",
666-
planType: planbuilder.PlanSelect,
680+
plan: fakeSelectPlan,
667681
tableName: "A",
668682
tabletType: topodata.TabletType_REPLICA,
669683
queryCount: 1,
@@ -678,12 +692,13 @@ func TestAddQueryStats(t *testing.T) {
678692
expectedQueryTimes: `{"A.Select": 10}`,
679693
expectedQueryRowsAffected: `{}`,
680694
expectedQueryRowsReturned: `{"A.Select": 15}`,
695+
expectedQueryTextCharsProcessed: `{"A.Select": 43}`,
681696
expectedQueryErrorCounts: `{"A.Select": 0}`,
682697
expectedQueryErrorCountsWithCode: `{}`,
683698
expectedQueryCountsWithTableType: `{"A.Select.REPLICA": 1}`,
684699
}, {
685700
name: "select into query",
686-
planType: planbuilder.PlanSelect,
701+
plan: fakeSelectPlan,
687702
tableName: "A",
688703
tabletType: topodata.TabletType_PRIMARY,
689704
queryCount: 1,
@@ -698,12 +713,13 @@ func TestAddQueryStats(t *testing.T) {
698713
expectedQueryTimes: `{"A.Select": 10}`,
699714
expectedQueryRowsAffected: `{"A.Select": 15}`,
700715
expectedQueryRowsReturned: `{"A.Select": 0}`,
716+
expectedQueryTextCharsProcessed: `{"A.Select": 43}`,
701717
expectedQueryErrorCounts: `{"A.Select": 0}`,
702718
expectedQueryErrorCountsWithCode: `{}`,
703719
expectedQueryCountsWithTableType: `{"A.Select.PRIMARY": 1}`,
704720
}, {
705721
name: "error",
706-
planType: planbuilder.PlanSelect,
722+
plan: fakeSelectPlan,
707723
tableName: "A",
708724
tabletType: topodata.TabletType_PRIMARY,
709725
queryCount: 1,
@@ -718,12 +734,13 @@ func TestAddQueryStats(t *testing.T) {
718734
expectedQueryTimes: `{"A.Select": 10}`,
719735
expectedQueryRowsAffected: `{}`,
720736
expectedQueryRowsReturned: `{"A.Select": 0}`,
737+
expectedQueryTextCharsProcessed: `{"A.Select": 43}`,
721738
expectedQueryErrorCounts: `{"A.Select": 1}`,
722739
expectedQueryErrorCountsWithCode: `{"A.Select.RESOURCE_EXHAUSTED": 1}`,
723740
expectedQueryCountsWithTableType: `{"A.Select.PRIMARY": 1}`,
724741
}, {
725742
name: "insert query",
726-
planType: planbuilder.PlanInsert,
743+
plan: fakeInsertPlan,
727744
tableName: "A",
728745
tabletType: topodata.TabletType_PRIMARY,
729746
queryCount: 1,
@@ -738,12 +755,13 @@ func TestAddQueryStats(t *testing.T) {
738755
expectedQueryTimes: `{"A.Insert": 10}`,
739756
expectedQueryRowsAffected: `{"A.Insert": 15}`,
740757
expectedQueryRowsReturned: `{}`,
758+
expectedQueryTextCharsProcessed: `{"A.Insert": 59}`,
741759
expectedQueryErrorCounts: `{"A.Insert": 0}`,
742760
expectedQueryErrorCountsWithCode: `{}`,
743761
expectedQueryCountsWithTableType: `{"A.Insert.PRIMARY": 1}`,
744762
}, {
745763
name: "select query with per workload metrics",
746-
planType: planbuilder.PlanSelect,
764+
plan: fakeSelectPlan,
747765
tableName: "A",
748766
tabletType: topodata.TabletType_PRIMARY,
749767
queryCount: 1,
@@ -758,12 +776,13 @@ func TestAddQueryStats(t *testing.T) {
758776
expectedQueryTimes: `{"A.Select.some-workload": 10}`,
759777
expectedQueryRowsAffected: `{}`,
760778
expectedQueryRowsReturned: `{"A.Select.some-workload": 15}`,
779+
expectedQueryTextCharsProcessed: `{"A.Select.some-workload": 43}`,
761780
expectedQueryErrorCounts: `{"A.Select.some-workload": 0}`,
762781
expectedQueryErrorCountsWithCode: `{}`,
763782
expectedQueryCountsWithTableType: `{"A.Select.PRIMARY": 1}`,
764783
}, {
765784
name: "select into query with per workload metrics",
766-
planType: planbuilder.PlanSelect,
785+
plan: fakeSelectPlan,
767786
tableName: "A",
768787
tabletType: topodata.TabletType_PRIMARY,
769788
queryCount: 1,
@@ -778,12 +797,13 @@ func TestAddQueryStats(t *testing.T) {
778797
expectedQueryTimes: `{"A.Select.some-workload": 10}`,
779798
expectedQueryRowsAffected: `{"A.Select.some-workload": 15}`,
780799
expectedQueryRowsReturned: `{"A.Select.some-workload": 0}`,
800+
expectedQueryTextCharsProcessed: `{"A.Select.some-workload": 43}`,
781801
expectedQueryErrorCounts: `{"A.Select.some-workload": 0}`,
782802
expectedQueryErrorCountsWithCode: `{}`,
783803
expectedQueryCountsWithTableType: `{"A.Select.PRIMARY": 1}`,
784804
}, {
785805
name: "error with per workload metrics",
786-
planType: planbuilder.PlanSelect,
806+
plan: fakeSelectPlan,
787807
tableName: "A",
788808
tabletType: topodata.TabletType_PRIMARY,
789809
queryCount: 1,
@@ -798,12 +818,13 @@ func TestAddQueryStats(t *testing.T) {
798818
expectedQueryTimes: `{"A.Select.some-workload": 10}`,
799819
expectedQueryRowsAffected: `{}`,
800820
expectedQueryRowsReturned: `{"A.Select.some-workload": 0}`,
821+
expectedQueryTextCharsProcessed: `{"A.Select.some-workload": 43}`,
801822
expectedQueryErrorCounts: `{"A.Select.some-workload": 1}`,
802823
expectedQueryErrorCountsWithCode: `{"A.Select.RESOURCE_EXHAUSTED": 1}`,
803824
expectedQueryCountsWithTableType: `{"A.Select.PRIMARY": 1}`,
804825
}, {
805826
name: "insert query with per workload metrics",
806-
planType: planbuilder.PlanInsert,
827+
plan: fakeInsertPlan,
807828
tableName: "A",
808829
tabletType: topodata.TabletType_PRIMARY,
809830
queryCount: 1,
@@ -818,6 +839,7 @@ func TestAddQueryStats(t *testing.T) {
818839
expectedQueryTimes: `{"A.Insert.some-workload": 10}`,
819840
expectedQueryRowsAffected: `{"A.Insert.some-workload": 15}`,
820841
expectedQueryRowsReturned: `{}`,
842+
expectedQueryTextCharsProcessed: `{"A.Insert.some-workload": 59}`,
821843
expectedQueryErrorCounts: `{"A.Insert.some-workload": 0}`,
822844
expectedQueryErrorCountsWithCode: `{}`,
823845
expectedQueryCountsWithTableType: `{"A.Insert.PRIMARY": 1}`,
@@ -833,12 +855,13 @@ func TestAddQueryStats(t *testing.T) {
833855
env := tabletenv.NewEnv(vtenv.NewTestEnv(), cfg, "TestAddQueryStats_"+testcase.name)
834856
se := schema.NewEngine(env)
835857
qe := NewQueryEngine(env, se)
836-
qe.AddStats(testcase.planType, testcase.tableName, testcase.workload, testcase.tabletType, testcase.queryCount, testcase.duration, testcase.mysqlTime, testcase.rowsAffected, testcase.rowsReturned, testcase.errorCount, testcase.errorCode)
858+
qe.AddStats(testcase.plan, testcase.tableName, testcase.workload, testcase.tabletType, testcase.queryCount, testcase.duration, testcase.mysqlTime, testcase.rowsAffected, testcase.rowsReturned, testcase.errorCount, testcase.errorCode)
837859
assert.Equal(t, testcase.expectedQueryCounts, qe.queryCounts.String())
838860
assert.Equal(t, testcase.expectedQueryCountsWithTableType, qe.queryCountsWithTabletType.String())
839861
assert.Equal(t, testcase.expectedQueryTimes, qe.queryTimes.String())
840862
assert.Equal(t, testcase.expectedQueryRowsAffected, qe.queryRowsAffected.String())
841863
assert.Equal(t, testcase.expectedQueryRowsReturned, qe.queryRowsReturned.String())
864+
assert.Equal(t, testcase.expectedQueryTextCharsProcessed, qe.queryTextCharsProcessed.String())
842865
assert.Equal(t, testcase.expectedQueryErrorCounts, qe.queryErrorCounts.String())
843866
assert.Equal(t, testcase.expectedQueryErrorCountsWithCode, qe.queryErrorCountsWithCode.String())
844867
})

‎go/vt/vttablet/tabletserver/query_executor.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -137,12 +137,12 @@ func (qre *QueryExecutor) Execute() (reply *sqltypes.Result, err error) {
137137
errCode = vtErrorCode.String()
138138

139139
if reply == nil {
140-
qre.tsv.qe.AddStats(qre.plan.PlanID, tableName, qre.options.GetWorkloadName(), qre.targetTabletType, 1, duration, mysqlTime, 0, 0, 1, errCode)
140+
qre.tsv.qe.AddStats(qre.plan, tableName, qre.options.GetWorkloadName(), qre.targetTabletType, 1, duration, mysqlTime, 0, 0, 1, errCode)
141141
qre.plan.AddStats(1, duration, mysqlTime, 0, 0, 1)
142142
return
143143
}
144144

145-
qre.tsv.qe.AddStats(qre.plan.PlanID, tableName, qre.options.GetWorkloadName(), qre.targetTabletType, 1, duration, mysqlTime, int64(reply.RowsAffected), int64(len(reply.Rows)), 0, errCode)
145+
qre.tsv.qe.AddStats(qre.plan, tableName, qre.options.GetWorkloadName(), qre.targetTabletType, 1, duration, mysqlTime, int64(reply.RowsAffected), int64(len(reply.Rows)), 0, errCode)
146146
qre.plan.AddStats(1, duration, mysqlTime, reply.RowsAffected, uint64(len(reply.Rows)), 0)
147147
qre.logStats.RowsAffected = int(reply.RowsAffected)
148148
qre.logStats.Rows = reply.Rows

0 commit comments

Comments
 (0)