Skip to content

Commit a0a9e11

Browse files
timvaillancourtshlomi-noachharshit-gangaldeepthi
authored
slack-19.0: v20 backports, pt. 2 (#459)
* `vtgate`: support filtering tablets by tablet-tags (vitessio#15911) Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com> Co-authored-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * Add support for sampling rate in `streamlog` (vitessio#15919) Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com> * Add sql text counts stats to `vtcombo`,`vtgate`+`vttablet` (vitessio#15897) Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com> Signed-off-by: Harshit Gangal <harshit@planetscale.com> Co-authored-by: Harshit Gangal <harshit@planetscale.com> Co-authored-by: Deepthi Sigireddi <deepthi.sigireddi@gmail.com> --------- Signed-off-by: Tim Vaillancourt <tim@timvaillancourt.com> Signed-off-by: Harshit Gangal <harshit@planetscale.com> Co-authored-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Co-authored-by: Harshit Gangal <harshit@planetscale.com> Co-authored-by: Deepthi Sigireddi <deepthi.sigireddi@gmail.com>
1 parent 1298a0f commit a0a9e11

File tree

13 files changed

+307
-38
lines changed

13 files changed

+307
-38
lines changed

‎go/flags/endtoend/vtcombo.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -271,6 +271,7 @@ Flags:
271271
--querylog-filter-tag string string that must be present in the query for it to be logged; if using a value as the tag, you need to disable query normalization
272272
--querylog-format string format for query logs ("text" or "json") (default "text")
273273
--querylog-row-threshold uint Number of rows a query has to return or affect before being logged; not useful for streaming queries. 0 means all queries will be logged.
274+
--querylog-sample-rate float Sample rate for logging queries. Value must be between 0.0 (no logging) and 1.0 (all queries)
274275
--queryserver-config-acl-exempt-acl string an acl that exempt from table acl checking (this acl is free to access any vitess tables).
275276
--queryserver-config-annotate-queries prefix queries to MySQL backend with comment indicating vtgate principal (user) and target tablet type
276277
--queryserver-config-enable-table-acl-dry-run If this flag is enabled, tabletserver will emit monitoring metrics and let the request pass regardless of table acl check results
@@ -337,6 +338,7 @@ Flags:
337338
--structured-logging enable structured logging
338339
--table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class
339340
--table_gc_lifecycle string States for a DROP TABLE garbage collection cycle. Default is 'hold,purge,evac,drop', use any subset ('drop' implicitly always included) (default "hold,purge,evac,drop")
341+
--tablet-filter-tags StringMap Specifies a comma-separated list of tablet tags (as key:value pairs) to filter the tablets to watch.
340342
--tablet_dir string The directory within the vtdataroot to store vttablet/mysql files. Defaults to being generated by the tablet uid.
341343
--tablet_filters strings Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch.
342344
--tablet_health_keep_alive duration close streaming tablet health connection if there are no requests for this long (default 5m0s)

‎go/flags/endtoend/vtgate.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -172,6 +172,7 @@ Flags:
172172
--querylog-filter-tag string string that must be present in the query for it to be logged; if using a value as the tag, you need to disable query normalization
173173
--querylog-format string format for query logs ("text" or "json") (default "text")
174174
--querylog-row-threshold uint Number of rows a query has to return or affect before being logged; not useful for streaming queries. 0 means all queries will be logged.
175+
--querylog-sample-rate float Sample rate for logging queries. Value must be between 0.0 (no logging) and 1.0 (all queries)
175176
--redact-debug-ui-queries redact full queries and bind variables from debug UI
176177
--remote_operation_timeout duration time to wait for a remote operation (default 15s)
177178
--retry-count int retry count (default 2)
@@ -194,6 +195,7 @@ Flags:
194195
--stream_buffer_size int the number of bytes sent from vtgate for each stream call. It's recommended to keep this value in sync with vttablet's query-server-config-stream-buffer-size. (default 32768)
195196
--structured-logging enable structured logging
196197
--table-refresh-interval int interval in milliseconds to refresh tables in status page with refreshRequired class
198+
--tablet-filter-tags StringMap Specifies a comma-separated list of tablet tags (as key:value pairs) to filter the tablets to watch.
197199
--tablet_filters strings Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch.
198200
--tablet_grpc_ca string the server ca to use to validate servers when connecting
199201
--tablet_grpc_cert string the cert to use to connect

‎go/flags/endtoend/vttablet.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -262,6 +262,7 @@ Flags:
262262
--querylog-filter-tag string string that must be present in the query for it to be logged; if using a value as the tag, you need to disable query normalization
263263
--querylog-format string format for query logs ("text" or "json") (default "text")
264264
--querylog-row-threshold uint Number of rows a query has to return or affect before being logged; not useful for streaming queries. 0 means all queries will be logged.
265+
--querylog-sample-rate float Sample rate for logging queries. Value must be between 0.0 (no logging) and 1.0 (all queries)
265266
--queryserver-config-acl-exempt-acl string an acl that exempt from table acl checking (this acl is free to access any vitess tables).
266267
--queryserver-config-annotate-queries prefix queries to MySQL backend with comment indicating vtgate principal (user) and target tablet type
267268
--queryserver-config-enable-table-acl-dry-run If this flag is enabled, tabletserver will emit monitoring metrics and let the request pass regardless of table acl check results

‎go/streamlog/streamlog.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ package streamlog
2020
import (
2121
"fmt"
2222
"io"
23+
rand "math/rand/v2"
2324
"net/http"
2425
"net/url"
2526
"os"
@@ -51,6 +52,7 @@ var (
5152
queryLogFilterTag string
5253
queryLogRowThreshold uint64
5354
queryLogFormat = "text"
55+
queryLogSampleRate float64
5456
)
5557

5658
func GetRedactDebugUIQueries() bool {
@@ -69,6 +71,10 @@ func SetQueryLogRowThreshold(newQueryLogRowThreshold uint64) {
6971
queryLogRowThreshold = newQueryLogRowThreshold
7072
}
7173

74+
func SetQueryLogSampleRate(sampleRate float64) {
75+
queryLogSampleRate = sampleRate
76+
}
77+
7278
func GetQueryLogFormat() string {
7379
return queryLogFormat
7480
}
@@ -96,6 +102,8 @@ func registerStreamLogFlags(fs *pflag.FlagSet) {
96102
// QueryLogRowThreshold only log queries returning or affecting this many rows
97103
fs.Uint64Var(&queryLogRowThreshold, "querylog-row-threshold", queryLogRowThreshold, "Number of rows a query has to return or affect before being logged; not useful for streaming queries. 0 means all queries will be logged.")
98104

105+
// QueryLogSampleRate causes a sample of queries to be logged
106+
fs.Float64Var(&queryLogSampleRate, "querylog-sample-rate", queryLogSampleRate, "Sample rate for logging queries. Value must be between 0.0 (no logging) and 1.0 (all queries)")
99107
}
100108

101109
const (
@@ -249,9 +257,22 @@ func GetFormatter[T any](logger *StreamLogger[T]) LogFormatter {
249257
}
250258
}
251259

260+
// shouldSampleQuery returns true if a query should be sampled based on queryLogSampleRate
261+
func shouldSampleQuery() bool {
262+
if queryLogSampleRate <= 0 {
263+
return false
264+
} else if queryLogSampleRate >= 1 {
265+
return true
266+
}
267+
return rand.Float64() <= queryLogSampleRate
268+
}
269+
252270
// ShouldEmitLog returns whether the log with the given SQL query
253271
// should be emitted or filtered
254272
func ShouldEmitLog(sql string, rowsAffected, rowsReturned uint64) bool {
273+
if shouldSampleQuery() {
274+
return true
275+
}
255276
if queryLogRowThreshold > max(rowsAffected, rowsReturned) && queryLogFilterTag == "" {
256277
return false
257278
}

‎go/streamlog/streamlog_test.go

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,9 @@ import (
2929
"testing"
3030
"time"
3131

32+
"github.com/stretchr/testify/assert"
33+
"github.com/stretchr/testify/require"
34+
3235
"vitess.io/vitess/go/vt/servenv"
3336
)
3437

@@ -260,3 +263,131 @@ func TestFile(t *testing.T) {
260263
t.Errorf("streamlog file: want %q got %q", want, got)
261264
}
262265
}
266+
267+
func TestShouldSampleQuery(t *testing.T) {
268+
queryLogSampleRate = -1
269+
assert.False(t, shouldSampleQuery())
270+
271+
queryLogSampleRate = 0
272+
assert.False(t, shouldSampleQuery())
273+
274+
// for test coverage, can't test a random result
275+
queryLogSampleRate = 0.5
276+
shouldSampleQuery()
277+
278+
queryLogSampleRate = 1.0
279+
assert.True(t, shouldSampleQuery())
280+
281+
queryLogSampleRate = 100.0
282+
assert.True(t, shouldSampleQuery())
283+
}
284+
285+
func TestShouldEmitLog(t *testing.T) {
286+
origQueryLogFilterTag := queryLogFilterTag
287+
origQueryLogRowThreshold := queryLogRowThreshold
288+
origQueryLogSampleRate := queryLogSampleRate
289+
defer func() {
290+
SetQueryLogFilterTag(origQueryLogFilterTag)
291+
SetQueryLogRowThreshold(origQueryLogRowThreshold)
292+
SetQueryLogSampleRate(origQueryLogSampleRate)
293+
}()
294+
295+
tests := []struct {
296+
sql string
297+
qLogFilterTag string
298+
qLogRowThreshold uint64
299+
qLogSampleRate float64
300+
rowsAffected uint64
301+
rowsReturned uint64
302+
ok bool
303+
}{
304+
{
305+
sql: "queryLogThreshold smaller than affected and returned",
306+
qLogFilterTag: "",
307+
qLogRowThreshold: 2,
308+
qLogSampleRate: 0.0,
309+
rowsAffected: 7,
310+
rowsReturned: 7,
311+
ok: true,
312+
},
313+
{
314+
sql: "queryLogThreshold greater than affected and returned",
315+
qLogFilterTag: "",
316+
qLogRowThreshold: 27,
317+
qLogSampleRate: 0.0,
318+
rowsAffected: 7,
319+
rowsReturned: 17,
320+
ok: false,
321+
},
322+
{
323+
sql: "this doesn't contains queryFilterTag: TAG",
324+
qLogFilterTag: "special tag",
325+
qLogRowThreshold: 10,
326+
qLogSampleRate: 0.0,
327+
rowsAffected: 7,
328+
rowsReturned: 17,
329+
ok: false,
330+
},
331+
{
332+
sql: "this contains queryFilterTag: TAG",
333+
qLogFilterTag: "TAG",
334+
qLogRowThreshold: 0,
335+
qLogSampleRate: 0.0,
336+
rowsAffected: 7,
337+
rowsReturned: 17,
338+
ok: true,
339+
},
340+
{
341+
sql: "this contains querySampleRate: 1.0",
342+
qLogFilterTag: "",
343+
qLogRowThreshold: 0,
344+
qLogSampleRate: 1.0,
345+
rowsAffected: 7,
346+
rowsReturned: 17,
347+
ok: true,
348+
},
349+
{
350+
sql: "this contains querySampleRate: 1.0 without expected queryFilterTag",
351+
qLogFilterTag: "TAG",
352+
qLogRowThreshold: 0,
353+
qLogSampleRate: 1.0,
354+
rowsAffected: 7,
355+
rowsReturned: 17,
356+
ok: true,
357+
},
358+
}
359+
360+
for _, tt := range tests {
361+
t.Run(tt.sql, func(t *testing.T) {
362+
SetQueryLogFilterTag(tt.qLogFilterTag)
363+
SetQueryLogRowThreshold(tt.qLogRowThreshold)
364+
SetQueryLogSampleRate(tt.qLogSampleRate)
365+
366+
require.Equal(t, tt.ok, ShouldEmitLog(tt.sql, tt.rowsAffected, tt.rowsReturned))
367+
})
368+
}
369+
}
370+
371+
func BenchmarkShouldEmitLog(b *testing.B) {
372+
b.Run("default", func(b *testing.B) {
373+
SetQueryLogSampleRate(0.0)
374+
for i := 0; i < b.N; i++ {
375+
ShouldEmitLog("select * from test where user='someone'", 0, 123)
376+
}
377+
})
378+
b.Run("filter_tag", func(b *testing.B) {
379+
SetQueryLogSampleRate(0.0)
380+
SetQueryLogFilterTag("LOG_QUERY")
381+
defer SetQueryLogFilterTag("")
382+
for i := 0; i < b.N; i++ {
383+
ShouldEmitLog("select /* LOG_QUERY=1 */ * from test where user='someone'", 0, 123)
384+
}
385+
})
386+
b.Run("50%_sample_rate", func(b *testing.B) {
387+
SetQueryLogSampleRate(0.5)
388+
defer SetQueryLogSampleRate(0.0)
389+
for i := 0; i < b.N; i++ {
390+
ShouldEmitLog("select * from test where user='someone'", 0, 123)
391+
}
392+
})
393+
}

‎go/vt/discovery/healthcheck.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ import (
4848
"github.com/spf13/pflag"
4949
"golang.org/x/sync/semaphore"
5050

51+
"vitess.io/vitess/go/flagutil"
5152
"vitess.io/vitess/go/netutil"
5253
"vitess.io/vitess/go/stats"
5354
"vitess.io/vitess/go/vt/log"
@@ -82,6 +83,9 @@ var (
8283
// tabletFilters are the keyspace|shard or keyrange filters to apply to the full set of tablets.
8384
tabletFilters []string
8485

86+
// tabletFilterTags are the tablet tag filters (as key:value pairs) to apply to the full set of tablets.
87+
tabletFilterTags flagutil.StringMapValue
88+
8589
// refreshInterval is the interval at which healthcheck refreshes its list of tablets from topo.
8690
refreshInterval = 1 * time.Minute
8791

@@ -164,6 +168,7 @@ func init() {
164168

165169
func registerDiscoveryFlags(fs *pflag.FlagSet) {
166170
fs.StringSliceVar(&tabletFilters, "tablet_filters", []string{}, "Specifies a comma-separated list of 'keyspace|shard_name or keyrange' values to filter the tablets to watch.")
171+
fs.Var(&tabletFilterTags, "tablet-filter-tags", "Specifies a comma-separated list of tablet tags (as key:value pairs) to filter the tablets to watch.")
167172
fs.Var((*topoproto.TabletTypeListFlag)(&AllowedTabletTypes), "allowed_tablet_types", "Specifies the tablet types this vtgate is allowed to route queries to. Should be provided as a comma-separated set of tablet types.")
168173
fs.StringSliceVar(&KeyspacesToWatch, "keyspaces_to_watch", []string{}, "Specifies which keyspaces this vtgate should have access to while routing queries or accessing the vschema.")
169174
}
@@ -337,13 +342,13 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur
337342
loadTabletsTrigger: make(chan struct{}),
338343
}
339344
var topoWatchers []*TopologyWatcher
340-
var filter TabletFilter
341345
cells := strings.Split(cellsToWatch, ",")
342346
if cellsToWatch == "" {
343347
cells = append(cells, localCell)
344348
}
345349

346350
for _, c := range cells {
351+
var filters TabletFilters
347352
log.Infof("Setting up healthcheck for cell: %v", c)
348353
if c == "" {
349354
continue
@@ -357,11 +362,14 @@ func NewHealthCheck(ctx context.Context, retryDelay, healthCheckTimeout time.Dur
357362
if err != nil {
358363
log.Exitf("Cannot parse tablet_filters parameter: %v", err)
359364
}
360-
filter = fbs
365+
filters = append(filters, fbs)
361366
} else if len(KeyspacesToWatch) > 0 {
362-
filter = NewFilterByKeyspace(KeyspacesToWatch)
367+
filters = append(filters, NewFilterByKeyspace(KeyspacesToWatch))
368+
}
369+
if len(tabletFilterTags) > 0 {
370+
filters = append(filters, NewFilterByTabletTags(tabletFilterTags))
363371
}
364-
topoWatchers = append(topoWatchers, NewTopologyWatcher(ctx, topoServer, hc, filter, c, refreshInterval, refreshKnownTablets, topo.DefaultConcurrency))
372+
topoWatchers = append(topoWatchers, NewTopologyWatcher(ctx, topoServer, hc, filters, c, refreshInterval, refreshKnownTablets, topo.DefaultConcurrency))
365373
}
366374

367375
hc.topoWatchers = topoWatchers

‎go/vt/discovery/topology_watcher.go

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -274,6 +274,19 @@ type TabletFilter interface {
274274
IsIncluded(tablet *topodata.Tablet) bool
275275
}
276276

277+
// TabletFilters contains filters for tablets.
278+
type TabletFilters []TabletFilter
279+
280+
// IsIncluded returns true if a tablet passes all filters.
281+
func (tf TabletFilters) IsIncluded(tablet *topodata.Tablet) bool {
282+
for _, filter := range tf {
283+
if !filter.IsIncluded(tablet) {
284+
return false
285+
}
286+
}
287+
return true
288+
}
289+
277290
// FilterByShard is a filter that filters tablets by
278291
// keyspace/shard.
279292
type FilterByShard struct {
@@ -375,3 +388,32 @@ func (fbk *FilterByKeyspace) IsIncluded(tablet *topodata.Tablet) bool {
375388
_, exist := fbk.keyspaces[tablet.Keyspace]
376389
return exist
377390
}
391+
392+
// FilterByTabletTags is a filter that filters tablets by tablet tag key/values.
393+
type FilterByTabletTags struct {
394+
tags map[string]string
395+
}
396+
397+
// NewFilterByTabletTags creates a new FilterByTabletTags. All tablets that match
398+
// all tablet tags will be forwarded to the TopologyWatcher's consumer.
399+
func NewFilterByTabletTags(tabletTags map[string]string) *FilterByTabletTags {
400+
return &FilterByTabletTags{
401+
tags: tabletTags,
402+
}
403+
}
404+
405+
// IsIncluded returns true if the tablet's tags match what we expect.
406+
func (fbtg *FilterByTabletTags) IsIncluded(tablet *topodata.Tablet) bool {
407+
if fbtg.tags == nil {
408+
return true
409+
}
410+
if tablet.Tags == nil {
411+
return false
412+
}
413+
for key, val := range fbtg.tags {
414+
if tabletVal, found := tablet.Tags[key]; !found || tabletVal != val {
415+
return false
416+
}
417+
}
418+
return true
419+
}

0 commit comments

Comments
 (0)