Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Tablet throttler: adding more stats #15224

Merged
merged 2 commits into from
Feb 15, 2024
Merged

Tablet throttler: adding more stats #15224

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d6b3e22
Throttler: record heartbeat requests in stats
shlomi-noach Feb 13, 2024
83458de
Tablet Throttler: adding more stats
shlomi-noach Feb 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions go/vt/vttablet/tabletmanager/rpc_throttler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package tabletmanager
import (
"context"

"vitess.io/vitess/go/stats"
tabletmanagerdatapb "vitess.io/vitess/go/vt/proto/tabletmanagerdata"
"vitess.io/vitess/go/vt/proto/vtrpc"
"vitess.io/vitess/go/vt/vterrors"
Expand All @@ -28,6 +29,7 @@ import (

// CheckThrottler executes a throttler check
func (tm *TabletManager) CheckThrottler(ctx context.Context, req *tabletmanagerdatapb.CheckThrottlerRequest) (*tabletmanagerdatapb.CheckThrottlerResponse, error) {
go stats.GetOrNewCounter("ThrottlerCheckRequest", "CheckThrottler requests").Add(1)
if req.AppName == "" {
req.AppName = throttlerapp.VitessName.String()
}
Expand Down
20 changes: 15 additions & 5 deletions go/vt/vttablet/tabletserver/throttle/throttler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@ import (

"vitess.io/vitess/go/constants/sidecar"
"vitess.io/vitess/go/protoutil"
"vitess.io/vitess/go/stats"

"vitess.io/vitess/go/textutil"
"vitess.io/vitess/go/timer"
Expand Down Expand Up @@ -443,7 +444,7 @@ func (throttler *Throttler) Enable() *sync.WaitGroup {
throttler.Operate(ctx, wg)

// Make a one-time request for a lease of heartbeats
go throttler.heartbeatWriter.RequestHeartbeats()
throttler.requestHeartbeats()

return wg
}
Expand Down Expand Up @@ -568,6 +569,13 @@ func (throttler *Throttler) Close() {
log.Infof("Throttler: finished execution of Close")
}

// requestHeartbeats sends a heartbeat lease request to the heartbeat writer.
// This action is recorded in stats.
func (throttler *Throttler) requestHeartbeats() {
go throttler.heartbeatWriter.RequestHeartbeats()
go stats.GetOrNewCounter("ThrottlerHeartbeatRequests", "heartbeat requests").Add(1)
}

func (throttler *Throttler) generateSelfMySQLThrottleMetricFunc(ctx context.Context, probe *mysql.Probe) func() *mysql.MySQLThrottleMetric {
f := func() *mysql.MySQLThrottleMetric {
return throttler.readSelfThrottleMetric(ctx, probe)
Expand Down Expand Up @@ -708,7 +716,7 @@ func (throttler *Throttler) Operate(ctx context.Context, wg *sync.WaitGroup) {
if transitionedIntoLeader {
// transitioned into leadership, let's speed up the next 'refresh' and 'collect' ticks
go mysqlRefreshTicker.TickNow()
go throttler.heartbeatWriter.RequestHeartbeats()
throttler.requestHeartbeats()
}
}()
case <-mysqlCollectTicker.C:
Expand Down Expand Up @@ -782,7 +790,8 @@ func (throttler *Throttler) generateTabletProbeFunction(ctx context.Context, clu
if resp.RecentlyChecked {
// We have just probed a tablet, and it reported back that someone just recently "check"ed it.
// We therefore renew the heartbeats lease.
go throttler.heartbeatWriter.RequestHeartbeats()
throttler.requestHeartbeats()
go stats.GetOrNewCounter("ThrottlerProbeRecentlyChecked", "probe recently checked").Add(1)
}
return mySQLThrottleMetric
}
Expand Down Expand Up @@ -1017,7 +1026,7 @@ func (throttler *Throttler) ThrottleApp(appName string, expireAt time.Time, rati
func (throttler *Throttler) UnthrottleApp(appName string) (appThrottle *base.AppThrottle) {
throttler.throttledApps.Delete(appName)
// the app is likely to check
go throttler.heartbeatWriter.RequestHeartbeats()
throttler.requestHeartbeats()
return base.NewAppThrottle(appName, time.Now(), 0, false)
}

Expand Down Expand Up @@ -1164,7 +1173,7 @@ func (throttler *Throttler) checkStore(ctx context.Context, appName string, stor
return okMetricCheckResult
}
if !flags.SkipRequestHeartbeats && !throttlerapp.VitessName.Equals(appName) {
go throttler.heartbeatWriter.RequestHeartbeats()
throttler.requestHeartbeats()
// This check was made by someone other than the throttler itself, i.e. this came from online-ddl or vreplication or other.
// We mark the fact that someone just made a check. If this is a REPLICA or RDONLY tables, this will be reported back
// to the PRIMARY so that it knows it must renew the heartbeat lease.
Expand All @@ -1178,6 +1187,7 @@ func (throttler *Throttler) checkStore(ctx context.Context, appName string, stor
// If this tablet is a REPLICA or RDONLY, we want to advertise to the PRIMARY that someone did a recent check,
// so that the PRIMARY knows it must renew the heartbeat lease.
checkResult.RecentlyChecked = true
go stats.GetOrNewCounter("ThrottlerRecentlyChecked", "recently checked").Add(1)
}

return checkResult
Expand Down
Loading