Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added more configurability around vote account tracking #75

Merged
merged 1 commit into from
Nov 15, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 69 additions & 48 deletions README.md

Large diffs are not rendered by default.

81 changes: 73 additions & 8 deletions cmd/solana-exporter/collector.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@ import (
"github.com/asymmetric-research/solana-exporter/pkg/slog"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
"slices"
)

const (
SkipStatusLabel = "status"
StateLabel = "state"
NodekeyLabel = "nodekey"
VotekeyLabel = "votekey"
VersionLabel = "version"
Expand All @@ -22,6 +24,9 @@ const (
StatusSkipped = "skipped"
StatusValid = "valid"

StateCurrent = "current"
StateDelinquent = "delinquent"

TransactionTypeVote = "vote"
TransactionTypeNonVote = "non_vote"
)
Expand All @@ -34,9 +39,13 @@ type SolanaCollector struct {

/// descriptors:
ValidatorActiveStake *GaugeDesc
ClusterActiveStake *GaugeDesc
ValidatorLastVote *GaugeDesc
ClusterLastVote *GaugeDesc
ValidatorRootSlot *GaugeDesc
ClusterRootSlot *GaugeDesc
ValidatorDelinquent *GaugeDesc
ClusterValidatorCount *GaugeDesc
AccountBalances *GaugeDesc
NodeVersion *GaugeDesc
NodeIsHealthy *GaugeDesc
Expand All @@ -55,21 +64,41 @@ func NewSolanaCollector(client *rpc.Client, config *ExporterConfig) *SolanaColle
fmt.Sprintf("Active stake (in SOL) per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel),
VotekeyLabel, NodekeyLabel,
),
ClusterActiveStake: NewGaugeDesc(
"solana_cluster_active_stake",
"Total active stake (in SOL) of the cluster",
),
ValidatorLastVote: NewGaugeDesc(
"solana_validator_last_vote",
fmt.Sprintf("Last voted-on slot per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel),
VotekeyLabel, NodekeyLabel,
),
ClusterLastVote: NewGaugeDesc(
"solana_cluster_last_vote",
"Most recent voted-on slot of the cluster",
),
ValidatorRootSlot: NewGaugeDesc(
"solana_validator_root_slot",
fmt.Sprintf("Root slot per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel),
VotekeyLabel, NodekeyLabel,
),
ClusterRootSlot: NewGaugeDesc(
"solana_cluster_root_slot",
"Max root slot of the cluster",
),
ValidatorDelinquent: NewGaugeDesc(
"solana_validator_delinquent",
fmt.Sprintf("Whether a validator (represented by %s and %s) is delinquent", VotekeyLabel, NodekeyLabel),
VotekeyLabel, NodekeyLabel,
),
ClusterValidatorCount: NewGaugeDesc(
"solana_cluster_validator_count",
fmt.Sprintf(
"Total number of validators in the cluster, grouped by %s ('%s' or '%s')",
StateLabel, StateCurrent, StateDelinquent,
),
StateLabel,
),
AccountBalances: NewGaugeDesc(
"solana_account_balance",
fmt.Sprintf("Solana account balances, grouped by %s", AddressLabel),
Expand Down Expand Up @@ -103,9 +132,13 @@ func NewSolanaCollector(client *rpc.Client, config *ExporterConfig) *SolanaColle
func (c *SolanaCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.NodeVersion.Desc
ch <- c.ValidatorActiveStake.Desc
ch <- c.ClusterActiveStake.Desc
ch <- c.ValidatorLastVote.Desc
ch <- c.ClusterLastVote.Desc
ch <- c.ValidatorRootSlot.Desc
ch <- c.ClusterRootSlot.Desc
ch <- c.ValidatorDelinquent.Desc
ch <- c.ClusterValidatorCount.Desc
ch <- c.AccountBalances.Desc
ch <- c.NodeIsHealthy.Desc
ch <- c.NodeNumSlotsBehind.Desc
Expand All @@ -123,26 +156,58 @@ func (c *SolanaCollector) collectVoteAccounts(ctx context.Context, ch chan<- pro
if err != nil {
c.logger.Errorf("failed to get vote accounts: %v", err)
ch <- c.ValidatorActiveStake.NewInvalidMetric(err)
ch <- c.ClusterActiveStake.NewInvalidMetric(err)
ch <- c.ValidatorLastVote.NewInvalidMetric(err)
ch <- c.ClusterLastVote.NewInvalidMetric(err)
ch <- c.ValidatorRootSlot.NewInvalidMetric(err)
ch <- c.ClusterRootSlot.NewInvalidMetric(err)
ch <- c.ValidatorDelinquent.NewInvalidMetric(err)
ch <- c.ClusterValidatorCount.NewInvalidMetric(err)
return
}

var (
totalStake float64
maxLastVote float64
maxRootSlot float64
)
for _, account := range append(voteAccounts.Current, voteAccounts.Delinquent...) {
accounts := []string{account.VotePubkey, account.NodePubkey}
ch <- c.ValidatorActiveStake.MustNewConstMetric(float64(account.ActivatedStake)/rpc.LamportsInSol, accounts...)
ch <- c.ValidatorLastVote.MustNewConstMetric(float64(account.LastVote), accounts...)
ch <- c.ValidatorRootSlot.MustNewConstMetric(float64(account.RootSlot), accounts...)
}
stake, lastVote, rootSlot :=
float64(account.ActivatedStake)/rpc.LamportsInSol,
float64(account.LastVote),
float64(account.RootSlot)

for _, account := range voteAccounts.Current {
ch <- c.ValidatorDelinquent.MustNewConstMetric(0, account.VotePubkey, account.NodePubkey)
if slices.Contains(c.config.NodeKeys, account.NodePubkey) || c.config.ComprehensiveVoteAccountTracking {
ch <- c.ValidatorActiveStake.MustNewConstMetric(stake, accounts...)
ch <- c.ValidatorLastVote.MustNewConstMetric(lastVote, accounts...)
ch <- c.ValidatorRootSlot.MustNewConstMetric(rootSlot, accounts...)
}

totalStake += stake
maxLastVote = max(maxLastVote, lastVote)
maxRootSlot = max(maxRootSlot, rootSlot)
}
for _, account := range voteAccounts.Delinquent {
ch <- c.ValidatorDelinquent.MustNewConstMetric(1, account.VotePubkey, account.NodePubkey)

{
for _, account := range voteAccounts.Current {
if slices.Contains(c.config.NodeKeys, account.NodePubkey) || c.config.ComprehensiveVoteAccountTracking {
ch <- c.ValidatorDelinquent.MustNewConstMetric(0, account.VotePubkey, account.NodePubkey)
}
}
for _, account := range voteAccounts.Delinquent {
if slices.Contains(c.config.NodeKeys, account.NodePubkey) || c.config.ComprehensiveVoteAccountTracking {
ch <- c.ValidatorDelinquent.MustNewConstMetric(1, account.VotePubkey, account.NodePubkey)
}
}
}

ch <- c.ClusterActiveStake.MustNewConstMetric(totalStake)
ch <- c.ClusterLastVote.MustNewConstMetric(maxLastVote)
ch <- c.ClusterRootSlot.MustNewConstMetric(maxRootSlot)
ch <- c.ClusterValidatorCount.MustNewConstMetric(float64(len(voteAccounts.Current)), StateCurrent)
ch <- c.ClusterValidatorCount.MustNewConstMetric(float64(len(voteAccounts.Delinquent)), StateDelinquent)

c.logger.Info("Vote accounts collected.")
}

Expand Down
34 changes: 27 additions & 7 deletions cmd/solana-exporter/collector_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ type (
Votekeys []string
FeeRewardLamports int
InflationRewardLamports int
LastVoteDistances map[string]int
RootSlotDistances map[string]int
}
)

Expand Down Expand Up @@ -86,6 +88,8 @@ func NewSimulator(t *testing.T, slot int) (*Simulator, *rpc.Client) {
Votekeys: votekeys,
InflationRewardLamports: inflationRewardLamports,
FeeRewardLamports: feeRewardLamports,
LastVoteDistances: map[string]int{"aaa": 1, "bbb": 2, "ccc": 3},
RootSlotDistances: map[string]int{"aaa": 4, "bbb": 5, "ccc": 6},
}
simulator.PopulateSlot(0)
if slot > 0 {
Expand Down Expand Up @@ -144,7 +148,8 @@ func (c *Simulator) PopulateSlot(slot int) {
for _, nodekey := range c.Nodekeys {
transactions = append(transactions, []string{nodekey, strings.ToUpper(nodekey), VoteProgram})
info := c.Server.GetValidatorInfo(nodekey)
info.LastVote = slot
info.LastVote = max(0, slot-c.LastVoteDistances[nodekey])
info.RootSlot = max(0, slot-c.RootSlotDistances[nodekey])
c.Server.SetOpt(rpc.ValidatorInfoOpt, nodekey, info)
}

Expand Down Expand Up @@ -199,6 +204,7 @@ func newTestConfig(simulator *Simulator, fast bool) *ExporterConfig {
nil,
true,
true,
true,
false,
pace,
}
Expand All @@ -218,21 +224,34 @@ func TestSolanaCollector(t *testing.T) {
NewLV(stake, "bbb", "BBB"),
NewLV(stake, "ccc", "CCC"),
),
collector.ClusterActiveStake.makeCollectionTest(
NewLV(3 * stake),
),
collector.ValidatorLastVote.makeCollectionTest(
NewLV(34, "aaa", "AAA"),
NewLV(34, "bbb", "BBB"),
NewLV(34, "ccc", "CCC"),
NewLV(33, "aaa", "AAA"),
NewLV(32, "bbb", "BBB"),
NewLV(31, "ccc", "CCC"),
),
collector.ClusterLastVote.makeCollectionTest(
NewLV(33),
),
collector.ValidatorRootSlot.makeCollectionTest(
NewLV(0, "aaa", "AAA"),
NewLV(0, "bbb", "BBB"),
NewLV(0, "ccc", "CCC"),
NewLV(30, "aaa", "AAA"),
NewLV(29, "bbb", "BBB"),
NewLV(28, "ccc", "CCC"),
),
collector.ClusterRootSlot.makeCollectionTest(
NewLV(30),
),
collector.ValidatorDelinquent.makeCollectionTest(
NewLV(0, "aaa", "AAA"),
NewLV(0, "bbb", "BBB"),
NewLV(0, "ccc", "CCC"),
),
collector.ClusterValidatorCount.makeCollectionTest(
NewLV(3, StateCurrent),
NewLV(0, StateDelinquent),
),
collector.NodeVersion.makeCollectionTest(
NewLV(1, "v1.0.0"),
),
Expand All @@ -258,6 +277,7 @@ func TestSolanaCollector(t *testing.T) {
),
}

fmt.Println(testCases[1].ExpectedResponse)
for _, test := range testCases {
t.Run(test.Name, func(t *testing.T) {
err := testutil.CollectAndCompare(collector, bytes.NewBufferString(test.ExpectedResponse), test.Name)
Expand Down
77 changes: 47 additions & 30 deletions cmd/solana-exporter/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,17 @@ type (
arrayFlags []string

ExporterConfig struct {
HttpTimeout time.Duration
RpcUrl string
ListenAddress string
NodeKeys []string
VoteKeys []string
BalanceAddresses []string
ComprehensiveSlotTracking bool
MonitorBlockSizes bool
LightMode bool
SlotPace time.Duration
HttpTimeout time.Duration
RpcUrl string
ListenAddress string
NodeKeys []string
VoteKeys []string
BalanceAddresses []string
ComprehensiveSlotTracking bool
ComprehensiveVoteAccountTracking bool
MonitorBlockSizes bool
LightMode bool
SlotPace time.Duration
}
)

Expand All @@ -43,6 +44,7 @@ func NewExporterConfig(
nodeKeys []string,
balanceAddresses []string,
comprehensiveSlotTracking bool,
comprehensiveVoteAccountTracking bool,
monitorBlockSizes bool,
lightMode bool,
slotPace time.Duration,
Expand All @@ -56,6 +58,7 @@ func NewExporterConfig(
"nodeKeys", nodeKeys,
"balanceAddresses", balanceAddresses,
"comprehensiveSlotTracking", comprehensiveSlotTracking,
"comprehensiveVoteAccountTracking", comprehensiveVoteAccountTracking,
"monitorBlockSizes", monitorBlockSizes,
"lightMode", lightMode,
)
Expand All @@ -64,6 +67,10 @@ func NewExporterConfig(
return nil, fmt.Errorf("'-light-mode' is incompatible with `-comprehensive-slot-tracking`")
}

if comprehensiveVoteAccountTracking {
return nil, fmt.Errorf("'-light-mode' is incompatible with '-comprehensive-vote-account-tracking'")
}

if monitorBlockSizes {
return nil, fmt.Errorf("'-light-mode' is incompatible with `-monitor-block-sizes`")
}
Expand All @@ -87,31 +94,33 @@ func NewExporterConfig(
}

config := ExporterConfig{
HttpTimeout: httpTimeout,
RpcUrl: rpcUrl,
ListenAddress: listenAddress,
NodeKeys: nodeKeys,
VoteKeys: voteKeys,
BalanceAddresses: balanceAddresses,
ComprehensiveSlotTracking: comprehensiveSlotTracking,
MonitorBlockSizes: monitorBlockSizes,
LightMode: lightMode,
SlotPace: slotPace,
HttpTimeout: httpTimeout,
RpcUrl: rpcUrl,
ListenAddress: listenAddress,
NodeKeys: nodeKeys,
VoteKeys: voteKeys,
BalanceAddresses: balanceAddresses,
ComprehensiveSlotTracking: comprehensiveSlotTracking,
ComprehensiveVoteAccountTracking: comprehensiveVoteAccountTracking,
MonitorBlockSizes: monitorBlockSizes,
LightMode: lightMode,
SlotPace: slotPace,
}
return &config, nil
}

func NewExporterConfigFromCLI(ctx context.Context) (*ExporterConfig, error) {
var (
httpTimeout int
rpcUrl string
listenAddress string
nodekeys arrayFlags
balanceAddresses arrayFlags
comprehensiveSlotTracking bool
monitorBlockSizes bool
lightMode bool
slotPace int
httpTimeout int
rpcUrl string
listenAddress string
nodekeys arrayFlags
balanceAddresses arrayFlags
comprehensiveSlotTracking bool
comprehensiveVoteAccountTracking bool
monitorBlockSizes bool
lightMode bool
slotPace int
)
flag.IntVar(
&httpTimeout,
Expand Down Expand Up @@ -147,9 +156,16 @@ func NewExporterConfigFromCLI(ctx context.Context) (*ExporterConfig, error) {
&comprehensiveSlotTracking,
"comprehensive-slot-tracking",
false,
"Set this flag to track solana_leader_slots_by_epoch for ALL validators. "+
"Set this flag to track solana_validator_leader_slots_by_epoch for all validators. "+
"Warning: this will lead to potentially thousands of new Prometheus metrics being created every epoch.",
)
flag.BoolVar(
&comprehensiveVoteAccountTracking,
"comprehensive-vote-account-tracking",
false,
"Set this flag to track vote-account metrics such as solana_validator_active_stake for all validators. "+
"Warning: this will lead to potentially thousands of Prometheus metrics.",
)
flag.BoolVar(
&monitorBlockSizes,
"monitor-block-sizes",
Expand Down Expand Up @@ -181,6 +197,7 @@ func NewExporterConfigFromCLI(ctx context.Context) (*ExporterConfig, error) {
nodekeys,
balanceAddresses,
comprehensiveSlotTracking,
comprehensiveVoteAccountTracking,
monitorBlockSizes,
lightMode,
time.Duration(slotPace)*time.Second,
Expand Down
Loading
Loading