Skip to content

Commit

Permalink
Merge pull request #75 from asymmetric-research/more-config
Browse files Browse the repository at this point in the history 
Added more configurability around vote account tracking
  • Loading branch information
@johnstonematt
johnstonematt authored Nov 15, 2024
2 parents 2577dc6 + de9da46 commit a084b30
Show file tree
Hide file tree
Showing 7 changed files with 277 additions and 148 deletions.
117 changes: 69 additions & 48 deletions README.md
View file

Large diffs are not rendered by default.

81 changes: 73 additions & 8 deletions cmd/solana-exporter/collector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,12 @@ import (
"github.com/asymmetric-research/solana-exporter/pkg/slog"
"github.com/prometheus/client_golang/prometheus"
"go.uber.org/zap"
"slices"
)

const (
SkipStatusLabel = "status"
StateLabel = "state"
NodekeyLabel = "nodekey"
VotekeyLabel = "votekey"
VersionLabel = "version"
Expand All @@ -22,6 +24,9 @@ const (
StatusSkipped = "skipped"
StatusValid = "valid"

StateCurrent = "current"
StateDelinquent = "delinquent"

TransactionTypeVote = "vote"
TransactionTypeNonVote = "non_vote"
)
Expand All @@ -34,9 +39,13 @@ type SolanaCollector struct {

/// descriptors:
ValidatorActiveStake *GaugeDesc
ClusterActiveStake *GaugeDesc
ValidatorLastVote *GaugeDesc
ClusterLastVote *GaugeDesc
ValidatorRootSlot *GaugeDesc
ClusterRootSlot *GaugeDesc
ValidatorDelinquent *GaugeDesc
ClusterValidatorCount *GaugeDesc
AccountBalances *GaugeDesc
NodeVersion *GaugeDesc
NodeIsHealthy *GaugeDesc
Expand All @@ -55,21 +64,41 @@ func NewSolanaCollector(client *rpc.Client, config *ExporterConfig) *SolanaColle
fmt.Sprintf("Active stake (in SOL) per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel),
VotekeyLabel, NodekeyLabel,
),
ClusterActiveStake: NewGaugeDesc(
"solana_cluster_active_stake",
"Total active stake (in SOL) of the cluster",
),
ValidatorLastVote: NewGaugeDesc(
"solana_validator_last_vote",
fmt.Sprintf("Last voted-on slot per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel),
VotekeyLabel, NodekeyLabel,
),
ClusterLastVote: NewGaugeDesc(
"solana_cluster_last_vote",
"Most recent voted-on slot of the cluster",
),
ValidatorRootSlot: NewGaugeDesc(
"solana_validator_root_slot",
fmt.Sprintf("Root slot per validator (represented by %s and %s)", VotekeyLabel, NodekeyLabel),
VotekeyLabel, NodekeyLabel,
),
ClusterRootSlot: NewGaugeDesc(
"solana_cluster_root_slot",
"Max root slot of the cluster",
),
ValidatorDelinquent: NewGaugeDesc(
"solana_validator_delinquent",
fmt.Sprintf("Whether a validator (represented by %s and %s) is delinquent", VotekeyLabel, NodekeyLabel),
VotekeyLabel, NodekeyLabel,
),
ClusterValidatorCount: NewGaugeDesc(
"solana_cluster_validator_count",
fmt.Sprintf(
"Total number of validators in the cluster, grouped by %s ('%s' or '%s')",
StateLabel, StateCurrent, StateDelinquent,
),
StateLabel,
),
AccountBalances: NewGaugeDesc(
"solana_account_balance",
fmt.Sprintf("Solana account balances, grouped by %s", AddressLabel),
Expand Down Expand Up @@ -103,9 +132,13 @@ func NewSolanaCollector(client *rpc.Client, config *ExporterConfig) *SolanaColle
func (c *SolanaCollector) Describe(ch chan<- *prometheus.Desc) {
ch <- c.NodeVersion.Desc
ch <- c.ValidatorActiveStake.Desc
ch <- c.ClusterActiveStake.Desc
ch <- c.ValidatorLastVote.Desc
ch <- c.ClusterLastVote.Desc
ch <- c.ValidatorRootSlot.Desc
ch <- c.ClusterRootSlot.Desc
ch <- c.ValidatorDelinquent.Desc
ch <- c.ClusterValidatorCount.Desc
ch <- c.AccountBalances.Desc
ch <- c.NodeIsHealthy.Desc
ch <- c.NodeNumSlotsBehind.Desc
Expand All @@ -123,26 +156,58 @@ func (c *SolanaCollector) collectVoteAccounts(ctx context.Context, ch chan<- pro
if err != nil {
c.logger.Errorf("failed to get vote accounts: %v", err)
ch <- c.ValidatorActiveStake.NewInvalidMetric(err)
ch <- c.ClusterActiveStake.NewInvalidMetric(err)
ch <- c.ValidatorLastVote.NewInvalidMetric(err)
ch <- c.ClusterLastVote.NewInvalidMetric(err)
ch <- c.ValidatorRootSlot.NewInvalidMetric(err)
ch <- c.ClusterRootSlot.NewInvalidMetric(err)
ch <- c.ValidatorDelinquent.NewInvalidMetric(err)
ch <- c.ClusterValidatorCount.NewInvalidMetric(err)
return
}

var (
totalStake float64
maxLastVote float64
maxRootSlot float64
)
for _, account := range append(voteAccounts.Current, voteAccounts.Delinquent...) {
accounts := []string{account.VotePubkey, account.NodePubkey}
ch <- c.ValidatorActiveStake.MustNewConstMetric(float64(account.ActivatedStake)/rpc.LamportsInSol, accounts...)
ch <- c.ValidatorLastVote.MustNewConstMetric(float64(account.LastVote), accounts...)
ch <- c.ValidatorRootSlot.MustNewConstMetric(float64(account.RootSlot), accounts...)
}
stake, lastVote, rootSlot :=
float64(account.ActivatedStake)/rpc.LamportsInSol,
float64(account.LastVote),
float64(account.RootSlot)

for _, account := range voteAccounts.Current {
ch <- c.ValidatorDelinquent.MustNewConstMetric(0, account.VotePubkey, account.NodePubkey)
if slices.Contains(c.config.NodeKeys, account.NodePubkey) || c.config.ComprehensiveVoteAccountTracking {
ch <- c.ValidatorActiveStake.MustNewConstMetric(stake, accounts...)
ch <- c.ValidatorLastVote.MustNewConstMetric(lastVote, accounts...)
ch <- c.ValidatorRootSlot.MustNewConstMetric(rootSlot, accounts...)
}

totalStake += stake
maxLastVote = max(maxLastVote, lastVote)
maxRootSlot = max(maxRootSlot, rootSlot)
}
for _, account := range voteAccounts.Delinquent {
ch <- c.ValidatorDelinquent.MustNewConstMetric(1, account.VotePubkey, account.NodePubkey)

{
for _, account := range voteAccounts.Current {
if slices.Contains(c.config.NodeKeys, account.NodePubkey) || c.config.ComprehensiveVoteAccountTracking {
ch <- c.ValidatorDelinquent.MustNewConstMetric(0, account.VotePubkey, account.NodePubkey)
}
}
for _, account := range voteAccounts.Delinquent {
if slices.Contains(c.config.NodeKeys, account.NodePubkey) || c.config.ComprehensiveVoteAccountTracking {
ch <- c.ValidatorDelinquent.MustNewConstMetric(1, account.VotePubkey, account.NodePubkey)
}
}
}

ch <- c.ClusterActiveStake.MustNewConstMetric(totalStake)
ch <- c.ClusterLastVote.MustNewConstMetric(maxLastVote)
ch <- c.ClusterRootSlot.MustNewConstMetric(maxRootSlot)
ch <- c.ClusterValidatorCount.MustNewConstMetric(float64(len(voteAccounts.Current)), StateCurrent)
ch <- c.ClusterValidatorCount.MustNewConstMetric(float64(len(voteAccounts.Delinquent)), StateDelinquent)

c.logger.Info("Vote accounts collected.")
}

Expand Down
34 changes: 27 additions & 7 deletions cmd/solana-exporter/collector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,8 @@ type (
Votekeys []string
FeeRewardLamports int
InflationRewardLamports int
LastVoteDistances map[string]int
RootSlotDistances map[string]int
}
)

Expand Down Expand Up @@ -86,6 +88,8 @@ func NewSimulator(t *testing.T, slot int) (*Simulator, *rpc.Client) {
Votekeys: votekeys,
InflationRewardLamports: inflationRewardLamports,
FeeRewardLamports: feeRewardLamports,
LastVoteDistances: map[string]int{"aaa": 1, "bbb": 2, "ccc": 3},
RootSlotDistances: map[string]int{"aaa": 4, "bbb": 5, "ccc": 6},
}
simulator.PopulateSlot(0)
if slot > 0 {
Expand Down Expand Up @@ -144,7 +148,8 @@ func (c *Simulator) PopulateSlot(slot int) {
for _, nodekey := range c.Nodekeys {
transactions = append(transactions, []string{nodekey, strings.ToUpper(nodekey), VoteProgram})
info := c.Server.GetValidatorInfo(nodekey)
info.LastVote = slot
info.LastVote = max(0, slot-c.LastVoteDistances[nodekey])
info.RootSlot = max(0, slot-c.RootSlotDistances[nodekey])
c.Server.SetOpt(rpc.ValidatorInfoOpt, nodekey, info)
}

Expand Down Expand Up @@ -199,6 +204,7 @@ func newTestConfig(simulator *Simulator, fast bool) *ExporterConfig {
nil,
true,
true,
true,
false,
pace,
}
Expand All @@ -218,21 +224,34 @@ func TestSolanaCollector(t *testing.T) {
NewLV(stake, "bbb", "BBB"),
NewLV(stake, "ccc", "CCC"),
),
collector.ClusterActiveStake.makeCollectionTest(
NewLV(3 * stake),
),
collector.ValidatorLastVote.makeCollectionTest(
NewLV(34, "aaa", "AAA"),
NewLV(34, "bbb", "BBB"),
NewLV(34, "ccc", "CCC"),
NewLV(33, "aaa", "AAA"),
NewLV(32, "bbb", "BBB"),
NewLV(31, "ccc", "CCC"),
),
collector.ClusterLastVote.makeCollectionTest(
NewLV(33),
),
collector.ValidatorRootSlot.makeCollectionTest(
NewLV(0, "aaa", "AAA"),
NewLV(0, "bbb", "BBB"),
NewLV(0, "ccc", "CCC"),
NewLV(30, "aaa", "AAA"),
NewLV(29, "bbb", "BBB"),
NewLV(28, "ccc", "CCC"),
),
collector.ClusterRootSlot.makeCollectionTest(
NewLV(30),
),
collector.ValidatorDelinquent.makeCollectionTest(
NewLV(0, "aaa", "AAA"),
NewLV(0, "bbb", "BBB"),
NewLV(0, "ccc", "CCC"),
),
collector.ClusterValidatorCount.makeCollectionTest(
NewLV(3, StateCurrent),
NewLV(0, StateDelinquent),
),
collector.NodeVersion.makeCollectionTest(
NewLV(1, "v1.0.0"),
),
Expand All @@ -258,6 +277,7 @@ func TestSolanaCollector(t *testing.T) {
),
}

fmt.Println(testCases[1].ExpectedResponse)
for _, test := range testCases {
t.Run(test.Name, func(t *testing.T) {
err := testutil.CollectAndCompare(collector, bytes.NewBufferString(test.ExpectedResponse), test.Name)
Expand Down
77 changes: 47 additions & 30 deletions cmd/solana-exporter/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,17 @@ type (
arrayFlags []string

ExporterConfig struct {
HttpTimeout time.Duration
RpcUrl string
ListenAddress string
NodeKeys []string
VoteKeys []string
BalanceAddresses []string
ComprehensiveSlotTracking bool
MonitorBlockSizes bool
LightMode bool
SlotPace time.Duration
HttpTimeout time.Duration
RpcUrl string
ListenAddress string
NodeKeys []string
VoteKeys []string
BalanceAddresses []string
ComprehensiveSlotTracking bool
ComprehensiveVoteAccountTracking bool
MonitorBlockSizes bool
LightMode bool
SlotPace time.Duration
}
)

Expand All @@ -43,6 +44,7 @@ func NewExporterConfig(
nodeKeys []string,
balanceAddresses []string,
comprehensiveSlotTracking bool,
comprehensiveVoteAccountTracking bool,
monitorBlockSizes bool,
lightMode bool,
slotPace time.Duration,
Expand All @@ -56,6 +58,7 @@ func NewExporterConfig(
"nodeKeys", nodeKeys,
"balanceAddresses", balanceAddresses,
"comprehensiveSlotTracking", comprehensiveSlotTracking,
"comprehensiveVoteAccountTracking", comprehensiveVoteAccountTracking,
"monitorBlockSizes", monitorBlockSizes,
"lightMode", lightMode,
)
Expand All @@ -64,6 +67,10 @@ func NewExporterConfig(
return nil, fmt.Errorf("'-light-mode' is incompatible with `-comprehensive-slot-tracking`")
}

if comprehensiveVoteAccountTracking {
return nil, fmt.Errorf("'-light-mode' is incompatible with '-comprehensive-vote-account-tracking'")
}

if monitorBlockSizes {
return nil, fmt.Errorf("'-light-mode' is incompatible with `-monitor-block-sizes`")
}
Expand All @@ -87,31 +94,33 @@ func NewExporterConfig(
}

config := ExporterConfig{
HttpTimeout: httpTimeout,
RpcUrl: rpcUrl,
ListenAddress: listenAddress,
NodeKeys: nodeKeys,
VoteKeys: voteKeys,
BalanceAddresses: balanceAddresses,
ComprehensiveSlotTracking: comprehensiveSlotTracking,
MonitorBlockSizes: monitorBlockSizes,
LightMode: lightMode,
SlotPace: slotPace,
HttpTimeout: httpTimeout,
RpcUrl: rpcUrl,
ListenAddress: listenAddress,
NodeKeys: nodeKeys,
VoteKeys: voteKeys,
BalanceAddresses: balanceAddresses,
ComprehensiveSlotTracking: comprehensiveSlotTracking,
ComprehensiveVoteAccountTracking: comprehensiveVoteAccountTracking,
MonitorBlockSizes: monitorBlockSizes,
LightMode: lightMode,
SlotPace: slotPace,
}
return &config, nil
}

func NewExporterConfigFromCLI(ctx context.Context) (*ExporterConfig, error) {
var (
httpTimeout int
rpcUrl string
listenAddress string
nodekeys arrayFlags
balanceAddresses arrayFlags
comprehensiveSlotTracking bool
monitorBlockSizes bool
lightMode bool
slotPace int
httpTimeout int
rpcUrl string
listenAddress string
nodekeys arrayFlags
balanceAddresses arrayFlags
comprehensiveSlotTracking bool
comprehensiveVoteAccountTracking bool
monitorBlockSizes bool
lightMode bool
slotPace int
)
flag.IntVar(
&httpTimeout,
Expand Down Expand Up @@ -147,9 +156,16 @@ func NewExporterConfigFromCLI(ctx context.Context) (*ExporterConfig, error) {
&comprehensiveSlotTracking,
"comprehensive-slot-tracking",
false,
"Set this flag to track solana_leader_slots_by_epoch for ALL validators. "+
"Set this flag to track solana_validator_leader_slots_by_epoch for all validators. "+
"Warning: this will lead to potentially thousands of new Prometheus metrics being created every epoch.",
)
flag.BoolVar(
&comprehensiveVoteAccountTracking,
"comprehensive-vote-account-tracking",
false,
"Set this flag to track vote-account metrics such as solana_validator_active_stake for all validators. "+
"Warning: this will lead to potentially thousands of Prometheus metrics.",
)
flag.BoolVar(
&monitorBlockSizes,
"monitor-block-sizes",
Expand Down Expand Up @@ -181,6 +197,7 @@ func NewExporterConfigFromCLI(ctx context.Context) (*ExporterConfig, error) {
nodekeys,
balanceAddresses,
comprehensiveSlotTracking,
comprehensiveVoteAccountTracking,
monitorBlockSizes,
lightMode,
time.Duration(slotPace)*time.Second,
Expand Down
Loading

0 comments on commit a084b30

Please sign in to comment.