Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Improve witness metric #51

Merged
merged 5 commits into from
Apr 24, 2024
Merged

Improve witness metric #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
6dd8cd9
avoid false positive + print list of validators who failed to witness
marcellorigotti Apr 24, 2024
f4f0f55
add new metric to carry over the list of failing validators
marcellorigotti Apr 24, 2024
1358e3f
add cf_event_extrinsic_failed with the inner Error as label
marcellorigotti Apr 24, 2024
4542656
lint & prettier
marcellorigotti Apr 24, 2024
3913fc3
use toJSON instead of toHuman to avoid parsing to a valid ASCII char
marcellorigotti Apr 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 15 additions & 30 deletions src/metrics/chainflip/countEvents.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,11 +12,11 @@ const metric: Counter = new promClient.Counter({
registers: [],
});

const metricNameBroadcast: string = 'cf_broadcast_timeout_count';
const metricBroadcast: Gauge = new promClient.Gauge({
name: metricNameBroadcast,
help: 'Count of the broadcastTimeout events, grouped by broadcastId',
labelNames: ['event', 'broadcastId'],
const metricExtrinsicFailedName: string = 'cf_event_extrinsic_failed';
const metricExtrinsicFailed: Counter = new promClient.Counter({
name: metricExtrinsicFailedName,
help: 'Count of failed extrinsics on chain',
labelNames: ['pallet', 'error'],
registers: [],
});

Expand All @@ -28,16 +28,6 @@ const metricSlash: Counter = new promClient.Counter({
registers: [],
});

const errorMap = new Map();
errorMap.set(
`{"module":{"index":35,"error":"0x03000000"}}`,
'liquidityPools.UnspecifiedOrderPrice',
);
errorMap.set(
`{"module":{"index":31,"error":"0x00000000"}}`,
`liquidityProvider.InsufficientBalance`,
);

export const countEvents = async (context: Context): Promise<void> => {
const { logger, registry, events, api } = context;
const config = context.config as FlipConfig;
Expand All @@ -46,8 +36,8 @@ export const countEvents = async (context: Context): Promise<void> => {
logger.debug(`Scraping ${metricName}`);

if (registry.getSingleMetric(metricName) === undefined) registry.registerMetric(metric);
if (registry.getSingleMetric(metricNameBroadcast) === undefined)
registry.registerMetric(metricBroadcast);
if (registry.getSingleMetric(metricExtrinsicFailedName) === undefined)
registry.registerMetric(metricExtrinsicFailed);
if (registry.getSingleMetric(metricNameSlashing) === undefined)
registry.registerMetric(metricSlash);

Expand All @@ -64,16 +54,17 @@ export const countEvents = async (context: Context): Promise<void> => {
}
metric.labels(`${event.section}:${event.method}`).inc(1);

let error;
if (event.method === 'ExtrinsicFailed') {
error = await getStateChainError(api, event.data.toHuman().dispatchError.Module);
const parsedError = error.data.name.split(':');
metricExtrinsicFailed.labels(`${parsedError[0]}`, `${parsedError[1]}`).inc();
}

if (config.eventLog) {
if (event.data.dispatchError) {
const error = await getStateChainError(
api,
event.data.toHuman().dispatchError.Module,
);

const metadata = { error };
logger.info('event_log', {
metadata,
error,
event: `${event.section}:${event.method}`,
data: event.data.toHuman(),
});
Expand All @@ -84,12 +75,6 @@ export const countEvents = async (context: Context): Promise<void> => {
});
}
}
// Set extra labels for specific events
if (event.method === 'BroadcastAttemptTimeout') {
metricBroadcast
.labels(`${event.section}:${event.method}`, event.data.broadcastId)
.set(event.data.attemptCount);
}
if (event.method === 'SlashingPerformed') {
for (const { ss58Address, alias } of accounts) {
const hex = `0x${Buffer.from(decodeAddress(ss58Address)).toString('hex')}`;
Expand Down
90 changes: 80 additions & 10 deletions src/metrics/chainflip/gaugeWitnessChainTracking.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,11 @@ import promClient, { Gauge } from 'prom-client';
import { Context } from '../../lib/interfaces';
import { blake2AsHex } from '@polkadot/util-crypto';
import { hex2bin, insertOrReplace } from '../../utils/utils';
import { customRpc } from '../../utils/makeRpcRequest';

const witnessHash10 = new Map<number, Set<string>>();
const witnessHash50 = new Map<number, Set<string>>();
const toDelete = new Map<string, number>();

const metricName: string = 'cf_chain_tracking_witness_count';
const metric: Gauge = new promClient.Gauge({
Expand All @@ -14,19 +16,40 @@ const metric: Gauge = new promClient.Gauge({
registers: [],
});

const metricFailureName: string = 'cf_chain_tracking_witness_failure';
const metricWitnessFailure: Gauge = new promClient.Gauge({
name: metricFailureName,
help: 'If 1 the number of witnesses is low, you can find the failing validators in the label `failing_validators`',
labelNames: ['extrinsic', 'failing_validators', 'witnessed_by'],
registers: [],
});

export const gaugeWitnessChainTracking = async (context: Context): Promise<void> => {
if (global.epochIndex) {
const { logger, api, registry, metricFailure } = context;
logger.debug(`Scraping ${metricName}`);

if (registry.getSingleMetric(metricName) === undefined) registry.registerMetric(metric);
if (registry.getSingleMetric(metricFailureName) === undefined)
registry.registerMetric(metricWitnessFailure);
metricFailure.labels({ metric: metricName }).set(0);
try {
const signedBlock = await api.rpc.chain.getBlock();
const currentBlockNumber = Number(
signedBlock.block.header.number.toHuman().replace(/,/g, ''),
);
global.currentBlock = currentBlockNumber;
toDelete.forEach((block, labels) => {
if (block <= currentBlockNumber) {
const values = JSON.parse(labels);
metricWitnessFailure.remove(
values.extrinsic,
values.validators,
values.witnessedBy,
);
toDelete.delete(labels);
}
});
for (const [blockNumber, set] of witnessHash10) {
if (currentBlockNumber - blockNumber > 10) {
const tmpSet = new Set(set);
Expand All @@ -35,19 +58,66 @@ export const gaugeWitnessChainTracking = async (context: Context): Promise<void>
const parsedObj = JSON.parse(hash);
api.query.witnesser
.votes(global.epochIndex, parsedObj.hash)
.then((votes: { toHuman: () => any }) => {
.then(async (votes: { toJSON: () => any }) => {
if (global.currentBlock === currentBlockNumber) {
const vote = votes.toHuman();
const vote = votes.toJSON();
if (vote) {
const binary = hex2bin(vote);
const number = binary.match(/1/g)?.length || 0;
let total = binary.match(/1/g)?.length || 0;
// check the previous epoch as well! could be a false positive after rotation!
if (total < global.currentAuthorities * 0.1) {
const previousEpochVote = (
await api.query.witnesser.votes(
global.epochIndex - 1,
parsedObj.hash,
)
).toJSON();
total +=
hex2bin(previousEpochVote).match(/1/g)?.length || 0;
}
metric.labels(parsedObj.type, '10').set(total);

metric.labels(parsedObj.type, '10').set(number);
// log the hash if not all the validator witnessed it so we can quickly look up the hash and check which validator failed to do so
if (number < global.currentAuthorities) {
logger.info(
`Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${number} validators after 10 blocks!`,
);
if (total < global.currentAuthorities) {
// log the list of validators if the total is below 90%
if (total <= global.currentAuthorities * 0.9) {
const votes = await customRpc(
api,
'witness_count',
parsedObj.hash,
);
const validators: string[] = [];
votes.validators.forEach(
([ss58address, vanity, witness]) => {
if (!witness) {
validators.push(ss58address);
}
},
);
logger.info(
`Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks!
Validators: [${validators}]`,
);
metricWitnessFailure
.labels(
`${parsedObj.type}`,
`${validators}`,
`${total}`,
)
.set(1);
toDelete.set(
JSON.stringify({
extrinsic: `${parsedObj.type}`,
validators: `${validators}`,
witnessedBy: `${total}`,
}),
currentBlockNumber + 50,
);
} else {
logger.info(
`Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks!`,
);
}
}
}
}
Expand All @@ -63,9 +133,9 @@ export const gaugeWitnessChainTracking = async (context: Context): Promise<void>
const parsedObj = JSON.parse(hash);
api.query.witnesser
.votes(global.epochIndex, parsedObj.hash)
.then((votes: { toHuman: () => any }) => {
.then((votes: { toJSON: () => any }) => {
if (global.currentBlock === currentBlockNumber) {
const vote = votes.toHuman();
const vote = votes.toJSON();
if (vote) {
const binary = hex2bin(vote);
const number = binary.match(/1/g)?.length || 0;
Expand Down
81 changes: 70 additions & 11 deletions src/metrics/chainflip/gaugeWitnessCount.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
import promClient, { Gauge } from 'prom-client';
import { Context } from '../../lib/interfaces';
import { hex2bin, insertOrReplace } from '../../utils/utils';
import { customRpc } from '../../utils/makeRpcRequest';

const witnessExtrinsicHash10 = new Map<number, Set<string>>();
const witnessExtrinsicHash50 = new Map<number, Set<string>>();
const toDelete = new Map<string, number>();

const metricName: string = 'cf_witness_count';
const metric: Gauge = new promClient.Gauge({
Expand All @@ -13,18 +15,40 @@ const metric: Gauge = new promClient.Gauge({
registers: [],
});

const metricFailureName: string = 'cf_witness_count_failure';
const metricWitnessFailure: Gauge = new promClient.Gauge({
name: metricFailureName,
help: 'If 1 the number of witnesses is low, you can find the failing validators in the label `failing_validators`',
labelNames: ['extrinsic', 'failing_validators', 'witnessed_by'],
registers: [],
});

export const gaugeWitnessCount = async (context: Context): Promise<void> => {
if (global.epochIndex) {
const { logger, api, registry, metricFailure, header } = context;
logger.debug(`Scraping ${metricName}`);

if (registry.getSingleMetric(metricName) === undefined) registry.registerMetric(metric);
if (registry.getSingleMetric(metricFailureName) === undefined)
registry.registerMetric(metricWitnessFailure);

metricFailure.labels({ metric: metricName }).set(0);
try {
const signedBlock = await api.rpc.chain.getBlock();
const currentBlockNumber = Number(
signedBlock.block.header.number.toHuman().replace(/,/g, ''),
);
toDelete.forEach((block, labels) => {
if (block <= currentBlockNumber) {
const values = JSON.parse(labels);
metricWitnessFailure.remove(
values.extrinsic,
values.validators,
values.witnessedBy,
);
toDelete.delete(labels);
}
});
for (const [blockNumber, set] of witnessExtrinsicHash10) {
if (currentBlockNumber - blockNumber > 10) {
const tmpSet = new Set(set);
Expand All @@ -33,18 +57,53 @@ export const gaugeWitnessCount = async (context: Context): Promise<void> => {
const parsedObj = JSON.parse(hash);
api.query.witnesser
.votes(global.epochIndex, parsedObj.hash)
.then((votes: { toHuman: () => any }) => {
const vote = votes.toHuman();
.then(async (votes: { toJSON: () => any }) => {
const vote = votes.toJSON();
if (vote) {
const binary = hex2bin(vote);
const number = binary.match(/1/g)?.length || 0;
const total = binary.match(/1/g)?.length || 0;

metric.labels(parsedObj.type, '10').set(number);
// log the hash if not all the validator witnessed it so we can quickly look up the hash and check which validator failed to do so
if (number < global.currentAuthorities) {
logger.info(
`Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${number} validators after 10 blocks!`,
);
metric.labels(parsedObj.type, '10').set(total);
if (total < global.currentAuthorities) {
// log the list of validators if the total is below 90%
if (total <= global.currentAuthorities * 0.67) {
const votes = await customRpc(
api,
'witness_count',
parsedObj.hash,
);
const validators: string[] = [];
votes.validators.forEach(
([ss58address, vanity, witness]) => {
if (!witness) {
validators.push(ss58address);
}
},
);
logger.info(
`Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks!
Validators: [${validators}]`,
);
metricWitnessFailure
.labels(
`${parsedObj.type}`,
`${validators}`,
`${total}`,
)
.set(1);
toDelete.set(
JSON.stringify({
extrinsic: `${parsedObj.type}`,
validators: `${validators}`,
witnessedBy: `${total}`,
}),
currentBlockNumber + 20,
);
} else {
logger.info(
`Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks!`,
);
}
}
}
});
Expand All @@ -59,8 +118,8 @@ export const gaugeWitnessCount = async (context: Context): Promise<void> => {
const parsedObj = JSON.parse(hash);
api.query.witnesser
.votes(global.epochIndex, parsedObj.hash)
.then((votes: { toHuman: () => any }) => {
const vote = votes.toHuman();
.then((votes: { toJSON: () => any }) => {
const vote = votes.toJSON();
if (vote) {
const binary = hex2bin(vote);
const number = binary.match(/1/g)?.length || 0;
Expand Down
4 changes: 4 additions & 0 deletions src/utils/chainTypes.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,10 @@ const stateChainTypes = {
},
Version: 'SemVer',
VoteCount: 'u32',
RpcFailingWitnessValidators: {
failing_count: 'u32',
validators: 'Vec<(ValidatorId, Vec<u8>, bool)>',
},
} as const;

export default stateChainTypes;
Loading
Loading