From 6dd8cd95b0b0e69350f785b8e0361b44948933b6 Mon Sep 17 00:00:00 2001 From: marcellorigotti Date: Wed, 24 Apr 2024 12:27:17 +0200 Subject: [PATCH 1/5] avoid false positive + print list of validators who failed to witness --- .../chainflip/gaugeWitnessChainTracking.ts | 47 ++++++++++++++++--- src/utils/chainTypes.ts | 4 ++ src/utils/customRpcSpecification.ts | 10 ++++ src/utils/makeRpcRequest.ts | 26 ++++++++++ 4 files changed, 80 insertions(+), 7 deletions(-) diff --git a/src/metrics/chainflip/gaugeWitnessChainTracking.ts b/src/metrics/chainflip/gaugeWitnessChainTracking.ts index ad0572b..cdce0d3 100644 --- a/src/metrics/chainflip/gaugeWitnessChainTracking.ts +++ b/src/metrics/chainflip/gaugeWitnessChainTracking.ts @@ -2,6 +2,7 @@ import promClient, { Gauge } from 'prom-client'; import { Context } from '../../lib/interfaces'; import { blake2AsHex } from '@polkadot/util-crypto'; import { hex2bin, insertOrReplace } from '../../utils/utils'; +import { customRpc } from '../../utils/makeRpcRequest'; const witnessHash10 = new Map>(); const witnessHash50 = new Map>(); @@ -35,19 +36,51 @@ export const gaugeWitnessChainTracking = async (context: Context): Promise const parsedObj = JSON.parse(hash); api.query.witnesser .votes(global.epochIndex, parsedObj.hash) - .then((votes: { toHuman: () => any }) => { + .then(async (votes: { toHuman: () => any }) => { if (global.currentBlock === currentBlockNumber) { const vote = votes.toHuman(); if (vote) { const binary = hex2bin(vote); - const number = binary.match(/1/g)?.length || 0; + let total = binary.match(/1/g)?.length || 0; + // check the previous epoch as well! could be a false positive after rotation! + if (total < global.currentAuthorities * 0.1) { + const previousEpochVote = ( + await api.query.witnesser.votes( + global.epochIndex - 1, + parsedObj.hash, + ) + ).toHuman(); + total += + hex2bin(previousEpochVote).match(/1/g)?.length || 0; + } + metric.labels(parsedObj.type, '10').set(total); - metric.labels(parsedObj.type, '10').set(number); // log the hash if not all the validator witnessed it so we can quickly look up the hash and check which validator failed to do so - if (number < global.currentAuthorities) { - logger.info( - `Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${number} validators after 10 blocks!`, - ); + if (total < global.currentAuthorities) { + // log the list of validators if the total is below 85% + if (total < global.currentAuthorities * 0.85) { + const votes = await customRpc( + api, + 'witness_count', + parsedObj.hash, + ); + const validators: string[] = []; + votes.validators.forEach( + ([ss58address, vanity, witness]) => { + if (!witness) { + validators.push(ss58address); + } + }, + ); + logger.info( + `Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks! + Validators: [${validators}]`, + ); + } else { + logger.info( + `Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks!`, + ); + } } } } diff --git a/src/utils/chainTypes.ts b/src/utils/chainTypes.ts index 8fbc64b..104642f 100644 --- a/src/utils/chainTypes.ts +++ b/src/utils/chainTypes.ts @@ -299,6 +299,10 @@ const stateChainTypes = { }, Version: 'SemVer', VoteCount: 'u32', + RpcFailingWitnessValidators: { + failing_count: 'u32', + validators: 'Vec<(ValidatorId, Vec, bool)>', + }, } as const; export default stateChainTypes; diff --git a/src/utils/customRpcSpecification.ts b/src/utils/customRpcSpecification.ts index 9a6df9d..daee8db 100644 --- a/src/utils/customRpcSpecification.ts +++ b/src/utils/customRpcSpecification.ts @@ -113,5 +113,15 @@ export const customRpcs = { type: 'RpcAuctionState', description: '', }, + witness_count: { + params: [ + { + name: 'hash', + type: 'String', + }, + ], + type: 'RpcFailingWitnessValidators', + description: '', + }, }, }; diff --git a/src/utils/makeRpcRequest.ts b/src/utils/makeRpcRequest.ts index c0312fd..52a5415 100644 --- a/src/utils/makeRpcRequest.ts +++ b/src/utils/makeRpcRequest.ts @@ -1,6 +1,8 @@ import { z } from 'zod'; import stateChainTypes from './chainTypes'; import { customRpcs } from './customRpcSpecification'; +import axios from 'axios'; +import { env } from '../config/getConfig'; const boolean = z.boolean(); const string = z.string(); @@ -76,6 +78,10 @@ const validators = { penalties: z.array(z.tuple([Offence, RpcPenalty])), suspensions: z.array(RpcSuspension), tx_fee_multiplier: Amount, + witness_count: z.object({ + failing_count: U32, + validators: z.array(z.tuple([string, string, boolean])), + }), } as const; type RpcParamsMap = { @@ -90,6 +96,7 @@ type RpcParamsMap = { eth_key_manager_address: []; eth_state_chain_gateway_address: []; flip_supply: []; + witness_count: [hash: string]; }; type RpcCall = keyof RpcParamsMap & keyof typeof validators & keyof typeof customRpcs.cf; @@ -110,3 +117,22 @@ export default async function makeRpcRequest( return parsed as RpcReturnValue[M]; } + +export async function customRpc( + apiPromise: CustomApiPromise, + method: M, + ...args: RpcParamsMap[M] +): Promise { + const url = env.CF_WS_ENDPOINT.split('wss'); + + const { data } = await axios.post(`https${url[1]}`, { + id: 1, + jsonrpc: '2.0', + method: `cf_${method}`, + params: args, + }); + + const parsed = validators[method].parse(data.result); + + return parsed as RpcReturnValue[M]; +} From f4f0f558645812ef015ac241566279d266920839 Mon Sep 17 00:00:00 2001 From: marcellorigotti Date: Wed, 24 Apr 2024 13:49:41 +0200 Subject: [PATCH 2/5] add new metric to carry over the list of failing validators --- .../chainflip/gaugeWitnessChainTracking.ts | 41 +++++++++- src/metrics/chainflip/gaugeWitnessCount.ts | 75 +++++++++++++++++-- 2 files changed, 106 insertions(+), 10 deletions(-) diff --git a/src/metrics/chainflip/gaugeWitnessChainTracking.ts b/src/metrics/chainflip/gaugeWitnessChainTracking.ts index cdce0d3..1e2c502 100644 --- a/src/metrics/chainflip/gaugeWitnessChainTracking.ts +++ b/src/metrics/chainflip/gaugeWitnessChainTracking.ts @@ -6,6 +6,7 @@ import { customRpc } from '../../utils/makeRpcRequest'; const witnessHash10 = new Map>(); const witnessHash50 = new Map>(); +const toDelete = new Map(); const metricName: string = 'cf_chain_tracking_witness_count'; const metric: Gauge = new promClient.Gauge({ @@ -15,12 +16,22 @@ const metric: Gauge = new promClient.Gauge({ registers: [], }); +const metricFailureName: string = 'cf_chain_tracking_witness_failure'; +const metricWitnessFailure: Gauge = new promClient.Gauge({ + name: metricFailureName, + help: 'If 1 the number of witnesses is low, you can find the failing validators in the label `failing_validators`', + labelNames: ['extrinsic', 'failing_validators', 'witnessed_by'], + registers: [], +}); + export const gaugeWitnessChainTracking = async (context: Context): Promise => { if (global.epochIndex) { const { logger, api, registry, metricFailure } = context; logger.debug(`Scraping ${metricName}`); if (registry.getSingleMetric(metricName) === undefined) registry.registerMetric(metric); + if (registry.getSingleMetric(metricFailureName) === undefined) + registry.registerMetric(metricWitnessFailure); metricFailure.labels({ metric: metricName }).set(0); try { const signedBlock = await api.rpc.chain.getBlock(); @@ -28,6 +39,17 @@ export const gaugeWitnessChainTracking = async (context: Context): Promise signedBlock.block.header.number.toHuman().replace(/,/g, ''), ); global.currentBlock = currentBlockNumber; + toDelete.forEach((block, labels) => { + if (block <= currentBlockNumber) { + const values = JSON.parse(labels); + metricWitnessFailure.remove( + values.extrinsic, + values.validators, + values.witnessedBy, + ); + toDelete.delete(labels); + } + }); for (const [blockNumber, set] of witnessHash10) { if (currentBlockNumber - blockNumber > 10) { const tmpSet = new Set(set); @@ -57,8 +79,8 @@ export const gaugeWitnessChainTracking = async (context: Context): Promise // log the hash if not all the validator witnessed it so we can quickly look up the hash and check which validator failed to do so if (total < global.currentAuthorities) { - // log the list of validators if the total is below 85% - if (total < global.currentAuthorities * 0.85) { + // log the list of validators if the total is below 90% + if (total <= global.currentAuthorities * 0.9) { const votes = await customRpc( api, 'witness_count', @@ -76,6 +98,21 @@ export const gaugeWitnessChainTracking = async (context: Context): Promise `Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks! Validators: [${validators}]`, ); + metricWitnessFailure + .labels( + `${parsedObj.type}`, + `${validators}`, + `${total}`, + ) + .set(1); + toDelete.set( + JSON.stringify({ + extrinsic: `${parsedObj.type}`, + validators: `${validators}`, + witnessedBy: `${total}`, + }), + currentBlockNumber + 50, + ); } else { logger.info( `Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks!`, diff --git a/src/metrics/chainflip/gaugeWitnessCount.ts b/src/metrics/chainflip/gaugeWitnessCount.ts index e258578..cd61b9f 100644 --- a/src/metrics/chainflip/gaugeWitnessCount.ts +++ b/src/metrics/chainflip/gaugeWitnessCount.ts @@ -1,9 +1,11 @@ import promClient, { Gauge } from 'prom-client'; import { Context } from '../../lib/interfaces'; import { hex2bin, insertOrReplace } from '../../utils/utils'; +import { customRpc } from '../../utils/makeRpcRequest'; const witnessExtrinsicHash10 = new Map>(); const witnessExtrinsicHash50 = new Map>(); +const toDelete = new Map(); const metricName: string = 'cf_witness_count'; const metric: Gauge = new promClient.Gauge({ @@ -13,18 +15,40 @@ const metric: Gauge = new promClient.Gauge({ registers: [], }); +const metricFailureName: string = 'cf_witness_count_failure'; +const metricWitnessFailure: Gauge = new promClient.Gauge({ + name: metricFailureName, + help: 'If 1 the number of witnesses is low, you can find the failing validators in the label `failing_validators`', + labelNames: ['extrinsic', 'failing_validators', 'witnessed_by'], + registers: [], +}); + export const gaugeWitnessCount = async (context: Context): Promise => { if (global.epochIndex) { const { logger, api, registry, metricFailure, header } = context; logger.debug(`Scraping ${metricName}`); if (registry.getSingleMetric(metricName) === undefined) registry.registerMetric(metric); + if (registry.getSingleMetric(metricFailureName) === undefined) + registry.registerMetric(metricWitnessFailure); + metricFailure.labels({ metric: metricName }).set(0); try { const signedBlock = await api.rpc.chain.getBlock(); const currentBlockNumber = Number( signedBlock.block.header.number.toHuman().replace(/,/g, ''), ); + toDelete.forEach((block, labels) => { + if (block <= currentBlockNumber) { + const values = JSON.parse(labels); + metricWitnessFailure.remove( + values.extrinsic, + values.validators, + values.witnessedBy, + ); + toDelete.delete(labels); + } + }); for (const [blockNumber, set] of witnessExtrinsicHash10) { if (currentBlockNumber - blockNumber > 10) { const tmpSet = new Set(set); @@ -33,18 +57,53 @@ export const gaugeWitnessCount = async (context: Context): Promise => { const parsedObj = JSON.parse(hash); api.query.witnesser .votes(global.epochIndex, parsedObj.hash) - .then((votes: { toHuman: () => any }) => { + .then(async (votes: { toHuman: () => any }) => { const vote = votes.toHuman(); if (vote) { const binary = hex2bin(vote); - const number = binary.match(/1/g)?.length || 0; + const total = binary.match(/1/g)?.length || 0; - metric.labels(parsedObj.type, '10').set(number); - // log the hash if not all the validator witnessed it so we can quickly look up the hash and check which validator failed to do so - if (number < global.currentAuthorities) { - logger.info( - `Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${number} validators after 10 blocks!`, - ); + metric.labels(parsedObj.type, '10').set(total); + if (total < global.currentAuthorities) { + // log the list of validators if the total is below 90% + if (total <= global.currentAuthorities * 0.67) { + const votes = await customRpc( + api, + 'witness_count', + parsedObj.hash, + ); + const validators: string[] = []; + votes.validators.forEach( + ([ss58address, vanity, witness]) => { + if (!witness) { + validators.push(ss58address); + } + }, + ); + logger.info( + `Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks! + Validators: [${validators}]`, + ); + metricWitnessFailure + .labels( + `${parsedObj.type}`, + `${validators}`, + `${total}`, + ) + .set(1); + toDelete.set( + JSON.stringify({ + extrinsic: `${parsedObj.type}`, + validators: `${validators}`, + witnessedBy: `${total}`, + }), + currentBlockNumber + 20, + ); + } else { + logger.info( + `Block ${blockNumber}: ${parsedObj.type} hash ${parsedObj.hash} witnesssed by ${total} validators after 10 blocks!`, + ); + } } } }); From 1358e3fc547f8d2311dbf1ba0101832d9792d08d Mon Sep 17 00:00:00 2001 From: marcellorigotti Date: Wed, 24 Apr 2024 15:10:44 +0200 Subject: [PATCH 3/5] add cf_event_extrinsic_failed with the inner Error as label --- src/metrics/chainflip/countEvents.ts | 48 +++++++++++----------------- src/utils/utils.ts | 2 +- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/src/metrics/chainflip/countEvents.ts b/src/metrics/chainflip/countEvents.ts index bc5a7c6..2054b83 100644 --- a/src/metrics/chainflip/countEvents.ts +++ b/src/metrics/chainflip/countEvents.ts @@ -12,11 +12,11 @@ const metric: Counter = new promClient.Counter({ registers: [], }); -const metricNameBroadcast: string = 'cf_broadcast_timeout_count'; -const metricBroadcast: Gauge = new promClient.Gauge({ - name: metricNameBroadcast, - help: 'Count of the broadcastTimeout events, grouped by broadcastId', - labelNames: ['event', 'broadcastId'], +const metricExtrinsicFailedName: string = 'cf_event_extrinsic_failed'; +const metricExtrinsicFailed: Counter = new promClient.Counter({ + name: metricExtrinsicFailedName, + help: 'Count of failed extrinsics on chain', + labelNames: ['pallet', 'error'], registers: [], }); @@ -28,16 +28,6 @@ const metricSlash: Counter = new promClient.Counter({ registers: [], }); -const errorMap = new Map(); -errorMap.set( - `{"module":{"index":35,"error":"0x03000000"}}`, - 'liquidityPools.UnspecifiedOrderPrice', -); -errorMap.set( - `{"module":{"index":31,"error":"0x00000000"}}`, - `liquidityProvider.InsufficientBalance`, -); - export const countEvents = async (context: Context): Promise => { const { logger, registry, events, api } = context; const config = context.config as FlipConfig; @@ -46,8 +36,8 @@ export const countEvents = async (context: Context): Promise => { logger.debug(`Scraping ${metricName}`); if (registry.getSingleMetric(metricName) === undefined) registry.registerMetric(metric); - if (registry.getSingleMetric(metricNameBroadcast) === undefined) - registry.registerMetric(metricBroadcast); + if (registry.getSingleMetric(metricExtrinsicFailedName) === undefined) + registry.registerMetric(metricExtrinsicFailed); if (registry.getSingleMetric(metricNameSlashing) === undefined) registry.registerMetric(metricSlash); @@ -64,16 +54,20 @@ export const countEvents = async (context: Context): Promise => { } metric.labels(`${event.section}:${event.method}`).inc(1); + let error; + if (event.method == 'ExtrinsicFailed') { + error = await getStateChainError( + api, + event.data.toHuman().dispatchError.Module, + ); + let parsedError = error.data.name.split(":"); + metricExtrinsicFailed.labels(`${parsedError[0]}`, `${parsedError[1]}`).inc(); + } + if (config.eventLog) { if (event.data.dispatchError) { - const error = await getStateChainError( - api, - event.data.toHuman().dispatchError.Module, - ); - - const metadata = { error }; logger.info('event_log', { - metadata, + error, event: `${event.section}:${event.method}`, data: event.data.toHuman(), }); @@ -84,12 +78,6 @@ export const countEvents = async (context: Context): Promise => { }); } } - // Set extra labels for specific events - if (event.method === 'BroadcastAttemptTimeout') { - metricBroadcast - .labels(`${event.section}:${event.method}`, event.data.broadcastId) - .set(event.data.attemptCount); - } if (event.method === 'SlashingPerformed') { for (const { ss58Address, alias } of accounts) { const hex = `0x${Buffer.from(decodeAddress(ss58Address)).toString('hex')}`; diff --git a/src/utils/utils.ts b/src/utils/utils.ts index 461e383..52a66ad 100644 --- a/src/utils/utils.ts +++ b/src/utils/utils.ts @@ -116,7 +116,7 @@ export const getStateChainError = async ( data: { palletIndex, errorIndex, - name: `${registryError.section}.${registryError.name}`, + name: `${registryError.section}:${registryError.name}`, docs: registryError.docs.join('\n').trim(), }, }; From 4542656d60fa9c951b8b61d7d321242b1802229f Mon Sep 17 00:00:00 2001 From: marcellorigotti Date: Wed, 24 Apr 2024 15:14:38 +0200 Subject: [PATCH 4/5] lint & prettier --- src/metrics/chainflip/countEvents.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/metrics/chainflip/countEvents.ts b/src/metrics/chainflip/countEvents.ts index 2054b83..d2ce727 100644 --- a/src/metrics/chainflip/countEvents.ts +++ b/src/metrics/chainflip/countEvents.ts @@ -55,12 +55,9 @@ export const countEvents = async (context: Context): Promise => { metric.labels(`${event.section}:${event.method}`).inc(1); let error; - if (event.method == 'ExtrinsicFailed') { - error = await getStateChainError( - api, - event.data.toHuman().dispatchError.Module, - ); - let parsedError = error.data.name.split(":"); + if (event.method === 'ExtrinsicFailed') { + error = await getStateChainError(api, event.data.toHuman().dispatchError.Module); + const parsedError = error.data.name.split(':'); metricExtrinsicFailed.labels(`${parsedError[0]}`, `${parsedError[1]}`).inc(); } From 3913fc3ec672ed48a4f4976816139c9cecfbb774 Mon Sep 17 00:00:00 2001 From: marcellorigotti Date: Wed, 24 Apr 2024 16:25:24 +0200 Subject: [PATCH 5/5] use toJSON instead of toHuman to avoid parsing to a valid ASCII char --- src/metrics/chainflip/gaugeWitnessChainTracking.ts | 10 +++++----- src/metrics/chainflip/gaugeWitnessCount.ts | 8 ++++---- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/metrics/chainflip/gaugeWitnessChainTracking.ts b/src/metrics/chainflip/gaugeWitnessChainTracking.ts index 1e2c502..6c26463 100644 --- a/src/metrics/chainflip/gaugeWitnessChainTracking.ts +++ b/src/metrics/chainflip/gaugeWitnessChainTracking.ts @@ -58,9 +58,9 @@ export const gaugeWitnessChainTracking = async (context: Context): Promise const parsedObj = JSON.parse(hash); api.query.witnesser .votes(global.epochIndex, parsedObj.hash) - .then(async (votes: { toHuman: () => any }) => { + .then(async (votes: { toJSON: () => any }) => { if (global.currentBlock === currentBlockNumber) { - const vote = votes.toHuman(); + const vote = votes.toJSON(); if (vote) { const binary = hex2bin(vote); let total = binary.match(/1/g)?.length || 0; @@ -71,7 +71,7 @@ export const gaugeWitnessChainTracking = async (context: Context): Promise global.epochIndex - 1, parsedObj.hash, ) - ).toHuman(); + ).toJSON(); total += hex2bin(previousEpochVote).match(/1/g)?.length || 0; } @@ -133,9 +133,9 @@ export const gaugeWitnessChainTracking = async (context: Context): Promise const parsedObj = JSON.parse(hash); api.query.witnesser .votes(global.epochIndex, parsedObj.hash) - .then((votes: { toHuman: () => any }) => { + .then((votes: { toJSON: () => any }) => { if (global.currentBlock === currentBlockNumber) { - const vote = votes.toHuman(); + const vote = votes.toJSON(); if (vote) { const binary = hex2bin(vote); const number = binary.match(/1/g)?.length || 0; diff --git a/src/metrics/chainflip/gaugeWitnessCount.ts b/src/metrics/chainflip/gaugeWitnessCount.ts index cd61b9f..dd793e5 100644 --- a/src/metrics/chainflip/gaugeWitnessCount.ts +++ b/src/metrics/chainflip/gaugeWitnessCount.ts @@ -57,8 +57,8 @@ export const gaugeWitnessCount = async (context: Context): Promise => { const parsedObj = JSON.parse(hash); api.query.witnesser .votes(global.epochIndex, parsedObj.hash) - .then(async (votes: { toHuman: () => any }) => { - const vote = votes.toHuman(); + .then(async (votes: { toJSON: () => any }) => { + const vote = votes.toJSON(); if (vote) { const binary = hex2bin(vote); const total = binary.match(/1/g)?.length || 0; @@ -118,8 +118,8 @@ export const gaugeWitnessCount = async (context: Context): Promise => { const parsedObj = JSON.parse(hash); api.query.witnesser .votes(global.epochIndex, parsedObj.hash) - .then((votes: { toHuman: () => any }) => { - const vote = votes.toHuman(); + .then((votes: { toJSON: () => any }) => { + const vote = votes.toJSON(); if (vote) { const binary = hex2bin(vote); const number = binary.match(/1/g)?.length || 0;