From 6ac8c07023bc72c7ef54a6ea66884790793eda21 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Tue, 5 Sep 2023 09:46:36 +0700 Subject: [PATCH] chore: benchmark bls signature deserialization (#5922) * chore: add benchmark for signature deserialization * chore: add reference to v1.11.0 profiles * feat: track time to deserialize and validate signature on main thread * chore: use timer?.() --- .../src/chain/bls/multithread/index.ts | 2 +- .../src/chain/bls/multithread/jobItem.ts | 22 ++++++++++++++----- .../src/metrics/metrics/lodestar.ts | 4 ++++ .../beacon-node/test/perf/bls/bls.test.ts | 15 +++++++++++++ 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/packages/beacon-node/src/chain/bls/multithread/index.ts b/packages/beacon-node/src/chain/bls/multithread/index.ts index 7a223a89ac3a..db8791e46e92 100644 --- a/packages/beacon-node/src/chain/bls/multithread/index.ts +++ b/packages/beacon-node/src/chain/bls/multithread/index.ts @@ -391,7 +391,7 @@ export class BlsMultiThreadWorkerPool implements IBlsVerifier { try { // Note: This can throw, must be handled per-job. // Pubkey and signature aggregation is defered here - workReq = jobItemWorkReq(job, this.format); + workReq = jobItemWorkReq(job, this.format, this.metrics); } catch (e) { this.metrics?.blsThreadPool.errorAggregateSignatureSetsCount.inc({type: job.type}); diff --git a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts index 5e64ba334360..4ae05cdab913 100644 --- a/packages/beacon-node/src/chain/bls/multithread/jobItem.ts +++ b/packages/beacon-node/src/chain/bls/multithread/jobItem.ts @@ -4,6 +4,7 @@ import {ISignatureSet, SignatureSetType} from "@lodestar/state-transition"; import {VerifySignatureOpts} from "../interface.js"; import {getAggregatedPubkey} from "../utils.js"; import {LinkedList} from "../../../util/array.js"; +import {Metrics} from "../../../metrics/metrics.js"; import {BlsWorkReq} from "./types.js"; export type JobQueueItem = JobQueueItemDefault | JobQueueItemSameMessage; @@ -48,7 +49,7 @@ export function jobItemSigSets(job: JobQueueItem): number { * Prepare BlsWorkReq from JobQueueItem * WARNING: May throw with untrusted user input */ -export function jobItemWorkReq(job: JobQueueItem, format: PointFormat): BlsWorkReq { +export function jobItemWorkReq(job: JobQueueItem, format: PointFormat, metrics: Metrics | null): BlsWorkReq { switch (job.type) { case JobQueueItemType.default: return { @@ -60,20 +61,29 @@ export function jobItemWorkReq(job: JobQueueItem, format: PointFormat): BlsWorkR message: set.signingRoot, })), }; - case JobQueueItemType.sameMessage: + case JobQueueItemType.sameMessage: { + // validate signature = true, this is slow code on main thread so should only run with network thread mode (useWorker=true) + // For a node subscribing to all subnets, with 1 signature per validator per epoch it takes around 80s + // to deserialize 750_000 signatures per epoch + // cpu profile on main thread has 250s idle so this only works until we reach 3M validators + // However, for normal node with only 2 to 7 subnet subscriptions per epoch this works until 27M validators + // and not a problem in the near future + // this is monitored on v1.11.0 https://github.com/ChainSafe/lodestar/pull/5912#issuecomment-1700320307 + const timer = metrics?.blsThreadPool.signatureDeserializationMainThreadDuration.startTimer(); + const signatures = job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true)); + timer?.(); + return { opts: job.opts, sets: [ { publicKey: bls.PublicKey.aggregate(job.sets.map((set) => set.publicKey)).toBytes(format), - signature: bls.Signature.aggregate( - // validate signature = true - job.sets.map((set) => bls.Signature.fromBytes(set.signature, CoordType.affine, true)) - ).toBytes(format), + signature: bls.Signature.aggregate(signatures).toBytes(format), message: job.message, }, ], }; + } } } diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 840d47dfbd06..8b8ce0f0c2bc 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -447,6 +447,10 @@ export function createLodestarMetrics( name: "lodestar_bls_thread_pool_batchable_sig_sets_total", help: "Count of total batchable signature sets", }), + signatureDeserializationMainThreadDuration: register.gauge({ + name: "lodestar_bls_thread_pool_signature_deserialization_main_thread_time_seconds", + help: "Total time spent deserializing signatures on main thread", + }), }, // BLS time on single thread mode diff --git a/packages/beacon-node/test/perf/bls/bls.test.ts b/packages/beacon-node/test/perf/bls/bls.test.ts index bd6a26a3f692..a982cc55e499 100644 --- a/packages/beacon-node/test/perf/bls/bls.test.ts +++ b/packages/beacon-node/test/perf/bls/bls.test.ts @@ -77,6 +77,21 @@ describe("BLS ops", function () { }); } + // this is total time we deserialize all signatures of validators per epoch + // ideally we want to track 700_000, 1_400_000, 2_100_000 validators but it takes too long + for (const numValidators of [10_000, 100_000]) { + const signatures = linspace(0, numValidators - 1).map((i) => getSet(i % 256).signature); + itBench({ + id: `BLS deserializing ${numValidators} signatures`, + fn: () => { + for (const signature of signatures) { + // true = validate signature + bls.Signature.fromBytes(signature, CoordType.affine, true); + } + }, + }); + } + // An aggregate and proof object has 3 signatures. // We may want to bundle up to 32 sets in a single batch. // TODO: figure out why it does not work with 256 or more