From edb6440537b9cadfe9c4affd96660f9f12199e74 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Sun, 31 Mar 2024 06:30:25 +0700 Subject: [PATCH 1/4] fix: store unknown root checkpoint states --- .../stateCache/persistentCheckpointsCache.ts | 67 ++++++++++++++++--- .../persistentCheckpointsCache.test.ts | 46 ++++++++++++- 2 files changed, 101 insertions(+), 12 deletions(-) diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index e33407a8dca6..8778e6e01fa2 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -87,8 +87,10 @@ export const DEFAULT_MAX_CP_STATE_EPOCHS_IN_MEMORY = 2; * ╚════════════════════════════════════╝═══════════════╝ * * The "in memory" checkpoint states are similar to the old implementation: we have both Previous Root Checkpoint State and Current Root Checkpoint State per epoch. - * However in the "persisted to db or fs" part, we usually only persist 1 checkpoint state per epoch, the one that could potentially be justified/finalized later - * based on the view of blocks. + * However in the "persisted to db or fs" part + * - if there is no reorg, we only store 1 checkpoint state per epoch, the one that could potentially be justified/finalized later based on the view of the state + * - if there is reorg, we may store >=2 checkpoint states per epoch, including any checkpoints with unknown roots to the processed state + * - the goal is to make sure we can regen any states later if needed, and we have the checkpoint state that could be justified/finalized later */ export class PersistentCheckpointStateCache implements CheckpointStateCache { private readonly cache: MapTracker; @@ -101,6 +103,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { private preComputedCheckpoint: string | null = null; private preComputedCheckpointHits: number | null = null; private readonly maxEpochsInMemory: number; + // only for testing, default false for production private readonly processLateBlock: boolean; private readonly datastore: CPStateDatastore; private readonly shufflingCache: ShufflingCache; @@ -476,9 +479,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - 1 then we'll persist {root: b1, epoch n-1} checkpoint state to disk. Note that at epoch n there is both {root: b0, epoch: n} and {root: c0, epoch: n} checkpoint states in memory * - 2 then we'll persist {root: b2, epoch n-2} checkpoint state to disk, there are also 2 checkpoint states in memory at epoch n, same to the above (maxEpochsInMemory=1) * - * As of Jan 2024, it takes 1.2s to persist a holesky state on fast server. TODO: - * - improve state serialization time - * - or research how to only store diff against the finalized state + * As of Mar 2024, it takes <=350ms to persist a holesky state on fast server */ async processState(blockRootHex: RootHex, state: CachedBeaconStateAllForks): Promise { let persistCount = 0; @@ -602,7 +603,28 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } /** - * Prune or persist checkpoint states in an epoch, see the description in `processState()` function + * Prune or persist checkpoint states in an epoch + * 1) If there is single checkpoint state in an epoch, persist it. This is when there is skipped slot at block 0 of epoch + * slot: n + * |-----------------------|-----------------------| + * root | + * + * 2) If there are any roots that unknown to this state, persist its cp state. This is to handle the current block is reorged later + * + * 3) If there are 2 checkpoint states, PRCS and CRCS and both roots are known to this state, persist CRCS. If the block is reorged, + * PRCS is regen and populated to this cache again. + * slot: n + * |-----------------------|-----------------------| + * root | + * root | + * + * 4) (derived from above) If there are 2 checkpoint states, PRCS and an unknown root, persist both. + * - PRCS is the checkpoint state that could be justified/finalized later based on the view of the state + * - unknown root checkpoint state is persisted to handle the reorg back to that branch later + * + * Performance note: + * - In normal condition, we persist 1 checkpoint state per epoch. + * - In reorged condition, we may persist >=2 (most likely 2) checkpoint states per epoch. */ private async processPastEpoch( blockRootHex: RootHex, @@ -615,13 +637,36 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { epochBoundarySlot === state.slot ? fromHexString(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot); const epochBoundaryHex = toHexString(epochBoundaryRoot); - // for each epoch, usually there are 2 rootHex respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State + // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State + const cpRootHexes = new Set(this.epochIndex.get(epoch) ?? []); + const persistedRootHexes = new Set(); + if (cpRootHexes.size === 1) { + // the slot at block 0 of epoch is skipped + persistedRootHexes.add(cpRootHexes.values().next().value); + } else if (cpRootHexes.size > 1) { + const prevEpochRoot = toHexString(getBlockRootAtSlot(state, epochBoundarySlot - 1)); + for (const rootHex of cpRootHexes) { + // rootHex is unknown to this state, persist it + if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) { + persistedRootHexes.add(rootHex); + } + } + + // if there are PRCS and CRCS, persist CRCS + if (prevEpochRoot !== epochBoundaryHex && cpRootHexes.has(prevEpochRoot) && cpRootHexes.has(epochBoundaryHex)) { + persistedRootHexes.add(epochBoundaryHex); + } + + // if there is no CRCS, persist PRCS (unknown root checkpoint state is persisted above) + if (prevEpochRoot === epochBoundaryHex && cpRootHexes.has(prevEpochRoot)) { + persistedRootHexes.add(prevEpochRoot); + } + } for (const rootHex of this.epochIndex.get(epoch) ?? []) { const cpKey = toCacheKey({epoch: epoch, rootHex}); const cacheItem = this.cache.get(cpKey); if (cacheItem !== undefined && isInMemoryCacheItem(cacheItem)) { - // this is state in memory, we don't care if the checkpoint state is already persisted let {persistedKey} = cacheItem; const {state} = cacheItem; const logMeta = { @@ -631,14 +676,14 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { persistedKey: persistedKey ? toHexString(persistedKey) : "", }; - if (rootHex === epochBoundaryHex) { + if (persistedRootHexes.has(rootHex)) { if (persistedKey) { - // no need to persist + // we don't care if the checkpoint state is already persisted this.logger.verbose("Pruned checkpoint state from memory but no need to persist", logMeta); } else { // persist and do not update epochIndex this.metrics?.statePersistSecFromSlot.observe(this.clock?.secFromSlot(this.clock?.currentSlot ?? 0) ?? 0); - const cpPersist = {epoch: epoch, root: epochBoundaryRoot}; + const cpPersist = {epoch: epoch, root: fromHexString(rootHex)}; { const timer = this.metrics?.stateSerializeDuration.startTimer(); // automatically free the buffer pool after this scope diff --git a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts index af7c118f5e99..9875ca6a8474 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts @@ -867,6 +867,11 @@ describe("PersistentCheckpointStateCache", function () { expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); }); + // Real mainnet scenario: root1b reorg root1a, and later on it's reorged back to root1a + // processState is skipped for root1a because it's late, it's only called for root1b + // we should persist both checkpoint states {0a, 20} and {0b, 20} in order to have finalized checkpoint states later + // - {0a, 20} is persisted because it's the view of root1b state + // - {0b, 20} is persisted because it's unknown in root1b state // epoch: 19 20 21 22 23 // |-----------|-----------|-----------|-----------| // ^^ ^ ^ @@ -874,7 +879,46 @@ describe("PersistentCheckpointStateCache", function () { // |0b --root1a| // | | // 0a---------root1b - it("reorg 1 epoch", async () => { + it("reorg 1 epoch, processState once", async () => { + fileApisBuffer = new Map(); + const datastore = getTestDatastore(fileApisBuffer); + cache = new PersistentCheckpointStateCache( + {datastore, logger: testLogger(), shufflingCache: new ShufflingCache()}, + {maxCPStateEpochsInMemory: 0, processLateBlock: true} + ); + + const root1a = Buffer.alloc(32, 100); + const state1a = states["cp0b"].clone(); + state1a.slot = 20 * SLOTS_PER_EPOCH + SLOTS_PER_EPOCH + 3; + state1a.blockRoots.set(state1a.slot % SLOTS_PER_HISTORICAL_ROOT, root1a); + // state transition add to cache + cache.add(cp0b, states["cp0b"]); + // do not processState root1a because it's late + + // no need to reload cp0b because it's available in block state + const root1b = Buffer.alloc(32, 101); + const state1b = states["cp0a"].clone(); + state1b.slot = state1a.slot + 1; + state1b.blockRoots.set(state1b.slot % SLOTS_PER_HISTORICAL_ROOT, root1b); + // state transition add to cache + cache.add(cp0a, states["cp0a"]); + + // need to persist 2 checkpoint states + expect(await cache.processState(toHexString(root1b), state1b)).toEqual(2); + // both are persisited + expect(await cache.getStateOrBytes(cp0bHex)).toEqual(stateBytes["cp0b"]); + expect(await cache.getStateOrBytes(cp0aHex)).toEqual(stateBytes["cp0a"]); + }); + + // Same to above, but we processState for both root1a and root1b + // epoch: 19 20 21 22 23 + // |-----------|-----------|-----------|-----------| + // ^^ ^ ^ + // || | | + // |0b --root1a| + // | | + // 0a---------root1b + it("reorg 1 epoch, processState twice", async () => { expect(await cache.processState(toHexString(root0b), states["cp0b"])).toEqual(1); await assertPersistedCheckpointState([cp0b], [stateBytes["cp0b"]]); expect(await cache.getStateOrBytes(cp0aHex)).toBeNull(); From 619b7a65c953cc55080401f795f36cf11a2b2f55 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Tue, 2 Apr 2024 14:21:02 +0700 Subject: [PATCH 2/4] chore: simplify logic --- .../stateCache/persistentCheckpointsCache.ts | 57 ++++++++++--------- 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 8778e6e01fa2..932459abc4c9 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -604,27 +604,40 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Prune or persist checkpoint states in an epoch - * 1) If there is single checkpoint state in an epoch, persist it. This is when there is skipped slot at block 0 of epoch + * 1) If there is 1 checkpoint state with known root, persist it. This is when there is skipped slot at block 0 of epoch * slot: n * |-----------------------|-----------------------| - * root | + * PRCS root | * - * 2) If there are any roots that unknown to this state, persist its cp state. This is to handle the current block is reorged later - * - * 3) If there are 2 checkpoint states, PRCS and CRCS and both roots are known to this state, persist CRCS. If the block is reorged, + * 2) If there are 2 checkpoint states, PRCS and CRCS and both roots are known to this state, persist CRCS. If the block is reorged, * PRCS is regen and populated to this cache again. * slot: n * |-----------------------|-----------------------| - * root | - * root | + * PRCS root - prune | + * CRCS root - persist | + * + * 3) If there are any roots that unknown to this state, persist their cp state. This is to handle the current block is reorged later * * 4) (derived from above) If there are 2 checkpoint states, PRCS and an unknown root, persist both. + * - In the example below block slot (n + 1) reorged n + * - If we process state n + 1, CRCS is unknown to it + * - we need to also store CRCS to handle the case (n+2) switches to n again + * + * PRCS - persist + * | processState() + * | | + * -------------n+1 + * / | + * n-1 ------n------------n+2 + * | + * CRCS - persist + * * - PRCS is the checkpoint state that could be justified/finalized later based on the view of the state * - unknown root checkpoint state is persisted to handle the reorg back to that branch later * * Performance note: * - In normal condition, we persist 1 checkpoint state per epoch. - * - In reorged condition, we may persist >=2 (most likely 2) checkpoint states per epoch. + * - In reorged condition, we may persist multiple (most likely 2) checkpoint states per epoch. */ private async processPastEpoch( blockRootHex: RootHex, @@ -636,32 +649,24 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { const epochBoundaryRoot = epochBoundarySlot === state.slot ? fromHexString(blockRootHex) : getBlockRootAtSlot(state, epochBoundarySlot); const epochBoundaryHex = toHexString(epochBoundaryRoot); + const prevEpochRoot = toHexString(getBlockRootAtSlot(state, epochBoundarySlot - 1)); // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State const cpRootHexes = new Set(this.epochIndex.get(epoch) ?? []); const persistedRootHexes = new Set(); - if (cpRootHexes.size === 1) { - // the slot at block 0 of epoch is skipped - persistedRootHexes.add(cpRootHexes.values().next().value); - } else if (cpRootHexes.size > 1) { - const prevEpochRoot = toHexString(getBlockRootAtSlot(state, epochBoundarySlot - 1)); - for (const rootHex of cpRootHexes) { - // rootHex is unknown to this state, persist it - if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) { - persistedRootHexes.add(rootHex); - } - } - // if there are PRCS and CRCS, persist CRCS - if (prevEpochRoot !== epochBoundaryHex && cpRootHexes.has(prevEpochRoot) && cpRootHexes.has(epochBoundaryHex)) { - persistedRootHexes.add(epochBoundaryHex); - } + // 1) if there is no CRCS, persist PRCS (block 0 of epoch is skipped). In this case prevEpochRoot === epochBoundaryHex + // 2) if there are PRCS and CRCS, persist CRCS => persist CRCS + // => this is simplified to always persist epochBoundaryHex + persistedRootHexes.add(epochBoundaryHex); - // if there is no CRCS, persist PRCS (unknown root checkpoint state is persisted above) - if (prevEpochRoot === epochBoundaryHex && cpRootHexes.has(prevEpochRoot)) { - persistedRootHexes.add(prevEpochRoot); + // 3) persist any states with unknown roots to this state + for (const rootHex of cpRootHexes) { + if (rootHex !== epochBoundaryHex && rootHex !== prevEpochRoot) { + persistedRootHexes.add(rootHex); } } + for (const rootHex of this.epochIndex.get(epoch) ?? []) { const cpKey = toCacheKey({epoch: epoch, rootHex}); const cacheItem = this.cache.get(cpKey); From 6fcc493ce788c93fe4ea576b5e3f7854ffc6dd73 Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Wed, 3 Apr 2024 11:20:23 +0700 Subject: [PATCH 3/4] fix: n-historical state e2e tests --- .../stateCache/nHistoricalStates.test.ts | 42 ++++++++++--------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts index fe6293519596..e21576907f34 100644 --- a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts +++ b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts @@ -57,8 +57,10 @@ describe( maxBlockStates: number; maxCPStateEpochsInMemory: number; reloadCount: number; + // total persist count, to compare to metrics persistCount: number; numStatesInMemory: number; + // number of states persisted at the end of test numStatesPersisted: number; numEpochsInMemory: number; numEpochsPersisted: number; @@ -162,7 +164,7 @@ describe( * 16 ^ ^ ^ ^ ^ * 23 24 25 26 27 * ^ - * PRCS at epoch 3 is persisted, CRCS is pruned + * both PRCS and CRCS are persisted */ { name: "maxCPStateEpochsInMemory=1, reorg last slot of previous epoch", @@ -172,12 +174,12 @@ describe( maxCPStateEpochsInMemory: 1, // PRCS at epoch 3 is available in memory so no need to reload reloadCount: 0, - // 1 cp state for epoch 0 1 2 3 - persistCount: 4, + // {root0, epoch: 0} {root8, epoch: 1} {root16, epoch: 2} {root23, epoch: 3} {root24, epoch: 3} + persistCount: 5, // epoch 4, one for Current Root Checkpoint State and one for Previous Root Checkpoint State numStatesInMemory: 2, - // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted - numStatesPersisted: 4, + // chain is not finalized, same to persistCount + numStatesPersisted: 5, // epoch 4 numEpochsInMemory: 1, // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted @@ -186,7 +188,7 @@ describe( skip: true, }, /** - * Block slot 28 has parent slot 23, block slot 24 25 26 and 27 are reorged + * Block slot 28 has parent slot 19, block slot 24 25 26 and 27 are reorged * --------------------------------|--- * / | ^ ^ ^ ^ * / | 28 29 32 33 @@ -194,7 +196,7 @@ describe( * 16 ^ ^ ^ ^ ^ ^ * 19 23 24 25 26 27 * ^ - * PRCS at epoch 3 is persisted, CRCS is pruned + * both PRCS and CRCS are persisted since their roots are unknown to block state 33 */ { name: "maxCPStateEpochsInMemory=1, reorg middle slot of previous epoch", @@ -204,12 +206,12 @@ describe( maxCPStateEpochsInMemory: 1, // reload CP state epoch 2 (slot = 16) reloadCount: 1, - // 1 cp state for epoch 0 1 2 3 - persistCount: 4, + // {root0, epoch: 0} {root8, epoch: 1} {root16, epoch: 2} {root23, epoch: 3} {root24, epoch: 3} {root19, epoch: 3} + persistCount: 6, // epoch 4, one for Current Root Checkpoint State and one for Previous Root Checkpoint State numStatesInMemory: 2, - // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted - numStatesPersisted: 4, + // chain is not finalized, same to persist count + numStatesPersisted: 6, // epoch 4 numEpochsInMemory: 1, // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted @@ -218,15 +220,15 @@ describe( skip: true, }, /** - * Block slot 28 has parent slot 23, block slot 24 25 26 and 27 are reorged + * Block slot 28 has parent slot 15, block slot 24 25 26 and 27 are reorged * --------------------------------------------|--- * / | ^ ^ ^ ^ * / | 28 29 32 33 - * |----------------|----------------|---------- - * ^ ^ 16 ^ ^ ^ ^ ^ ^ + * |----------------|----------------|---------- ^ + * ^ ^ 16 ^ ^ ^ ^ ^ ^ test end * 8 15 19 23 24 25 26 27 *reload ^ - * PRCS at epoch 3 is persisted, CRCS is pruned + * both PRCS and CRCS are persisted because roots are unknown to block 28 */ { name: "maxCPStateEpochsInMemory=1, reorg 2 epochs", @@ -236,17 +238,17 @@ describe( maxCPStateEpochsInMemory: 1, // reload CP state epoch 2 (slot = 16) reloadCount: 1, - // 1 cp state for epoch 0 1, 2 CP states for epoch 2, 1 cp state for epoch 3 - persistCount: 5, + // {root0, epoch: 0} {root8, epoch: 1} {root16, epoch: 2} {root15, epoch: 2} {root23, epoch: 3} {root24, epoch: 3} {root15, epoch: 3} + persistCount: 7, // epoch 4, one for Current Root Checkpoint State and one for Previous Root Checkpoint State numStatesInMemory: 2, - // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted, epoch 2 has 2 CP states - numStatesPersisted: 5, + // chain is not finalized, so same number to persistCount + numStatesPersisted: 7, // epoch 4 numEpochsInMemory: 1, // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted numEpochsPersisted: 4, - // chain is NOT finalized end of test + // chain is NOT finalized end of test, }, ]; From 16f8c7caa8ca03190ce13ad621a43e4996b3028b Mon Sep 17 00:00:00 2001 From: Tuyen Nguyen Date: Fri, 5 Apr 2024 16:01:23 +0700 Subject: [PATCH 4/4] chore: reuse variable --- .../src/chain/stateCache/persistentCheckpointsCache.ts | 4 ++-- .../test/e2e/chain/stateCache/nHistoricalStates.test.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 932459abc4c9..46f0b59334e8 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -652,7 +652,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { const prevEpochRoot = toHexString(getBlockRootAtSlot(state, epochBoundarySlot - 1)); // for each epoch, usually there are 2 rootHexes respective to the 2 checkpoint states: Previous Root Checkpoint State and Current Root Checkpoint State - const cpRootHexes = new Set(this.epochIndex.get(epoch) ?? []); + const cpRootHexes = this.epochIndex.get(epoch) ?? []; const persistedRootHexes = new Set(); // 1) if there is no CRCS, persist PRCS (block 0 of epoch is skipped). In this case prevEpochRoot === epochBoundaryHex @@ -667,7 +667,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { } } - for (const rootHex of this.epochIndex.get(epoch) ?? []) { + for (const rootHex of cpRootHexes) { const cpKey = toCacheKey({epoch: epoch, rootHex}); const cacheItem = this.cache.get(cpKey); diff --git a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts index e21576907f34..0c506a8b1425 100644 --- a/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts +++ b/packages/beacon-node/test/e2e/chain/stateCache/nHistoricalStates.test.ts @@ -248,7 +248,7 @@ describe( numEpochsInMemory: 1, // chain is not finalized, epoch 4 is in-memory so CP state at epoch 0 1 2 3 are persisted numEpochsPersisted: 4, - // chain is NOT finalized end of test, + // chain is NOT finalized end of test }, ];