From 53f8f99c643698b6c7f2c47b7a9dba200ab44b7b Mon Sep 17 00:00:00 2001 From: Faybian Byrd <48360446+FaybianB@users.noreply.github.com> Date: Wed, 21 Feb 2024 07:47:25 -0500 Subject: [PATCH 01/34] refactor: update retry function (#6451) * refactor: update retry function * chore: subtract 1 from retry values * Keep same number of retries in download test * Update retry function * Fix jsonRpcHttpClient tests * Restore comment --------- Co-authored-by: Nico Flaig --- .../src/eth1/provider/jsonRpcHttpClient.ts | 2 +- .../beacon-node/src/execution/builder/http.ts | 2 +- .../beacon-node/src/execution/engine/http.ts | 4 ++-- .../test/e2e/eth1/jsonRpcHttpClient.test.ts | 24 +++++++++---------- packages/utils/src/retry.ts | 10 ++++++-- 5 files changed, 24 insertions(+), 18 deletions(-) diff --git a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts index 9655aba5fc6b..4b31fbe3f668 100644 --- a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts +++ b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts @@ -162,7 +162,7 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { return this.fetchJson({jsonrpc: "2.0", id: this.id++, ...payload}, opts); }, { - retries: opts?.retryAttempts ?? this.opts?.retryAttempts ?? 1, + retries: opts?.retryAttempts ?? this.opts?.retryAttempts ?? 0, retryDelay: opts?.retryDelay ?? this.opts?.retryDelay ?? 0, shouldRetry: opts?.shouldRetry, signal: this.opts?.signal, diff --git a/packages/beacon-node/src/execution/builder/http.ts b/packages/beacon-node/src/execution/builder/http.ts index b4013c384f81..ca37bfa4072c 100644 --- a/packages/beacon-node/src/execution/builder/http.ts +++ b/packages/beacon-node/src/execution/builder/http.ts @@ -118,7 +118,7 @@ export class ExecutionBuilderHttp implements IExecutionBuilder { async submitBlindedBlock( signedBlindedBlock: allForks.SignedBlindedBeaconBlock ): Promise { - const res = await this.api.submitBlindedBlock(signedBlindedBlock, {retryAttempts: 3}); + const res = await this.api.submitBlindedBlock(signedBlindedBlock, {retryAttempts: 2}); ApiError.assert(res, "execution.builder.submitBlindedBlock"); const {data} = res.response; diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index 91ceabaf2770..b609defb442c 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -72,7 +72,7 @@ export const defaultExecutionEngineHttpOpts: ExecutionEngineHttpOpts = { * port/url, one can override this and skip providing a jwt secret. */ urls: ["http://localhost:8551"], - retryAttempts: 3, + retryAttempts: 2, retryDelay: 2000, timeout: 12000, }; @@ -305,7 +305,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { // If we are just fcUing and not asking execution for payload, retry is not required // and we can move on, as the next fcU will be issued soon on the new slot const fcUReqOpts = - payloadAttributes !== undefined ? forkchoiceUpdatedV1Opts : {...forkchoiceUpdatedV1Opts, retryAttempts: 1}; + payloadAttributes !== undefined ? forkchoiceUpdatedV1Opts : {...forkchoiceUpdatedV1Opts, retryAttempts: 0}; const request = this.rpcFetchQueue.push({ method, diff --git a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts index cf6d769ed9a3..277073bf10a8 100644 --- a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts +++ b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts @@ -225,10 +225,10 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { }); it("should retry 404", async function () { - let retryCount = 0; + let requestCount = 0; const server = http.createServer((req, res) => { - retryCount++; + requestCount++; res.statusCode = 404; res.end(); }); @@ -251,14 +251,14 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const controller = new AbortController(); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retryAttempts})).rejects.toThrow("Not Found"); - expect(retryCount).toBeWithMessage(retryAttempts, "404 responses should be retried before failing"); + expect(requestCount).toBeWithMessage(retryAttempts + 1, "404 responses should be retried before failing"); }); it("should retry timeout", async function () { - let retryCount = 0; + let requestCount = 0; const server = http.createServer(async () => { - retryCount++; + requestCount++; }); await new Promise((resolve) => server.listen(port, resolve)); @@ -284,13 +284,13 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retryAttempts, timeout})).rejects.toThrow( "Timeout request" ); - expect(retryCount).toBeWithMessage(retryAttempts, "Timeout request should be retried before failing"); + expect(requestCount).toBeWithMessage(retryAttempts + 1, "Timeout request should be retried before failing"); }); it("should retry aborted", async function () { - let retryCount = 0; + let requestCount = 0; const server = http.createServer(() => { - retryCount++; + requestCount++; // leave the request open until timeout }); @@ -314,14 +314,14 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { setTimeout(() => controller.abort(), 50); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retryAttempts, timeout})).rejects.toThrow("Aborted"); - expect(retryCount).toBeWithMessage(1, "Aborted request should be retried before failing"); + expect(requestCount).toBeWithMessage(1, "Aborted request should be retried before failing"); }); it("should not retry payload error", async function () { - let retryCount = 0; + let requestCount = 0; const server = http.createServer((req, res) => { - retryCount++; + requestCount++; res.setHeader("Content-Type", "application/json"); res.end(JSON.stringify({jsonrpc: "2.0", id: 83, error: noMethodError})); }); @@ -344,6 +344,6 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const controller = new AbortController(); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retryAttempts})).rejects.toThrow("Method not found"); - expect(retryCount).toBeWithMessage(1, "Payload error (non-network error) should not be retried"); + expect(requestCount).toBeWithMessage(1, "Payload error (non-network error) should not be retried"); }); }, {timeout: 10_000}); diff --git a/packages/utils/src/retry.ts b/packages/utils/src/retry.ts index 21d3b9a805e4..d875e81c75d9 100644 --- a/packages/utils/src/retry.ts +++ b/packages/utils/src/retry.ts @@ -32,11 +32,13 @@ export type RetryOptions = { */ export async function retry(fn: (attempt: number) => A | Promise, opts?: RetryOptions): Promise { const maxRetries = opts?.retries ?? 5; + // Number of retries + the initial attempt + const maxAttempts = maxRetries + 1; const shouldRetry = opts?.shouldRetry; const onRetry = opts?.onRetry; let lastError: Error = Error("RetryError"); - for (let i = 1; i <= maxRetries; i++) { + for (let i = 1; i <= maxAttempts; i++) { try { // If not the first attempt, invoke right before retrying if (i > 1) onRetry?.(lastError, i); @@ -44,7 +46,11 @@ export async function retry(fn: (attempt: number) => A | Promise, opts?: R return await fn(i); } catch (e) { lastError = e as Error; - if (shouldRetry && !shouldRetry(lastError)) { + + if (i === maxAttempts) { + // Reached maximum number of attempts, there's no need to check if we should retry + break; + } else if (shouldRetry && !shouldRetry(lastError)) { break; } else if (opts?.retryDelay !== undefined) { await sleep(opts?.retryDelay, opts?.signal); From c999e4a353d87c2b525721665cf623c07b15dad0 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 22 Feb 2024 15:04:13 +0100 Subject: [PATCH 02/34] chore: use term 'retries' instead of 'retryAttempts' consistently (#6465) --- packages/api/src/utils/client/client.ts | 6 ++-- packages/api/src/utils/client/httpClient.ts | 6 ++-- .../src/eth1/provider/jsonRpcHttpClient.ts | 10 +++--- .../beacon-node/src/execution/builder/http.ts | 2 +- .../beacon-node/src/execution/engine/http.ts | 6 ++-- .../test/e2e/eth1/jsonRpcHttpClient.test.ts | 32 +++++++++---------- .../test/sim/merge-interop.test.ts | 4 +-- .../test/sim/withdrawal-interop.test.ts | 4 +-- .../test/unit/executionEngine/http.test.ts | 2 +- .../unit/executionEngine/httpRetry.test.ts | 4 +-- .../options/beaconNodeOptions/execution.ts | 11 ++++--- .../unit/options/beaconNodeOptions.test.ts | 4 +-- 12 files changed, 46 insertions(+), 45 deletions(-) diff --git a/packages/api/src/utils/client/client.ts b/packages/api/src/utils/client/client.ts index 430d302a070c..59a06aa024a2 100644 --- a/packages/api/src/utils/client/client.ts +++ b/packages/api/src/utils/client/client.ts @@ -7,7 +7,7 @@ import {HttpStatusCode} from "./httpStatusCode.js"; /* eslint-disable @typescript-eslint/no-explicit-any */ -type ExtraOpts = {retryAttempts?: number}; +type ExtraOpts = {retries?: number}; type ParametersWithOptionalExtraOpts any> = [...Parameters, ExtraOpts] | Parameters; export type ApiWithExtraOpts> = { @@ -78,8 +78,8 @@ export function generateGenericJsonClient< // const argLen = (args as any[])?.length ?? 0; const lastArg = (args as any[])[argLen] as ExtraOpts | undefined; - const retryAttempts = lastArg?.retryAttempts; - const extraOpts = {retryAttempts}; + const retries = lastArg?.retries; + const extraOpts = {retries}; if (returnType) { // open extraOpts first if some serializer wants to add some overriding param diff --git a/packages/api/src/utils/client/httpClient.ts b/packages/api/src/utils/client/httpClient.ts index ba2ea0c15498..411d986ef6a6 100644 --- a/packages/api/src/utils/client/httpClient.ts +++ b/packages/api/src/utils/client/httpClient.ts @@ -70,7 +70,7 @@ export type FetchOpts = { /** Optional, for metrics */ routeId?: string; timeoutMs?: number; - retryAttempts?: number; + retries?: number; }; export interface IHttpClient { @@ -183,7 +183,7 @@ export class HttpClient implements IHttpClient { opts: FetchOpts, getBody: (res: Response) => Promise ): Promise<{status: HttpStatusCode; body: T}> { - if (opts.retryAttempts !== undefined) { + if (opts.retries !== undefined) { const routeId = opts.routeId ?? DEFAULT_ROUTE_ID; return retry( @@ -191,7 +191,7 @@ export class HttpClient implements IHttpClient { return this.requestWithBodyWithFallbacks(opts, getBody); }, { - retries: opts.retryAttempts, + retries: opts.retries, retryDelay: 200, onRetry: (e, attempt) => { this.logger?.debug("Retrying request", {routeId, attempt, lastError: e.message}); diff --git a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts index 4b31fbe3f668..68c924741f35 100644 --- a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts +++ b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts @@ -51,7 +51,7 @@ export type ReqOpts = { // To label request metrics routeId?: string; // retry opts - retryAttempts?: number; + retries?: number; retryDelay?: number; shouldRetry?: (lastError: Error) => boolean; }; @@ -108,9 +108,9 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { jwtId?: string; /** If jwtSecret and jwtVersion are provided, jwtVersion will be included in JwtClaim.clv. */ jwtVersion?: string; - /** Retry attempts */ - retryAttempts?: number; - /** Retry delay, only relevant with retry attempts */ + /** Number of retries per request */ + retries?: number; + /** Retry delay, only relevant if retries > 0 */ retryDelay?: number; /** Metrics for retry, could be expanded later */ metrics?: JsonRpcHttpClientMetrics | null; @@ -162,7 +162,7 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { return this.fetchJson({jsonrpc: "2.0", id: this.id++, ...payload}, opts); }, { - retries: opts?.retryAttempts ?? this.opts?.retryAttempts ?? 0, + retries: opts?.retries ?? this.opts?.retries ?? 0, retryDelay: opts?.retryDelay ?? this.opts?.retryDelay ?? 0, shouldRetry: opts?.shouldRetry, signal: this.opts?.signal, diff --git a/packages/beacon-node/src/execution/builder/http.ts b/packages/beacon-node/src/execution/builder/http.ts index ca37bfa4072c..986a5f343bfb 100644 --- a/packages/beacon-node/src/execution/builder/http.ts +++ b/packages/beacon-node/src/execution/builder/http.ts @@ -118,7 +118,7 @@ export class ExecutionBuilderHttp implements IExecutionBuilder { async submitBlindedBlock( signedBlindedBlock: allForks.SignedBlindedBeaconBlock ): Promise { - const res = await this.api.submitBlindedBlock(signedBlindedBlock, {retryAttempts: 2}); + const res = await this.api.submitBlindedBlock(signedBlindedBlock, {retries: 2}); ApiError.assert(res, "execution.builder.submitBlindedBlock"); const {data} = res.response; diff --git a/packages/beacon-node/src/execution/engine/http.ts b/packages/beacon-node/src/execution/engine/http.ts index b609defb442c..15337b371526 100644 --- a/packages/beacon-node/src/execution/engine/http.ts +++ b/packages/beacon-node/src/execution/engine/http.ts @@ -45,7 +45,7 @@ export type ExecutionEngineModules = { export type ExecutionEngineHttpOpts = { urls: string[]; - retryAttempts: number; + retries: number; retryDelay: number; timeout?: number; /** @@ -72,7 +72,7 @@ export const defaultExecutionEngineHttpOpts: ExecutionEngineHttpOpts = { * port/url, one can override this and skip providing a jwt secret. */ urls: ["http://localhost:8551"], - retryAttempts: 2, + retries: 2, retryDelay: 2000, timeout: 12000, }; @@ -305,7 +305,7 @@ export class ExecutionEngineHttp implements IExecutionEngine { // If we are just fcUing and not asking execution for payload, retry is not required // and we can move on, as the next fcU will be issued soon on the new slot const fcUReqOpts = - payloadAttributes !== undefined ? forkchoiceUpdatedV1Opts : {...forkchoiceUpdatedV1Opts, retryAttempts: 0}; + payloadAttributes !== undefined ? forkchoiceUpdatedV1Opts : {...forkchoiceUpdatedV1Opts, retries: 0}; const request = this.rpcFetchQueue.push({ method, diff --git a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts index 277073bf10a8..26062b13af51 100644 --- a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts +++ b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts @@ -185,13 +185,13 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const url = "https://goerli.fake-website.io"; const payload = {method: "get", params: []}; - const retryAttempts = 2; + const retries = 2; const controller = new AbortController(); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); await expect( eth1JsonRpcClient.fetchWithRetries(payload, { - retryAttempts, + retries, shouldRetry: () => { // using the shouldRetry function to keep tab of the retried requests retryCount++; @@ -199,7 +199,7 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { }, }) ).rejects.toThrow("getaddrinfo ENOTFOUND"); - expect(retryCount).toBeWithMessage(retryAttempts, "ENOTFOUND should be retried before failing"); + expect(retryCount).toBeWithMessage(retries, "ENOTFOUND should be retried before failing"); }); it("should retry ECONNREFUSED", async function () { @@ -207,13 +207,13 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const url = `http://localhost:${port + 1}`; const payload = {method: "get", params: []}; - const retryAttempts = 2; + const retries = 2; const controller = new AbortController(); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); await expect( eth1JsonRpcClient.fetchWithRetries(payload, { - retryAttempts, + retries, shouldRetry: () => { // using the shouldRetry function to keep tab of the retried requests retryCount++; @@ -221,7 +221,7 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { }, }) ).rejects.toThrow(expect.objectContaining({code: "ECONNREFUSED"})); - expect(retryCount).toBeWithMessage(retryAttempts, "code ECONNREFUSED should be retried before failing"); + expect(retryCount).toBeWithMessage(retries, "code ECONNREFUSED should be retried before failing"); }); it("should retry 404", async function () { @@ -246,12 +246,12 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const url = `http://localhost:${port}`; const payload = {method: "get", params: []}; - const retryAttempts = 2; + const retries = 2; const controller = new AbortController(); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); - await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retryAttempts})).rejects.toThrow("Not Found"); - expect(requestCount).toBeWithMessage(retryAttempts + 1, "404 responses should be retried before failing"); + await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retries})).rejects.toThrow("Not Found"); + expect(requestCount).toBeWithMessage(retries + 1, "404 responses should be retried before failing"); }); it("should retry timeout", async function () { @@ -276,15 +276,15 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const url = `http://localhost:${port}`; const payload = {method: "get", params: []}; - const retryAttempts = 2; + const retries = 2; const timeout = 2000; const controller = new AbortController(); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); - await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retryAttempts, timeout})).rejects.toThrow( + await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retries, timeout})).rejects.toThrow( "Timeout request" ); - expect(requestCount).toBeWithMessage(retryAttempts + 1, "Timeout request should be retried before failing"); + expect(requestCount).toBeWithMessage(retries + 1, "Timeout request should be retried before failing"); }); it("should retry aborted", async function () { @@ -307,13 +307,13 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const url = `http://localhost:${port}`; const payload = {method: "get", params: []}; - const retryAttempts = 2; + const retries = 2; const timeout = 2000; const controller = new AbortController(); setTimeout(() => controller.abort(), 50); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); - await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retryAttempts, timeout})).rejects.toThrow("Aborted"); + await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retries, timeout})).rejects.toThrow("Aborted"); expect(requestCount).toBeWithMessage(1, "Aborted request should be retried before failing"); }); @@ -339,11 +339,11 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const url = `http://localhost:${port}`; const payload = {method: "get", params: []}; - const retryAttempts = 2; + const retries = 2; const controller = new AbortController(); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); - await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retryAttempts})).rejects.toThrow("Method not found"); + await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retries})).rejects.toThrow("Method not found"); expect(requestCount).toBeWithMessage(1, "Payload error (non-network error) should not be retried"); }); }, {timeout: 10_000}); diff --git a/packages/beacon-node/test/sim/merge-interop.test.ts b/packages/beacon-node/test/sim/merge-interop.test.ts index 59bb28879b12..65c3381f94fb 100644 --- a/packages/beacon-node/test/sim/merge-interop.test.ts +++ b/packages/beacon-node/test/sim/merge-interop.test.ts @@ -49,7 +49,7 @@ import {shell} from "./shell.js"; const terminalTotalDifficultyPreMerge = 10; const TX_SCENARIOS = process.env.TX_SCENARIOS?.split(",") || []; const jwtSecretHex = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; -const retryAttempts = defaultExecutionEngineHttpOpts.retryAttempts; +const retries = defaultExecutionEngineHttpOpts.retries; const retryDelay = defaultExecutionEngineHttpOpts.retryDelay; describe("executionEngine / ExecutionEngineHttp", function () { @@ -110,7 +110,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { //const controller = new AbortController(); const executionEngine = initializeExecutionEngine( - {mode: "http", urls: [engineRpcUrl], jwtSecretHex, retryAttempts, retryDelay}, + {mode: "http", urls: [engineRpcUrl], jwtSecretHex, retries, retryDelay}, {signal: controller.signal, logger: testLogger("Node-A-Engine")} ); diff --git a/packages/beacon-node/test/sim/withdrawal-interop.test.ts b/packages/beacon-node/test/sim/withdrawal-interop.test.ts index 4f8efa71dce3..4ba0dc2136e3 100644 --- a/packages/beacon-node/test/sim/withdrawal-interop.test.ts +++ b/packages/beacon-node/test/sim/withdrawal-interop.test.ts @@ -33,7 +33,7 @@ import {shell} from "./shell.js"; /* eslint-disable no-console, @typescript-eslint/naming-convention */ const jwtSecretHex = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; -const retryAttempts = defaultExecutionEngineHttpOpts.retryAttempts; +const retries = defaultExecutionEngineHttpOpts.retries; const retryDelay = defaultExecutionEngineHttpOpts.retryDelay; describe("executionEngine / ExecutionEngineHttp", function () { @@ -82,7 +82,7 @@ describe("executionEngine / ExecutionEngineHttp", function () { //const controller = new AbortController(); const executionEngine = initializeExecutionEngine( - {mode: "http", urls: [engineRpcUrl], jwtSecretHex, retryAttempts, retryDelay}, + {mode: "http", urls: [engineRpcUrl], jwtSecretHex, retries, retryDelay}, {signal: controller.signal, logger: testLogger("executionEngine")} ); diff --git a/packages/beacon-node/test/unit/executionEngine/http.test.ts b/packages/beacon-node/test/unit/executionEngine/http.test.ts index 250b433214ce..27e9b86887ee 100644 --- a/packages/beacon-node/test/unit/executionEngine/http.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/http.test.ts @@ -45,7 +45,7 @@ describe("ExecutionEngine / http", () => { { mode: "http", urls: [baseUrl], - retryAttempts: defaultExecutionEngineHttpOpts.retryAttempts, + retries: defaultExecutionEngineHttpOpts.retries, retryDelay: defaultExecutionEngineHttpOpts.retryDelay, }, {signal: controller.signal, logger: console as unknown as Logger} diff --git a/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts b/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts index 63b220cb3382..b75dc8048283 100644 --- a/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts +++ b/packages/beacon-node/test/unit/executionEngine/httpRetry.test.ts @@ -49,7 +49,7 @@ describe("ExecutionEngine / http ", () => { { mode: "http", urls: [baseUrl], - retryAttempts: defaultExecutionEngineHttpOpts.retryAttempts, + retries: defaultExecutionEngineHttpOpts.retries, retryDelay: defaultExecutionEngineHttpOpts.retryDelay, }, {signal: controller.signal, logger: console as unknown as Logger} @@ -86,7 +86,7 @@ describe("ExecutionEngine / http ", () => { }); it("notifyForkchoiceUpdate with retry when pay load attributes", async function () { - errorResponsesBeforeSuccess = defaultExecutionEngineHttpOpts.retryAttempts - 1; + errorResponsesBeforeSuccess = defaultExecutionEngineHttpOpts.retries - 1; const forkChoiceHeadData = { headBlockHash: "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", safeBlockHash: "0xb084c10440f05f5a23a55d1d7ebcb1b3892935fb56f23cdc9a7f42c348eed174", diff --git a/packages/cli/src/options/beaconNodeOptions/execution.ts b/packages/cli/src/options/beaconNodeOptions/execution.ts index 23f9e6e0706c..58b372d26e52 100644 --- a/packages/cli/src/options/beaconNodeOptions/execution.ts +++ b/packages/cli/src/options/beaconNodeOptions/execution.ts @@ -5,7 +5,7 @@ import {CliCommandOptions, extractJwtHexSecret} from "../../util/index.js"; export type ExecutionEngineArgs = { "execution.urls": string[]; "execution.timeout"?: number; - "execution.retryAttempts": number; + "execution.retries": number; "execution.retryDelay": number; "execution.engineMock"?: boolean; jwtSecret?: string; @@ -23,7 +23,7 @@ export function parseArgs(args: ExecutionEngineArgs): IBeaconNodeOptions["execut return { urls: args["execution.urls"], timeout: args["execution.timeout"], - retryAttempts: args["execution.retryAttempts"], + retries: args["execution.retries"], retryDelay: args["execution.retryDelay"], /** * jwtSecret is parsed as hex instead of bytes because the merge with defaults @@ -55,10 +55,11 @@ export const options: CliCommandOptions = { group: "execution", }, - "execution.retryAttempts": { - description: "Number of retry attempts when calling execution engine API", + "execution.retries": { + alias: ["execution.retryAttempts"], + description: "Number of retries when calling execution engine API", type: "number", - default: defaultExecutionEngineHttpOpts.retryAttempts, + default: defaultExecutionEngineHttpOpts.retries, group: "execution", }, diff --git a/packages/cli/test/unit/options/beaconNodeOptions.test.ts b/packages/cli/test/unit/options/beaconNodeOptions.test.ts index 22faea094314..1f36b3c9751c 100644 --- a/packages/cli/test/unit/options/beaconNodeOptions.test.ts +++ b/packages/cli/test/unit/options/beaconNodeOptions.test.ts @@ -51,7 +51,7 @@ describe("options / beaconNodeOptions", () => { "execution.urls": ["http://localhost:8551"], "execution.timeout": 12000, "execution.retryDelay": 2000, - "execution.retryAttempts": 1, + "execution.retries": 1, builder: false, "builder.url": "http://localhost:8661", @@ -153,7 +153,7 @@ describe("options / beaconNodeOptions", () => { }, executionEngine: { urls: ["http://localhost:8551"], - retryAttempts: 1, + retries: 1, retryDelay: 2000, timeout: 12000, }, From eeaa7dabf3078e8daedec4389a83dc6d252aa73a Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 22 Feb 2024 15:12:25 +0100 Subject: [PATCH 03/34] fix: update SSE payload attributes to be spec compliant (#6471) --- packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts | 2 +- packages/types/src/bellatrix/sszTypes.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index b25b71514a71..a80dc03d4cc0 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -519,7 +519,7 @@ export async function getPayloadAttributesForSSE( const ssePayloadAttributes: allForks.SSEPayloadAttributes = { proposerIndex: prepareState.epochCtx.getBeaconProposer(prepareSlot), proposalSlot: prepareSlot, - proposalBlockNumber: prepareState.latestExecutionPayloadHeader.blockNumber + 1, + parentBlockNumber: prepareState.latestExecutionPayloadHeader.blockNumber, parentBlockRoot, parentBlockHash: parentHash, payloadAttributes, diff --git a/packages/types/src/bellatrix/sszTypes.ts b/packages/types/src/bellatrix/sszTypes.ts index 08f0378ef92a..53e6d436c012 100644 --- a/packages/types/src/bellatrix/sszTypes.ts +++ b/packages/types/src/bellatrix/sszTypes.ts @@ -222,7 +222,7 @@ export const SSEPayloadAttributesCommon = new ContainerType( { proposerIndex: UintNum64, proposalSlot: Slot, - proposalBlockNumber: UintNum64, + parentBlockNumber: UintNum64, parentBlockRoot: Root, parentBlockHash: Root, }, From 8959bda79bd3d86610efd88c99af7b1f7b4207f0 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 22 Feb 2024 15:26:43 +0100 Subject: [PATCH 04/34] refactor: reuse command types/utils across packages (#6441) * refactor: reuse command types/utils across packages * Allow as an alternative to setting a default * Update prover cmd options * Update comment * Fix comment * Remove unused import * Demand option if arg must be provided * Update handler --- packages/cli/docsgen/markdown.ts | 2 +- packages/cli/src/cli.ts | 2 +- packages/cli/src/cmds/beacon/index.ts | 2 +- packages/cli/src/cmds/beacon/options.ts | 2 +- packages/cli/src/cmds/bootnode/index.ts | 2 +- packages/cli/src/cmds/bootnode/options.ts | 2 +- packages/cli/src/cmds/dev/index.ts | 2 +- packages/cli/src/cmds/dev/options.ts | 2 +- packages/cli/src/cmds/index.ts | 2 +- packages/cli/src/cmds/lightclient/handler.ts | 9 +--- packages/cli/src/cmds/lightclient/index.ts | 2 +- packages/cli/src/cmds/lightclient/options.ts | 8 +-- .../cmds/validator/blsToExecutionChange.ts | 24 ++++----- packages/cli/src/cmds/validator/import.ts | 3 +- packages/cli/src/cmds/validator/index.ts | 2 +- packages/cli/src/cmds/validator/list.ts | 2 +- packages/cli/src/cmds/validator/options.ts | 3 +- .../validator/slashingProtection/export.ts | 6 +-- .../validator/slashingProtection/import.ts | 5 +- .../validator/slashingProtection/index.ts | 2 +- .../validator/slashingProtection/options.ts | 2 +- .../cli/src/cmds/validator/voluntaryExit.ts | 4 +- .../cli/src/options/beaconNodeOptions/api.ts | 2 +- .../src/options/beaconNodeOptions/builder.ts | 3 +- .../src/options/beaconNodeOptions/chain.ts | 2 +- .../cli/src/options/beaconNodeOptions/eth1.ts | 3 +- .../options/beaconNodeOptions/execution.ts | 3 +- .../src/options/beaconNodeOptions/metrics.ts | 2 +- .../options/beaconNodeOptions/monitoring.ts | 2 +- .../src/options/beaconNodeOptions/network.ts | 3 +- .../cli/src/options/beaconNodeOptions/sync.ts | 2 +- packages/cli/src/options/globalOptions.ts | 3 +- packages/cli/src/options/logOptions.ts | 3 +- packages/cli/src/options/paramsOptions.ts | 3 +- packages/cli/src/util/index.ts | 1 - packages/flare/src/cli.ts | 2 +- packages/flare/src/cmds/index.ts | 2 +- packages/flare/src/cmds/selfSlashAttester.ts | 3 +- packages/flare/src/cmds/selfSlashProposer.ts | 3 +- packages/flare/src/util/command.ts | 52 ------------------- packages/flare/src/util/deriveSecretKeys.ts | 2 +- packages/prover/src/cli/cli.ts | 2 +- packages/prover/src/cli/cmds/index.ts | 2 +- packages/prover/src/cli/cmds/start/index.ts | 2 +- packages/prover/src/cli/cmds/start/options.ts | 11 ++-- packages/prover/src/cli/options.ts | 20 ++++--- packages/prover/src/utils/command.ts | 52 ------------------- packages/utils/package.json | 1 + .../{cli/src/util => utils/src}/command.ts | 12 ++--- packages/utils/src/index.ts | 1 + 50 files changed, 95 insertions(+), 194 deletions(-) delete mode 100644 packages/flare/src/util/command.ts delete mode 100644 packages/prover/src/utils/command.ts rename packages/{cli/src/util => utils/src}/command.ts (85%) diff --git a/packages/cli/docsgen/markdown.ts b/packages/cli/docsgen/markdown.ts index e7fbcab7ad4b..4ca0d5b21e99 100644 --- a/packages/cli/docsgen/markdown.ts +++ b/packages/cli/docsgen/markdown.ts @@ -1,4 +1,4 @@ -import {CliOptionDefinition, CliCommand, CliExample, CliCommandOptions} from "../src/util/index.js"; +import {CliOptionDefinition, CliCommand, CliExample, CliCommandOptions} from "@lodestar/utils"; import {toKebab} from "./changeCase.js"; const DEFAULT_SEPARATOR = "\n\n"; diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index bfc5372cc3e8..e8f4aeba8ebc 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -1,9 +1,9 @@ // Must not use `* as yargs`, see https://github.com/yargs/yargs/issues/1131 import yargs from "yargs"; import {hideBin} from "yargs/helpers"; +import {registerCommandToYargs} from "@lodestar/utils"; import {cmds} from "./cmds/index.js"; import {globalOptions, rcConfigOption} from "./options/index.js"; -import {registerCommandToYargs} from "./util/index.js"; import {getVersionData} from "./util/version.js"; const {version} = getVersionData(); diff --git a/packages/cli/src/cmds/beacon/index.ts b/packages/cli/src/cmds/beacon/index.ts index 38d1d4cad221..b6d5c26f6fed 100644 --- a/packages/cli/src/cmds/beacon/index.ts +++ b/packages/cli/src/cmds/beacon/index.ts @@ -1,4 +1,4 @@ -import {CliCommand, CliCommandOptions} from "../../util/index.js"; +import {CliCommand, CliCommandOptions} from "@lodestar/utils"; import {GlobalArgs} from "../../options/index.js"; import {beaconOptions, BeaconArgs} from "./options.js"; import {beaconHandler} from "./handler.js"; diff --git a/packages/cli/src/cmds/beacon/options.ts b/packages/cli/src/cmds/beacon/options.ts index a1d5b35fe5de..1ec508696a13 100644 --- a/packages/cli/src/cmds/beacon/options.ts +++ b/packages/cli/src/cmds/beacon/options.ts @@ -1,6 +1,6 @@ +import {CliCommandOptions, CliOptionDefinition} from "@lodestar/utils"; import {beaconNodeOptions, paramsOptions, BeaconNodeArgs} from "../../options/index.js"; import {LogArgs, logOptions} from "../../options/logOptions.js"; -import {CliCommandOptions, CliOptionDefinition} from "../../util/index.js"; import {defaultBeaconPaths, BeaconPaths} from "./paths.js"; type BeaconExtraArgs = { diff --git a/packages/cli/src/cmds/bootnode/index.ts b/packages/cli/src/cmds/bootnode/index.ts index 4030c4a73b0f..c429f42b1fbe 100644 --- a/packages/cli/src/cmds/bootnode/index.ts +++ b/packages/cli/src/cmds/bootnode/index.ts @@ -1,4 +1,4 @@ -import {CliCommand, CliCommandOptions} from "../../util/index.js"; +import {CliCommand, CliCommandOptions} from "@lodestar/utils"; import {GlobalArgs} from "../../options/index.js"; import {bootnodeOptions, BootnodeArgs} from "./options.js"; import {bootnodeHandler} from "./handler.js"; diff --git a/packages/cli/src/cmds/bootnode/options.ts b/packages/cli/src/cmds/bootnode/options.ts index ab92ec00e155..dd597e6a22ef 100644 --- a/packages/cli/src/cmds/bootnode/options.ts +++ b/packages/cli/src/cmds/bootnode/options.ts @@ -1,5 +1,5 @@ +import {CliOptionDefinition, CliCommandOptions} from "@lodestar/utils"; import {LogArgs, logOptions} from "../../options/logOptions.js"; -import {CliOptionDefinition, CliCommandOptions} from "../../util/index.js"; import {MetricsArgs, options as metricsOptions} from "../../options/beaconNodeOptions/metrics.js"; import {defaultListenAddress, defaultP2pPort, defaultP2pPort6} from "../../options/beaconNodeOptions/network.js"; diff --git a/packages/cli/src/cmds/dev/index.ts b/packages/cli/src/cmds/dev/index.ts index d213c8b3218d..6c0f73327816 100644 --- a/packages/cli/src/cmds/dev/index.ts +++ b/packages/cli/src/cmds/dev/index.ts @@ -1,4 +1,4 @@ -import {CliCommand, CliCommandOptions} from "../../util/index.js"; +import {CliCommand, CliCommandOptions} from "@lodestar/utils"; import {GlobalArgs} from "../../options/index.js"; import {devOptions, IDevArgs} from "./options.js"; import {devHandler} from "./handler.js"; diff --git a/packages/cli/src/cmds/dev/options.ts b/packages/cli/src/cmds/dev/options.ts index 4665fe529776..c484150e58d7 100644 --- a/packages/cli/src/cmds/dev/options.ts +++ b/packages/cli/src/cmds/dev/options.ts @@ -1,4 +1,4 @@ -import {CliCommandOptions, CliOptionDefinition} from "../../util/index.js"; +import {CliCommandOptions, CliOptionDefinition} from "@lodestar/utils"; import {beaconOptions, BeaconArgs} from "../beacon/options.js"; import {NetworkName} from "../../networks/index.js"; import {beaconNodeOptions, globalOptions} from "../../options/index.js"; diff --git a/packages/cli/src/cmds/index.ts b/packages/cli/src/cmds/index.ts index 849cb23d9af7..7f701379b097 100644 --- a/packages/cli/src/cmds/index.ts +++ b/packages/cli/src/cmds/index.ts @@ -1,4 +1,4 @@ -import {CliCommand} from "../util/index.js"; +import {CliCommand} from "@lodestar/utils"; import {GlobalArgs} from "../options/index.js"; import {beacon} from "./beacon/index.js"; import {dev} from "./dev/index.js"; diff --git a/packages/cli/src/cmds/lightclient/handler.ts b/packages/cli/src/cmds/lightclient/handler.ts index 1aaaac5075a1..02bb98cb4d1d 100644 --- a/packages/cli/src/cmds/lightclient/handler.ts +++ b/packages/cli/src/cmds/lightclient/handler.ts @@ -7,7 +7,6 @@ import {getNodeLogger} from "@lodestar/logger/node"; import {getBeaconConfigFromArgs} from "../../config/beaconParams.js"; import {getGlobalPaths} from "../../paths/global.js"; import {parseLoggerArgs} from "../../util/logger.js"; -import {YargsError} from "../../util/errors.js"; import {GlobalArgs} from "../../options/index.js"; import {ILightClientArgs} from "./options.js"; @@ -19,11 +18,7 @@ export async function lightclientHandler(args: ILightClientArgs & GlobalArgs): P parseLoggerArgs(args, {defaultLogFilepath: path.join(globalPaths.dataDir, "lightclient.log")}, config) ); - const {beaconApiUrl, checkpointRoot} = args; - if (!beaconApiUrl) throw new YargsError("must provide beaconApiUrl arg"); - if (!checkpointRoot) throw new YargsError("must provide checkpointRoot arg"); - - const api = getClient({baseUrl: beaconApiUrl}, {config}); + const api = getClient({baseUrl: args.beaconApiUrl}, {config}); const res = await api.beacon.getGenesis(); ApiError.assert(res, "Can not fetch genesis data"); @@ -34,7 +29,7 @@ export async function lightclientHandler(args: ILightClientArgs & GlobalArgs): P genesisTime: Number(res.response.data.genesisTime), genesisValidatorsRoot: res.response.data.genesisValidatorsRoot, }, - checkpointRoot: fromHexString(checkpointRoot), + checkpointRoot: fromHexString(args.checkpointRoot), transport: new LightClientRestTransport(api), }); diff --git a/packages/cli/src/cmds/lightclient/index.ts b/packages/cli/src/cmds/lightclient/index.ts index 1fceb3823154..e896a49abc56 100644 --- a/packages/cli/src/cmds/lightclient/index.ts +++ b/packages/cli/src/cmds/lightclient/index.ts @@ -1,4 +1,4 @@ -import {CliCommand} from "../../util/index.js"; +import {CliCommand} from "@lodestar/utils"; import {GlobalArgs} from "../../options/index.js"; import {ILightClientArgs, lightclientOptions} from "./options.js"; import {lightclientHandler} from "./handler.js"; diff --git a/packages/cli/src/cmds/lightclient/options.ts b/packages/cli/src/cmds/lightclient/options.ts index 1dd1ddab8f00..d8a3f2f99861 100644 --- a/packages/cli/src/cmds/lightclient/options.ts +++ b/packages/cli/src/cmds/lightclient/options.ts @@ -1,9 +1,9 @@ +import {CliCommandOptions} from "@lodestar/utils"; import {LogArgs, logOptions} from "../../options/logOptions.js"; -import {CliCommandOptions} from "../../util/index.js"; export type ILightClientArgs = LogArgs & { - beaconApiUrl?: string; - checkpointRoot?: string; + beaconApiUrl: string; + checkpointRoot: string; }; export const lightclientOptions: CliCommandOptions = { @@ -11,9 +11,11 @@ export const lightclientOptions: CliCommandOptions = { beaconApiUrl: { description: "Url to a beacon node that support lightclient API", type: "string", + demandOption: true, }, checkpointRoot: { description: "Checkpoint root hex string to sync the lightclient from, start with 0x", type: "string", + demandOption: true, }, }; diff --git a/packages/cli/src/cmds/validator/blsToExecutionChange.ts b/packages/cli/src/cmds/validator/blsToExecutionChange.ts index ec81a9370bd3..7452840e1c71 100644 --- a/packages/cli/src/cmds/validator/blsToExecutionChange.ts +++ b/packages/cli/src/cmds/validator/blsToExecutionChange.ts @@ -6,8 +6,8 @@ import {DOMAIN_BLS_TO_EXECUTION_CHANGE, ForkName} from "@lodestar/params"; import {createBeaconConfig} from "@lodestar/config"; import {ssz, capella} from "@lodestar/types"; import {ApiError, getClient} from "@lodestar/api"; +import {CliCommand} from "@lodestar/utils"; -import {CliCommand, YargsError} from "../../util/index.js"; import {GlobalArgs} from "../../options/index.js"; import {getBeaconConfigFromArgs} from "../../config/index.js"; import {IValidatorCliArgs} from "./options.js"; @@ -15,9 +15,9 @@ import {IValidatorCliArgs} from "./options.js"; /* eslint-disable no-console */ type BlsToExecutionChangeArgs = { - publicKey?: string; - fromBlsPrivkey?: string; - toExecutionAddress?: string; + publicKey: string; + fromBlsPrivkey: string; + toExecutionAddress: string; }; export const blsToExecutionChange: CliCommand = { @@ -39,26 +39,22 @@ like to choose for BLS To Execution Change.", publicKey: { description: "Validator public key for which to set withdrawal address hence enabling withdrawals", type: "string", - string: true, + demandOption: true, }, fromBlsPrivkey: { description: "Bls withdrawals private key to sign the message", type: "string", - string: true, + demandOption: true, }, toExecutionAddress: { description: "Address to which the validator's balances will be set to be withdrawn.", type: "string", - string: true, + demandOption: true, }, }, handler: async (args) => { - const {publicKey, fromBlsPrivkey, toExecutionAddress} = args; - if (!publicKey) throw new YargsError("must provide publicKey arg"); - if (!fromBlsPrivkey) throw new YargsError("must provide fromBlsPrivkey arg"); - if (!toExecutionAddress) throw new YargsError("must provide toExecutionAddress arg"); - + const {publicKey} = args; // Fetch genesisValidatorsRoot always from beacon node as anyway beacon node is needed for // submitting the signed message const {config: chainForkConfig} = getBeaconConfigFromArgs(args); @@ -76,13 +72,13 @@ like to choose for BLS To Execution Change.", throw new Error(`Validator pubkey ${publicKey} not found in state`); } - const blsPrivkey = bls.SecretKey.fromBytes(fromHexString(fromBlsPrivkey)); + const blsPrivkey = bls.SecretKey.fromBytes(fromHexString(args.fromBlsPrivkey)); const fromBlsPubkey = blsPrivkey.toPublicKey().toBytes(PointFormat.compressed); const blsToExecutionChange: capella.BLSToExecutionChange = { validatorIndex: stateValidator.index, fromBlsPubkey, - toExecutionAddress: fromHexString(toExecutionAddress), + toExecutionAddress: fromHexString(args.toExecutionAddress), }; const signatureFork = ForkName.phase0; diff --git a/packages/cli/src/cmds/validator/import.ts b/packages/cli/src/cmds/validator/import.ts index a39dfcc16f74..58ae8a033de6 100644 --- a/packages/cli/src/cmds/validator/import.ts +++ b/packages/cli/src/cmds/validator/import.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import {Keystore} from "@chainsafe/bls-keystore"; -import {YargsError, CliCommand, getPubkeyHexFromKeystore} from "../../util/index.js"; +import {CliCommand} from "@lodestar/utils"; +import {YargsError, getPubkeyHexFromKeystore} from "../../util/index.js"; import {getBeaconConfigFromArgs} from "../../config/beaconParams.js"; import {GlobalArgs} from "../../options/index.js"; import {validatorOptions, IValidatorCliArgs} from "./options.js"; diff --git a/packages/cli/src/cmds/validator/index.ts b/packages/cli/src/cmds/validator/index.ts index 49c7211c740d..c8b55bf4600a 100644 --- a/packages/cli/src/cmds/validator/index.ts +++ b/packages/cli/src/cmds/validator/index.ts @@ -1,4 +1,4 @@ -import {CliCommand} from "../../util/index.js"; +import {CliCommand} from "@lodestar/utils"; import {GlobalArgs} from "../../options/index.js"; import {getAccountPaths} from "./paths.js"; import {slashingProtection} from "./slashingProtection/index.js"; diff --git a/packages/cli/src/cmds/validator/list.ts b/packages/cli/src/cmds/validator/list.ts index ae713bcbdecb..4e867b042b37 100644 --- a/packages/cli/src/cmds/validator/list.ts +++ b/packages/cli/src/cmds/validator/list.ts @@ -1,4 +1,4 @@ -import {CliCommand} from "../../util/index.js"; +import {CliCommand} from "@lodestar/utils"; import {getBeaconConfigFromArgs} from "../../config/beaconParams.js"; import {GlobalArgs} from "../../options/index.js"; import {IValidatorCliArgs} from "./options.js"; diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index a5b4044f6867..7fdcdec59e86 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -1,6 +1,7 @@ import {defaultOptions} from "@lodestar/validator"; +import {CliCommandOptions} from "@lodestar/utils"; import {LogArgs, logOptions} from "../../options/logOptions.js"; -import {ensure0xPrefix, CliCommandOptions} from "../../util/index.js"; +import {ensure0xPrefix} from "../../util/index.js"; import {keymanagerRestApiServerOptsDefault} from "./keymanager/server.js"; import {defaultAccountPaths, defaultValidatorPaths} from "./paths.js"; diff --git a/packages/cli/src/cmds/validator/slashingProtection/export.ts b/packages/cli/src/cmds/validator/slashingProtection/export.ts index 0e5b7a17833e..fb694feb058a 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/export.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/export.ts @@ -2,7 +2,8 @@ import path from "node:path"; import {toHexString} from "@chainsafe/ssz"; import {InterchangeFormatVersion} from "@lodestar/validator"; import {getNodeLogger} from "@lodestar/logger/node"; -import {CliCommand, YargsError, ensure0xPrefix, isValidatePubkeyHex, writeFile600Perm} from "../../../util/index.js"; +import {CliCommand} from "@lodestar/utils"; +import {YargsError, ensure0xPrefix, isValidatePubkeyHex, writeFile600Perm} from "../../../util/index.js"; import {parseLoggerArgs} from "../../../util/logger.js"; import {GlobalArgs} from "../../../options/index.js"; import {LogArgs} from "../../../options/logOptions.js"; @@ -13,7 +14,7 @@ import {getGenesisValidatorsRoot, getSlashingProtection} from "./utils.js"; import {ISlashingProtectionArgs} from "./options.js"; type ExportArgs = { - file?: string; + file: string; pubkeys?: string[]; }; @@ -51,7 +52,6 @@ export const exportCmd: CliCommand { const {file} = args; - if (!file) throw new YargsError("must provide file arg"); const {config, network} = getBeaconConfigFromArgs(args); const validatorPaths = getValidatorPaths(args, network); diff --git a/packages/cli/src/cmds/validator/slashingProtection/import.ts b/packages/cli/src/cmds/validator/slashingProtection/import.ts index de7f1bd48bd7..20c37550526d 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/import.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/import.ts @@ -2,7 +2,7 @@ import fs from "node:fs"; import path from "node:path"; import {Interchange} from "@lodestar/validator"; import {getNodeLogger} from "@lodestar/logger/node"; -import {CliCommand, YargsError} from "../../../util/index.js"; +import {CliCommand} from "@lodestar/utils"; import {parseLoggerArgs} from "../../../util/logger.js"; import {GlobalArgs} from "../../../options/index.js"; import {LogArgs} from "../../../options/logOptions.js"; @@ -13,7 +13,7 @@ import {getGenesisValidatorsRoot, getSlashingProtection} from "./utils.js"; import {ISlashingProtectionArgs} from "./options.js"; type ImportArgs = { - file?: string; + file: string; }; export const importCmd: CliCommand = @@ -39,7 +39,6 @@ export const importCmd: CliCommand { const {file} = args; - if (!file) throw new YargsError("must provide file arg"); const {config, network} = getBeaconConfigFromArgs(args); const validatorPaths = getValidatorPaths(args, network); diff --git a/packages/cli/src/cmds/validator/slashingProtection/index.ts b/packages/cli/src/cmds/validator/slashingProtection/index.ts index 5644b3e1126f..9c180c59c378 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/index.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/index.ts @@ -1,4 +1,4 @@ -import {CliCommand} from "../../../util/index.js"; +import {CliCommand} from "@lodestar/utils"; import {AccountValidatorArgs} from "../options.js"; import {ISlashingProtectionArgs, slashingProtectionOptions} from "./options.js"; import {importCmd} from "./import.js"; diff --git a/packages/cli/src/cmds/validator/slashingProtection/options.ts b/packages/cli/src/cmds/validator/slashingProtection/options.ts index ff2f109d7d4c..741d4c87742d 100644 --- a/packages/cli/src/cmds/validator/slashingProtection/options.ts +++ b/packages/cli/src/cmds/validator/slashingProtection/options.ts @@ -1,4 +1,4 @@ -import {CliCommandOptions} from "../../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; import {IValidatorCliArgs, validatorOptions} from "../options.js"; export type ISlashingProtectionArgs = Pick & { diff --git a/packages/cli/src/cmds/validator/voluntaryExit.ts b/packages/cli/src/cmds/validator/voluntaryExit.ts index c3c0360a8264..4676e94f7547 100644 --- a/packages/cli/src/cmds/validator/voluntaryExit.ts +++ b/packages/cli/src/cmds/validator/voluntaryExit.ts @@ -8,10 +8,10 @@ import { } from "@lodestar/state-transition"; import {createBeaconConfig, BeaconConfig} from "@lodestar/config"; import {phase0, ssz, ValidatorIndex, Epoch} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; +import {CliCommand, toHex} from "@lodestar/utils"; import {externalSignerPostSignature, SignableMessageType, Signer, SignerType} from "@lodestar/validator"; import {Api, ApiError, getClient} from "@lodestar/api"; -import {CliCommand, ensure0xPrefix, YargsError, wrapError} from "../../util/index.js"; +import {ensure0xPrefix, YargsError, wrapError} from "../../util/index.js"; import {GlobalArgs} from "../../options/index.js"; import {getBeaconConfigFromArgs} from "../../config/index.js"; import {IValidatorCliArgs} from "./options.js"; diff --git a/packages/cli/src/options/beaconNodeOptions/api.ts b/packages/cli/src/options/beaconNodeOptions/api.ts index ab3ceeff945d..996136f262ec 100644 --- a/packages/cli/src/options/beaconNodeOptions/api.ts +++ b/packages/cli/src/options/beaconNodeOptions/api.ts @@ -1,5 +1,5 @@ import {defaultOptions, IBeaconNodeOptions, allNamespaces} from "@lodestar/beacon-node"; -import {CliCommandOptions} from "../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; const enabledAll = "*"; diff --git a/packages/cli/src/options/beaconNodeOptions/builder.ts b/packages/cli/src/options/beaconNodeOptions/builder.ts index 96388ddfe2dd..2c89cbad89d2 100644 --- a/packages/cli/src/options/beaconNodeOptions/builder.ts +++ b/packages/cli/src/options/beaconNodeOptions/builder.ts @@ -1,5 +1,6 @@ import {defaultExecutionBuilderHttpOpts, IBeaconNodeOptions} from "@lodestar/beacon-node"; -import {CliCommandOptions, YargsError} from "../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; +import {YargsError} from "../../util/index.js"; export type ExecutionBuilderArgs = { builder: boolean; diff --git a/packages/cli/src/options/beaconNodeOptions/chain.ts b/packages/cli/src/options/beaconNodeOptions/chain.ts index a324e3060e1f..390ffb3ad2f6 100644 --- a/packages/cli/src/options/beaconNodeOptions/chain.ts +++ b/packages/cli/src/options/beaconNodeOptions/chain.ts @@ -1,6 +1,6 @@ import * as path from "node:path"; import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; -import {CliCommandOptions} from "../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; export type ChainArgs = { suggestedFeeRecipient: string; diff --git a/packages/cli/src/options/beaconNodeOptions/eth1.ts b/packages/cli/src/options/beaconNodeOptions/eth1.ts index 196deb59161f..46654cca6b2e 100644 --- a/packages/cli/src/options/beaconNodeOptions/eth1.ts +++ b/packages/cli/src/options/beaconNodeOptions/eth1.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; -import {CliCommandOptions, extractJwtHexSecret} from "../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; +import {extractJwtHexSecret} from "../../util/index.js"; import {ExecutionEngineArgs} from "./execution.js"; export type Eth1Args = { diff --git a/packages/cli/src/options/beaconNodeOptions/execution.ts b/packages/cli/src/options/beaconNodeOptions/execution.ts index 58b372d26e52..f2f1b42fb2bf 100644 --- a/packages/cli/src/options/beaconNodeOptions/execution.ts +++ b/packages/cli/src/options/beaconNodeOptions/execution.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; import {defaultExecutionEngineHttpOpts, IBeaconNodeOptions} from "@lodestar/beacon-node"; -import {CliCommandOptions, extractJwtHexSecret} from "../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; +import {extractJwtHexSecret} from "../../util/index.js"; export type ExecutionEngineArgs = { "execution.urls": string[]; diff --git a/packages/cli/src/options/beaconNodeOptions/metrics.ts b/packages/cli/src/options/beaconNodeOptions/metrics.ts index dc328cfa5685..ba12a7546eae 100644 --- a/packages/cli/src/options/beaconNodeOptions/metrics.ts +++ b/packages/cli/src/options/beaconNodeOptions/metrics.ts @@ -1,5 +1,5 @@ import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; -import {CliCommandOptions} from "../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; export type MetricsArgs = { metrics: boolean; diff --git a/packages/cli/src/options/beaconNodeOptions/monitoring.ts b/packages/cli/src/options/beaconNodeOptions/monitoring.ts index f9224dca684f..2143277df2ae 100644 --- a/packages/cli/src/options/beaconNodeOptions/monitoring.ts +++ b/packages/cli/src/options/beaconNodeOptions/monitoring.ts @@ -1,5 +1,5 @@ import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; -import {CliCommandOptions} from "../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; export type MonitoringArgs = { "monitoring.endpoint"?: string; diff --git a/packages/cli/src/options/beaconNodeOptions/network.ts b/packages/cli/src/options/beaconNodeOptions/network.ts index 79ec3d710d44..59d74a5cfa48 100644 --- a/packages/cli/src/options/beaconNodeOptions/network.ts +++ b/packages/cli/src/options/beaconNodeOptions/network.ts @@ -1,7 +1,8 @@ import {multiaddr} from "@multiformats/multiaddr"; import {ENR} from "@chainsafe/enr"; import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; -import {CliCommandOptions, YargsError} from "../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; +import {YargsError} from "../../util/index.js"; export const defaultListenAddress = "0.0.0.0"; export const defaultP2pPort = 9000; diff --git a/packages/cli/src/options/beaconNodeOptions/sync.ts b/packages/cli/src/options/beaconNodeOptions/sync.ts index 7130b835b987..789307781ed0 100644 --- a/packages/cli/src/options/beaconNodeOptions/sync.ts +++ b/packages/cli/src/options/beaconNodeOptions/sync.ts @@ -1,5 +1,5 @@ import {defaultOptions, IBeaconNodeOptions} from "@lodestar/beacon-node"; -import {CliCommandOptions} from "../../util/index.js"; +import {CliCommandOptions} from "@lodestar/utils"; export type SyncArgs = { "sync.isSingleNode"?: boolean; diff --git a/packages/cli/src/options/globalOptions.ts b/packages/cli/src/options/globalOptions.ts index 30e515de49c0..52a5090c6794 100644 --- a/packages/cli/src/options/globalOptions.ts +++ b/packages/cli/src/options/globalOptions.ts @@ -1,6 +1,7 @@ import {ACTIVE_PRESET} from "@lodestar/params"; +import {CliCommandOptions} from "@lodestar/utils"; import {NetworkName, networkNames} from "../networks/index.js"; -import {CliCommandOptions, readFile} from "../util/index.js"; +import {readFile} from "../util/index.js"; import {paramsOptions, IParamsArgs} from "./paramsOptions.js"; type GlobalSingleArgs = { diff --git a/packages/cli/src/options/logOptions.ts b/packages/cli/src/options/logOptions.ts index 687057d6ec1e..b45057a4532f 100644 --- a/packages/cli/src/options/logOptions.ts +++ b/packages/cli/src/options/logOptions.ts @@ -1,6 +1,5 @@ -import {LogLevels} from "@lodestar/utils"; +import {LogLevels, CliCommandOptions} from "@lodestar/utils"; import {LogLevel, logFormats} from "@lodestar/logger"; -import {CliCommandOptions} from "../util/command.js"; import {LOG_FILE_DISABLE_KEYWORD} from "../util/logger.js"; export type LogArgs = { diff --git a/packages/cli/src/options/paramsOptions.ts b/packages/cli/src/options/paramsOptions.ts index 643fb991bc61..b35a15e3c9b6 100644 --- a/packages/cli/src/options/paramsOptions.ts +++ b/packages/cli/src/options/paramsOptions.ts @@ -1,6 +1,7 @@ import {ChainConfig, chainConfigTypes} from "@lodestar/config"; +import {CliCommandOptions, CliOptionDefinition} from "@lodestar/utils"; import {IBeaconParamsUnparsed} from "../config/types.js"; -import {ObjectKeys, CliCommandOptions, CliOptionDefinition} from "../util/index.js"; +import {ObjectKeys} from "../util/index.js"; // No options are statically declared // If an arbitrary key notation is used, it removes type safety on most of this CLI arg parsing code. diff --git a/packages/cli/src/util/index.ts b/packages/cli/src/util/index.ts index 4d2be3cc92f4..3d94977f5fb7 100644 --- a/packages/cli/src/util/index.ts +++ b/packages/cli/src/util/index.ts @@ -1,4 +1,3 @@ -export * from "./command.js"; export * from "./errors.js"; export * from "./ethers.js"; export * from "./file.js"; diff --git a/packages/flare/src/cli.ts b/packages/flare/src/cli.ts index 4da4eb4e158c..91c4ef83ca09 100644 --- a/packages/flare/src/cli.ts +++ b/packages/flare/src/cli.ts @@ -1,8 +1,8 @@ // Must not use `* as yargs`, see https://github.com/yargs/yargs/issues/1131 import yargs from "yargs"; import {hideBin} from "yargs/helpers"; +import {registerCommandToYargs} from "@lodestar/utils"; import {cmds} from "./cmds/index.js"; -import {registerCommandToYargs} from "./util/command.js"; const topBanner = `Beacon chain multi-purpose and debugging tool. diff --git a/packages/flare/src/cmds/index.ts b/packages/flare/src/cmds/index.ts index 12a989ae6b27..63e1f316b3b6 100644 --- a/packages/flare/src/cmds/index.ts +++ b/packages/flare/src/cmds/index.ts @@ -1,4 +1,4 @@ -import {CliCommand} from "../util/command.js"; +import {CliCommand} from "@lodestar/utils"; import {selfSlashProposer} from "./selfSlashProposer.js"; import {selfSlashAttester} from "./selfSlashAttester.js"; diff --git a/packages/flare/src/cmds/selfSlashAttester.ts b/packages/flare/src/cmds/selfSlashAttester.ts index 3fa3414f5012..beaa14ed9291 100644 --- a/packages/flare/src/cmds/selfSlashAttester.ts +++ b/packages/flare/src/cmds/selfSlashAttester.ts @@ -5,9 +5,8 @@ import {phase0, ssz} from "@lodestar/types"; import {config as chainConfig} from "@lodestar/config/default"; import {createBeaconConfig, BeaconConfig} from "@lodestar/config"; import {DOMAIN_BEACON_ATTESTER, MAX_VALIDATORS_PER_COMMITTEE} from "@lodestar/params"; -import {toHexString} from "@lodestar/utils"; +import {CliCommand, toHexString} from "@lodestar/utils"; import {computeSigningRoot} from "@lodestar/state-transition"; -import {CliCommand} from "../util/command.js"; import {deriveSecretKeys, SecretKeysArgs, secretKeysOptions} from "../util/deriveSecretKeys.js"; /* eslint-disable no-console */ diff --git a/packages/flare/src/cmds/selfSlashProposer.ts b/packages/flare/src/cmds/selfSlashProposer.ts index 49675bb802de..ba8a85bc8e71 100644 --- a/packages/flare/src/cmds/selfSlashProposer.ts +++ b/packages/flare/src/cmds/selfSlashProposer.ts @@ -4,9 +4,8 @@ import {phase0, ssz} from "@lodestar/types"; import {config as chainConfig} from "@lodestar/config/default"; import {createBeaconConfig, BeaconConfig} from "@lodestar/config"; import {DOMAIN_BEACON_PROPOSER} from "@lodestar/params"; -import {toHexString} from "@lodestar/utils"; +import {CliCommand, toHexString} from "@lodestar/utils"; import {computeSigningRoot} from "@lodestar/state-transition"; -import {CliCommand} from "../util/command.js"; import {deriveSecretKeys, SecretKeysArgs, secretKeysOptions} from "../util/deriveSecretKeys.js"; /* eslint-disable no-console */ diff --git a/packages/flare/src/util/command.ts b/packages/flare/src/util/command.ts deleted file mode 100644 index f01d9f7ab17b..000000000000 --- a/packages/flare/src/util/command.ts +++ /dev/null @@ -1,52 +0,0 @@ -import {Options, Argv} from "yargs"; - -export interface CliExample { - command: string; - title?: string; - description?: string; -} - -export interface CliOptionDefinition extends Options { - example?: CliExample; -} - -export type CliCommandOptions = Required<{[key in keyof OwnArgs]: CliOptionDefinition}>; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export interface CliCommand, ParentArgs = Record, R = any> { - command: string; - describe: string; - examples?: {command: string; description: string}[]; - options?: CliCommandOptions; - // 1st arg: any = free own sub command options - // 2nd arg: subcommand parent options is = to this command options + parent options - // eslint-disable-next-line @typescript-eslint/no-explicit-any - subcommands?: CliCommand[]; - handler?: (args: OwnArgs & ParentArgs) => Promise; -} - -/** - * Register a CliCommand type to yargs. Recursively registers subcommands too. - * @param yargs - * @param cliCommand - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function registerCommandToYargs(yargs: Argv, cliCommand: CliCommand): void { - yargs.command({ - command: cliCommand.command, - describe: cliCommand.describe, - builder: (yargsBuilder) => { - yargsBuilder.options(cliCommand.options || {}); - for (const subcommand of cliCommand.subcommands || []) { - registerCommandToYargs(yargsBuilder, subcommand); - } - if (cliCommand.examples) { - for (const example of cliCommand.examples) { - yargsBuilder.example(`$0 ${example.command}`, example.description); - } - } - return yargs; - }, - handler: cliCommand.handler || function emptyHandler(): void {}, - }); -} diff --git a/packages/flare/src/util/deriveSecretKeys.ts b/packages/flare/src/util/deriveSecretKeys.ts index 9660f86624a2..272cf87c09c4 100644 --- a/packages/flare/src/util/deriveSecretKeys.ts +++ b/packages/flare/src/util/deriveSecretKeys.ts @@ -2,9 +2,9 @@ import bls from "@chainsafe/bls"; import type {SecretKey} from "@chainsafe/bls/types"; import {deriveEth2ValidatorKeys, deriveKeyFromMnemonic} from "@chainsafe/bls-keygen"; import {interopSecretKey} from "@lodestar/state-transition"; +import {CliCommandOptions} from "@lodestar/utils"; import {YargsError} from "./errors.js"; import {parseRange} from "./format.js"; -import {CliCommandOptions} from "./command.js"; export type SecretKeysArgs = { mnemonic?: string; diff --git a/packages/prover/src/cli/cli.ts b/packages/prover/src/cli/cli.ts index 2d1475f3d39d..5e084a551536 100644 --- a/packages/prover/src/cli/cli.ts +++ b/packages/prover/src/cli/cli.ts @@ -1,7 +1,7 @@ // Must not use `* as yargs`, see https://github.com/yargs/yargs/issues/1131 import yargs from "yargs"; import {hideBin} from "yargs/helpers"; -import {registerCommandToYargs} from "../utils/command.js"; +import {registerCommandToYargs} from "@lodestar/utils"; import {getVersionData} from "../utils/version.js"; import {cmds, proverProxyStartCommand} from "./cmds/index.js"; import {globalOptions} from "./options.js"; diff --git a/packages/prover/src/cli/cmds/index.ts b/packages/prover/src/cli/cmds/index.ts index ecd2dae1da99..310f541cf591 100644 --- a/packages/prover/src/cli/cmds/index.ts +++ b/packages/prover/src/cli/cmds/index.ts @@ -1,4 +1,4 @@ -import {CliCommand} from "../../utils/command.js"; +import {CliCommand} from "@lodestar/utils"; import {GlobalArgs} from "../options.js"; import {proverProxyStartCommand} from "./start/index.js"; export {proverProxyStartCommand} from "./start/index.js"; diff --git a/packages/prover/src/cli/cmds/start/index.ts b/packages/prover/src/cli/cmds/start/index.ts index 2b49a6466a61..f69c3acebe94 100644 --- a/packages/prover/src/cli/cmds/start/index.ts +++ b/packages/prover/src/cli/cmds/start/index.ts @@ -1,4 +1,4 @@ -import {CliCommand} from "../../../utils/command.js"; +import {CliCommand} from "@lodestar/utils"; import {GlobalArgs} from "../../options.js"; import {proverProxyStartHandler} from "./handler.js"; import {StartArgs, startOptions} from "./options.js"; diff --git a/packages/prover/src/cli/cmds/start/options.ts b/packages/prover/src/cli/cmds/start/options.ts index 53ff5957765b..f63ee974be44 100644 --- a/packages/prover/src/cli/cmds/start/options.ts +++ b/packages/prover/src/cli/cmds/start/options.ts @@ -1,12 +1,12 @@ +import {CliCommandOptions} from "@lodestar/utils"; import {DEFAULT_PROXY_REQUEST_TIMEOUT} from "../../../constants.js"; import {LCTransport} from "../../../interfaces.js"; -import {CliCommandOptions} from "../../../utils/command.js"; import {alwaysAllowedMethods} from "../../../utils/process.js"; export type StartArgs = { port: number; executionRpcUrl: string; - beaconUrls?: string[]; + beaconUrls: string[]; wsCheckpoint?: string; unverifiedWhitelist?: string[]; requestTimeout: number; @@ -53,9 +53,12 @@ export const startOptions: CliCommandOptions = { beaconUrls: { description: "Urls of the beacon nodes to connect to.", - type: "string", + type: "array", + string: true, + coerce: (urls: string[]): string[] => + // Parse ["url1,url2"] to ["url1", "url2"] + urls.map((item) => item.split(",")).flat(), demandOption: true, - array: true, group: "beacon", }, diff --git a/packages/prover/src/cli/options.ts b/packages/prover/src/cli/options.ts index cb6ba1aaeca2..c37250070056 100644 --- a/packages/prover/src/cli/options.ts +++ b/packages/prover/src/cli/options.ts @@ -1,14 +1,14 @@ import {NetworkName, networksChainConfig} from "@lodestar/config/networks"; -import {LogLevel, LogLevels} from "@lodestar/utils"; +import {CliCommandOptions, LogLevel, LogLevels} from "@lodestar/utils"; import {ACTIVE_PRESET} from "@lodestar/params"; -import {CliCommandOptions} from "../utils/command.js"; +import {YargsError} from "../utils/errors.js"; export type GlobalArgs = { - network: string; + network?: string; logLevel: string; presetFile?: string; preset: string; - paramsFile: string; + paramsFile?: string; }; export type GlobalOptions = { @@ -62,8 +62,12 @@ export function parseGlobalArgs(args: GlobalArgs): GlobalOptions { }; } - return { - logLevel: args.logLevel as LogLevel, - paramsFile: args.paramsFile, - }; + if (args.paramsFile) { + return { + logLevel: args.logLevel as LogLevel, + paramsFile: args.paramsFile, + }; + } + + throw new YargsError("Either --network or --paramsFile must be provided"); } diff --git a/packages/prover/src/utils/command.ts b/packages/prover/src/utils/command.ts deleted file mode 100644 index f01d9f7ab17b..000000000000 --- a/packages/prover/src/utils/command.ts +++ /dev/null @@ -1,52 +0,0 @@ -import {Options, Argv} from "yargs"; - -export interface CliExample { - command: string; - title?: string; - description?: string; -} - -export interface CliOptionDefinition extends Options { - example?: CliExample; -} - -export type CliCommandOptions = Required<{[key in keyof OwnArgs]: CliOptionDefinition}>; - -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export interface CliCommand, ParentArgs = Record, R = any> { - command: string; - describe: string; - examples?: {command: string; description: string}[]; - options?: CliCommandOptions; - // 1st arg: any = free own sub command options - // 2nd arg: subcommand parent options is = to this command options + parent options - // eslint-disable-next-line @typescript-eslint/no-explicit-any - subcommands?: CliCommand[]; - handler?: (args: OwnArgs & ParentArgs) => Promise; -} - -/** - * Register a CliCommand type to yargs. Recursively registers subcommands too. - * @param yargs - * @param cliCommand - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -export function registerCommandToYargs(yargs: Argv, cliCommand: CliCommand): void { - yargs.command({ - command: cliCommand.command, - describe: cliCommand.describe, - builder: (yargsBuilder) => { - yargsBuilder.options(cliCommand.options || {}); - for (const subcommand of cliCommand.subcommands || []) { - registerCommandToYargs(yargsBuilder, subcommand); - } - if (cliCommand.examples) { - for (const example of cliCommand.examples) { - yargsBuilder.example(`$0 ${example.command}`, example.description); - } - } - return yargs; - }, - handler: cliCommand.handler || function emptyHandler(): void {}, - }); -} diff --git a/packages/utils/package.json b/packages/utils/package.json index 06c89f6593ca..3d888bb4e38b 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -49,6 +49,7 @@ "devDependencies": { "@types/js-yaml": "^4.0.5", "@types/triple-beam": "^1.3.2", + "@types/yargs": "^17.0.24", "prom-client": "^15.1.0", "triple-beam": "^1.3.0" }, diff --git a/packages/cli/src/util/command.ts b/packages/utils/src/command.ts similarity index 85% rename from packages/cli/src/util/command.ts rename to packages/utils/src/command.ts index ccc8f47e71a9..3bfb372bd78f 100644 --- a/packages/cli/src/util/command.ts +++ b/packages/utils/src/command.ts @@ -1,4 +1,4 @@ -import {Options, Argv} from "yargs"; +import type {Options, Argv} from "yargs"; export interface CliExample { command: string; @@ -24,8 +24,8 @@ export interface CliOptionDefinition extends Options { export type CliCommandOptions = Required<{ [K in keyof OwnArgs]: undefined extends OwnArgs[K] ? CliOptionDefinition - : // If arg cannot be undefined it must specify a default value - CliOptionDefinition & Required>; + : // If arg cannot be undefined it must specify a default value or be provided by the user + CliOptionDefinition & (Required> | {demandOption: true}); }>; // eslint-disable-next-line @typescript-eslint/no-explicit-any @@ -57,8 +57,8 @@ export function registerCommandToYargs(yargs: Argv, cliCommand: CliCommand { - yargsBuilder.options(cliCommand.options || {}); - for (const subcommand of cliCommand.subcommands || []) { + yargsBuilder.options(cliCommand.options ?? {}); + for (const subcommand of cliCommand.subcommands ?? []) { registerCommandToYargs(yargsBuilder, subcommand); } if (cliCommand.examples) { @@ -68,6 +68,6 @@ export function registerCommandToYargs(yargs: Argv, cliCommand: CliCommand Date: Thu, 22 Feb 2024 22:37:48 +0100 Subject: [PATCH 05/34] ci: fix chrome download in browser tests (#6473) ci: fix downloading chrome in browser tests --- package.json | 1 + yarn.lock | 144 +++++++++++++++++++++++++++++++-------------------- 2 files changed, 89 insertions(+), 56 deletions(-) diff --git a/package.json b/package.json index 38c78a7456f7..4c1316942c33 100644 --- a/package.json +++ b/package.json @@ -86,6 +86,7 @@ "webdriverio": "^8.28.0" }, "resolutions": { + "@puppeteer/browsers": "^2.1.0", "dns-over-http-resolver": "^2.1.1", "loupe": "^2.3.6", "vite": "^5.0.0" diff --git a/yarn.lock b/yarn.lock index 669492e94737..e11de5621cd1 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2476,29 +2476,17 @@ resolved "https://registry.yarnpkg.com/@protobufjs/utf8/-/utf8-1.1.0.tgz#a777360b5b39a1a2e5106f8e858f2fd2d060c570" integrity sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw== -"@puppeteer/browsers@1.4.6": - version "1.4.6" - resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-1.4.6.tgz#1f70fd23d5d2ccce9d29b038e5039d7a1049ca77" - integrity sha512-x4BEjr2SjOPowNeiguzjozQbsc6h437ovD/wu+JpaenxVLm3jkgzHY2xOslMTp50HoTvQreMjiexiGQw1sqZlQ== - dependencies: - debug "4.3.4" - extract-zip "2.0.1" - progress "2.0.3" - proxy-agent "6.3.0" - tar-fs "3.0.4" - unbzip2-stream "1.4.3" - yargs "17.7.1" - -"@puppeteer/browsers@^1.6.0": - version "1.8.0" - resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-1.8.0.tgz#fb6ee61de15e7f0e67737aea9f9bab1512dbd7d8" - integrity sha512-TkRHIV6k2D8OlUe8RtG+5jgOF/H98Myx0M6AOafC8DdNVOFiBSFa5cpRDtpm8LXOa9sVwe0+e6Q3FC56X/DZfg== +"@puppeteer/browsers@1.4.6", "@puppeteer/browsers@^1.6.0", "@puppeteer/browsers@^2.1.0": + version "2.1.0" + resolved "https://registry.yarnpkg.com/@puppeteer/browsers/-/browsers-2.1.0.tgz#2683d3c908ecfc9af6b63111b5037679d3cebfd8" + integrity sha512-xloWvocjvryHdUjDam/ZuGMh7zn4Sn3ZAaV4Ah2e2EwEt90N3XphZlSsU3n0VDc1F7kggCjMuH0UuxfPQ5mD9w== dependencies: debug "4.3.4" extract-zip "2.0.1" progress "2.0.3" - proxy-agent "6.3.1" - tar-fs "3.0.4" + proxy-agent "6.4.0" + semver "7.6.0" + tar-fs "3.0.5" unbzip2-stream "1.4.3" yargs "17.7.2" @@ -4034,6 +4022,33 @@ balanced-match@^1.0.0: resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.2.tgz#e83e3a7e3f300b34cb9d87f615fa0cbf357690ee" integrity sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw== +bare-events@^2.0.0, bare-events@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/bare-events/-/bare-events-2.2.0.tgz#a7a7263c107daf8b85adf0b64f908503454ab26e" + integrity sha512-Yyyqff4PIFfSuthCZqLlPISTWHmnQxoPuAvkmgzsJEmG3CesdIv6Xweayl0JkCZJSB2yYIdJyEz97tpxNhgjbg== + +bare-fs@^2.1.1: + version "2.1.5" + resolved "https://registry.yarnpkg.com/bare-fs/-/bare-fs-2.1.5.tgz#55aae5f1c7701a83d7fbe62b0a57cfbee89a1726" + integrity sha512-5t0nlecX+N2uJqdxe9d18A98cp2u9BETelbjKpiVgQqzzmVNFYWEAjQHqS+2Khgto1vcwhik9cXucaj5ve2WWA== + dependencies: + bare-events "^2.0.0" + bare-os "^2.0.0" + bare-path "^2.0.0" + streamx "^2.13.0" + +bare-os@^2.0.0, bare-os@^2.1.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/bare-os/-/bare-os-2.2.0.tgz#24364692984d0bd507621754781b31d7872736b2" + integrity sha512-hD0rOPfYWOMpVirTACt4/nK8mC55La12K5fY1ij8HAdfQakD62M+H4o4tpfKzVGLgRDTuk3vjA4GqGXXCeFbag== + +bare-path@^2.0.0, bare-path@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/bare-path/-/bare-path-2.1.0.tgz#830f17fd39842813ca77d211ebbabe238a88cb4c" + integrity sha512-DIIg7ts8bdRKwJRJrUMy/PICEaQZaPGZ26lsSx9MJSwIhSrcdHn7/C8W+XmnG/rKi6BaRcz+JO00CjZteybDtw== + dependencies: + bare-os "^2.1.0" + base64-js@^1.0.2, base64-js@^1.3.1: version "1.5.1" resolved "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz" @@ -7187,6 +7202,14 @@ http-proxy-agent@^7.0.0: agent-base "^7.1.0" debug "^4.3.4" +http-proxy-agent@^7.0.1: + version "7.0.2" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz#9a8b1f246866c028509486585f62b8f2c18c270e" + integrity sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig== + dependencies: + agent-base "^7.1.0" + debug "^4.3.4" + http-proxy@^1.18.1: version "1.18.1" resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549" @@ -7225,7 +7248,7 @@ https-proxy-agent@^5.0.0: agent-base "6" debug "4" -https-proxy-agent@^7.0.0, https-proxy-agent@^7.0.1, https-proxy-agent@^7.0.2: +https-proxy-agent@^7.0.1, https-proxy-agent@^7.0.2: version "7.0.2" resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.2.tgz#e2645b846b90e96c6e6f347fb5b2e41f1590b09b" integrity sha512-NmLNjm6ucYwtcUmL7JQC1ZQ57LmHP4lT15FQ8D61nak1rO6DH+fz5qNK2Ap5UN4ZapYICE3/0KodcLYSPsPbaA== @@ -7233,6 +7256,14 @@ https-proxy-agent@^7.0.0, https-proxy-agent@^7.0.1, https-proxy-agent@^7.0.2: agent-base "^7.0.2" debug "4" +https-proxy-agent@^7.0.3: + version "7.0.4" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-7.0.4.tgz#8e97b841a029ad8ddc8731f26595bad868cb4168" + integrity sha512-wlwpilI7YdjSkWaQ/7omYBMTliDcmCN8OLihO6I9B86g06lMyAoqgoDpV0XqoaPOKj+0DIdAvnsWfyAAhmimcg== + dependencies: + agent-base "^7.0.2" + debug "4" + human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" @@ -10032,7 +10063,7 @@ p-waterfall@2.1.1: dependencies: p-reduce "^2.0.0" -pac-proxy-agent@^7.0.0, pac-proxy-agent@^7.0.1: +pac-proxy-agent@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/pac-proxy-agent/-/pac-proxy-agent-7.0.1.tgz#6b9ddc002ec3ff0ba5fdf4a8a21d363bcc612d75" integrity sha512-ASV8yU4LLKBAjqIPMbrgtaKIvxQri/yh2OpI+S6hVa9JRkUI3Y3NPFbfngDtY7oFtSMD3w31Xns89mDa3Feo5A== @@ -10488,29 +10519,15 @@ proxy-addr@^2.0.7: forwarded "0.2.0" ipaddr.js "1.9.1" -proxy-agent@6.3.0: - version "6.3.0" - resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.3.0.tgz#72f7bb20eb06049db79f7f86c49342c34f9ba08d" - integrity sha512-0LdR757eTj/JfuU7TL2YCuAZnxWXu3tkJbg4Oq3geW/qFNT/32T0sp2HnZ9O0lMR4q3vwAt0+xCA8SR0WAD0og== - dependencies: - agent-base "^7.0.2" - debug "^4.3.4" - http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.0" - lru-cache "^7.14.1" - pac-proxy-agent "^7.0.0" - proxy-from-env "^1.1.0" - socks-proxy-agent "^8.0.1" - -proxy-agent@6.3.1: - version "6.3.1" - resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.3.1.tgz#40e7b230552cf44fd23ffaf7c59024b692612687" - integrity sha512-Rb5RVBy1iyqOtNl15Cw/llpeLH8bsb37gM1FUfKQ+Wck6xHlbAhWGUFiTRHtkjqGTA5pSHz6+0hrPW/oECihPQ== +proxy-agent@6.4.0: + version "6.4.0" + resolved "https://registry.yarnpkg.com/proxy-agent/-/proxy-agent-6.4.0.tgz#b4e2dd51dee2b377748aef8d45604c2d7608652d" + integrity sha512-u0piLU+nCOHMgGjRbimiXmA9kM/L9EHh3zL81xCdp7m+Y2pHIsnmbdDoEDoAz5geaonNR6q6+yOPQs6n4T6sBQ== dependencies: agent-base "^7.0.2" debug "^4.3.4" - http-proxy-agent "^7.0.0" - https-proxy-agent "^7.0.2" + http-proxy-agent "^7.0.1" + https-proxy-agent "^7.0.3" lru-cache "^7.14.1" pac-proxy-agent "^7.0.1" proxy-from-env "^1.1.0" @@ -11157,6 +11174,13 @@ semver@7.5.3: dependencies: lru-cache "^6.0.0" +semver@7.6.0: + version "7.6.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" + integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== + dependencies: + lru-cache "^6.0.0" + semver@^6.1.0, semver@^6.2.0, semver@^6.3.1: version "6.3.1" resolved "https://registry.yarnpkg.com/semver/-/semver-6.3.1.tgz#556d2ef8689146e46dcea4bfdd095f3434dffcb4" @@ -11377,7 +11401,7 @@ socks-proxy-agent@^7.0.0: debug "^4.3.3" socks "^2.6.2" -socks-proxy-agent@^8.0.1, socks-proxy-agent@^8.0.2: +socks-proxy-agent@^8.0.2: version "8.0.2" resolved "https://registry.yarnpkg.com/socks-proxy-agent/-/socks-proxy-agent-8.0.2.tgz#5acbd7be7baf18c46a3f293a840109a430a640ad" integrity sha512-8zuqoLv1aP/66PHF5TqwJ7Czm3Yv32urJQHrVyhD7mmA6d61Zv8cIXQYPTWwmg6qlupnPvs/QKDmfa4P/qct2g== @@ -11597,6 +11621,16 @@ stream-to-it@^0.2.2: dependencies: get-iterator "^1.0.2" +streamx@^2.13.0: + version "2.16.1" + resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.16.1.tgz#2b311bd34832f08aa6bb4d6a80297c9caef89614" + integrity sha512-m9QYj6WygWyWa3H1YY69amr4nVgy61xfjys7xO7kviL5rfIEc2naf+ewFiOA+aEJD7y0JO3h2GoiUv4TDwEGzQ== + dependencies: + fast-fifo "^1.1.0" + queue-tick "^1.0.1" + optionalDependencies: + bare-events "^2.2.0" + streamx@^2.15.0: version "2.15.1" resolved "https://registry.yarnpkg.com/streamx/-/streamx-2.15.1.tgz#396ad286d8bc3eeef8f5cea3f029e81237c024c6" @@ -11832,7 +11866,18 @@ tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== -tar-fs@3.0.4, tar-fs@^3.0.4: +tar-fs@3.0.5: + version "3.0.5" + resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.5.tgz#f954d77767e4e6edf973384e1eb95f8f81d64ed9" + integrity sha512-JOgGAmZyMgbqpLwct7ZV8VzkEB6pxXFBVErLtb+XCOqzc6w1xiWKI9GVd6bwk68EX7eJ4DWmfXVmq8K2ziZTGg== + dependencies: + pump "^3.0.0" + tar-stream "^3.1.5" + optionalDependencies: + bare-fs "^2.1.1" + bare-path "^2.1.0" + +tar-fs@^3.0.4: version "3.0.4" resolved "https://registry.yarnpkg.com/tar-fs/-/tar-fs-3.0.4.tgz#a21dc60a2d5d9f55e0089ccd78124f1d3771dbbf" integrity sha512-5AFQU8b9qLfZCX9zp2duONhPmZv0hGYiBPJsyUdqMjzq/mqVpy/rEUSeHk1+YitmxugaptgBh5oDGU3VsAJq4w== @@ -13299,19 +13344,6 @@ yargs@16.2.0, yargs@^16.2.0: y18n "^5.0.5" yargs-parser "^20.2.2" -yargs@17.7.1: - version "17.7.1" - resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.1.tgz#34a77645201d1a8fc5213ace787c220eabbd0967" - integrity sha512-cwiTb08Xuv5fqF4AovYacTFNxk62th7LKJ6BL9IGUpTJrWoU7/7WdQGTP2SjKf1dUNBGzDd28p/Yfs/GI6JrLw== - dependencies: - cliui "^8.0.1" - escalade "^3.1.1" - get-caller-file "^2.0.5" - require-directory "^2.1.1" - string-width "^4.2.3" - y18n "^5.0.5" - yargs-parser "^21.1.1" - yargs@17.7.2, yargs@^17.1.1, yargs@^17.5.1, yargs@^17.6.2, yargs@^17.7.1: version "17.7.2" resolved "https://registry.yarnpkg.com/yargs/-/yargs-17.7.2.tgz#991df39aca675a192b816e1e0363f9d75d2aa269" From f47cc18b9cfd08b173d4cdfa39a0c51cff05b70c Mon Sep 17 00:00:00 2001 From: NC Date: Fri, 23 Feb 2024 05:56:21 +0800 Subject: [PATCH 06/34] feat: add endpoint for Altair block reward (#6178) * Add block rewards api * Add test * Add unit test * Lint * Address comment * Reduce code redundancy * Read reward cache first before calculate * Lint * Partially address comments * Accept optional postState to get the reward cache * Update test * lint * Update packages/beacon-node/src/chain/rewards/blockRewards.ts Co-authored-by: Nico Flaig * Update packages/beacon-node/src/chain/rewards/blockRewards.ts Co-authored-by: Nico Flaig * Update packages/beacon-node/src/chain/rewards/blockRewards.ts Co-authored-by: Nico Flaig * Update packages/beacon-node/src/chain/rewards/blockRewards.ts Co-authored-by: Nico Flaig * Rename proposerRewards to blockRewards. Fix import * Remove getBlockRewards from api ignore list * Fix test * Rename state to preState * Add description to fields in BlockRewards * Clean up imports * Use jsdoc to document properties * Apply suggestions from code review * Add `getPreStateSync()` * fix: clone states to compute block rewards --------- Co-authored-by: Nico Flaig Co-authored-by: Cayman Co-authored-by: Tuyen Nguyen --- .../api/src/beacon/routes/beacon/index.ts | 9 +- .../api/src/beacon/routes/beacon/rewards.ts | 97 ++++++++++ .../api/test/unit/beacon/oapiSpec.test.ts | 2 +- .../api/test/unit/beacon/testData/beacon.ts | 17 ++ .../beacon-node/src/api/impl/beacon/index.ts | 3 + .../src/api/impl/beacon/rewards/index.ts | 13 ++ packages/beacon-node/src/chain/chain.ts | 12 ++ packages/beacon-node/src/chain/interface.ts | 3 + .../beacon-node/src/chain/regen/interface.ts | 1 + .../beacon-node/src/chain/regen/queued.ts | 64 ++++--- .../src/chain/rewards/blockRewards.ts | 137 +++++++++++++ .../unit/chain/rewards/blockRewards.test.ts | 181 ++++++++++++++++++ packages/state-transition/src/index.ts | 2 + 13 files changed, 512 insertions(+), 29 deletions(-) create mode 100644 packages/api/src/beacon/routes/beacon/rewards.ts create mode 100644 packages/beacon-node/src/api/impl/beacon/rewards/index.ts create mode 100644 packages/beacon-node/src/chain/rewards/blockRewards.ts create mode 100644 packages/beacon-node/test/unit/chain/rewards/blockRewards.test.ts diff --git a/packages/api/src/beacon/routes/beacon/index.ts b/packages/api/src/beacon/routes/beacon/index.ts index 4d0c8186fd22..92fcc2093188 100644 --- a/packages/api/src/beacon/routes/beacon/index.ts +++ b/packages/api/src/beacon/routes/beacon/index.ts @@ -6,6 +6,7 @@ import {RoutesData, ReturnTypes, reqEmpty, ContainerData} from "../../../utils/i import * as block from "./block.js"; import * as pool from "./pool.js"; import * as state from "./state.js"; +import * as rewards from "./rewards.js"; // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes @@ -15,9 +16,11 @@ import * as state from "./state.js"; export * as block from "./block.js"; export * as pool from "./pool.js"; export * as state from "./state.js"; +export * as rewards from "./rewards.js"; export {BroadcastValidation} from "./block.js"; export type {BlockId, BlockHeaderResponse} from "./block.js"; export type {AttestationFilters} from "./pool.js"; +export type {BlockRewards} from "./rewards.js"; // TODO: Review if re-exporting all these types is necessary export type { StateId, @@ -34,7 +37,8 @@ export type { export type Api = block.Api & pool.Api & - state.Api & { + state.Api & + rewards.Api & { getGenesis(): Promise>; }; @@ -43,6 +47,7 @@ export const routesData: RoutesData = { ...block.routesData, ...pool.routesData, ...state.routesData, + ...rewards.routesData, }; export type ReqTypes = { @@ -56,6 +61,7 @@ export function getReqSerializers(config: ChainForkConfig) { ...block.getReqSerializers(config), ...pool.getReqSerializers(), ...state.getReqSerializers(), + ...rewards.getReqSerializers(), }; } @@ -65,5 +71,6 @@ export function getReturnTypes(): ReturnTypes { ...block.getReturnTypes(), ...pool.getReturnTypes(), ...state.getReturnTypes(), + ...rewards.getReturnTypes(), }; } diff --git a/packages/api/src/beacon/routes/beacon/rewards.ts b/packages/api/src/beacon/routes/beacon/rewards.ts new file mode 100644 index 000000000000..42dced7d5c3f --- /dev/null +++ b/packages/api/src/beacon/routes/beacon/rewards.ts @@ -0,0 +1,97 @@ +import {ContainerType} from "@chainsafe/ssz"; +import {ssz, ValidatorIndex} from "@lodestar/types"; + +import { + RoutesData, + ReturnTypes, + Schema, + ReqSerializers, + ContainerDataExecutionOptimistic, +} from "../../../utils/index.js"; +import {HttpStatusCode} from "../../../utils/client/httpStatusCode.js"; +import {ApiClientResponse} from "../../../interfaces.js"; +import {BlockId} from "./block.js"; + +// See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes + +/** + * True if the response references an unverified execution payload. Optimistic information may be invalidated at + * a later time. If the field is not present, assume the False value. + */ +export type ExecutionOptimistic = boolean; + +/** + * Rewards info for a single block. Every reward value is in Gwei. + */ +export type BlockRewards = { + /** Proposer of the block, the proposer index who receives these rewards */ + proposerIndex: ValidatorIndex; + /** Total block reward, equal to attestations + sync_aggregate + proposer_slashings + attester_slashings */ + total: number; + /** Block reward component due to included attestations */ + attestations: number; + /** Block reward component due to included sync_aggregate */ + syncAggregate: number; + /** Block reward component due to included proposer_slashings */ + proposerSlashings: number; + /** Block reward component due to included attester_slashings */ + attesterSlashings: number; +}; + +export type Api = { + /** + * Get block rewards + * Returns the info of rewards received by the block proposer + * + * @param blockId Block identifier. + * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \, \. + */ + getBlockRewards( + blockId: BlockId + ): Promise< + ApiClientResponse< + {[HttpStatusCode.OK]: {data: BlockRewards; executionOptimistic: ExecutionOptimistic}}, + HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND + > + >; +}; + +/** + * Define javascript values for each route + */ +export const routesData: RoutesData = { + getBlockRewards: {url: "/eth/v1/beacon/rewards/blocks/{block_id}", method: "GET"}, +}; + +export type ReqTypes = { + /* eslint-disable @typescript-eslint/naming-convention */ + getBlockRewards: {params: {block_id: string}}; +}; + +export function getReqSerializers(): ReqSerializers { + return { + getBlockRewards: { + writeReq: (block_id) => ({params: {block_id: String(block_id)}}), + parseReq: ({params}) => [params.block_id], + schema: {params: {block_id: Schema.StringRequired}}, + }, + }; +} + +export function getReturnTypes(): ReturnTypes { + const BlockRewardsResponse = new ContainerType( + { + proposerIndex: ssz.ValidatorIndex, + total: ssz.UintNum64, + attestations: ssz.UintNum64, + syncAggregate: ssz.UintNum64, + proposerSlashings: ssz.UintNum64, + attesterSlashings: ssz.UintNum64, + }, + {jsonCase: "eth2"} + ); + + return { + getBlockRewards: ContainerDataExecutionOptimistic(BlockRewardsResponse), + }; +} diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index f7d6cb9a077c..4d036fb2dd8d 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -88,7 +88,6 @@ const ignoredOperations = [ /* missing route */ /* https://github.com/ChainSafe/lodestar/issues/5694 */ "getSyncCommitteeRewards", - "getBlockRewards", "getAttestationsRewards", "getDepositSnapshot", // Won't fix for now, see https://github.com/ChainSafe/lodestar/issues/5697 "getBlindedBlock", // https://github.com/ChainSafe/lodestar/issues/5699 @@ -123,6 +122,7 @@ const ignoredProperties: Record = { getBlockRoot: {response: ["finalized"]}, getBlockAttestations: {response: ["finalized"]}, getStateV2: {response: ["finalized"]}, + getBlockRewards: {response: ["finalized"]}, /* https://github.com/ChainSafe/lodestar/issues/6168 diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index 6d6bc6576f56..9c39849906de 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -168,6 +168,23 @@ export const testData: GenericServerTestCases = { res: {executionOptimistic: true, data: {validators: [1300], validatorAggregates: [[1300]]}}, }, + // reward + + getBlockRewards: { + args: ["head"], + res: { + executionOptimistic: true, + data: { + proposerIndex: 0, + total: 15, + attestations: 8, + syncAggregate: 4, + proposerSlashings: 2, + attesterSlashings: 1, + }, + }, + }, + // - getGenesis: { diff --git a/packages/beacon-node/src/api/impl/beacon/index.ts b/packages/beacon-node/src/api/impl/beacon/index.ts index d613e3c2d394..492e2f8ff8b1 100644 --- a/packages/beacon-node/src/api/impl/beacon/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/index.ts @@ -3,6 +3,7 @@ import {ApiModules} from "../types.js"; import {getBeaconBlockApi} from "./blocks/index.js"; import {getBeaconPoolApi} from "./pool/index.js"; import {getBeaconStateApi} from "./state/index.js"; +import {getBeaconRewardsApi} from "./rewards/index.js"; export function getBeaconApi( modules: Pick @@ -10,6 +11,7 @@ export function getBeaconApi( const block = getBeaconBlockApi(modules); const pool = getBeaconPoolApi(modules); const state = getBeaconStateApi(modules); + const rewards = getBeaconRewardsApi(modules); const {chain, config} = modules; @@ -17,6 +19,7 @@ export function getBeaconApi( ...block, ...pool, ...state, + ...rewards, async getGenesis() { return { diff --git a/packages/beacon-node/src/api/impl/beacon/rewards/index.ts b/packages/beacon-node/src/api/impl/beacon/rewards/index.ts new file mode 100644 index 000000000000..03a182359d90 --- /dev/null +++ b/packages/beacon-node/src/api/impl/beacon/rewards/index.ts @@ -0,0 +1,13 @@ +import {routes, ServerApi} from "@lodestar/api"; +import {ApiModules} from "../../types.js"; +import {resolveBlockId} from "../blocks/utils.js"; + +export function getBeaconRewardsApi({chain}: Pick): ServerApi { + return { + async getBlockRewards(blockId) { + const {block, executionOptimistic} = await resolveBlockId(chain, blockId); + const data = await chain.getBlockRewards(block.message); + return {data, executionOptimistic}; + }, + }; +} diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index f20bc0dbffa2..20a6ca343565 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -76,6 +76,7 @@ import {BlockAttributes, produceBlockBody, produceCommonBlockBody} from "./produ import {computeNewStateRoot} from "./produceBlock/computeNewStateRoot.js"; import {BlockInput} from "./blocks/types.js"; import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; +import {BlockRewards, computeBlockRewards} from "./rewards/blockRewards.js"; import {ShufflingCache} from "./shufflingCache.js"; import {StateContextCache} from "./stateCache/stateContextCache.js"; import {SeenGossipBlockInput} from "./seenCache/index.js"; @@ -991,4 +992,15 @@ export class BeaconChain implements IBeaconChain { } } } + + async getBlockRewards(block: allForks.FullOrBlindedBeaconBlock): Promise { + const preState = this.regen.getPreStateSync(block); + const postState = this.regen.getStateSync(toHexString(block.stateRoot)) ?? undefined; + + if (preState === null) { + throw Error(`Pre-state is unavailable given block's parent root ${toHexString(block.parentRoot)}`); + } + + return computeBlockRewards(block, preState.clone(), postState?.clone()); + } } diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 3939457a8ac3..99c1b7ea0c4a 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -52,6 +52,7 @@ import {AssembledBlockType, BlockAttributes, BlockType} from "./produceBlock/pro import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; import {SeenGossipBlockInput} from "./seenCache/index.js"; import {ShufflingCache} from "./shufflingCache.js"; +import {BlockRewards} from "./rewards/blockRewards.js"; export {BlockType, type AssembledBlockType}; export {type ProposerPreparationData}; @@ -198,6 +199,8 @@ export interface IBeaconChain { regenCanAcceptWork(): boolean; blsThreadPoolCanAcceptWork(): boolean; + + getBlockRewards(blockRef: allForks.FullOrBlindedBeaconBlock): Promise; } export type SSZObjectType = diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index be481de9abc8..b861378ff440 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -35,6 +35,7 @@ export interface IStateRegenerator extends IStateRegeneratorInternal { dropCache(): void; dumpCacheSummary(): routes.lodestar.StateCacheItem[]; getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null; + getPreStateSync(block: allForks.BeaconBlock): CachedBeaconStateAllForks | null; getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null; getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null; pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void; diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 928c2e399b9a..a65c462227f3 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -71,6 +71,40 @@ export class QueuedStateRegenerator implements IStateRegenerator { return this.stateCache.get(stateRoot); } + getPreStateSync(block: allForks.BeaconBlock): CachedBeaconStateAllForks | null { + const parentRoot = toHexString(block.parentRoot); + const parentBlock = this.forkChoice.getBlockHex(parentRoot); + if (!parentBlock) { + throw new RegenError({ + code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE, + blockRoot: block.parentRoot, + }); + } + + const parentEpoch = computeEpochAtSlot(parentBlock.slot); + const blockEpoch = computeEpochAtSlot(block.slot); + + // Check the checkpoint cache (if the pre-state is a checkpoint state) + if (parentEpoch < blockEpoch) { + const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch); + if (checkpointState && computeEpochAtSlot(checkpointState.slot) === blockEpoch) { + return checkpointState; + } + } + + // Check the state cache, only if the state doesn't need to go through an epoch transition. + // Otherwise the state transition may not be cached and wasted. Queue for regen since the + // work required will still be significant. + if (parentEpoch === blockEpoch) { + const state = this.stateCache.get(parentBlock.stateRoot); + if (state) { + return state; + } + } + + return null; + } + getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null { return this.checkpointStateCache.get(cp); } @@ -137,34 +171,10 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getPreState}); // First attempt to fetch the state from caches before queueing - const parentRoot = toHexString(block.parentRoot); - const parentBlock = this.forkChoice.getBlockHex(parentRoot); - if (!parentBlock) { - throw new RegenError({ - code: RegenErrorCode.BLOCK_NOT_IN_FORKCHOICE, - blockRoot: block.parentRoot, - }); - } + const cachedState = this.getPreStateSync(block); - const parentEpoch = computeEpochAtSlot(parentBlock.slot); - const blockEpoch = computeEpochAtSlot(block.slot); - - // Check the checkpoint cache (if the pre-state is a checkpoint state) - if (parentEpoch < blockEpoch) { - const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch); - if (checkpointState && computeEpochAtSlot(checkpointState.slot) === blockEpoch) { - return checkpointState; - } - } - - // Check the state cache, only if the state doesn't need to go through an epoch transition. - // Otherwise the state transition may not be cached and wasted. Queue for regen since the - // work required will still be significant. - if (parentEpoch === blockEpoch) { - const state = this.stateCache.get(parentBlock.stateRoot); - if (state) { - return state; - } + if (cachedState !== null) { + return cachedState; } // The state is not immediately available in the caches, enqueue the job diff --git a/packages/beacon-node/src/chain/rewards/blockRewards.ts b/packages/beacon-node/src/chain/rewards/blockRewards.ts new file mode 100644 index 000000000000..bd8bf3537582 --- /dev/null +++ b/packages/beacon-node/src/chain/rewards/blockRewards.ts @@ -0,0 +1,137 @@ +import { + CachedBeaconStateAllForks, + CachedBeaconStateAltair, + CachedBeaconStatePhase0, + getAttesterSlashableIndices, + processAttestationsAltair, +} from "@lodestar/state-transition"; +import {allForks, altair, phase0} from "@lodestar/types"; +import {ForkName, WHISTLEBLOWER_REWARD_QUOTIENT} from "@lodestar/params"; +import {routes} from "@lodestar/api"; + +export type BlockRewards = routes.beacon.BlockRewards; +type SubRewardValue = number; // All reward values should be integer + +/** + * Calculate total proposer block rewards given block and the beacon state of the same slot before the block is applied (preState) + * postState can be passed in to read reward cache if available + * Standard (Non MEV) rewards for proposing a block consists of: + * 1) Including attestations from (beacon) committee + * 2) Including attestations from sync committee + * 3) Reporting slashable behaviours from proposer and attester + */ +export async function computeBlockRewards( + block: allForks.BeaconBlock, + preState: CachedBeaconStateAllForks, + postState?: CachedBeaconStateAllForks +): Promise { + const fork = preState.config.getForkName(block.slot); + const {attestations: cachedAttestationsReward = 0, syncAggregate: cachedSyncAggregateReward = 0} = + postState?.proposerRewards ?? {}; + let blockAttestationReward = cachedAttestationsReward; + let syncAggregateReward = cachedSyncAggregateReward; + + if (blockAttestationReward === 0) { + blockAttestationReward = + fork === ForkName.phase0 + ? computeBlockAttestationRewardPhase0(block as phase0.BeaconBlock, preState as CachedBeaconStatePhase0) + : computeBlockAttestationRewardAltair(block as altair.BeaconBlock, preState as CachedBeaconStateAltair); + } + + if (syncAggregateReward === 0) { + syncAggregateReward = computeSyncAggregateReward(block as altair.BeaconBlock, preState as CachedBeaconStateAltair); + } + + const blockProposerSlashingReward = computeBlockProposerSlashingReward(block, preState); + const blockAttesterSlashingReward = computeBlockAttesterSlashingReward(block, preState); + + const total = + blockAttestationReward + syncAggregateReward + blockProposerSlashingReward + blockAttesterSlashingReward; + + return { + proposerIndex: block.proposerIndex, + total, + attestations: blockAttestationReward, + syncAggregate: syncAggregateReward, + proposerSlashings: blockProposerSlashingReward, + attesterSlashings: blockAttesterSlashingReward, + }; +} + +/** + * TODO: Calculate rewards received by block proposer for including attestations. + */ +function computeBlockAttestationRewardPhase0( + _block: phase0.BeaconBlock, + _preState: CachedBeaconStatePhase0 +): SubRewardValue { + throw new Error("Unsupported fork! Block attestation reward calculation is not available in phase0"); +} + +/** + * Calculate rewards received by block proposer for including attestations since Altair. + * Reuses `processAttestationsAltair()`. Has dependency on RewardCache + */ +function computeBlockAttestationRewardAltair( + block: altair.BeaconBlock, + preState: CachedBeaconStateAltair +): SubRewardValue { + const fork = preState.config.getForkSeq(block.slot); + const {attestations} = block.body; + + processAttestationsAltair(fork, preState, attestations, false); + + return preState.proposerRewards.attestations; +} + +function computeSyncAggregateReward(block: altair.BeaconBlock, preState: CachedBeaconStateAltair): SubRewardValue { + if (block.body.syncAggregate !== undefined) { + const {syncCommitteeBits} = block.body.syncAggregate; + const {syncProposerReward} = preState.epochCtx; + + return syncCommitteeBits.getTrueBitIndexes().length * Math.floor(syncProposerReward); // syncProposerReward should already be integer + } else { + return 0; // phase0 block does not have syncAggregate + } +} + +/** + * Calculate rewards received by block proposer for including proposer slashings. + * All proposer slashing rewards go to block proposer and none to whistleblower as of Deneb + */ +function computeBlockProposerSlashingReward( + block: allForks.BeaconBlock, + state: CachedBeaconStateAllForks +): SubRewardValue { + let proposerSlashingReward = 0; + + for (const proposerSlashing of block.body.proposerSlashings) { + const offendingProposerIndex = proposerSlashing.signedHeader1.message.proposerIndex; + const offendingProposerBalance = state.validators.getReadonly(offendingProposerIndex).effectiveBalance; + + proposerSlashingReward += Math.floor(offendingProposerBalance / WHISTLEBLOWER_REWARD_QUOTIENT); + } + + return proposerSlashingReward; +} + +/** + * Calculate rewards received by block proposer for including attester slashings. + * All attester slashing rewards go to block proposer and none to whistleblower as of Deneb + */ +function computeBlockAttesterSlashingReward( + block: allForks.BeaconBlock, + preState: CachedBeaconStateAllForks +): SubRewardValue { + let attesterSlashingReward = 0; + + for (const attesterSlashing of block.body.attesterSlashings) { + for (const offendingAttesterIndex of getAttesterSlashableIndices(attesterSlashing)) { + const offendingAttesterBalance = preState.validators.getReadonly(offendingAttesterIndex).effectiveBalance; + + attesterSlashingReward += Math.floor(offendingAttesterBalance / WHISTLEBLOWER_REWARD_QUOTIENT); + } + } + + return attesterSlashingReward; +} diff --git a/packages/beacon-node/test/unit/chain/rewards/blockRewards.test.ts b/packages/beacon-node/test/unit/chain/rewards/blockRewards.test.ts new file mode 100644 index 000000000000..f0d85ce3220f --- /dev/null +++ b/packages/beacon-node/test/unit/chain/rewards/blockRewards.test.ts @@ -0,0 +1,181 @@ +import {describe, it, expect} from "vitest"; +import {SYNC_COMMITTEE_SIZE} from "@lodestar/params"; +import {ssz} from "@lodestar/types"; +import { + CachedBeaconStateAllForks, + DataAvailableStatus, + ExecutionPayloadStatus, + stateTransition, +} from "@lodestar/state-transition"; +import { + generatePerfTestCachedStateAltair, + cachedStateAltairPopulateCaches, + // eslint-disable-next-line import/no-relative-packages +} from "../../../../../state-transition/test/perf/util.js"; +// eslint-disable-next-line import/no-relative-packages +import {BlockAltairOpts, getBlockAltair} from "../../../../../state-transition/test/perf/block/util.js"; +import {computeBlockRewards} from "../../../../src/chain/rewards/blockRewards.js"; + +describe("chain / rewards / blockRewards", () => { + const testCases: {id: string; opts: BlockAltairOpts}[] = [ + { + id: "Normal case", + opts: { + proposerSlashingLen: 1, + attesterSlashingLen: 2, + attestationLen: 90, + depositsLen: 0, + voluntaryExitLen: 0, + bitsLen: 90, + syncCommitteeBitsLen: Math.round(SYNC_COMMITTEE_SIZE * 0.7), + }, + }, + { + id: "Attestation only", + opts: { + proposerSlashingLen: 0, + attesterSlashingLen: 0, + attestationLen: 90, + depositsLen: 0, + voluntaryExitLen: 0, + bitsLen: 90, + syncCommitteeBitsLen: 0, + }, + }, + { + id: "Sync aggregate only", + opts: { + proposerSlashingLen: 0, + attesterSlashingLen: 0, + attestationLen: 0, + depositsLen: 0, + voluntaryExitLen: 0, + bitsLen: 90, + syncCommitteeBitsLen: Math.round(SYNC_COMMITTEE_SIZE * 0.7), + }, + }, + { + id: "Proposer slashing only", + opts: { + proposerSlashingLen: 2, + attesterSlashingLen: 0, + attestationLen: 0, + depositsLen: 0, + voluntaryExitLen: 0, + bitsLen: 90, + syncCommitteeBitsLen: 0, + }, + }, + { + id: "Attester slashing only", + opts: { + proposerSlashingLen: 0, + attesterSlashingLen: 5, + attestationLen: 0, + depositsLen: 0, + voluntaryExitLen: 0, + bitsLen: 90, + syncCommitteeBitsLen: 0, + }, + }, + ]; + + for (const {id, opts} of testCases) { + it(`${id}`, async () => { + const state = generatePerfTestCachedStateAltair(); + const block = getBlockAltair(state, opts); + // Populate permanent root caches of the block + ssz.altair.BeaconBlock.hashTreeRoot(block.message); + // Populate tree root caches of the state + state.hashTreeRoot(); + cachedStateAltairPopulateCaches(state); + const calculatedBlockReward = await computeBlockRewards(block.message, state as CachedBeaconStateAllForks); + const {proposerIndex, total, attestations, syncAggregate, proposerSlashings, attesterSlashings} = + calculatedBlockReward; + + // Sanity check + expect(proposerIndex).toBe(block.message.proposerIndex); + expect(total).toBe(attestations + syncAggregate + proposerSlashings + attesterSlashings); + if (opts.syncCommitteeBitsLen === 0) { + expect(syncAggregate).toBe(0); + } + if (opts.attestationLen === 0) { + expect(attestations).toBe(0); + } + if (opts.proposerSlashingLen === 0) { + expect(proposerSlashings).toBe(0); + } + if (opts.attesterSlashingLen === 0) { + expect(attesterSlashings).toBe(0); + } + + const postState = stateTransition(state as CachedBeaconStateAllForks, block, { + executionPayloadStatus: ExecutionPayloadStatus.valid, + dataAvailableStatus: DataAvailableStatus.available, + verifyProposer: false, + verifySignatures: false, + verifyStateRoot: false, + }); + + // Cross check with rewardCache + const rewardCache = postState.proposerRewards; + expect(total).toBe(rewardCache.attestations + rewardCache.syncAggregate + rewardCache.slashing); + expect(attestations).toBe(rewardCache.attestations); + expect(syncAggregate).toBe(rewardCache.syncAggregate); + expect(proposerSlashings + attesterSlashings).toBe(rewardCache.slashing); + }); + } + + // Check if `computeBlockRewards` consults reward cache in the post state first + it("Check reward cache", async () => { + const preState = generatePerfTestCachedStateAltair(); + const {opts} = testCases[0]; // Use opts of `normal case` + const block = getBlockAltair(preState, testCases[0].opts); + // Populate permanent root caches of the block + ssz.altair.BeaconBlock.hashTreeRoot(block.message); + // Populate tree root caches of the state + preState.hashTreeRoot(); + cachedStateAltairPopulateCaches(preState); + + const postState = stateTransition(preState as CachedBeaconStateAllForks, block, { + executionPayloadStatus: ExecutionPayloadStatus.valid, + dataAvailableStatus: DataAvailableStatus.available, + verifyProposer: false, + verifySignatures: false, + verifyStateRoot: false, + }); + + // Set postState's reward cache + const rewardCache = postState.proposerRewards; // Grab original reward cache before overwritten + postState.proposerRewards = {attestations: 1000, syncAggregate: 1001, slashing: 1002}; + + const calculatedBlockReward = await computeBlockRewards( + block.message, + preState as CachedBeaconStateAllForks, + postState + ); + const {proposerIndex, total, attestations, syncAggregate, proposerSlashings, attesterSlashings} = + calculatedBlockReward; + + expect(proposerIndex).toBe(block.message.proposerIndex); + expect(total).toBe(attestations + syncAggregate + proposerSlashings + attesterSlashings); + if (opts.syncCommitteeBitsLen === 0) { + expect(syncAggregate).toBe(0); + } + if (opts.attestationLen === 0) { + expect(attestations).toBe(0); + } + if (opts.proposerSlashingLen === 0) { + expect(proposerSlashings).toBe(0); + } + if (opts.attesterSlashingLen === 0) { + expect(attesterSlashings).toBe(0); + } + + // Cross check with rewardCache + expect(attestations).toBe(1000); + expect(syncAggregate).toBe(1001); + expect(proposerSlashings + attesterSlashings).not.toBe(1002); + expect(proposerSlashings + attesterSlashings).toBe(rewardCache.slashing); + }); +}); diff --git a/packages/state-transition/src/index.ts b/packages/state-transition/src/index.ts index 8786c0f6e358..0ef460e784af 100644 --- a/packages/state-transition/src/index.ts +++ b/packages/state-transition/src/index.ts @@ -61,3 +61,5 @@ export {ExecutionPayloadStatus, DataAvailableStatus, type BlockExternalData} fro export {becomesNewEth1Data} from "./block/processEth1Data.js"; // Withdrawals for new blocks export {getExpectedWithdrawals} from "./block/processWithdrawals.js"; + +export {getAttestationParticipationStatus, processAttestationsAltair} from "./block/processAttestationsAltair.js"; From c3523e1b8c065a067c0fb0a3b78f8b92d5c54cc5 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 22 Feb 2024 22:57:59 +0100 Subject: [PATCH 07/34] chore: clean up prettier-ignore in tests (#6464) * chore: clean up prettier-ignore in tests * setConfig needs to be called in describe * Set test timeout as hookTimeout * Rename variable * Remove unrelated change * Better name for min capella time variable --- .../beacon/genericServerTest/debug.test.ts | 78 ++++--- .../api/impl/beacon/node/endpoints.test.ts | 9 +- .../test/e2e/api/lodestar/lodestar.test.ts | 9 +- .../test/e2e/chain/lightclient.test.ts | 8 +- .../test/e2e/eth1/jsonRpcHttpClient.test.ts | 18 +- .../test/e2e/sync/finalizedSync.test.ts | 204 +++++++++--------- .../test/e2e/sync/unknownBlockSync.test.ts | 8 +- .../test/e2e/web3_batch_request.test.ts | 9 +- .../prover/test/e2e/web3_provider.test.ts | 9 +- packages/prover/test/utils/e2e_env.ts | 4 +- 10 files changed, 175 insertions(+), 181 deletions(-) diff --git a/packages/api/test/unit/beacon/genericServerTest/debug.test.ts b/packages/api/test/unit/beacon/genericServerTest/debug.test.ts index 9a4c4bd71ef1..3a8ccd0afe25 100644 --- a/packages/api/test/unit/beacon/genericServerTest/debug.test.ts +++ b/packages/api/test/unit/beacon/genericServerTest/debug.test.ts @@ -1,4 +1,4 @@ -import {describe, it, expect, MockInstance, beforeAll, afterAll} from "vitest"; +import {describe, it, expect, MockInstance, beforeAll, afterAll, vi} from "vitest"; import {toHexString} from "@chainsafe/ssz"; import {FastifyInstance} from "fastify"; import {ssz} from "@lodestar/types"; @@ -12,51 +12,49 @@ import {registerRoute} from "../../../../src/utils/server/registerRoute.js"; import {HttpClient} from "../../../../src/utils/client/httpClient.js"; import {testData} from "../testData/debug.js"; -describe( - "beacon / debug", - () => { - runGenericServerTest(config, getClient, getRoutes, testData); +describe("beacon / debug", () => { + // Extend timeout since states are very big + vi.setConfig({testTimeout: 30_000}); - // Get state by SSZ + runGenericServerTest(config, getClient, getRoutes, testData); - describe("getState() in SSZ format", () => { - const mockApi = getMockApi(routesData); - let baseUrl: string; - let server: FastifyInstance; + // Get state by SSZ - beforeAll(async () => { - const res = getTestServer(); - server = res.server; - for (const route of Object.values(getRoutes(config, mockApi))) { - registerRoute(server, route); - } - baseUrl = await res.start(); - }); + describe("getState() in SSZ format", () => { + const mockApi = getMockApi(routesData); + let baseUrl: string; + let server: FastifyInstance; - afterAll(async () => { - if (server !== undefined) await server.close(); - }); + beforeAll(async () => { + const res = getTestServer(); + server = res.server; + for (const route of Object.values(getRoutes(config, mockApi))) { + registerRoute(server, route); + } + baseUrl = await res.start(); + }); - for (const method of ["getState" as const, "getStateV2" as const]) { - it(method, async () => { - const state = ssz.phase0.BeaconState.defaultValue(); - const stateSerialized = ssz.phase0.BeaconState.serialize(state); - (mockApi[method] as MockInstance).mockResolvedValue(stateSerialized); + afterAll(async () => { + if (server !== undefined) await server.close(); + }); - const httpClient = new HttpClient({baseUrl}); - const client = getClient(config, httpClient); + for (const method of ["getState" as const, "getStateV2" as const]) { + it(method, async () => { + const state = ssz.phase0.BeaconState.defaultValue(); + const stateSerialized = ssz.phase0.BeaconState.serialize(state); + (mockApi[method] as MockInstance).mockResolvedValue(stateSerialized); - const res = await client[method]("head", "ssz"); + const httpClient = new HttpClient({baseUrl}); + const client = getClient(config, httpClient); - expect(res.ok).toBe(true); + const res = await client[method]("head", "ssz"); - if (res.ok) { - expect(toHexString(res.response)).toBe(toHexString(stateSerialized)); - } - }); - } - }); - }, - // Extend timeout since states are very big - {timeout: 30 * 1000} -); + expect(res.ok).toBe(true); + + if (res.ok) { + expect(toHexString(res.response)).toBe(toHexString(stateSerialized)); + } + }); + } + }); +}); diff --git a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts index e496f3ad1ef7..89d98902676b 100644 --- a/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/beacon/node/endpoints.test.ts @@ -1,4 +1,4 @@ -import {describe, beforeAll, afterAll, it, expect} from "vitest"; +import {describe, beforeAll, afterAll, it, expect, vi} from "vitest"; import {createBeaconConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; import {Api, getClient} from "@lodestar/api/beacon"; @@ -10,6 +10,8 @@ import {BeaconNode} from "../../../../../../src/node/nodejs.js"; import {getAndInitDevValidators} from "../../../../../utils/node/validator.js"; describe("beacon node api", function () { + vi.setConfig({testTimeout: 60_000}); + const restPort = 9596; const config = createBeaconConfig(chainConfigDef, Buffer.alloc(32, 0xaa)); const validatorCount = 8; @@ -62,8 +64,6 @@ describe("beacon node api", function () { expect(res.response.data.elOffline).toEqual(false); }); - // To make the code review easy for code block below - /* prettier-ignore */ it("should return 'el_offline' as 'true' when EL not available", async () => { const portElOffline = 9597; const bnElOffline = await getDevBeaconNode({ @@ -109,8 +109,7 @@ describe("beacon node api", function () { await Promise.all(validators.map((v) => v.close())); await bnElOffline.close(); - }, - {timeout: 60_000}); + }); }); describe("getHealth", () => { diff --git a/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts b/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts index 4bb8f76ef39a..e9d02beb6835 100644 --- a/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts +++ b/packages/beacon-node/test/e2e/api/lodestar/lodestar.test.ts @@ -1,4 +1,4 @@ -import {describe, it, afterEach, expect} from "vitest"; +import {describe, it, afterEach, expect, vi} from "vitest"; import {createBeaconConfig, ChainConfig} from "@lodestar/config"; import {chainConfig as chainConfigDef} from "@lodestar/config/default"; import {phase0} from "@lodestar/types"; @@ -11,6 +11,8 @@ import {ClockEvent} from "../../../../src/util/clock.js"; import {BeaconNode} from "../../../../src/index.js"; describe("api / impl / validator", function () { + vi.setConfig({testTimeout: 60_000}); + describe("getLiveness endpoint", function () { let bn: BeaconNode | undefined; const SECONDS_PER_SLOT = 2; @@ -74,8 +76,6 @@ describe("api / impl / validator", function () { }); }); - // To make the code review easy for code block below - /* prettier-ignore */ it("Should return only for previous, current and next epoch", async function () { const chainConfig: ChainConfig = {...chainConfigDef, SECONDS_PER_SLOT, ALTAIR_FORK_EPOCH}; const genesisValidatorsRoot = Buffer.alloc(32, 0xaa); @@ -128,7 +128,6 @@ describe("api / impl / validator", function () { `Request epoch ${currentEpoch - 2} is more than one epoch before or after the current epoch ${currentEpoch}` ) ); - }, - {timeout: 60_000}); + }); }); }); diff --git a/packages/beacon-node/test/e2e/chain/lightclient.test.ts b/packages/beacon-node/test/e2e/chain/lightclient.test.ts index 834b0ca0e729..6857ff8824ec 100644 --- a/packages/beacon-node/test/e2e/chain/lightclient.test.ts +++ b/packages/beacon-node/test/e2e/chain/lightclient.test.ts @@ -1,4 +1,4 @@ -import {describe, it, expect, afterEach} from "vitest"; +import {describe, it, expect, afterEach, vi} from "vitest"; import {JsonPath, toHexString, fromHexString} from "@chainsafe/ssz"; import {computeDescriptor, TreeOffsetProof} from "@chainsafe/persistent-merkle-tree"; import {ChainConfig} from "@lodestar/config"; @@ -14,9 +14,9 @@ import {getDevBeaconNode} from "../../utils/node/beacon.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; import {HeadEventData} from "../../../src/chain/index.js"; -// To make the code review easy for code block below -/* prettier-ignore */ describe("chain / lightclient", function () { + vi.setConfig({testTimeout: 600_000}); + /** * Max distance between beacon node head and lightclient head * If SECONDS_PER_SLOT === 1, there should be some margin for slow blocks, @@ -178,7 +178,7 @@ describe("chain / lightclient", function () { const head = await bn.db.block.get(fromHexString(headSummary.blockRoot)); if (!head) throw Error("First beacon node has no head block"); }); -}, {timeout: 600_000}); +}); // TODO: Re-incorporate for REST-only light-client async function getHeadStateProof( diff --git a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts index 26062b13af51..437157931a4a 100644 --- a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts +++ b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts @@ -1,15 +1,15 @@ import crypto from "node:crypto"; import http from "node:http"; -import {describe, it, expect, afterEach} from "vitest"; +import {describe, it, expect, afterEach, vi} from "vitest"; import {FetchError} from "@lodestar/api"; import {sleep} from "@lodestar/utils"; import {JsonRpcHttpClient} from "../../../src/eth1/provider/jsonRpcHttpClient.js"; import {getGoerliRpcUrl} from "../../testParams.js"; import {RpcPayload} from "../../../src/eth1/interface.js"; -// To make the code review easy for code block below -/* prettier-ignore */ describe("eth1 / jsonRpcHttpClient", function () { + vi.setConfig({testTimeout: 10_000}); + const port = 36421; const noMethodError = {code: -32601, message: "Method not found"}; const notInSpecError = "JSON RPC Error not in spec"; @@ -160,11 +160,11 @@ describe("eth1 / jsonRpcHttpClient", function () { expect.assertions(1); }); } -}, {timeout: 10_000}); +}); -// To make the code review easy for code block below -/* prettier-ignore */ describe("eth1 / jsonRpcHttpClient - with retries", function () { + vi.setConfig({testTimeout: 10_000}); + const port = 36421; const noMethodError = {code: -32601, message: "Method not found"}; const afterHooks: (() => Promise)[] = []; @@ -281,9 +281,7 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const controller = new AbortController(); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); - await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retries, timeout})).rejects.toThrow( - "Timeout request" - ); + await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retries, timeout})).rejects.toThrow("Timeout request"); expect(requestCount).toBeWithMessage(retries + 1, "Timeout request should be retried before failing"); }); @@ -346,4 +344,4 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retries})).rejects.toThrow("Method not found"); expect(requestCount).toBeWithMessage(1, "Payload error (non-network error) should not be retried"); }); -}, {timeout: 10_000}); +}); diff --git a/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts b/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts index fcfe3b5156dc..01293285d756 100644 --- a/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/finalizedSync.test.ts @@ -1,4 +1,4 @@ -import {describe, it, afterEach} from "vitest"; +import {describe, it, afterEach, vi} from "vitest"; import {assert} from "chai"; import {fromHexString} from "@chainsafe/ssz"; import {ChainConfig} from "@lodestar/config"; @@ -14,110 +14,108 @@ import {ChainEvent} from "../../../src/chain/index.js"; import {connect, onPeerConnect} from "../../utils/network.js"; import {testLogger, LogLevel, TestLoggerOpts} from "../../utils/logger.js"; -describe( - "sync / finalized sync", - function () { - const validatorCount = 8; - const testParams: Pick = { - // eslint-disable-next-line @typescript-eslint/naming-convention - SECONDS_PER_SLOT: 2, +describe("sync / finalized sync", function () { + // chain is finalized at slot 32, plus 4 slots for genesis delay => ~72s it should sync pretty fast + vi.setConfig({testTimeout: 90_000}); + + const validatorCount = 8; + const testParams: Pick = { + // eslint-disable-next-line @typescript-eslint/naming-convention + SECONDS_PER_SLOT: 2, + }; + + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); + + it("should do a finalized sync from another BN", async function () { + // single node at beginning, use main thread to verify bls + const genesisSlotsDelay = 4; + const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; + + const testLoggerOpts: TestLoggerOpts = { + level: LogLevel.info, + timestampFormat: { + format: TimestampFormatCode.EpochSlot, + genesisTime, + slotsPerEpoch: SLOTS_PER_EPOCH, + secondsPerSlot: testParams.SECONDS_PER_SLOT, + }, }; - const afterEachCallbacks: (() => Promise | void)[] = []; - afterEach(async () => { - while (afterEachCallbacks.length > 0) { - const callback = afterEachCallbacks.pop(); - if (callback) await callback(); - } + const loggerNodeA = testLogger("FinalizedSync-Node-A", testLoggerOpts); + const loggerNodeB = testLogger("FinalizedSync-Node-B", testLoggerOpts); + + const bn = await getDevBeaconNode({ + params: testParams, + options: { + sync: {isSingleNode: true}, + network: {allowPublishToZeroPeers: true, useWorker: false}, + chain: {blsVerifyAllMainThread: true}, + }, + validatorCount, + genesisTime, + logger: loggerNodeA, }); - it("should do a finalized sync from another BN", async function () { - // single node at beginning, use main thread to verify bls - const genesisSlotsDelay = 4; - const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; - - const testLoggerOpts: TestLoggerOpts = { - level: LogLevel.info, - timestampFormat: { - format: TimestampFormatCode.EpochSlot, - genesisTime, - slotsPerEpoch: SLOTS_PER_EPOCH, - secondsPerSlot: testParams.SECONDS_PER_SLOT, - }, - }; - - const loggerNodeA = testLogger("FinalizedSync-Node-A", testLoggerOpts); - const loggerNodeB = testLogger("FinalizedSync-Node-B", testLoggerOpts); - - const bn = await getDevBeaconNode({ - params: testParams, - options: { - sync: {isSingleNode: true}, - network: {allowPublishToZeroPeers: true, useWorker: false}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - genesisTime, - logger: loggerNodeA, - }); - - afterEachCallbacks.push(() => bn.close()); - - const {validators} = await getAndInitDevValidators({ - node: bn, - logPrefix: "FinalizedSyncVc", - validatorsPerClient: validatorCount, - validatorClientCount: 1, - startIndex: 0, - useRestApi: false, - testLoggerOpts, - }); - - afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.close()))); - - // stop beacon node after validators - afterEachCallbacks.push(() => bn.close()); - - await waitForEvent(bn.chain.emitter, ChainEvent.forkChoiceFinalized, 240000); - loggerNodeA.info("Node A emitted finalized checkpoint event"); - - const bn2 = await getDevBeaconNode({ - params: testParams, - options: { - api: {rest: {enabled: false}}, - network: {useWorker: false}, - chain: {blsVerifyAllMainThread: true}, - }, - validatorCount, - genesisTime, - logger: loggerNodeB, - }); - loggerNodeA.info("Node B created"); - - afterEachCallbacks.push(() => bn2.close()); - afterEachCallbacks.push(() => bn2.close()); - - const headSummary = bn.chain.forkChoice.getHead(); - const head = await bn.db.block.get(fromHexString(headSummary.blockRoot)); - if (!head) throw Error("First beacon node has no head block"); - const waitForSynced = waitForEvent( - bn2.chain.emitter, - routes.events.EventType.head, - 100000, - ({block}) => block === headSummary.blockRoot - ); - - await Promise.all([connect(bn2.network, bn.network), onPeerConnect(bn2.network), onPeerConnect(bn.network)]); - loggerNodeA.info("Node A connected to Node B"); - - try { - await waitForSynced; - loggerNodeB.info("Node B synced to Node A, received head block", {slot: head.message.slot}); - } catch (e) { - assert.fail("Failed to sync to other node in time"); - } + afterEachCallbacks.push(() => bn.close()); + + const {validators} = await getAndInitDevValidators({ + node: bn, + logPrefix: "FinalizedSyncVc", + validatorsPerClient: validatorCount, + validatorClientCount: 1, + startIndex: 0, + useRestApi: false, + testLoggerOpts, }); - }, - // chain is finalized at slot 32, plus 4 slots for genesis delay => ~72s it should sync pretty fast - {timeout: 90000} -); + + afterEachCallbacks.push(() => Promise.all(validators.map((validator) => validator.close()))); + + // stop beacon node after validators + afterEachCallbacks.push(() => bn.close()); + + await waitForEvent(bn.chain.emitter, ChainEvent.forkChoiceFinalized, 240000); + loggerNodeA.info("Node A emitted finalized checkpoint event"); + + const bn2 = await getDevBeaconNode({ + params: testParams, + options: { + api: {rest: {enabled: false}}, + network: {useWorker: false}, + chain: {blsVerifyAllMainThread: true}, + }, + validatorCount, + genesisTime, + logger: loggerNodeB, + }); + loggerNodeA.info("Node B created"); + + afterEachCallbacks.push(() => bn2.close()); + afterEachCallbacks.push(() => bn2.close()); + + const headSummary = bn.chain.forkChoice.getHead(); + const head = await bn.db.block.get(fromHexString(headSummary.blockRoot)); + if (!head) throw Error("First beacon node has no head block"); + const waitForSynced = waitForEvent( + bn2.chain.emitter, + routes.events.EventType.head, + 100000, + ({block}) => block === headSummary.blockRoot + ); + + await Promise.all([connect(bn2.network, bn.network), onPeerConnect(bn2.network), onPeerConnect(bn.network)]); + loggerNodeA.info("Node A connected to Node B"); + + try { + await waitForSynced; + loggerNodeB.info("Node B synced to Node A, received head block", {slot: head.message.slot}); + } catch (e) { + assert.fail("Failed to sync to other node in time"); + } + }); +}); diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts index e64adfc94888..a4e390533b29 100644 --- a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts @@ -1,4 +1,4 @@ -import {describe, it, afterEach} from "vitest"; +import {describe, it, afterEach, vi} from "vitest"; import {fromHexString} from "@chainsafe/ssz"; import {ChainConfig} from "@lodestar/config"; import {phase0} from "@lodestar/types"; @@ -17,9 +17,9 @@ import {testLogger, LogLevel, TestLoggerOpts} from "../../utils/logger.js"; import {BlockError, BlockErrorCode} from "../../../src/chain/errors/index.js"; import {BlockSource, getBlockInput} from "../../../src/chain/blocks/types.js"; -// To make the code review easy for code block below -/* prettier-ignore */ describe("sync / unknown block sync", function () { + vi.setConfig({testTimeout: 40_000}); + const validatorCount = 8; const testParams: Pick = { // eslint-disable-next-line @typescript-eslint/naming-convention @@ -152,4 +152,4 @@ describe("sync / unknown block sync", function () { await waitForSynced; }); } -}, {timeout: 40_000}); +}); diff --git a/packages/prover/test/e2e/web3_batch_request.test.ts b/packages/prover/test/e2e/web3_batch_request.test.ts index fc99abea4bdd..e232208a15b3 100644 --- a/packages/prover/test/e2e/web3_batch_request.test.ts +++ b/packages/prover/test/e2e/web3_batch_request.test.ts @@ -1,12 +1,13 @@ -import {describe, it, expect, beforeAll} from "vitest"; +import {describe, it, expect, beforeAll, vi} from "vitest"; import {Web3} from "web3"; import {LCTransport} from "../../src/interfaces.js"; import {createVerifiedExecutionProvider} from "../../src/web3_provider.js"; -import {rpcUrl, beaconUrl, config, waitForCapellaFork} from "../utils/e2e_env.js"; +import {rpcUrl, beaconUrl, config, waitForCapellaFork, minCapellaTimeMs} from "../utils/e2e_env.js"; import {getVerificationFailedMessage} from "../../src/utils/json_rpc.js"; -/* prettier-ignore */ describe("web3_batch_requests", function () { + vi.setConfig({hookTimeout: minCapellaTimeMs}); + let web3: Web3; beforeAll(async () => { @@ -69,4 +70,4 @@ describe("web3_batch_requests", function () { await expect(errorRequest).rejects.toThrow(getVerificationFailedMessage("eth_getBlockByHash")); }); }); -}, {timeout: 10_000}); +}); diff --git a/packages/prover/test/e2e/web3_provider.test.ts b/packages/prover/test/e2e/web3_provider.test.ts index 21ed13b8787a..ad00ae71b9fc 100644 --- a/packages/prover/test/e2e/web3_provider.test.ts +++ b/packages/prover/test/e2e/web3_provider.test.ts @@ -1,12 +1,13 @@ -import {describe, it, expect, beforeAll} from "vitest"; +import {describe, it, expect, beforeAll, vi} from "vitest"; import {Web3} from "web3"; import {ethers} from "ethers"; import {LCTransport} from "../../src/interfaces.js"; import {createVerifiedExecutionProvider} from "../../src/web3_provider.js"; -import {waitForCapellaFork, testTimeout, rpcUrl, beaconUrl, config} from "../utils/e2e_env.js"; +import {waitForCapellaFork, minCapellaTimeMs, rpcUrl, beaconUrl, config} from "../utils/e2e_env.js"; -/* prettier-ignore */ describe("web3_provider", function () { + vi.setConfig({hookTimeout: minCapellaTimeMs}); + beforeAll(async () => { await waitForCapellaFork(); }); @@ -43,4 +44,4 @@ describe("web3_provider", function () { }); }); }); -}, {timeout: testTimeout}); +}); diff --git a/packages/prover/test/utils/e2e_env.ts b/packages/prover/test/utils/e2e_env.ts index 1968fb841090..b63d276daa5f 100644 --- a/packages/prover/test/utils/e2e_env.ts +++ b/packages/prover/test/utils/e2e_env.ts @@ -14,8 +14,8 @@ const bellatrixForkEpoch = 2; const capellaForkEpoch = 3; const genesisDelaySeconds = 30 * secondsPerSlot; -// Wait for at least teh capella fork to be started -export const testTimeout = (capellaForkEpoch + 2) * 8 * 4 * 1000; +// Wait for at least the capella fork to be started +export const minCapellaTimeMs = (capellaForkEpoch + 2) * 8 * 4 * 1000; export const config = { ALTAIR_FORK_EPOCH: altairForkEpoch, From 2f71c3284c9049d35aed5db2cb901527329ae76f Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 22 Feb 2024 22:59:25 +0100 Subject: [PATCH 08/34] chore: add eslint rule to prevent unused expressions (#6445) * chore: add eslint rule to prevent unused expressions * Fix unit tests --- .eslintrc.js | 8 ++++++++ packages/beacon-node/src/eth1/utils/eth1Vote.ts | 1 - packages/beacon-node/src/network/peers/discover.ts | 2 +- .../beacon-node/test/unit/monitoring/service.test.ts | 2 +- packages/cli/test/utils/simulation/TableReporter.ts | 10 +++++----- packages/params/test/e2e/overridePresetError.ts | 6 ++---- packages/params/test/e2e/setPresetError.ts | 6 ++---- 7 files changed, 19 insertions(+), 16 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index c4f0d45fc42d..e38300c2caa9 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -80,6 +80,7 @@ module.exports = { "@typescript-eslint/no-unsafe-call": "error", "@typescript-eslint/no-unsafe-member-access": "error", "@typescript-eslint/no-unsafe-return": "error", + "@typescript-eslint/no-unused-expressions": "error", "@typescript-eslint/no-unused-vars": ["error", {varsIgnorePattern: "^_", argsIgnorePattern: "^_"}], "@typescript-eslint/no-use-before-define": "off", "@typescript-eslint/restrict-template-expressions": [ @@ -209,6 +210,13 @@ module.exports = { "import/no-named-as-default-member": "off", }, }, + { + files: ["**/perf/**/*.ts"], + rules: { + // A lot of benchmarks just need to execute expressions without using the result + "@typescript-eslint/no-unused-expressions": "off", + }, + }, { files: ["**/test/**/*.test.ts"], plugins: ["vitest"], diff --git a/packages/beacon-node/src/eth1/utils/eth1Vote.ts b/packages/beacon-node/src/eth1/utils/eth1Vote.ts index ebe5ce4b1d5b..3940ccb27bae 100644 --- a/packages/beacon-node/src/eth1/utils/eth1Vote.ts +++ b/packages/beacon-node/src/eth1/utils/eth1Vote.ts @@ -80,7 +80,6 @@ export function pickEth1Vote(state: BeaconStateAllForks, votesToConsider: phase0 else { const latestMostVotedRoot = eth1DataVotesOrder[Math.max(...eth1DataRootsMaxVotes.map((root) => eth1DataVotesOrder.indexOf(root)))]; - eth1DataHashToEth1Data; return eth1DataHashToEth1Data.get(latestMostVotedRoot) ?? state.eth1Data; } } diff --git a/packages/beacon-node/src/network/peers/discover.ts b/packages/beacon-node/src/network/peers/discover.ts index 34ea959d4ec7..1cb084846f61 100644 --- a/packages/beacon-node/src/network/peers/discover.ts +++ b/packages/beacon-node/src/network/peers/discover.ts @@ -506,6 +506,6 @@ function formatLibp2pDialError(e: Error): void { e.message.includes("stream ended before 1 bytes became available") || e.message.includes("The operation was aborted") ) { - e.stack === undefined; + e.stack = undefined; } } diff --git a/packages/beacon-node/test/unit/monitoring/service.test.ts b/packages/beacon-node/test/unit/monitoring/service.test.ts index c5911ba4b113..43936ff1756d 100644 --- a/packages/beacon-node/test/unit/monitoring/service.test.ts +++ b/packages/beacon-node/test/unit/monitoring/service.test.ts @@ -257,7 +257,7 @@ describe("monitoring / service", () => { await waitForInterval(); afterAll(() => { - service.close; + service.close(); }); return service; diff --git a/packages/cli/test/utils/simulation/TableReporter.ts b/packages/cli/test/utils/simulation/TableReporter.ts index 4a49470cc15b..fbf401ddddba 100644 --- a/packages/cli/test/utils/simulation/TableReporter.ts +++ b/packages/cli/test/utils/simulation/TableReporter.ts @@ -89,19 +89,19 @@ export class TableReporter extends SimulationReporter for (const node of nodes) { const finalized = stores["finalized"][node.beacon.id][slot]; - !isNullish(finalized) && finalizedSlots.push(finalized); + if (!isNullish(finalized)) finalizedSlots.push(finalized); const inclusionDelay = stores["inclusionDelay"][node.beacon.id][slot]; - !isNullish(inclusionDelay) && inclusionDelays.push(inclusionDelay); + if (!isNullish(inclusionDelay)) inclusionDelays.push(inclusionDelay); const attestationsCount = stores["attestationsCount"][node.beacon.id][slot]; - !isNullish(attestationsCount) && attestationCounts.push(attestationsCount); + if (!isNullish(attestationsCount)) attestationCounts.push(attestationsCount); const head = stores["head"][node.beacon.id][slot]; - !isNullish(head) && heads.push(head); + if (!isNullish(head)) heads.push(head); const connectedPeerCount = stores["connectedPeerCount"][node.beacon.id][slot]; - !isNullish(connectedPeerCount) && peersCount.push(connectedPeerCount); + if (!isNullish(connectedPeerCount)) peersCount.push(connectedPeerCount); } const head0 = heads.length > 0 ? heads[0] : null; diff --git a/packages/params/test/e2e/overridePresetError.ts b/packages/params/test/e2e/overridePresetError.ts index 14a985dfb10a..d5a665bece1f 100644 --- a/packages/params/test/e2e/overridePresetError.ts +++ b/packages/params/test/e2e/overridePresetError.ts @@ -1,11 +1,9 @@ // This script is should be run in an e2e !! -// It demostrates how NOT to change the Lodestar preset +// It demonstrates how NOT to change the Lodestar preset // 1. Import from not only @lodestar/params/setPreset will trigger an error -import {SLOTS_PER_EPOCH} from "../../lib/index.js"; +import "../../lib/index.js"; import {setActivePreset, PresetName} from "../../lib/setPreset.js"; // This line should throw // eslint-disable-next-line @typescript-eslint/naming-convention setActivePreset(PresetName.minimal, {SLOTS_PER_EPOCH: 2}); - -SLOTS_PER_EPOCH; diff --git a/packages/params/test/e2e/setPresetError.ts b/packages/params/test/e2e/setPresetError.ts index ec2c12607fca..debbb47f013d 100644 --- a/packages/params/test/e2e/setPresetError.ts +++ b/packages/params/test/e2e/setPresetError.ts @@ -1,10 +1,8 @@ // This script is should be run in an e2e !! -// It demostrates how NOT to change the Lodestar preset +// It demonstrates how NOT to change the Lodestar preset // 1. Import from not only @lodestar/params/setPreset will trigger an error -import {SLOTS_PER_EPOCH} from "../../lib/index.js"; +import "../../lib/index.js"; import {setActivePreset, PresetName} from "../../lib/setPreset.js"; // This line should throw setActivePreset(PresetName.minimal); - -SLOTS_PER_EPOCH; From 202c1d76f1e1d5ee5a38b3abe9c9829848de4843 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 22 Feb 2024 23:01:32 +0100 Subject: [PATCH 09/34] fix: request retries are abortable (#6466) * Request retries are abortable * Reduce timeouts in tests * Abort retry if not first attempt --- packages/api/src/utils/client/httpClient.ts | 1 + .../src/eth1/provider/jsonRpcHttpClient.ts | 2 +- .../test/e2e/eth1/jsonRpcHttpClient.test.ts | 8 ++++---- packages/utils/src/retry.ts | 16 +++++++++++++--- 4 files changed, 19 insertions(+), 8 deletions(-) diff --git a/packages/api/src/utils/client/httpClient.ts b/packages/api/src/utils/client/httpClient.ts index 411d986ef6a6..9cbadde1cca5 100644 --- a/packages/api/src/utils/client/httpClient.ts +++ b/packages/api/src/utils/client/httpClient.ts @@ -193,6 +193,7 @@ export class HttpClient implements IHttpClient { { retries: opts.retries, retryDelay: 200, + signal: this.getAbortSignal?.(), onRetry: (e, attempt) => { this.logger?.debug("Retrying request", {routeId, attempt, lastError: e.message}); }, diff --git a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts index 68c924741f35..42938fa24b09 100644 --- a/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts +++ b/packages/beacon-node/src/eth1/provider/jsonRpcHttpClient.ts @@ -163,7 +163,7 @@ export class JsonRpcHttpClient implements IJsonRpcHttpClient { }, { retries: opts?.retries ?? this.opts?.retries ?? 0, - retryDelay: opts?.retryDelay ?? this.opts?.retryDelay ?? 0, + retryDelay: opts?.retryDelay ?? this.opts?.retryDelay, shouldRetry: opts?.shouldRetry, signal: this.opts?.signal, onRetry: () => { diff --git a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts index 437157931a4a..4e290c6d6318 100644 --- a/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts +++ b/packages/beacon-node/test/e2e/eth1/jsonRpcHttpClient.test.ts @@ -277,7 +277,7 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const url = `http://localhost:${port}`; const payload = {method: "get", params: []}; const retries = 2; - const timeout = 2000; + const timeout = 200; const controller = new AbortController(); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); @@ -285,7 +285,7 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { expect(requestCount).toBeWithMessage(retries + 1, "Timeout request should be retried before failing"); }); - it("should retry aborted", async function () { + it("should not retry aborted", async function () { let requestCount = 0; const server = http.createServer(() => { requestCount++; @@ -306,13 +306,13 @@ describe("eth1 / jsonRpcHttpClient - with retries", function () { const url = `http://localhost:${port}`; const payload = {method: "get", params: []}; const retries = 2; - const timeout = 2000; + const timeout = 200; const controller = new AbortController(); setTimeout(() => controller.abort(), 50); const eth1JsonRpcClient = new JsonRpcHttpClient([url], {signal: controller.signal}); await expect(eth1JsonRpcClient.fetchWithRetries(payload, {retries, timeout})).rejects.toThrow("Aborted"); - expect(requestCount).toBeWithMessage(1, "Aborted request should be retried before failing"); + expect(requestCount).toBeWithMessage(1, "Aborted request should not be retried"); }); it("should not retry payload error", async function () { diff --git a/packages/utils/src/retry.ts b/packages/utils/src/retry.ts index d875e81c75d9..6c5e63deca42 100644 --- a/packages/utils/src/retry.ts +++ b/packages/utils/src/retry.ts @@ -1,3 +1,4 @@ +import {ErrorAborted} from "./errors.js"; import {sleep} from "./sleep.js"; export type RetryOptions = { @@ -21,6 +22,9 @@ export type RetryOptions = { * Milliseconds to wait before retrying again */ retryDelay?: number; + /** + * Abort signal to stop retrying + */ signal?: AbortSignal; }; @@ -39,10 +43,16 @@ export async function retry(fn: (attempt: number) => A | Promise, opts?: R let lastError: Error = Error("RetryError"); for (let i = 1; i <= maxAttempts; i++) { - try { - // If not the first attempt, invoke right before retrying - if (i > 1) onRetry?.(lastError, i); + // If not the first attempt + if (i > 1) { + if (opts?.signal?.aborted) { + throw new ErrorAborted("retry"); + } + // Invoke right before retrying + onRetry?.(lastError, i); + } + try { return await fn(i); } catch (e) { lastError = e as Error; From 8943ab686dc3744cabcfc115f5e801b13d2f6f0c Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 22 Feb 2024 23:02:05 +0100 Subject: [PATCH 10/34] ci: remove duplicate debug logs upload step (#6474) --- .github/workflows/test-sim-merge.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml index 641d5ab2732e..599df818a4bc 100644 --- a/.github/workflows/test-sim-merge.yml +++ b/.github/workflows/test-sim-merge.yml @@ -32,7 +32,7 @@ jobs: with: node-version: 20 check-latest: true - cache: yarn + cache: yarn - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT @@ -90,13 +90,6 @@ jobs: ENGINE_PORT: 8551 ETH_PORT: 8661 - - name: Upload debug log test files - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: debug-test-logs - path: packages/beacon-node/test-logs - - name: Pull geth withdrawals run: docker pull $GETH_WITHDRAWALS_IMAGE From 5de9a7419430190c69898386c74689c5a0f6acd8 Mon Sep 17 00:00:00 2001 From: g11tech Date: Sat, 24 Feb 2024 17:15:45 +0530 Subject: [PATCH 11/34] fix: increase bodyLimit of the beacon api server (#6476) * fix: increase bodyLimit of the beacon api server * update limit --- packages/beacon-node/src/api/rest/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/api/rest/index.ts b/packages/beacon-node/src/api/rest/index.ts index 47488a2e3d9a..fa4a52c9c014 100644 --- a/packages/beacon-node/src/api/rest/index.ts +++ b/packages/beacon-node/src/api/rest/index.ts @@ -21,7 +21,7 @@ export const beaconRestApiServerOpts: BeaconRestApiServerOpts = { port: 9596, cors: "*", // beacon -> validator API is trusted, and for large amounts of keys the payload is multi-MB - bodyLimit: 10 * 1024 * 1024, // 10MB + bodyLimit: 20 * 1024 * 1024, // 20MB for big block + blobs }; export type BeaconRestApiServerModules = RestApiServerModules & { From 06210efab54433a65f275de367c6f8e17dbd4a62 Mon Sep 17 00:00:00 2001 From: Harikrishnan Shaji Date: Sat, 24 Feb 2024 20:31:36 +0530 Subject: [PATCH 12/34] chore: update yarn links to point to classic yarn (#6477) --- CONTRIBUTING.md | 2 +- README.md | 2 +- packages/api/README.md | 2 +- packages/beacon-node/README.md | 2 +- packages/prover/README.md | 2 +- packages/reqresp/README.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 009eaa850367..cb81ae78ad6f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Thanks for your interest in contributing to Lodestar. It's people like you that ## Prerequisites - :gear: [NodeJS](https://nodejs.org/) (LTS) -- :toolbox: [Yarn](https://yarnpkg.com/) +- :toolbox: [Yarn](https://classic.yarnpkg.com/lang/en/) ### MacOS Specifics diff --git a/README.md b/README.md index b5accf7a2c61..6803706c5d75 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ ## Prerequisites - :gear: [NodeJS](https://nodejs.org/) (LTS) -- :toolbox: [Yarn](https://yarnpkg.com/) +- :toolbox: [Yarn](https://classic.yarnpkg.com/lang/en/) ###### Developer Quickstart: diff --git a/packages/api/README.md b/packages/api/README.md index e8837961159b..16596e91b316 100644 --- a/packages/api/README.md +++ b/packages/api/README.md @@ -36,7 +36,7 @@ api.beacon ## Prerequisites - [NodeJS](https://nodejs.org/) (LTS) -- [Yarn](https://yarnpkg.com/) +- [Yarn](https://classic.yarnpkg.com/lang/en/) ## What you need diff --git a/packages/beacon-node/README.md b/packages/beacon-node/README.md index 12d1aca05e95..8a016236462d 100644 --- a/packages/beacon-node/README.md +++ b/packages/beacon-node/README.md @@ -9,7 +9,7 @@ ## Prerequisites -- [Yarn](https://yarnpkg.com/) +- [Yarn](https://classic.yarnpkg.com/lang/en/) ## What you need diff --git a/packages/prover/README.md b/packages/prover/README.md index 21f167fc43de..8d43fd861473 100644 --- a/packages/prover/README.md +++ b/packages/prover/README.md @@ -114,7 +114,7 @@ lodestar-prover proxy \ ## Prerequisites - [NodeJS](https://nodejs.org/) (LTS) -- [Yarn](https://yarnpkg.com/) +- [Yarn](https://classic.yarnpkg.com/lang/en/) ## What you need diff --git a/packages/reqresp/README.md b/packages/reqresp/README.md index 6f8aea550ca6..8298a15187ad 100644 --- a/packages/reqresp/README.md +++ b/packages/reqresp/README.md @@ -44,7 +44,7 @@ async function getReqResp(libp2p: Libp2p, logger: Logger): Promise { ## Prerequisites - [NodeJS](https://nodejs.org/) (LTS) -- [Yarn](https://yarnpkg.com/) +- [Yarn](https://classic.yarnpkg.com/lang/en/) ## What you need From 2bb4ae6c9ee2fa464cca63084491b199143e6106 Mon Sep 17 00:00:00 2001 From: g11tech Date: Mon, 26 Feb 2024 13:20:33 +0530 Subject: [PATCH 13/34] fix: remove extra validation from builder response in builder proposal (#6478) remove extra validation from builder response in builder proposal --- .../beacon-node/src/execution/builder/http.ts | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/beacon-node/src/execution/builder/http.ts b/packages/beacon-node/src/execution/builder/http.ts index 986a5f343bfb..9e26faf90b63 100644 --- a/packages/beacon-node/src/execution/builder/http.ts +++ b/packages/beacon-node/src/execution/builder/http.ts @@ -1,5 +1,4 @@ -import {byteArrayEquals, toHexString} from "@chainsafe/ssz"; -import {allForks, bellatrix, Slot, Root, BLSPubkey, ssz, deneb, Wei} from "@lodestar/types"; +import {allForks, bellatrix, Slot, Root, BLSPubkey, deneb, Wei} from "@lodestar/types"; import {parseExecutionPayloadAndBlobsBundle, reconstructFullBlockOrContents} from "@lodestar/state-transition"; import {ChainForkConfig} from "@lodestar/config"; import {Logger} from "@lodestar/logger"; @@ -123,17 +122,12 @@ export class ExecutionBuilderHttp implements IExecutionBuilder { const {data} = res.response; const {executionPayload, blobsBundle} = parseExecutionPayloadAndBlobsBundle(data); - // some validations for execution payload - const expectedTransactionsRoot = signedBlindedBlock.message.body.executionPayloadHeader.transactionsRoot; - const actualTransactionsRoot = ssz.bellatrix.Transactions.hashTreeRoot(executionPayload.transactions); - if (!byteArrayEquals(expectedTransactionsRoot, actualTransactionsRoot)) { - throw Error( - `Invalid transactionsRoot of the builder payload, expected=${toHexString( - expectedTransactionsRoot - )}, actual=${toHexString(actualTransactionsRoot)}` - ); - } + // for the sake of timely proposals we can skip matching the payload with payloadHeader + // if the roots (transactions, withdrawals) don't match, this will likely lead to a block with + // invalid signature, but there is no recourse to this anyway so lets just proceed and will + // probably need diagonis if this block turns out to be invalid because of some bug + // const contents = blobsBundle ? {blobs: blobsBundle.blobs, kzgProofs: blobsBundle.proofs} : null; return reconstructFullBlockOrContents(signedBlindedBlock, {executionPayload, contents}); } From e9a3f07a64ffaf98227c88d9ff0c439d036c129f Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Mon, 26 Feb 2024 17:10:19 +0100 Subject: [PATCH 14/34] test: consolidate sim-merge tests (#6344) * Add new assertions for multi-fork sim tests * Remove redundant sim-merge test * Remove the unused script file and update docs * Fix an edge case for multi-client tests * Remove unused ci tasks * Enable all nodes --- .github/workflows/test-sim-merge.yml | 19 - .../contribution/testing/integration-tests.md | 2 - packages/beacon-node/package.json | 1 - .../test/sim/merge-interop.test.ts | 458 ------------------ packages/cli/test/sim/mixed_client.test.ts | 3 +- packages/cli/test/sim/multi_fork.test.ts | 21 + .../assertions/accountBalanceAssertion.ts | 74 +++ .../assertions/executionHeadAssertion.ts | 51 ++ .../cli/test/utils/simulation/constants.ts | 3 + .../simulation/execution_clients/geth.ts | 20 +- 10 files changed, 162 insertions(+), 490 deletions(-) delete mode 100644 packages/beacon-node/test/sim/merge-interop.test.ts create mode 100644 packages/cli/test/utils/simulation/assertions/accountBalanceAssertion.ts create mode 100644 packages/cli/test/utils/simulation/assertions/executionHeadAssertion.ts diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml index 599df818a4bc..7ae64466a45d 100644 --- a/.github/workflows/test-sim-merge.yml +++ b/.github/workflows/test-sim-merge.yml @@ -55,28 +55,9 @@ jobs: - name: Pull Geth run: docker pull $GETH_IMAGE - - name: Test Lodestar <> Geth interop - run: yarn test:sim:merge-interop - working-directory: packages/beacon-node - env: - EL_BINARY_DIR: ${{ env.GETH_IMAGE }} - EL_SCRIPT_DIR: gethdocker - ENGINE_PORT: 8551 - ETH_PORT: 8545 - TX_SCENARIOS: simple - - name: Pull Nethermind run: docker pull $NETHERMIND_IMAGE - - name: Test Lodestar <> Nethermind interop - run: yarn test:sim:merge-interop - working-directory: packages/beacon-node - env: - EL_BINARY_DIR: ${{ env.NETHERMIND_IMAGE }} - EL_SCRIPT_DIR: netherminddocker - ENGINE_PORT: 8551 - ETH_PORT: 8545 - - name: Pull mergemock run: docker pull $MERGEMOCK_IMAGE diff --git a/docs/pages/contribution/testing/integration-tests.md b/docs/pages/contribution/testing/integration-tests.md index dcf0201e4949..c93cb635afca 100644 --- a/docs/pages/contribution/testing/integration-tests.md +++ b/docs/pages/contribution/testing/integration-tests.md @@ -20,8 +20,6 @@ The images used by this test during CI are: - `GETH_WITHDRAWALS_IMAGE: g11tech/geth:withdrawalsfeb8` - `ETHEREUMJS_WITHDRAWALS_IMAGE: g11tech/ethereumjs:blobs-b6b63` -#### `test:sim:merge-interop` - #### `test:sim:mergemock` #### `yarn test:sim:blobs` diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 7608035cb440..4908b2f4ea23 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -80,7 +80,6 @@ "test:unit": "wrapper() { yarn test:unit:minimal $@ && yarn test:unit:mainnet $@; }; wrapper", "test:e2e": "LODESTAR_PRESET=minimal vitest --run --segfaultRetry 3 --config vitest.e2e.config.ts --dir test/e2e", "test:sim": "vitest --run test/sim/**/*.test.ts", - "test:sim:merge-interop": "vitest --run test/sim/merge-interop.test.ts", "test:sim:mergemock": "vitest --run test/sim/mergemock.test.ts", "test:sim:withdrawals": "vitest --run test/sim/withdrawal-interop.test.ts", "test:sim:blobs": "vitest --run test/sim/4844-interop.test.ts", diff --git a/packages/beacon-node/test/sim/merge-interop.test.ts b/packages/beacon-node/test/sim/merge-interop.test.ts deleted file mode 100644 index 65c3381f94fb..000000000000 --- a/packages/beacon-node/test/sim/merge-interop.test.ts +++ /dev/null @@ -1,458 +0,0 @@ -import fs from "node:fs"; -import {describe, it, afterAll, afterEach, vi} from "vitest"; -import {fromHexString} from "@chainsafe/ssz"; -import {isExecutionStateType, isMergeTransitionComplete} from "@lodestar/state-transition"; -import {LogLevel, sleep} from "@lodestar/utils"; -import {TimestampFormatCode} from "@lodestar/logger"; -import {ForkName, SLOTS_PER_EPOCH} from "@lodestar/params"; -import {ChainConfig} from "@lodestar/config"; -import {routes} from "@lodestar/api"; -import {Epoch} from "@lodestar/types"; -import {ValidatorProposerConfig} from "@lodestar/validator"; - -import {ExecutionPayloadStatus, PayloadAttributes} from "../../src/execution/engine/interface.js"; -import {initializeExecutionEngine} from "../../src/execution/index.js"; -import {ClockEvent} from "../../src/util/clock.js"; -import {testLogger, TestLoggerOpts} from "../utils/logger.js"; -import {getDevBeaconNode} from "../utils/node/beacon.js"; -import {BeaconRestApiServerOpts} from "../../src/api/index.js"; -import {simTestInfoTracker} from "../utils/node/simTest.js"; -import {getAndInitDevValidators} from "../utils/node/validator.js"; -import {Eth1Provider} from "../../src/index.js"; -import {ZERO_HASH} from "../../src/constants/index.js"; -import {bytesToData, dataToBytes, quantityToNum} from "../../src/eth1/provider/utils.js"; -import {defaultExecutionEngineHttpOpts} from "../../src/execution/engine/http.js"; -import {runEL, ELStartMode, ELClient, sendTransaction, getBalance} from "../utils/runEl.js"; -import {logFilesDir} from "./params.js"; -import {shell} from "./shell.js"; - -// NOTE: Must specify -// EL_BINARY_DIR: File path to locate the EL executable -// EL_SCRIPT_DIR: Directory in packages/beacon-node for the EL client, from where to -// execute post-merge/pre-merge EL scenario scripts -// ETH_PORT: EL port on localhost hosting non auth protected eth_ methods -// ENGINE_PORT: Specify the port on which an jwt auth protected engine api is being hosted, -// typically by default at 8551 for geth. Some ELs could host it as same port as eth_ apis, -// but just with the engine_ methods protected. In that case this param can be skipped -// TX_SCENARIOS: comma seprated transaction scenarios this EL client build supports -// Example: -// ``` -// $ EL_BINARY_DIR=/home/lion/Code/eth2.0/merge-interop/go-ethereum/build/bin \ -// EL_SCRIPT_DIR=geth ETH_PORT=8545 ENGINE_PORT=8551 TX_SCENARIOS=simple \ -// ../../node_modules/.bin/vitest --run test/sim/merge.test.ts -// ``` - -/* eslint-disable no-console, @typescript-eslint/naming-convention, quotes */ - -// BELLATRIX_EPOCH will happen at 2 sec * 8 slots = 16 sec -// 10 ttd / 2 difficulty per block = 5 blocks * 5 sec = 25 sec -const terminalTotalDifficultyPreMerge = 10; -const TX_SCENARIOS = process.env.TX_SCENARIOS?.split(",") || []; -const jwtSecretHex = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; -const retries = defaultExecutionEngineHttpOpts.retries; -const retryDelay = defaultExecutionEngineHttpOpts.retryDelay; - -describe("executionEngine / ExecutionEngineHttp", function () { - if (!process.env.EL_BINARY_DIR || !process.env.EL_SCRIPT_DIR) { - throw Error( - `EL ENV must be provided, EL_BINARY_DIR: ${process.env.EL_BINARY_DIR}, EL_SCRIPT_DIR: ${process.env.EL_SCRIPT_DIR}` - ); - } - vi.setConfig({testTimeout: 10 * 60 * 1000}); - - const dataPath = fs.mkdtempSync("lodestar-test-merge-interop"); - const elSetupConfig = { - elScriptDir: process.env.EL_SCRIPT_DIR, - elBinaryDir: process.env.EL_BINARY_DIR, - }; - const elRunOptions = { - dataPath, - jwtSecretHex, - enginePort: parseInt(process.env.ENGINE_PORT ?? "8551"), - ethPort: parseInt(process.env.ETH_PORT ?? "8545"), - }; - - const controller = new AbortController(); - afterAll(async () => { - controller?.abort(); - await shell(`sudo rm -rf ${dataPath}`); - }); - - const afterEachCallbacks: (() => Promise | void)[] = []; - afterEach(async () => { - while (afterEachCallbacks.length > 0) { - const callback = afterEachCallbacks.pop(); - if (callback) await callback(); - } - }); - - it("Send stub payloads to EL", async () => { - const {elClient, tearDownCallBack} = await runEL( - {...elSetupConfig, mode: ELStartMode.PostMerge}, - {...elRunOptions, ttd: BigInt(0)}, - controller.signal - ); - afterEachCallbacks.push(() => tearDownCallBack()); - const {genesisBlockHash, engineRpcUrl, ethRpcUrl} = elClient; - - if (TX_SCENARIOS.includes("simple")) { - await sendTransaction(ethRpcUrl, { - from: "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - to: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - gas: "0x76c0", - gasPrice: "0x9184e72a000", - value: "0x9184e72a", - }); - - const balance = await getBalance(ethRpcUrl, "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - if (balance != "0x0") throw new Error("Invalid Balance: " + balance); - } - - //const controller = new AbortController(); - const executionEngine = initializeExecutionEngine( - {mode: "http", urls: [engineRpcUrl], jwtSecretHex, retries, retryDelay}, - {signal: controller.signal, logger: testLogger("Node-A-Engine")} - ); - - // 1. Prepare a payload - - /** - * curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"engine_forkchoiceUpdatedV1","params":[{"headBlockHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a", "safeBlockHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a", "finalizedBlockHash":"0x0000000000000000000000000000000000000000000000000000000000000000"}, {"timestamp":"0x5", "prevRandao":"0x0000000000000000000000000000000000000000000000000000000000000000", "feeRecipient":"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"}],"id":67}' http://localhost:8550 - **/ - - const preparePayloadParams: PayloadAttributes = { - // Note: this is created with a pre-defined genesis.json - timestamp: quantityToNum("0x5"), - prevRandao: dataToBytes("0x0000000000000000000000000000000000000000000000000000000000000000", 32), - suggestedFeeRecipient: "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - }; - - const finalizedBlockHash = "0x0000000000000000000000000000000000000000000000000000000000000000"; - - const payloadId = await executionEngine.notifyForkchoiceUpdate( - ForkName.bellatrix, - genesisBlockHash, - //use finalizedBlockHash as safeBlockHash - finalizedBlockHash, - finalizedBlockHash, - preparePayloadParams - ); - - if (!payloadId) throw Error("InvalidPayloadId"); - - // 2. Get the payload - /** - * curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"engine_getPayloadV1","params":["0xa247243752eb10b4"],"id":67}' http://localhost:8550 - **/ - - const {executionPayload: payload} = await executionEngine.getPayload(ForkName.bellatrix, payloadId); - if (TX_SCENARIOS.includes("simple")) { - if (payload.transactions.length !== 1) - throw new Error("Expected a simple transaction to be in the fetched payload"); - const balance = await getBalance(ethRpcUrl, "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - if (balance != "0x0") throw new Error("Invalid Balance: " + balance); - } - - // 3. Execute the payload - /** - * curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"engine_newPayloadV1","params":[{"parentHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a","coinbase":"0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b","stateRoot":"0xca3149fa9e37db08d1cd49c9061db1002ef1cd58db2210f2115c8c989b2bdf45","receiptRoot":"0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421","logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","prevRandao":"0x0000000000000000000000000000000000000000000000000000000000000000","blockNumber":"0x1","gasLimit":"0x1c9c380","gasUsed":"0x0","timestamp":"0x5","extraData":"0x","baseFeePerGas":"0x7","blockHash":"0x3559e851470f6e7bbed1db474980683e8c315bfce99b2a6ef47c057c04de7858","transactions":[]}],"id":67}' http://localhost:8550 - **/ - - const payloadResult = await executionEngine.notifyNewPayload(ForkName.bellatrix, payload); - if (payloadResult.status !== ExecutionPayloadStatus.VALID) { - throw Error("getPayload returned payload that notifyNewPayload deems invalid"); - } - - // 4. Update the fork choice - - /** - * curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"engine_forkchoiceUpdatedV1","params":[{"headBlockHash":"0x3559e851470f6e7bbed1db474980683e8c315bfce99b2a6ef47c057c04de7858", "safeBlockHash":"0x3559e851470f6e7bbed1db474980683e8c315bfce99b2a6ef47c057c04de7858", "finalizedBlockHash":"0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a"}, null],"id":67}' http://localhost:8550 - **/ - - await executionEngine.notifyForkchoiceUpdate( - ForkName.bellatrix, - bytesToData(payload.blockHash), - genesisBlockHash, - genesisBlockHash - ); - - if (TX_SCENARIOS.includes("simple")) { - const balance = await getBalance(ethRpcUrl, "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - if (balance !== "0x9184e72a") throw new Error("Invalid Balance"); - } - - // Error cases - // 1. unknown payload - - /** - * curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"engine_getPayload", - * "params":["0x123"] - * ,"id":67}' http://localhost:8545 - */ - - // await executionEngine.getPayload(1234567); - - // 2. unknown header - - /** - * curl -X POST -H "Content-Type: application/json" --data '{"jsonrpc":"2.0","method":"engine_consensusValidated","params":[{ - * "blockHash":"0x0000000000000000000000000000000000000000000000000000000000000000", - * "status":"VALID" - * }],"id":67}' http://localhost:8545 - */ - }); - - it("Post-merge, run for a few blocks", async function () { - console.log("\n\nPost-merge, run for a few blocks\n\n"); - const {elClient, tearDownCallBack} = await runEL( - {...elSetupConfig, mode: ELStartMode.PostMerge}, - {...elRunOptions, ttd: BigInt(0)}, - controller.signal - ); - afterEachCallbacks.push(() => tearDownCallBack()); - - await runNodeWithEL({ - elClient, - bellatrixEpoch: 0, - testName: "post-merge", - }); - }); - - it("Pre-merge, run for a few blocks", async function () { - console.log("\n\nPre-merge, run for a few blocks\n\n"); - const {elClient, tearDownCallBack} = await runEL( - {...elSetupConfig, mode: ELStartMode.PreMerge}, - {...elRunOptions, ttd: BigInt(terminalTotalDifficultyPreMerge)}, - controller.signal - ); - afterEachCallbacks.push(() => tearDownCallBack()); - - await runNodeWithEL({ - elClient, - bellatrixEpoch: 1, - testName: "pre-merge", - }); - }); - - async function runNodeWithEL({ - elClient, - bellatrixEpoch, - testName, - }: { - elClient: ELClient; - bellatrixEpoch: Epoch; - testName: string; - }): Promise { - const {genesisBlockHash, ttd, engineRpcUrl, ethRpcUrl} = elClient; - const validatorClientCount = 1; - const validatorsPerClient = 32; - - const testParams: Pick = { - SECONDS_PER_SLOT: 2, - }; - - // Should reach justification in 6 epochs max. - // Merge block happens at epoch 2 slot 4. Then 4 epochs to finalize - const expectedEpochsToFinish = 6; - // 1 epoch of margin of error - const epochsOfMargin = 1; - const timeoutSetupMargin = 30 * 1000; // Give extra 30 seconds of margin - - // delay a bit so regular sync sees it's up to date and sync is completed from the beginning - const genesisSlotsDelay = 8; - - const timeout = - ((epochsOfMargin + expectedEpochsToFinish) * SLOTS_PER_EPOCH + genesisSlotsDelay) * - testParams.SECONDS_PER_SLOT * - 1000; - - vi.setConfig({testTimeout: timeout + 2 * timeoutSetupMargin}); - - const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; - - const testLoggerOpts: TestLoggerOpts = { - level: LogLevel.info, - file: { - filepath: `${logFilesDir}/merge-interop-${testName}.log`, - level: LogLevel.debug, - }, - timestampFormat: { - format: TimestampFormatCode.EpochSlot, - genesisTime, - slotsPerEpoch: SLOTS_PER_EPOCH, - secondsPerSlot: testParams.SECONDS_PER_SLOT, - }, - }; - const loggerNodeA = testLogger("Node-A", testLoggerOpts); - - const bn = await getDevBeaconNode({ - params: { - ...testParams, - ALTAIR_FORK_EPOCH: 0, - BELLATRIX_FORK_EPOCH: bellatrixEpoch, - TERMINAL_TOTAL_DIFFICULTY: ttd, - }, - options: { - api: {rest: {enabled: true} as BeaconRestApiServerOpts}, - sync: {isSingleNode: true}, - network: {allowPublishToZeroPeers: true, discv5: null}, - // Now eth deposit/merge tracker methods directly available on engine endpoints - eth1: {enabled: true, providerUrls: [engineRpcUrl], jwtSecretHex}, - executionEngine: {urls: [engineRpcUrl], jwtSecretHex}, - chain: {suggestedFeeRecipient: "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"}, - }, - validatorCount: validatorClientCount * validatorsPerClient, - logger: loggerNodeA, - genesisTime, - eth1BlockHash: fromHexString(genesisBlockHash), - }); - - afterEachCallbacks.push(async function () { - await bn.close(); - await sleep(1000); - }); - - const stopInfoTracker = simTestInfoTracker(bn, loggerNodeA); - const valProposerConfig = { - proposerConfig: { - "0xa99a76ed7796f7be22d5b7e85deeb7c5677e88e511e0b337618f8c4eb61349b4bf2d153f649f7b53359fe8b94a38e44c": { - graffiti: "graffiti", - strictFeeRecipientCheck: true, - feeRecipient: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - builder: { - gasLimit: 30000000, - builderSelection: "executiononly", - }, - }, - "0xa4855c83d868f772a579133d9f23818008417b743e8447e235d8eb78b1d8f8a9f63f98c551beb7de254400f89592314d": { - feeRecipient: "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb", - builder: { - gasLimit: 35000000, - }, - }, - }, - defaultConfig: { - graffiti: "default graffiti", - strictFeeRecipientCheck: true, - feeRecipient: "0xcccccccccccccccccccccccccccccccccccccccc", - builder: { - gasLimit: 30000000, - }, - }, - } as ValidatorProposerConfig; - - const {validators} = await getAndInitDevValidators({ - logPrefix: "Node-A", - node: bn, - validatorsPerClient, - validatorClientCount, - startIndex: 0, - // At least one sim test must use the REST API for beacon <-> validator comms - useRestApi: true, - testLoggerOpts, - valProposerConfig, - }); - - afterEachCallbacks.push(async function () { - await Promise.all(validators.map((v) => v.close())); - }); - - if (TX_SCENARIOS.includes("simple")) { - // If bellatrixEpoch > 0, this is the case of pre-merge transaction submission on EL pow - await sendTransaction(ethRpcUrl, { - from: "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - to: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - gas: "0x76c0", - gasPrice: "0x9184e72a000", - value: "0x9184e72a", - }); - } - - await new Promise((resolve, reject) => { - // Play TX_SCENARIOS - bn.chain.clock.on(ClockEvent.slot, async (slot) => { - if (slot < 2) return; - switch (slot) { - // If bellatrixEpoch > 0, this is the case of pre-merge transaction confirmation on EL pow - case 2: - if (TX_SCENARIOS.includes("simple")) { - const balance = await getBalance(ethRpcUrl, "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - if (balance !== "0x9184e72a") reject("Invalid Balance"); - } - break; - - // By this slot, ttd should be reached and merge complete - case Number(ttd) + 3: { - const headState = bn.chain.getHeadState(); - if (!(isExecutionStateType(headState) && isMergeTransitionComplete(headState))) { - reject("Merge not completed"); - } - - // Send another tx post-merge, total amount in destination account should be double after this is included in chain - if (TX_SCENARIOS.includes("simple")) { - await sendTransaction(ethRpcUrl, { - from: "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b", - to: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", - gas: "0x76c0", - gasPrice: "0x9184e72a000", - value: "0x9184e72a", - }); - } - break; - } - - default: - } - }); - - bn.chain.emitter.on(routes.events.EventType.finalizedCheckpoint, (checkpoint) => { - // Resolve only if the finalized checkpoint includes execution payload - const finalizedBlock = bn.chain.forkChoice.getBlockHex(checkpoint.block); - if (finalizedBlock?.executionPayloadBlockHash !== null) { - console.log(`\nGot finalized event, stopping validators and nodes\n`); - resolve(); - } - }); - }); - - // Stop chain and un-subscribe events so the execution engine won't update it's head - // Allow some time to broadcast finalized events and complete the importBlock routine - await Promise.all(validators.map((v) => v.close())); - await bn.close(); - await sleep(500); - - if (bn.chain.beaconProposerCache.get(1) !== "0xcccccccccccccccccccccccccccccccccccccccc") { - throw Error("Invalid feeRecipient set at BN"); - } - - // Assertions to make sure the end state is good - // 1. The proper head is set - const rpc = new Eth1Provider({DEPOSIT_CONTRACT_ADDRESS: ZERO_HASH}, {providerUrls: [engineRpcUrl], jwtSecretHex}); - const consensusHead = bn.chain.forkChoice.getHead(); - const executionHeadBlockNumber = await rpc.getBlockNumber(); - const executionHeadBlock = await rpc.getBlockByNumber(executionHeadBlockNumber); - if (!executionHeadBlock) throw Error("Execution has not head block"); - if (consensusHead.executionPayloadBlockHash !== executionHeadBlock.hash) { - throw Error( - "Consensus head not equal to execution head: " + - JSON.stringify({ - executionHeadBlockNumber, - executionHeadBlockHash: executionHeadBlock.hash, - consensusHeadExecutionPayloadBlockHash: consensusHead.executionPayloadBlockHash, - consensusHeadSlot: consensusHead.slot, - }) - ); - } - - if (TX_SCENARIOS.includes("simple")) { - const balance = await getBalance(ethRpcUrl, "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); - // 0x12309ce54 = 2 * 0x9184e72a - if (balance !== "0x12309ce54") throw Error("Invalid Balance"); - } - - // wait for 1 slot to print current epoch stats - await sleep(1 * bn.config.SECONDS_PER_SLOT * 1000); - stopInfoTracker(); - console.log("\n\nDone\n\n"); - } -}); diff --git a/packages/cli/test/sim/mixed_client.test.ts b/packages/cli/test/sim/mixed_client.test.ts index 80c20471ede5..9d56349457e6 100644 --- a/packages/cli/test/sim/mixed_client.test.ts +++ b/packages/cli/test/sim/mixed_client.test.ts @@ -70,6 +70,7 @@ env.tracker.register({ await env.start({runTimeoutMs: estimatedTimeoutMs}); await connectAllNodes(env.nodes); -await waitForSlot(env.clock.getLastSlotOfEpoch(capellaForkEpoch + 1), env.nodes, {env, silent: true}); +// Stopping at last slot usually cause assertion to fail because of missing data as node are shutting down +await waitForSlot(env.clock.getLastSlotOfEpoch(capellaForkEpoch + 1) + 2, env.nodes, {env, silent: true}); await env.stop(); diff --git a/packages/cli/test/sim/multi_fork.test.ts b/packages/cli/test/sim/multi_fork.test.ts index 1302ad98bc82..d816e0ddae54 100644 --- a/packages/cli/test/sim/multi_fork.test.ts +++ b/packages/cli/test/sim/multi_fork.test.ts @@ -17,6 +17,8 @@ import { import {nodeAssertion} from "../utils/simulation/assertions/nodeAssertion.js"; import {mergeAssertion} from "../utils/simulation/assertions/mergeAssertion.js"; import {createForkAssertion} from "../utils/simulation/assertions/forkAssertion.js"; +import {createAccountBalanceAssertion} from "../utils/simulation/assertions/accountBalanceAssertion.js"; +import {createExecutionHeadAssertion} from "../utils/simulation/assertions/executionHeadAssertion.js"; const altairForkEpoch = 2; const bellatrixForkEpoch = 4; @@ -132,6 +134,25 @@ env.tracker.register({ }, }); +env.tracker.register( + createAccountBalanceAssertion({ + address: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + sendTransactionsAtSlot: [ + env.clock.getFirstSlotOfEpoch(altairForkEpoch) + 4, + env.clock.getFirstSlotOfEpoch(bellatrixForkEpoch) + 4, + ], + validateTotalBalanceAt: [env.clock.getFirstSlotOfEpoch(bellatrixForkEpoch + 1) + 4], + targetNode: env.nodes[0], + }) +); + +env.tracker.register( + createExecutionHeadAssertion({ + // Second last slot of second bellatrix epoch + checkForSlot: [env.clock.getLastSlotOfEpoch(bellatrixForkEpoch + 1) - 1], + }) +); + await env.start({runTimeoutMs: estimatedTimeoutMs}); await connectAllNodes(env.nodes); diff --git a/packages/cli/test/utils/simulation/assertions/accountBalanceAssertion.ts b/packages/cli/test/utils/simulation/assertions/accountBalanceAssertion.ts new file mode 100644 index 000000000000..2367763b0a2f --- /dev/null +++ b/packages/cli/test/utils/simulation/assertions/accountBalanceAssertion.ts @@ -0,0 +1,74 @@ +import {EL_GENESIS_ACCOUNT} from "../constants.js"; +import {AssertionMatch, AssertionResult, NodePair, SimulationAssertion} from "../interfaces.js"; + +function hexToBigInt(num: string): bigint { + return num.startsWith("0x") ? BigInt(num) : BigInt(`0x${num}`); +} + +function bigIntToHex(num: bigint): string { + return `0x${num.toString(16)}`; +} + +const transactionAmount = BigInt(2441406250); + +export function createAccountBalanceAssertion({ + address, + sendTransactionsAtSlot, + validateTotalBalanceAt, + targetNode, +}: { + address: string; + sendTransactionsAtSlot: number[]; + validateTotalBalanceAt: number[]; + targetNode: NodePair; +}): SimulationAssertion<`accountBalance_${typeof address}`, bigint> { + return { + id: `accountBalance_${address}`, + match({slot, node}) { + if (sendTransactionsAtSlot.includes(slot) && node.id === targetNode.id) return AssertionMatch.Capture; + if (validateTotalBalanceAt.includes(slot) && node.id === targetNode.id) return AssertionMatch.Assert; + return AssertionMatch.None; + }, + async capture({node}) { + await node.execution.provider?.getRpc().fetch({ + method: "eth_sendTransaction", + params: [ + { + to: address, + from: EL_GENESIS_ACCOUNT, + gas: "0x76c0", + gasPrice: "0x9184e72a000", + value: bigIntToHex(transactionAmount), + }, + ], + }); + + // Capture the value transferred to account + return transactionAmount; + }, + async assert({node, store, slot}) { + const errors: AssertionResult[] = []; + + const expectedCaptureSlots = sendTransactionsAtSlot.filter((s) => s <= slot); + if (expectedCaptureSlots.length === 0) errors.push(`No transaction was sent to account ${address}`); + + let expectedBalanceAtCurrentSlot = BigInt(0); + for (const captureSlot of expectedCaptureSlots) { + expectedBalanceAtCurrentSlot += BigInt(store[captureSlot]); + } + + const balance = hexToBigInt( + (await node.execution.provider?.getRpc().fetch({method: "eth_getBalance", params: [address, "latest"]})) ?? + "0x0" + ); + + if (balance !== expectedBalanceAtCurrentSlot) { + errors.push( + `Account balance for ${address} does not match. Expected: ${expectedBalanceAtCurrentSlot}, got: ${balance}` + ); + } + + return errors; + }, + }; +} diff --git a/packages/cli/test/utils/simulation/assertions/executionHeadAssertion.ts b/packages/cli/test/utils/simulation/assertions/executionHeadAssertion.ts new file mode 100644 index 000000000000..595f98aea701 --- /dev/null +++ b/packages/cli/test/utils/simulation/assertions/executionHeadAssertion.ts @@ -0,0 +1,51 @@ +import {ApiError} from "@lodestar/api"; +import {toHex} from "@lodestar/utils"; +import {bellatrix} from "@lodestar/types"; +import {AssertionMatch, AssertionResult, SimulationAssertion} from "../interfaces.js"; + +export function createExecutionHeadAssertion({ + checkForSlot, +}: { + checkForSlot: number[]; +}): SimulationAssertion< + "executionHead", + {executionHead: {hash: string}; consensusHead: {executionPayload: {blockHash: string}}} +> { + return { + id: "executionHead", + match({slot}) { + if (checkForSlot.includes(slot)) return AssertionMatch.Capture | AssertionMatch.Assert; + return AssertionMatch.None; + }, + async capture({node}) { + const blockNumber = await node.execution.provider?.getBlockNumber(); + if (blockNumber == null) throw new Error("Execution provider not available"); + const executionHeadBlock = await node.execution.provider?.getBlockByNumber(blockNumber); + + const consensusHead = await node.beacon.api.beacon.getBlockV2("head"); + ApiError.assert(consensusHead); + + return { + executionHead: {hash: executionHeadBlock?.hash ?? "0x0"}, + consensusHead: { + executionPayload: { + blockHash: toHex( + (consensusHead.response.data.message as bellatrix.BeaconBlock).body.executionPayload.blockHash + ), + }, + }, + }; + }, + async assert({store, slot}) { + const errors: AssertionResult[] = []; + + if (store[slot].executionHead.hash !== store[slot].consensusHead.executionPayload.blockHash) { + errors.push( + `Execution head does not match consensus head. Expected: ${store[slot].consensusHead.executionPayload.blockHash}, got: ${store[slot].executionHead.hash}` + ); + } + + return errors; + }, + }; +} diff --git a/packages/cli/test/utils/simulation/constants.ts b/packages/cli/test/utils/simulation/constants.ts index 1d3c0f0f2c2b..b248f5109ffa 100644 --- a/packages/cli/test/utils/simulation/constants.ts +++ b/packages/cli/test/utils/simulation/constants.ts @@ -23,3 +23,6 @@ export const LODESTAR_BINARY_PATH = `${__dirname}/../../../bin/lodestar.js`; export const MOCK_ETH1_GENESIS_HASH = "0xfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfbfb"; export const SHARED_JWT_SECRET = "0xdc6457099f127cf0bac78de8b297df04951281909db4f58b43def7c7151e765d"; export const SHARED_VALIDATOR_PASSWORD = "passwrod"; +export const EL_GENESIS_SECRET_KEY = "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8"; +export const EL_GENESIS_PASSWORD = "12345678"; +export const EL_GENESIS_ACCOUNT = "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"; diff --git a/packages/cli/test/utils/simulation/execution_clients/geth.ts b/packages/cli/test/utils/simulation/execution_clients/geth.ts index 2ed9a2a91e1c..4c1b8196ab17 100644 --- a/packages/cli/test/utils/simulation/execution_clients/geth.ts +++ b/packages/cli/test/utils/simulation/execution_clients/geth.ts @@ -3,16 +3,18 @@ import {writeFile} from "node:fs/promises"; import path from "node:path"; import got from "got"; import {ZERO_HASH} from "@lodestar/state-transition"; -import {SHARED_JWT_SECRET, SIM_ENV_NETWORK_ID} from "../constants.js"; +import { + EL_GENESIS_ACCOUNT, + EL_GENESIS_PASSWORD, + EL_GENESIS_SECRET_KEY, + SHARED_JWT_SECRET, + SIM_ENV_NETWORK_ID, +} from "../constants.js"; import {Eth1ProviderWithAdmin} from "../Eth1ProviderWithAdmin.js"; import {ExecutionClient, ExecutionNodeGenerator, ExecutionStartMode, JobOptions, RunnerType} from "../interfaces.js"; import {getNodeMountedPaths} from "../utils/paths.js"; import {getNodePorts} from "../utils/ports.js"; -const SECRET_KEY = "45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8"; -const PASSWORD = "12345678"; -const GENESIS_ACCOUNT = "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b"; - export const generateGethNode: ExecutionNodeGenerator = (opts, runner) => { if (!process.env.GETH_BINARY_DIR && !process.env.GETH_DOCKER_IMAGE) { throw new Error("GETH_BINARY_DIR or GETH_DOCKER_IMAGE must be provided"); @@ -74,8 +76,8 @@ export const generateGethNode: ExecutionNodeGenerator = (o } : undefined, bootstrap: async () => { - await writeFile(skPath, SECRET_KEY); - await writeFile(passwordPath, PASSWORD); + await writeFile(skPath, EL_GENESIS_SECRET_KEY); + await writeFile(passwordPath, EL_GENESIS_PASSWORD); }, cli: { command: binaryPath, @@ -132,7 +134,7 @@ export const generateGethNode: ExecutionNodeGenerator = (o rootDirMounted, "--allow-insecure-unlock", "--unlock", - GENESIS_ACCOUNT, + EL_GENESIS_ACCOUNT, "--password", passwordPathMounted, "--syncmode", @@ -142,7 +144,7 @@ export const generateGethNode: ExecutionNodeGenerator = (o // Logging verbosity: 0=silent, 1=error, 2=warn, 3=info, 4=debug, 5=detail "--verbosity", "5", - ...(mining ? ["--mine", "--miner.etherbase", GENESIS_ACCOUNT] : []), + ...(mining ? ["--mine", "--miner.etherbase", EL_GENESIS_ACCOUNT] : []), ...(mode == ExecutionStartMode.PreMerge ? ["--nodiscover"] : []), ...clientOptions, ], From f62bc13d892c674646e70a22fc4a5fa3266b6186 Mon Sep 17 00:00:00 2001 From: Julien Date: Mon, 26 Feb 2024 14:53:15 -0800 Subject: [PATCH 15/34] chore: cleanup older dependencies (#6482) * chore: cleanup older dependencies * chore: some more removed dependencies --- packages/beacon-node/package.json | 11 +- packages/cli/package.json | 7 -- packages/db/package.json | 1 - packages/logger/package.json | 1 - packages/prover/package.json | 1 - packages/state-transition/package.json | 4 +- packages/test-utils/package.json | 4 +- packages/utils/package.json | 5 +- packages/validator/package.json | 1 - yarn.lock | 147 +++---------------------- 10 files changed, 19 insertions(+), 163 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 4908b2f4ea23..88951d1ab597 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -92,7 +92,6 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/as-chacha20poly1305": "^0.1.0", "@chainsafe/as-sha256": "^0.4.1", "@chainsafe/bls": "7.1.3", "@chainsafe/blst": "^0.2.9", @@ -132,8 +131,6 @@ "@lodestar/utils": "^1.16.0", "@lodestar/validator": "^1.16.0", "@multiformats/multiaddr": "^12.1.3", - "@types/datastore-level": "^3.0.0", - "buffer-xor": "^2.0.2", "c-kzg": "^2.1.2", "datastore-core": "^9.1.1", "datastore-level": "^10.1.1", @@ -142,6 +139,7 @@ "interface-datastore": "^8.2.7", "it-all": "^3.0.4", "it-pipe": "^3.0.1", + "lodash": "^4.17.21", "jwt-simple": "0.5.6", "libp2p": "1.1.1", "multiformats": "^11.0.1", @@ -150,18 +148,15 @@ "snappyjs": "^0.7.0", "strict-event-emitter-types": "^2.0.0", "systeminformation": "^5.17.12", - "uint8-varint": "^2.0.2", "uint8arraylist": "^2.4.7", - "uint8arrays": "^5.0.1", "xxhash-wasm": "1.0.2" }, "devDependencies": { - "@types/eventsource": "^1.1.11", + "@types/datastore-level": "^3.0.0", "@types/leveldown": "^4.0.3", + "@types/lodash": "^4.14.192", "@types/qs": "^6.9.7", - "@types/supertest": "^2.0.12", "@types/tmp": "^0.2.3", - "eventsource": "^2.0.2", "it-drain": "^3.0.3", "it-pair": "^2.0.6", "leveldown": "^6.1.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 0906047fd232..00cc5bd9bc1f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -51,7 +51,6 @@ "blockchain" ], "dependencies": { - "@chainsafe/as-sha256": "^0.4.1", "@chainsafe/bls": "7.1.3", "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/bls-keystore": "^3.0.1", @@ -76,30 +75,24 @@ "@lodestar/utils": "^1.16.0", "@lodestar/validator": "^1.16.0", "@multiformats/multiaddr": "^12.1.3", - "bip39": "^3.1.0", "deepmerge": "^4.3.1", "ethers": "^6.7.0", - "expand-tilde": "^2.0.2", "find-up": "^6.3.0", "got": "^11.8.6", "inquirer": "^9.1.5", "js-yaml": "^4.1.0", - "lodash": "^4.17.21", "prom-client": "^15.1.0", "proper-lockfile": "^4.1.2", "rimraf": "^4.4.1", "source-map-support": "^0.5.21", "uint8arrays": "^5.0.1", - "uuidv4": "^6.2.13", "yargs": "^17.7.1" }, "devDependencies": { "@lodestar/test-utils": "^1.16.0", "@types/debug": "^4.1.7", - "@types/expand-tilde": "^2.0.0", "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", - "@types/lodash": "^4.14.192", "@types/proper-lockfile": "^4.1.4", "@types/yargs": "^17.0.24" } diff --git a/packages/db/package.json b/packages/db/package.json index 50a1531d4b8c..631bbbc37e7c 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -38,7 +38,6 @@ "@chainsafe/ssz": "^0.14.0", "@lodestar/config": "^1.16.0", "@lodestar/utils": "^1.16.0", - "@types/levelup": "^4.3.3", "it-all": "^3.0.4", "level": "^8.0.0" }, diff --git a/packages/logger/package.json b/packages/logger/package.json index e0b69b0a9a7d..e0d773e8fe25 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -75,7 +75,6 @@ "@chainsafe/threads": "^1.11.1", "@lodestar/test-utils": "^1.16.0", "@types/triple-beam": "^1.3.2", - "rimraf": "^4.4.1", "triple-beam": "^1.3.0" }, "keywords": [ diff --git a/packages/prover/package.json b/packages/prover/package.json index a170b39dc7e7..a0b105bef18e 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -64,7 +64,6 @@ "@ethereumjs/block": "^4.2.2", "@ethereumjs/blockchain": "^6.2.2", "@ethereumjs/common": "^3.1.2", - "@ethereumjs/evm": "^1.3.2", "@ethereumjs/rlp": "^4.0.1", "@ethereumjs/trie": "^5.0.5", "@ethereumjs/tx": "^4.1.2", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 7c3e02a878b9..f861792e5c38 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -73,9 +73,7 @@ }, "devDependencies": { "@chainsafe/blst": "^0.2.9", - "@types/buffer-xor": "^2.0.0", - "@types/mockery": "^1.4.30", - "mockery": "^2.1.0" + "@types/buffer-xor": "^2.0.0" }, "keywords": [ "ethereum", diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 9d8e8c306e37..b46e56303e08 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -67,9 +67,7 @@ "vitest": "^1.2.1" }, "devDependencies": { - "@types/dockerode": "^3.3.19", - "@types/yargs": "^17.0.24", - "yargs": "^17.7.1" + "@types/yargs": "^17.0.24" }, "peerDependencies": { "vitest": "^1.2.1" diff --git a/packages/utils/package.json b/packages/utils/package.json index 3d888bb4e38b..6c3f372c0a1b 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -43,15 +43,12 @@ "any-signal": "3.0.1", "bigint-buffer": "^1.1.5", "case": "^1.6.3", - "chalk": "^5.2.0", "js-yaml": "^4.1.0" }, "devDependencies": { "@types/js-yaml": "^4.0.5", - "@types/triple-beam": "^1.3.2", "@types/yargs": "^17.0.24", - "prom-client": "^15.1.0", - "triple-beam": "^1.3.0" + "prom-client": "^15.1.0" }, "keywords": [ "ethereum", diff --git a/packages/validator/package.json b/packages/validator/package.json index c415d44c5a46..1eebe5706b9a 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -54,7 +54,6 @@ "@lodestar/state-transition": "^1.16.0", "@lodestar/types": "^1.16.0", "@lodestar/utils": "^1.16.0", - "bigint-buffer": "^1.1.5", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { diff --git a/yarn.lock b/yarn.lock index e11de5621cd1..ad12c793647e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1864,7 +1864,7 @@ resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.1.2.tgz#e9e035b9b166ca0af657a7848eb2718f0f22f183" integrity sha512-KYRCASVTv6aeUi1tsF8/vpyR7zpfs3FUzy2Jqm+MU+LmUKhQ0y2FpfwqkCcxSg2ua4GALJd8k2R76WxwZGbQpA== -"@noble/hashes@1.3.3", "@noble/hashes@^1.0.0", "@noble/hashes@^1.2.0", "@noble/hashes@^1.3.0", "@noble/hashes@^1.3.1", "@noble/hashes@^1.3.3", "@noble/hashes@~1.3.2": +"@noble/hashes@1.3.3", "@noble/hashes@^1.0.0", "@noble/hashes@^1.3.0", "@noble/hashes@^1.3.1", "@noble/hashes@^1.3.3", "@noble/hashes@~1.3.2": version "1.3.3" resolved "https://registry.yarnpkg.com/@noble/hashes/-/hashes-1.3.3.tgz#39908da56a4adc270147bb07968bf3b16cfe1699" integrity sha512-V7/fPHgl+jsVPXqqeOzT8egNj2iBIVt+ECeMMG8TdcnTikP3oaBtUVqpT/gYCR68aEBJSF+XbYUxStjbFMqIIA== @@ -2799,9 +2799,9 @@ integrity sha512-wYxU3kp5zItbxKmeRYCEplS2MW7DzyBnxPGj+GJVHZEUZiK/nn5Ei1sUFgURDh+X051+zsGe28iud3oHjrYWQQ== "@types/buffer-xor@^2.0.0": - version "2.0.0" - resolved "https://registry.npmjs.org/@types/buffer-xor/-/buffer-xor-2.0.0.tgz" - integrity sha512-NEJkIKUE/xboduuIAJmdtUvbXgUOfMkjOo6lWsiBVUIWBC5fVWGT+50yEw9W1Xp4ga76khg57pHELXw5Xm3Y+A== + version "2.0.2" + resolved "https://registry.yarnpkg.com/@types/buffer-xor/-/buffer-xor-2.0.2.tgz#d8c463583b8fbb322ea824562dc78a0c3cea2ca6" + integrity sha512-OqdCua7QCTupPnJgmyGJUpxWgbuOi0IMIVslXTSePS2o+qDrDB6f2Pg44zRyqhUA5GbFAf39U8z0+mH4WG0fLQ== dependencies: "@types/node" "*" @@ -2815,11 +2815,6 @@ "@types/node" "*" "@types/responselike" "^1.0.0" -"@types/cookiejar@*": - version "2.1.2" - resolved "https://registry.npmjs.org/@types/cookiejar/-/cookiejar-2.1.2.tgz" - integrity sha512-t73xJJrvdTjXrn4jLS9VSGRbz0nUY3cl2DMGDU48lKl+HR9dbbjW2A9r3g40VA++mQpy6uuHg33gy7du2BKpog== - "@types/datastore-level@^3.0.0": version "3.0.0" resolved "https://registry.npmjs.org/@types/datastore-level/-/datastore-level-3.0.0.tgz" @@ -2841,22 +2836,6 @@ dependencies: "@types/node" "*" -"@types/docker-modem@*": - version "3.0.3" - resolved "https://registry.yarnpkg.com/@types/docker-modem/-/docker-modem-3.0.3.tgz#28e1d4971fc88073bbd03c989b40c978af693def" - integrity sha512-i1A2Etnav7uHizZ87vUf4EqwJehY3JOcTfBS0pGBlO+HQ0jg2lUMCaJRg9VQM8ldZkpYdIfsenxcTOCpwxPXEg== - dependencies: - "@types/node" "*" - "@types/ssh2" "*" - -"@types/dockerode@^3.3.19": - version "3.3.19" - resolved "https://registry.yarnpkg.com/@types/dockerode/-/dockerode-3.3.19.tgz#59eb07550a102b397a9504083a6c50d811eed04c" - integrity sha512-7CC5yIpQi+bHXwDK43b/deYXteP3Lem9gdocVVHJPSRJJLMfbiOchQV3rDmAPkMw+n3GIVj7m1six3JW+VcwwA== - dependencies: - "@types/docker-modem" "*" - "@types/node" "*" - "@types/estree@^1.0.0": version "1.0.1" resolved "https://registry.yarnpkg.com/@types/estree/-/estree-1.0.1.tgz#aa22750962f3bf0e79d753d3cc067f010c95f194" @@ -2867,11 +2846,6 @@ resolved "https://registry.yarnpkg.com/@types/eventsource/-/eventsource-1.1.11.tgz#a2c0bfd0436b7db42ed1b2b2117f7ec2e8478dc7" integrity sha512-L7wLDZlWm5mROzv87W0ofIYeQP5K2UhoFnnUyEWLKM6UBb0ZNRgAqp98qE5DkgfBXdWfc2kYmw9KZm4NLjRbsw== -"@types/expand-tilde@^2.0.0": - version "2.0.0" - resolved "https://registry.npmjs.org/@types/expand-tilde/-/expand-tilde-2.0.0.tgz" - integrity sha512-17h/6MRHoetV2QVUVnUfrmaFCXNIFJ3uDJmXlklX2xDtlEb1W0OXLgP+qwND2Ibg/PtQfQi0vx19KGuPayjLiw== - "@types/got@^9.6.12": version "9.6.12" resolved "https://registry.yarnpkg.com/@types/got/-/got-9.6.12.tgz#fd42a6e1f5f64cd6bb422279b08c30bb5a15a56f" @@ -2933,11 +2907,6 @@ dependencies: "@types/node" "*" -"@types/level-errors@*": - version "3.0.0" - resolved "https://registry.npmjs.org/@types/level-errors/-/level-errors-3.0.0.tgz" - integrity sha512-/lMtoq/Cf/2DVOm6zE6ORyOM+3ZVm/BvzEZVxUhf6bgh8ZHglXlBqxbxSlJeVp8FCbD3IVvk/VbsaNmDjrQvqQ== - "@types/leveldown@^4.0.3": version "4.0.3" resolved "https://registry.yarnpkg.com/@types/leveldown/-/leveldown-4.0.3.tgz#4b868fd747808d378df6ffb27de7f889cae46aad" @@ -2946,19 +2915,10 @@ "@types/abstract-leveldown" "*" "@types/node" "*" -"@types/levelup@^4.3.3": - version "4.3.3" - resolved "https://registry.npmjs.org/@types/levelup/-/levelup-4.3.3.tgz" - integrity sha512-K+OTIjJcZHVlZQN1HmU64VtrC0jC3dXWQozuEIR9zVvltIk90zaGPM2AgT+fIkChpzHhFE3YnvFLCbLtzAmexA== - dependencies: - "@types/abstract-leveldown" "*" - "@types/level-errors" "*" - "@types/node" "*" - "@types/lodash@^4.14.192": - version "4.14.192" - resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.192.tgz#5790406361a2852d332d41635d927f1600811285" - integrity sha512-km+Vyn3BYm5ytMO13k9KTp27O75rbQ0NFw+U//g+PX7VZyjCioXaRFisqSIJRECljcTv73G3i6BpglNGHgUQ5A== + version "4.14.202" + resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.202.tgz#f09dbd2fb082d507178b2f2a5c7e74bd72ff98f8" + integrity sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ== "@types/minimatch@^3.0.3": version "3.0.5" @@ -2975,11 +2935,6 @@ resolved "https://registry.yarnpkg.com/@types/mocha/-/mocha-10.0.6.tgz#818551d39113081048bdddbef96701b4e8bb9d1b" integrity sha512-dJvrYWxP/UcXm36Qn36fxhUKu8A/xMRXVT2cliFF1Z7UA9liG5Psj3ezNSZw+5puH2czDXRLcXQxf8JbJt0ejg== -"@types/mockery@^1.4.30": - version "1.4.30" - resolved "https://registry.yarnpkg.com/@types/mockery/-/mockery-1.4.30.tgz#25f07fa7340371c7ee0fb9239511a34e0a19d5b7" - integrity sha512-uv53RrNdhbkV/3VmVCtfImfYCWC3GTTRn3R11Whni3EJ+gb178tkZBVNj2edLY5CMrB749dQi+SJkg87jsN8UQ== - "@types/ms@*": version "0.7.31" resolved "https://registry.yarnpkg.com/@types/ms/-/ms-0.7.31.tgz#31b7ca6407128a3d2bbc27fe2d21b345397f6197" @@ -3084,13 +3039,6 @@ dependencies: "@types/node" "*" -"@types/ssh2@*": - version "1.11.13" - resolved "https://registry.yarnpkg.com/@types/ssh2/-/ssh2-1.11.13.tgz#e6224da936abec0541bf26aa826b1cc37ea70d69" - integrity sha512-08WbG68HvQ2YVi74n2iSUnYHYpUdFc/s2IsI0BHBdJwaqYJpWlVv9elL0tYShTv60yr0ObdxJR5NrCRiGJ/0CQ== - dependencies: - "@types/node" "^18.11.18" - "@types/ssh2@^0.5.48": version "0.5.52" resolved "https://registry.yarnpkg.com/@types/ssh2/-/ssh2-0.5.52.tgz#9dbd8084e2a976e551d5e5e70b978ed8b5965741" @@ -3109,21 +3057,6 @@ resolved "https://registry.npmjs.org/@types/strip-json-comments/-/strip-json-comments-0.0.30.tgz" integrity sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ== -"@types/superagent@*": - version "4.1.10" - resolved "https://registry.npmjs.org/@types/superagent/-/superagent-4.1.10.tgz" - integrity sha512-xAgkb2CMWUMCyVc/3+7iQfOEBE75NvuZeezvmixbUw3nmENf2tCnQkW5yQLTYqvXUQ+R6EXxdqKKbal2zM5V/g== - dependencies: - "@types/cookiejar" "*" - "@types/node" "*" - -"@types/supertest@^2.0.12": - version "2.0.12" - resolved "https://registry.yarnpkg.com/@types/supertest/-/supertest-2.0.12.tgz#ddb4a0568597c9aadff8dbec5b2e8fddbe8692fc" - integrity sha512-X3HPWTwXRerBZS7Mo1k6vMVR1Z6zmJcDVn5O/31whe0tnjE4te6ZJSJGq1RiqHPjzPdMTfjCFogDJmwng9xHaQ== - dependencies: - "@types/superagent" "*" - "@types/tar@^6.1.4": version "6.1.4" resolved "https://registry.yarnpkg.com/@types/tar/-/tar-6.1.4.tgz#cf8497e1ebdc09212fd51625cd2eb5ca18365ad1" @@ -3161,11 +3094,6 @@ dependencies: "@types/node" "*" -"@types/uuid@8.3.4": - version "8.3.4" - resolved "https://registry.yarnpkg.com/@types/uuid/-/uuid-8.3.4.tgz#bd86a43617df0594787d38b735f55c805becf1bc" - integrity sha512-c/I8ZRb51j+pYGAu5CrFMRxqZ2ke4y2grEBO5AUjgSkSk+qT2Ea+OdWElz/OiMf5MNpn2b17kuVBwZLQJXzihw== - "@types/which@^2.0.1": version "2.0.2" resolved "https://registry.yarnpkg.com/@types/which/-/which-2.0.2.tgz#54541d02d6b1daee5ec01ac0d1b37cecf37db1ae" @@ -4126,13 +4054,6 @@ bintrees@1.0.1: resolved "https://registry.npmjs.org/bintrees/-/bintrees-1.0.1.tgz" integrity sha1-DmVcm5wkNeqraL9AJyJtK1WjRSQ= -bip39@^3.1.0: - version "3.1.0" - resolved "https://registry.yarnpkg.com/bip39/-/bip39-3.1.0.tgz#c55a418deaf48826a6ceb34ac55b3ee1577e18a3" - integrity sha512-c9kiwdk45Do5GL0vJMe7tS95VjCii65mYAH7DfWl3uW8AVzXKQVUm64i3hzVybBDMp9r7j9iNxR85+ul8MdN/A== - dependencies: - "@noble/hashes" "^1.2.0" - bl@^4.0.3, bl@^4.1.0: version "4.1.0" resolved "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz" @@ -6190,13 +6111,6 @@ execa@^8.0.1: signal-exit "^4.1.0" strip-final-newline "^3.0.0" -expand-tilde@^2.0.2: - version "2.0.2" - resolved "https://registry.npmjs.org/expand-tilde/-/expand-tilde-2.0.2.tgz" - integrity sha1-l+gBqgUt8CRU3kawK/YhZCzchQI= - dependencies: - homedir-polyfill "^1.0.1" - exponential-backoff@^3.1.1: version "3.1.1" resolved "https://registry.yarnpkg.com/exponential-backoff/-/exponential-backoff-3.1.1.tgz#64ac7526fe341ab18a39016cd22c787d01e00bf6" @@ -7115,13 +7029,6 @@ hmac-drbg@^1.0.1: minimalistic-assert "^1.0.0" minimalistic-crypto-utils "^1.0.1" -homedir-polyfill@^1.0.1: - version "1.0.3" - resolved "https://registry.npmjs.org/homedir-polyfill/-/homedir-polyfill-1.0.3.tgz" - integrity sha512-eSmmWE5bZTK2Nou4g0AI3zZ9rswp7GRKoKXS1BLUkvPviOqs4YTN1djQIqrXy9k5gEtdLPy86JjRwsNM9tnDcA== - dependencies: - parse-passwd "^1.0.0" - hosted-git-info@^2.1.4: version "2.8.9" resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" @@ -7972,17 +7879,7 @@ it-pipe@^3.0.0, it-pipe@^3.0.1: it-pushable "^3.1.2" it-stream-types "^2.0.1" -it-protobuf-stream@^1.0.2: - version "1.1.1" - resolved "https://registry.yarnpkg.com/it-protobuf-stream/-/it-protobuf-stream-1.1.1.tgz#cc303ac31b9de768d24288b0898c18ebd7624868" - integrity sha512-H7fiC+m85AAz84I8SQOKHKZTDREFrsYfKxEhWTlhAdySoUyiC72Xe2ocqBFy3zUWCGYq6rCTMGnCbTKntSlcog== - dependencies: - it-length-prefixed-stream "^1.0.0" - it-stream-types "^2.0.1" - protons-runtime "^5.0.0" - uint8arraylist "^2.4.1" - -it-protobuf-stream@^1.1.1: +it-protobuf-stream@^1.0.2, it-protobuf-stream@^1.1.1: version "1.1.2" resolved "https://registry.yarnpkg.com/it-protobuf-stream/-/it-protobuf-stream-1.1.2.tgz#4444d78fcae0fce949b4cbea622bf1d92667e64f" integrity sha512-epZBuG+7cPaTxCR/Lf3ApshBdA9qfflGPQLfLLrp9VQ0w67Z2xo4H+SLLetav57/29oPtAXwVaoyemg99JOWzA== @@ -9176,11 +9073,6 @@ mocha@^10.2.0: yargs-parser "20.2.4" yargs-unparser "2.0.0" -mockery@^2.1.0: - version "2.1.0" - resolved "https://registry.npmjs.org/mockery/-/mockery-2.1.0.tgz" - integrity sha512-9VkOmxKlWXoDO/h1jDZaS4lH33aWfRiJiNT/tKj+8OGzrcFDLo8d0syGdbsc3Bc4GvRXPb+NMMvojotmuGJTvA== - modify-values@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022" @@ -10151,11 +10043,6 @@ parse-json@^5.0.0, parse-json@^5.2.0: json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" -parse-passwd@^1.0.0: - version "1.0.0" - resolved "https://registry.npmjs.org/parse-passwd/-/parse-passwd-1.0.0.tgz" - integrity sha1-bVuTSkVpk7I9N/QKOC1vFmao5cY= - parse-path@^7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/parse-path/-/parse-path-7.0.0.tgz#605a2d58d0a749c8594405d8cc3a2bf76d16099b" @@ -12565,29 +12452,21 @@ uuid@3.3.2: resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.3.2.tgz#1b4af4955eb3077c501c23872fc6513811587131" integrity sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA== -uuid@8.3.2, uuid@^8.3.0, uuid@^8.3.2: - version "8.3.2" - resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" - integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== - uuid@^3.3.2, uuid@^3.3.3: version "3.4.0" resolved "https://registry.npmjs.org/uuid/-/uuid-3.4.0.tgz" integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A== +uuid@^8.3.0, uuid@^8.3.2: + version "8.3.2" + resolved "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz" + integrity sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg== + uuid@^9.0.0, uuid@^9.0.1: version "9.0.1" resolved "https://registry.yarnpkg.com/uuid/-/uuid-9.0.1.tgz#e188d4c8853cc722220392c424cd637f32293f30" integrity sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA== -uuidv4@^6.2.13: - version "6.2.13" - resolved "https://registry.yarnpkg.com/uuidv4/-/uuidv4-6.2.13.tgz#8f95ec5ef22d1f92c8e5d4c70b735d1c89572cb7" - integrity sha512-AXyzMjazYB3ovL3q051VLH06Ixj//Knx7QnUSi1T//Ie3io6CpsPu9nVMOx5MoLWh6xV0B9J0hIaxungxXUbPQ== - dependencies: - "@types/uuid" "8.3.4" - uuid "8.3.2" - v8-compile-cache-lib@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/v8-compile-cache-lib/-/v8-compile-cache-lib-3.0.1.tgz#6336e8d71965cb3d35a1bbb7868445a7c05264bf" From 645d491d49c584a096ee05728e047711c33599f5 Mon Sep 17 00:00:00 2001 From: Julien Date: Tue, 27 Feb 2024 07:22:44 -0800 Subject: [PATCH 16/34] test: update beacon api spec to v2.5.0 (#6354) * fix: update events test data * fix: reflect typo fix * fix: remove workarounds * chore: remove now irrelevan dropOneOf option * Fix parent block number in SSE payload attributes * Tests pass against latest spec version * Update comment * Lint * Update beacon api spec to v2.5.0 --------- Co-authored-by: Nico Flaig --- .../api/test/unit/beacon/oapiSpec.test.ts | 34 +++++------------- .../api/test/unit/beacon/testData/events.ts | 36 ++++++++++++++++--- packages/api/test/utils/checkAgainstSpec.ts | 26 ++------------ packages/api/test/utils/parseOpenApiSpec.ts | 24 +++---------- 4 files changed, 45 insertions(+), 75 deletions(-) diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index 4d036fb2dd8d..7256d3f4ed63 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -23,7 +23,7 @@ import {testData as validatorTestData} from "./testData/validator.js"; // eslint-disable-next-line @typescript-eslint/naming-convention const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const version = "v2.4.2"; +const version = "v2.5.0"; const openApiFile: OpenApiFile = { url: `https://github.com/ethereum/beacon-APIs/releases/download/${version}/beacon-node-oapi.json`, filepath: path.join(__dirname, "../../../oapi-schemas/beacon-node-oapi.json"), @@ -89,11 +89,14 @@ const ignoredOperations = [ /* https://github.com/ChainSafe/lodestar/issues/5694 */ "getSyncCommitteeRewards", "getAttestationsRewards", + /* https://github.com/ChainSafe/lodestar/issues/6058 */ + "postStateValidators", + "postStateValidatorBalances", "getDepositSnapshot", // Won't fix for now, see https://github.com/ChainSafe/lodestar/issues/5697 "getBlindedBlock", // https://github.com/ChainSafe/lodestar/issues/5699 "getNextWithdrawals", // https://github.com/ChainSafe/lodestar/issues/5696 "getDebugForkChoice", // https://github.com/ChainSafe/lodestar/issues/5700 - /* https://github.com/ChainSafe/lodestar/issues/6080 */ + /* Ensure operationId matches spec value, blocked by https://github.com/ChainSafe/lodestar/pull/6080 */ "getLightClientBootstrap", "getLightClientUpdatesByRange", "getLightClientFinalityUpdate", @@ -145,37 +148,16 @@ runTestCheckAgainstSpec( reqSerializers, returnTypes, testDatas, - { - // TODO: Investigate why schema validation fails otherwise (see https://github.com/ChainSafe/lodestar/issues/6187) - routesDropOneOf: [ - "produceBlockV2", - "produceBlockV3", - "produceBlindedBlock", - "publishBlindedBlock", - "publishBlindedBlockV2", - ], - }, ignoredOperations, ignoredProperties ); const ignoredTopics = [ /* - https://github.com/ChainSafe/lodestar/issues/6167 - eventTestData[bls_to_execution_change] does not match spec's example + https://github.com/ChainSafe/lodestar/issues/6470 + topic block_gossip not implemented */ - "bls_to_execution_change", - /* - https://github.com/ChainSafe/lodestar/issues/6170 - Error: Invalid slot=0 fork=phase0 for lightclient fork types - */ - "light_client_finality_update", - "light_client_optimistic_update", - /* - https://github.com/ethereum/beacon-APIs/pull/379 - SyntaxError: Unexpected non-whitespace character after JSON at position 629 (line 1 column 630) - */ - "payload_attributes", + "block_gossip", ]; // eventstream types are defined as comments in the description of "examples". diff --git a/packages/api/test/unit/beacon/testData/events.ts b/packages/api/test/unit/beacon/testData/events.ts index f4962bc1827a..1ac101f32f4d 100644 --- a/packages/api/test/unit/beacon/testData/events.ts +++ b/packages/api/test/unit/beacon/testData/events.ts @@ -114,8 +114,8 @@ export const eventTestData: EventData = { message: { validator_index: "1", from_bls_pubkey: - "0x9048a71944feba4695ef870dfb5745c934d81c5efd934c0250a12942fcc2a2dfd6b20d53314379dec7aae5ca5fe9e9c4", - to_execution_address: "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "0x933ad9491b62059dd065b560d256d8957a8c402cc6e8d8ee7290ae11e8f7329267a8811c397529dac52ae1342ba58c95", + to_execution_address: "0x9Be8d619c56699667c1feDCD15f6b14D8B067F72", }, signature: "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", @@ -213,8 +213,34 @@ export const eventTestData: EventData = { }), }, [EventType.payloadAttributes]: { - version: ForkName.bellatrix, - data: ssz.bellatrix.SSEPayloadAttributes.defaultValue(), + version: ForkName.capella, + data: ssz.capella.SSEPayloadAttributes.fromJson({ + proposer_index: "123", + proposal_slot: "10", + parent_block_number: "9", + parent_block_root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", + parent_block_hash: "0x9a2fefd2fdb57f74993c7780ea5b9030d2897b615b89f808011ca5aebed54eaf", + payload_attributes: { + timestamp: "123456", + prev_randao: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", + suggested_fee_recipient: "0x0000000000000000000000000000000000000000", + withdrawals: [ + { + index: "5", + validator_index: "10", + address: "0x0000000000000000000000000000000000000000", + amount: "15640", + }, + ], + }, + }), }, - [EventType.blobSidecar]: blobSidecarSSE.defaultValue(), + [EventType.blobSidecar]: blobSidecarSSE.fromJson({ + block_root: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", + index: "1", + kzg_commitment: + "0x1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505cc411d61252fb6cb3fa0017b679f8bb2305b26a285fa2737f175668d0dff91cc1b66ac1fb663c9bc59509846d6ec05345bd908eda73e670af888da41af171505", + slot: "1", + versioned_hash: "0xcf8e0d4e9587369b2301d0790347320302cc0943d5a1884560367e8208d920f2", + }), }; diff --git a/packages/api/test/utils/checkAgainstSpec.ts b/packages/api/test/utils/checkAgainstSpec.ts index c887f66e95e6..354ae53b2358 100644 --- a/packages/api/test/utils/checkAgainstSpec.ts +++ b/packages/api/test/utils/checkAgainstSpec.ts @@ -1,13 +1,11 @@ import Ajv, {ErrorObject} from "ajv"; import {expect, describe, beforeAll, it} from "vitest"; import {ReqGeneric, ReqSerializer, ReturnTypes, RouteDef} from "../../src/utils/types.js"; -import {applyRecursively, JsonSchema, OpenApiJson, parseOpenApiSpec, ParseOpenApiSpecOpts} from "./parseOpenApiSpec.js"; +import {applyRecursively, JsonSchema, OpenApiJson, parseOpenApiSpec} from "./parseOpenApiSpec.js"; import {GenericServerTestCases} from "./genericServerTest.js"; const ajv = new Ajv({ strict: true, - strictTypes: false, // TODO Enable once beacon-APIs is fixed. See https://github.com/ChainSafe/lodestar/issues/6206 - allErrors: true, }); // Ensure embedded schema 'example' do not fail validation @@ -68,11 +66,10 @@ export function runTestCheckAgainstSpec( reqSerializers: Record>, returnTypes: Record[string]>, testDatas: Record[string]>, - opts?: ParseOpenApiSpecOpts, ignoredOperations: string[] = [], ignoredProperties: Record = {} ): void { - const openApiSpec = parseOpenApiSpec(openApiJson, opts); + const openApiSpec = parseOpenApiSpec(openApiJson); for (const [operationId, routeSpec] of openApiSpec.entries()) { const isIgnored = ignoredOperations.some((id) => id === operationId); @@ -106,15 +103,6 @@ export function runTestCheckAgainstSpec( it(`${operationId}_request`, function () { const reqJson = reqSerializers[routeId].writeReq(...(testData.args as [never])) as unknown; - if (operationId === "publishBlock" || operationId === "publishBlindedBlock") { - // For some reason AJV invalidates valid blocks if multiple forks are defined with oneOf - // `.data - should match exactly one schema in oneOf` - // Dropping all definitions except (phase0) pases the validation - if (routeSpec.requestSchema?.oneOf) { - routeSpec.requestSchema = routeSpec.requestSchema?.oneOf[0]; - } - } - // Stringify param and query to simulate rendering in HTTP query // TODO: Review conversions in fastify and other servers stringifyProperties((reqJson as ReqGeneric).params ?? {}); @@ -137,16 +125,6 @@ export function runTestCheckAgainstSpec( it(`${operationId}_response`, function () { const resJson = returnTypes[operationId].toJson(testData.res as any); - // Patch for getBlockV2 - if (operationId === "getBlockV2" || operationId === "getStateV2") { - // For some reason AJV invalidates valid blocks if multiple forks are defined with oneOf - // `.data - should match exactly one schema in oneOf` - // Dropping all definitions except (phase0) pases the validation - if (responseOkSchema.properties?.data.oneOf) { - responseOkSchema.properties.data = responseOkSchema.properties.data.oneOf[1]; - } - } - const ignoredProperties = ignoredProperty?.response; if (ignoredProperties) { // Remove ignored properties from schema validation diff --git a/packages/api/test/utils/parseOpenApiSpec.ts b/packages/api/test/utils/parseOpenApiSpec.ts index 84b024e5950e..2672b381eea6 100644 --- a/packages/api/test/utils/parseOpenApiSpec.ts +++ b/packages/api/test/utils/parseOpenApiSpec.ts @@ -82,23 +82,17 @@ enum ContentType { json = "application/json", } -export type ParseOpenApiSpecOpts = { - routesDropOneOf?: string[]; -}; - -export function parseOpenApiSpec(openApiJson: OpenApiJson, opts?: ParseOpenApiSpecOpts): Map { +export function parseOpenApiSpec(openApiJson: OpenApiJson): Map { const routes = new Map(); for (const [routeUrl, routesByMethod] of Object.entries(openApiJson.paths)) { for (const [httpMethod, routeDefinition] of Object.entries(routesByMethod)) { const responseOkSchema = routeDefinition.responses[StatusCode.ok]?.content?.[ContentType.json]?.schema; - const dropOneOf = opts?.routesDropOneOf?.includes(routeDefinition.operationId); - // Force all properties to have required, else ajv won't validate missing properties if (responseOkSchema) { try { - preprocessSchema(responseOkSchema, {dropOneOf}); + preprocessSchema(responseOkSchema); } catch (e) { // eslint-disable-next-line no-console console.log(responseOkSchema); @@ -107,7 +101,7 @@ export function parseOpenApiSpec(openApiJson: OpenApiJson, opts?: ParseOpenApiSp } const requestSchema = buildReqSchema(routeDefinition); - preprocessSchema(requestSchema, {dropOneOf}); + preprocessSchema(requestSchema); routes.set(routeDefinition.operationId, { url: routeUrl, @@ -121,7 +115,7 @@ export function parseOpenApiSpec(openApiJson: OpenApiJson, opts?: ParseOpenApiSp return routes; } -function preprocessSchema(schema: JsonSchema, opts?: {dropOneOf?: boolean}): void { +function preprocessSchema(schema: JsonSchema): void { // Require all properties applyRecursively(schema, (obj) => { if (obj.type === "object" && obj.properties && !obj.required) { @@ -141,16 +135,6 @@ function preprocessSchema(schema: JsonSchema, opts?: {dropOneOf?: boolean}): voi } }); - if (opts?.dropOneOf) { - // Pick single oneOf, AJV has trouble validating against blocks and states - applyRecursively(schema, (obj) => { - if (obj.oneOf) { - // splice(1) = mutate array in place to drop all items after index 1 (included) - obj.oneOf.splice(1); - } - }); - } - // Remove non-intersecting allOf enum applyRecursively(schema, (obj) => { if (obj.allOf && obj.allOf.every((s) => s.enum)) { From 86d28c932b564b77b1cba77f135a128b95eb24b1 Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Tue, 27 Feb 2024 18:09:24 +0100 Subject: [PATCH 17/34] deps: update typescript to 5.3.3 (#6491) * Update typescript * Fix lint errors --- package.json | 6 ++--- packages/prover/src/utils/consensus.ts | 2 +- packages/utils/src/objects.ts | 2 +- yarn.lock | 37 +++++++++++++++++++++++--- 4 files changed, 39 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 4c1316942c33..0ad7cd81ee7c 100644 --- a/package.json +++ b/package.json @@ -49,7 +49,7 @@ "@chainsafe/eslint-plugin-node": "^11.2.3", "@dapplion/benchmark": "^0.2.4", "@types/mocha": "^10.0.6", - "@types/node": "^20.6.5", + "@types/node": "^20.11.20", "@typescript-eslint/eslint-plugin": "6.21.0", "@typescript-eslint/parser": "6.21.0", "@vitest/coverage-v8": "^1.2.1", @@ -75,8 +75,8 @@ "stream-browserify": "^3.0.0", "stream-http": "^3.2.0", "supertest": "^6.3.3", - "ts-node": "^10.9.1", - "typescript": "^5.2.2", + "ts-node": "^10.9.2", + "typescript": "^5.3.3", "typescript-docs-verifier": "^2.5.0", "vite-plugin-node-polyfills": "^0.19.0", "vite-plugin-top-level-await": "^1.4.1", diff --git a/packages/prover/src/utils/consensus.ts b/packages/prover/src/utils/consensus.ts index d008a8e42459..58f6d7f97701 100644 --- a/packages/prover/src/utils/consensus.ts +++ b/packages/prover/src/utils/consensus.ts @@ -13,7 +13,7 @@ export async function fetchNearestBlock( ): Promise { const res = await api.beacon.getBlockV2(slot); - if (res.ok) return res.response.data; + if (res.ok) return res.response.data as capella.SignedBeaconBlock; if (!res.ok && res.error.code === 404) { return fetchNearestBlock(api, direction === "down" ? slot - 1 : slot + 1); diff --git a/packages/utils/src/objects.ts b/packages/utils/src/objects.ts index 67d360a6b0c2..ad09d36b0ecc 100644 --- a/packages/utils/src/objects.ts +++ b/packages/utils/src/objects.ts @@ -1,6 +1,6 @@ import Case from "case"; -/* eslint-disable @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-assignment */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment */ export type KeyCase = | "snake" diff --git a/yarn.lock b/yarn.lock index ad12c793647e..0e1fa22a42e5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2956,7 +2956,7 @@ "@types/node" "*" form-data "^3.0.0" -"@types/node@*", "@types/node@>=13.7.0", "@types/node@^20.6.5": +"@types/node@*", "@types/node@>=13.7.0": version "20.6.5" resolved "https://registry.yarnpkg.com/@types/node/-/node-20.6.5.tgz#4c6a79adf59a8e8193ac87a0e522605b16587258" integrity sha512-2qGq5LAOTh9izcc0+F+dToFigBWiK1phKPt7rNhOqJSr35y8rlIBjDwGtFSgAI6MGIhjwOVNSQZVdJsZJ2uR1w== @@ -2978,6 +2978,13 @@ dependencies: undici-types "~5.26.4" +"@types/node@^20.11.20": + version "20.11.20" + resolved "https://registry.yarnpkg.com/@types/node/-/node-20.11.20.tgz#f0a2aee575215149a62784210ad88b3a34843659" + integrity sha512-7/rR21OS+fq8IyHTgtLkDK949uzsa6n8BkziAKtPVpugIkO6D+/ooXMvzXxDnZrmtXVfjb1bKQafYpb8s89LOg== + dependencies: + undici-types "~5.26.4" + "@types/normalize-package-data@^2.4.0": version "2.4.1" resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" @@ -12034,7 +12041,7 @@ ts-api-utils@^1.0.1: resolved "https://registry.yarnpkg.com/ts-api-utils/-/ts-api-utils-1.0.1.tgz#8144e811d44c749cd65b2da305a032510774452d" integrity sha512-lC/RGlPmwdrIBFTX59wwNzqh7aR2otPNPR/5brHZm/XKFYKsfqxihXUe9pU3JI+3vGkl+vyCoNNnPhJn3aLK1A== -ts-node@^10.8.1, ts-node@^10.9.1: +ts-node@^10.8.1: version "10.9.1" resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.1.tgz#e73de9102958af9e1f0b168a6ff320e25adcff4b" integrity sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw== @@ -12053,6 +12060,25 @@ ts-node@^10.8.1, ts-node@^10.9.1: v8-compile-cache-lib "^3.0.1" yn "3.1.1" +ts-node@^10.9.2: + version "10.9.2" + resolved "https://registry.yarnpkg.com/ts-node/-/ts-node-10.9.2.tgz#70f021c9e185bccdca820e26dc413805c101c71f" + integrity sha512-f0FFpIdcHgn8zcPSbf1dRevwt047YMnaiJM3u2w2RewrB+fob/zePZcrOyQoLMMO7aBIddLcQIEK5dYjkLnGrQ== + dependencies: + "@cspotcode/source-map-support" "^0.8.0" + "@tsconfig/node10" "^1.0.7" + "@tsconfig/node12" "^1.0.7" + "@tsconfig/node14" "^1.0.0" + "@tsconfig/node16" "^1.0.2" + acorn "^8.4.1" + acorn-walk "^8.1.1" + arg "^4.1.0" + create-require "^1.1.0" + diff "^4.0.1" + make-error "^1.1.1" + v8-compile-cache-lib "^3.0.1" + yn "3.1.1" + tsconfig-paths@^3.15.0: version "3.15.0" resolved "https://registry.yarnpkg.com/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz#5299ec605e55b1abb23ec939ef15edaf483070d4" @@ -12245,11 +12271,16 @@ typescript-docs-verifier@^2.5.0: tsconfig "^7.0.0" yargs "^17.5.1" -"typescript@>=3 < 6", typescript@^5.2.2: +"typescript@>=3 < 6": version "5.2.2" resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.2.2.tgz#5ebb5e5a5b75f085f22bc3f8460fba308310fa78" integrity sha512-mI4WrpHsbCIcwT9cF4FZvr80QUeKvsUsUvKDoR+X/7XHQH98xYD8YHZg7ANtz2GtZt/CBq2QJ0thkGJMHfqc1w== +typescript@^5.3.3: + version "5.3.3" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.3.3.tgz#b3ce6ba258e72e6305ba66f5c9b452aaee3ffe37" + integrity sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw== + ufo@^1.3.0: version "1.3.1" resolved "https://registry.yarnpkg.com/ufo/-/ufo-1.3.1.tgz#e085842f4627c41d4c1b60ebea1f75cdab4ce86b" From 40b8924f23ffe012745b924dca3dda1b17d2acb1 Mon Sep 17 00:00:00 2001 From: Julien Date: Wed, 28 Feb 2024 03:37:59 -0800 Subject: [PATCH 18/34] chore: improve missing docker engine detection (#6495) * chore: detect when docker is not installed * chore: improve e2e tests requirement --- CONTRIBUTING.md | 2 +- scripts/run_e2e_env.sh | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index cb81ae78ad6f..bfede5f41f36 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -32,7 +32,7 @@ To run tests: - :test_tube: Run `yarn check-types` to check TypeScript types. - :test_tube: Run `yarn lint` to run the linter (ESLint). -Note that to run `test:e2e`, first ensure that the environment is correctly setup by running the `run_e2e_env.sh` script. +Note that to run `test:e2e`, first ensure that the environment is correctly setup by running the `run_e2e_env.sh` script. This script requires a running docker engine. ```sh ./scripts/run_e2e_env.sh start diff --git a/scripts/run_e2e_env.sh b/scripts/run_e2e_env.sh index e81eb501f407..b6742302aec5 100755 --- a/scripts/run_e2e_env.sh +++ b/scripts/run_e2e_env.sh @@ -16,7 +16,11 @@ function stop_app() { kill -s TERM $(cat test-logs/e2e-test-env/simulation.pid) } - +docker version > /dev/null 2>&1 +if [ $? -ne 0 ]; then + echo "Docker is not running. Please start Docker and try again." + exit 1 +fi case "$1" in start) start_app ;; From 6675739aba0aacd904109bf06faaf74a7b41ec6d Mon Sep 17 00:00:00 2001 From: Nazar Hussain Date: Wed, 28 Feb 2024 14:23:24 +0100 Subject: [PATCH 19/34] deps: upgrade the lint and prettier deps (#6496) * Upgrade eslint and prettier * Fix the lint errors --- package.json | 8 +- packages/api/src/builder/routes.ts | 4 +- .../chain/produceBlock/produceBlockBody.ts | 4 +- .../src/db/repositories/blockArchive.ts | 4 +- .../src/eth1/provider/jsonRpcHttpClient.ts | 4 +- .../beacon-node/src/execution/engine/http.ts | 12 +- .../beacon-node/src/execution/engine/utils.ts | 4 +- .../src/metrics/validatorMonitor.ts | 6 +- .../cli/src/cmds/validator/keymanager/impl.ts | 4 +- packages/cli/src/util/logger.ts | 18 +-- .../utils/simulation/SimulationEnvironment.ts | 8 +- .../test/utils/simulation/TableReporter.ts | 4 +- .../cli/test/utils/simulation/interfaces.ts | 7 +- packages/config/src/chainConfig/types.ts | 12 +- .../src/block/slashValidator.ts | 4 +- .../src/epoch/processSlashings.ts | 4 +- packages/utils/src/command.ts | 12 +- packages/utils/src/types.ts | 4 +- yarn.lock | 124 ++++++++++++------ 19 files changed, 146 insertions(+), 101 deletions(-) diff --git a/package.json b/package.json index 0ad7cd81ee7c..5b76d19a8760 100644 --- a/package.json +++ b/package.json @@ -50,14 +50,14 @@ "@dapplion/benchmark": "^0.2.4", "@types/mocha": "^10.0.6", "@types/node": "^20.11.20", - "@typescript-eslint/eslint-plugin": "6.21.0", - "@typescript-eslint/parser": "6.21.0", + "@typescript-eslint/eslint-plugin": "^7.1.0", + "@typescript-eslint/parser": "^7.1.0", "@vitest/coverage-v8": "^1.2.1", "@vitest/browser": "^1.2.1", "crypto-browserify": "^3.12.0", "dotenv": "^16.4.1", "electron": "^26.2.2", - "eslint": "^8.56.0", + "eslint": "^8.57.0", "eslint-import-resolver-typescript": "^3.6.1", "eslint-plugin-import": "^2.29.1", "eslint-plugin-prettier": "^5.1.3", @@ -70,7 +70,7 @@ "node-gyp": "^9.4.0", "npm-run-all": "^4.1.5", "path-browserify": "^1.0.1", - "prettier": "^3.0.3", + "prettier": "^3.2.5", "process": "^0.11.10", "stream-browserify": "^3.0.0", "stream-http": "^3.2.0", diff --git a/packages/api/src/builder/routes.ts b/packages/api/src/builder/routes.ts index 6f5a55f0dcff..ca4c81a9fade 100644 --- a/packages/api/src/builder/routes.ts +++ b/packages/api/src/builder/routes.ts @@ -92,8 +92,8 @@ export function getReturnTypes(): ReturnTypes { isForkBlobs(fork) ? ssz.allForksBlobs[fork].ExecutionPayloadAndBlobsBundle : isForkExecution(fork) - ? ssz.allForksExecution[fork].ExecutionPayload - : ssz.bellatrix.ExecutionPayload + ? ssz.allForksExecution[fork].ExecutionPayload + : ssz.bellatrix.ExecutionPayload ), }; } diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index a80dc03d4cc0..d598ddcb9688 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -158,8 +158,8 @@ export async function produceBlockBody( const feeRecipientType = requestedFeeRecipient ? "requested" : this.beaconProposerCache.get(proposerIndex) - ? "cached" - : "default"; + ? "cached" + : "default"; Object.assign(logMeta, {feeRecipientType, feeRecipient}); diff --git a/packages/beacon-node/src/db/repositories/blockArchive.ts b/packages/beacon-node/src/db/repositories/blockArchive.ts index 091784783d18..59775e770b41 100644 --- a/packages/beacon-node/src/db/repositories/blockArchive.ts +++ b/packages/beacon-node/src/db/repositories/blockArchive.ts @@ -155,8 +155,8 @@ export class BlockArchiveRepository extends Repository= ForkSeq.deneb ? "engine_newPayloadV3" : ForkSeq[fork] >= ForkSeq.capella - ? "engine_newPayloadV2" - : "engine_newPayloadV1"; + ? "engine_newPayloadV2" + : "engine_newPayloadV1"; const serializedExecutionPayload = serializeExecutionPayload(fork, executionPayload); @@ -299,8 +299,8 @@ export class ExecutionEngineHttp implements IExecutionEngine { ForkSeq[fork] >= ForkSeq.deneb ? "engine_forkchoiceUpdatedV3" : ForkSeq[fork] >= ForkSeq.capella - ? "engine_forkchoiceUpdatedV2" - : "engine_forkchoiceUpdatedV1"; + ? "engine_forkchoiceUpdatedV2" + : "engine_forkchoiceUpdatedV1"; const payloadAttributesRpc = payloadAttributes ? serializePayloadAttributes(payloadAttributes) : undefined; // If we are just fcUing and not asking execution for payload, retry is not required // and we can move on, as the next fcU will be issued soon on the new slot @@ -373,8 +373,8 @@ export class ExecutionEngineHttp implements IExecutionEngine { ForkSeq[fork] >= ForkSeq.deneb ? "engine_getPayloadV3" : ForkSeq[fork] >= ForkSeq.capella - ? "engine_getPayloadV2" - : "engine_getPayloadV1"; + ? "engine_getPayloadV2" + : "engine_getPayloadV1"; const payloadResponse = await this.rpc.fetchWithRetries< EngineApiRpcReturnTypes[typeof method], EngineApiRpcParamTypes[typeof method] diff --git a/packages/beacon-node/src/execution/engine/utils.ts b/packages/beacon-node/src/execution/engine/utils.ts index e661af8daf70..b56f62bf602b 100644 --- a/packages/beacon-node/src/execution/engine/utils.ts +++ b/packages/beacon-node/src/execution/engine/utils.ts @@ -121,8 +121,8 @@ export function getExecutionEngineState 0 && remote ? {type: "remote", secretKeys: interopKeys} : interopKeys.length > 0 - ? {type: "local", secretKeys: interopKeys} - : {type: "no-keys"}; + ? {type: "local", secretKeys: interopKeys} + : {type: "no-keys"}; const commonOptions: GeneratorOptions = { id, @@ -287,8 +287,8 @@ export class SimulationEnvironment { typeof validator === "object" ? validator.type : validator === undefined - ? getValidatorForBeaconNode(beaconType) - : validator; + ? getValidatorForBeaconNode(beaconType) + : validator; const validatorOptions = typeof validator === "object" ? validator.options : {}; const beaconUrls = [ // As lodestar is running on host machine, need to connect through docker named host diff --git a/packages/cli/test/utils/simulation/TableReporter.ts b/packages/cli/test/utils/simulation/TableReporter.ts index fbf401ddddba..e2c77d961e5c 100644 --- a/packages/cli/test/utils/simulation/TableReporter.ts +++ b/packages/cli/test/utils/simulation/TableReporter.ts @@ -122,8 +122,8 @@ export class TableReporter extends SimulationReporter finalizedSlots.length === 0 ? "---" : isSingletonArray(finalizedSlots) - ? finalizedSlots[0] - : finalizedSlots.join(","), + ? finalizedSlots[0] + : finalizedSlots.join(","), peers: peersCount.length === 0 ? "---" : isSingletonArray(peersCount) ? peersCount[0] : peersCount.join(","), attCount: attestationCounts.length > 0 && isSingletonArray(attestationCounts) ? attestationCounts[0] : "---", incDelay: inclusionDelays.length > 0 && isSingletonArray(inclusionDelays) ? inclusionDelays[0].toFixed(2) : "---", diff --git a/packages/cli/test/utils/simulation/interfaces.ts b/packages/cli/test/utils/simulation/interfaces.ts index 639aa287e6d8..d8708c199eb1 100644 --- a/packages/cli/test/utils/simulation/interfaces.ts +++ b/packages/cli/test/utils/simulation/interfaces.ts @@ -349,11 +349,8 @@ export enum AssertionMatch { Remove = 1 << 2, } export type AssertionMatcher = (input: SimulationMatcherInput) => AssertionMatch; -export type ExtractAssertionType = T extends SimulationAssertion - ? A extends I - ? B - : never - : never; +export type ExtractAssertionType = + T extends SimulationAssertion ? (A extends I ? B : never) : never; export type ExtractAssertionId = T extends SimulationAssertion ? A : never; export type StoreType = Record< AssertionId, diff --git a/packages/config/src/chainConfig/types.ts b/packages/config/src/chainConfig/types.ts index 20e8119332f3..3e0844118290 100644 --- a/packages/config/src/chainConfig/types.ts +++ b/packages/config/src/chainConfig/types.ts @@ -131,12 +131,12 @@ export type SpecValue = number | bigint | Uint8Array | string; export type SpecValueType = V extends number ? "number" : V extends bigint - ? "bigint" - : V extends Uint8Array - ? "bytes" - : V extends string - ? "string" - : never; + ? "bigint" + : V extends Uint8Array + ? "bytes" + : V extends string + ? "string" + : never; /** All possible type names for a SpecValue */ export type SpecValueTypeName = SpecValueType; diff --git a/packages/state-transition/src/block/slashValidator.ts b/packages/state-transition/src/block/slashValidator.ts index 133041d36869..9f3eb2947644 100644 --- a/packages/state-transition/src/block/slashValidator.ts +++ b/packages/state-transition/src/block/slashValidator.ts @@ -51,8 +51,8 @@ export function slashValidator( fork === ForkSeq.phase0 ? MIN_SLASHING_PENALTY_QUOTIENT : fork === ForkSeq.altair - ? MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR - : MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX; + ? MIN_SLASHING_PENALTY_QUOTIENT_ALTAIR + : MIN_SLASHING_PENALTY_QUOTIENT_BELLATRIX; decreaseBalance(state, slashedIndex, Math.floor(effectiveBalance / minSlashingPenaltyQuotient)); // apply proposer and whistleblower rewards diff --git a/packages/state-transition/src/epoch/processSlashings.ts b/packages/state-transition/src/epoch/processSlashings.ts index 7f4403dc027a..ba4b483dffc2 100644 --- a/packages/state-transition/src/epoch/processSlashings.ts +++ b/packages/state-transition/src/epoch/processSlashings.ts @@ -41,8 +41,8 @@ export function processSlashings( fork === ForkSeq.phase0 ? PROPORTIONAL_SLASHING_MULTIPLIER : fork === ForkSeq.altair - ? PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR - : PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX; + ? PROPORTIONAL_SLASHING_MULTIPLIER_ALTAIR + : PROPORTIONAL_SLASHING_MULTIPLIER_BELLATRIX; const {effectiveBalanceIncrements} = state.epochCtx; const adjustedTotalSlashingBalanceByIncrement = Math.min( diff --git a/packages/utils/src/command.ts b/packages/utils/src/command.ts index 3bfb372bd78f..89929a6c41ef 100644 --- a/packages/utils/src/command.ts +++ b/packages/utils/src/command.ts @@ -13,12 +13,12 @@ export interface CliOptionDefinition extends Options { type: T extends string ? "string" : T extends number - ? "number" - : T extends boolean - ? "boolean" - : T extends Array - ? "array" - : never; + ? "number" + : T extends boolean + ? "boolean" + : T extends Array + ? "array" + : never; } export type CliCommandOptions = Required<{ diff --git a/packages/utils/src/types.ts b/packages/utils/src/types.ts index 5b46d65053ef..935c13cda2c1 100644 --- a/packages/utils/src/types.ts +++ b/packages/utils/src/types.ts @@ -6,8 +6,8 @@ export type RecursivePartial = { [P in keyof T]?: T[P] extends (infer U)[] ? RecursivePartial[] : T[P] extends Readonly[] - ? Readonly>[] - : RecursivePartial; + ? Readonly>[] + : RecursivePartial; }; /** Type safe wrapper for Number constructor that takes 'any' */ diff --git a/yarn.lock b/yarn.lock index 0e1fa22a42e5..b96e4de79135 100644 --- a/yarn.lock +++ b/yarn.lock @@ -671,10 +671,10 @@ minimatch "^3.1.2" strip-json-comments "^3.1.1" -"@eslint/js@8.56.0": - version "8.56.0" - resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.56.0.tgz#ef20350fec605a7f7035a01764731b2de0f3782b" - integrity sha512-gMsVel9D7f2HLkBma9VbtzZRehRogVRfbr++f06nL2vnCGCNlzOD+/MUov/F4p8myyAHspEhVobgjpX64q5m6A== +"@eslint/js@8.57.0": + version "8.57.0" + resolved "https://registry.yarnpkg.com/@eslint/js/-/js-8.57.0.tgz#a5417ae8427873f1dd08b70b3574b453e67b5f7f" + integrity sha512-Ys+3g2TaW7gADOJzPt83SJtCDhMjndcDMFVQ/Tj9iA1BfJzFKD9mAUXT3OenpuPHbI6P/myECxRJrofUsDx/5g== "@ethereumjs/block@^4.2.2": version "4.2.2" @@ -1252,7 +1252,7 @@ resolved "https://registry.npmjs.org/@gar/promisify/-/promisify-1.1.3.tgz" integrity sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw== -"@humanwhocodes/config-array@^0.11.13": +"@humanwhocodes/config-array@^0.11.14": version "0.11.14" resolved "https://registry.yarnpkg.com/@humanwhocodes/config-array/-/config-array-0.11.14.tgz#d78e481a039f7566ecc9660b4ea7fe6b1fec442b" integrity sha512-3T8LkOmg45BV5FICb15QQMsyUSWrQ8AygVfC7ZG32zOalnqrilm018ZVCw0eapXux8FtA33q8PSRSstjee3jSg== @@ -3132,16 +3132,16 @@ dependencies: "@types/node" "*" -"@typescript-eslint/eslint-plugin@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-6.21.0.tgz#30830c1ca81fd5f3c2714e524c4303e0194f9cd3" - integrity sha512-oy9+hTPCUFpngkEZUSzbf9MxI65wbKFoQYsgPdILTfbUldp5ovUuphZVe4i30emU9M/kP+T64Di0mxl7dSw3MA== +"@typescript-eslint/eslint-plugin@^7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/eslint-plugin/-/eslint-plugin-7.1.0.tgz#22bb999a8d59893c0ea07923e8a21f9d985ad740" + integrity sha512-j6vT/kCulhG5wBmGtstKeiVr1rdXE4nk+DT1k6trYkwlrvW9eOF5ZbgKnd/YR6PcM4uTEXa0h6Fcvf6X7Dxl0w== dependencies: "@eslint-community/regexpp" "^4.5.1" - "@typescript-eslint/scope-manager" "6.21.0" - "@typescript-eslint/type-utils" "6.21.0" - "@typescript-eslint/utils" "6.21.0" - "@typescript-eslint/visitor-keys" "6.21.0" + "@typescript-eslint/scope-manager" "7.1.0" + "@typescript-eslint/type-utils" "7.1.0" + "@typescript-eslint/utils" "7.1.0" + "@typescript-eslint/visitor-keys" "7.1.0" debug "^4.3.4" graphemer "^1.4.0" ignore "^5.2.4" @@ -3149,15 +3149,15 @@ semver "^7.5.4" ts-api-utils "^1.0.1" -"@typescript-eslint/parser@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-6.21.0.tgz#af8fcf66feee2edc86bc5d1cf45e33b0630bf35b" - integrity sha512-tbsV1jPne5CkFQCgPBcDOt30ItF7aJoZL997JSF7MhGQqOeT3svWRYxiqlfA5RUdlHN6Fi+EI9bxqbdyAUZjYQ== +"@typescript-eslint/parser@^7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/parser/-/parser-7.1.0.tgz#b89dab90840f7d2a926bf4c23b519576e8c31970" + integrity sha512-V1EknKUubZ1gWFjiOZhDSNToOjs63/9O0puCgGS8aDOgpZY326fzFu15QAUjwaXzRZjf/qdsdBrckYdv9YxB8w== dependencies: - "@typescript-eslint/scope-manager" "6.21.0" - "@typescript-eslint/types" "6.21.0" - "@typescript-eslint/typescript-estree" "6.21.0" - "@typescript-eslint/visitor-keys" "6.21.0" + "@typescript-eslint/scope-manager" "7.1.0" + "@typescript-eslint/types" "7.1.0" + "@typescript-eslint/typescript-estree" "7.1.0" + "@typescript-eslint/visitor-keys" "7.1.0" debug "^4.3.4" "@typescript-eslint/scope-manager@6.21.0": @@ -3168,13 +3168,21 @@ "@typescript-eslint/types" "6.21.0" "@typescript-eslint/visitor-keys" "6.21.0" -"@typescript-eslint/type-utils@6.21.0": - version "6.21.0" - resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-6.21.0.tgz#6473281cfed4dacabe8004e8521cee0bd9d4c01e" - integrity sha512-rZQI7wHfao8qMX3Rd3xqeYSMCL3SoiSQLBATSiVKARdFGCYSRvmViieZjqc58jKgs8Y8i9YvVVhRbHSTA4VBag== +"@typescript-eslint/scope-manager@7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/scope-manager/-/scope-manager-7.1.0.tgz#e4babaa39a3d612eff0e3559f3e99c720a2b4a54" + integrity sha512-6TmN4OJiohHfoOdGZ3huuLhpiUgOGTpgXNUPJgeZOZR3DnIpdSgtt83RS35OYNNXxM4TScVlpVKC9jyQSETR1A== dependencies: - "@typescript-eslint/typescript-estree" "6.21.0" - "@typescript-eslint/utils" "6.21.0" + "@typescript-eslint/types" "7.1.0" + "@typescript-eslint/visitor-keys" "7.1.0" + +"@typescript-eslint/type-utils@7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/type-utils/-/type-utils-7.1.0.tgz#372dfa470df181bcee0072db464dc778b75ed722" + integrity sha512-UZIhv8G+5b5skkcuhgvxYWHjk7FW7/JP5lPASMEUoliAPwIH/rxoUSQPia2cuOj9AmDZmwUl1usKm85t5VUMew== + dependencies: + "@typescript-eslint/typescript-estree" "7.1.0" + "@typescript-eslint/utils" "7.1.0" debug "^4.3.4" ts-api-utils "^1.0.1" @@ -3183,6 +3191,11 @@ resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-6.21.0.tgz#205724c5123a8fef7ecd195075fa6e85bac3436d" integrity sha512-1kFmZ1rOm5epu9NZEZm1kckCDGj5UJEf7P1kliH4LKu/RkwpsfqqGmY2OOcUs18lSlQBKLDYBOGxRVtrMN5lpg== +"@typescript-eslint/types@7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/types/-/types-7.1.0.tgz#52a86d6236fda646e7e5fe61154991dc0dc433ef" + integrity sha512-qTWjWieJ1tRJkxgZYXx6WUYtWlBc48YRxgY2JN1aGeVpkhmnopq+SUC8UEVGNXIvWH7XyuTjwALfG6bFEgCkQA== + "@typescript-eslint/typescript-estree@6.21.0": version "6.21.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-6.21.0.tgz#c47ae7901db3b8bddc3ecd73daff2d0895688c46" @@ -3197,7 +3210,34 @@ semver "^7.5.4" ts-api-utils "^1.0.1" -"@typescript-eslint/utils@6.21.0", "@typescript-eslint/utils@^6.21.0": +"@typescript-eslint/typescript-estree@7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/typescript-estree/-/typescript-estree-7.1.0.tgz#419b1310f061feee6df676c5bed460537310c593" + integrity sha512-k7MyrbD6E463CBbSpcOnwa8oXRdHzH1WiVzOipK3L5KSML92ZKgUBrTlehdi7PEIMT8k0bQixHUGXggPAlKnOQ== + dependencies: + "@typescript-eslint/types" "7.1.0" + "@typescript-eslint/visitor-keys" "7.1.0" + debug "^4.3.4" + globby "^11.1.0" + is-glob "^4.0.3" + minimatch "9.0.3" + semver "^7.5.4" + ts-api-utils "^1.0.1" + +"@typescript-eslint/utils@7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-7.1.0.tgz#710ecda62aff4a3c8140edabf3c5292d31111ddd" + integrity sha512-WUFba6PZC5OCGEmbweGpnNJytJiLG7ZvDBJJoUcX4qZYf1mGZ97mO2Mps6O2efxJcJdRNpqweCistDbZMwIVHw== + dependencies: + "@eslint-community/eslint-utils" "^4.4.0" + "@types/json-schema" "^7.0.12" + "@types/semver" "^7.5.0" + "@typescript-eslint/scope-manager" "7.1.0" + "@typescript-eslint/types" "7.1.0" + "@typescript-eslint/typescript-estree" "7.1.0" + semver "^7.5.4" + +"@typescript-eslint/utils@^6.21.0": version "6.21.0" resolved "https://registry.yarnpkg.com/@typescript-eslint/utils/-/utils-6.21.0.tgz#4714e7a6b39e773c1c8e97ec587f520840cd8134" integrity sha512-NfWVaC8HP9T8cbKQxHcsJBY5YE1O33+jpMwN45qzWWaPDZgLIbo12toGMWnmhvCpd3sIxkpDw3Wv1B3dYrbDQQ== @@ -3218,6 +3258,14 @@ "@typescript-eslint/types" "6.21.0" eslint-visitor-keys "^3.4.1" +"@typescript-eslint/visitor-keys@7.1.0": + version "7.1.0" + resolved "https://registry.yarnpkg.com/@typescript-eslint/visitor-keys/-/visitor-keys-7.1.0.tgz#576c4ad462ca1378135a55e2857d7aced96ce0a0" + integrity sha512-FhUqNWluiGNzlvnDZiXad4mZRhtghdoKW6e98GoEOYSu5cND+E39rG5KwJMUzeENwm1ztYBRqof8wMLP+wNPIA== + dependencies: + "@typescript-eslint/types" "7.1.0" + eslint-visitor-keys "^3.4.1" + "@ungap/structured-clone@^1.2.0": version "1.2.0" resolved "https://registry.yarnpkg.com/@ungap/structured-clone/-/structured-clone-1.2.0.tgz#756641adb587851b5ccb3e095daf27ae581c8406" @@ -5877,16 +5925,16 @@ eslint-visitor-keys@^3.3.0, eslint-visitor-keys@^3.4.1, eslint-visitor-keys@^3.4 resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.4.3.tgz#0cd72fe8550e3c2eae156a96a4dddcd1c8ac5800" integrity sha512-wpc+LXeiyiisxPlEkUzU6svyS1frIO3Mgxj1fdy7Pm8Ygzguax2N3Fa/D/ag1WqbOprdI+uY6wMUl8/a2G+iag== -eslint@^8.56.0: - version "8.56.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.56.0.tgz#4957ce8da409dc0809f99ab07a1b94832ab74b15" - integrity sha512-Go19xM6T9puCOWntie1/P997aXxFsOi37JIHRWI514Hc6ZnaHGKY9xFhrU65RT6CcBEzZoGG1e6Nq+DT04ZtZQ== +eslint@^8.57.0: + version "8.57.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.57.0.tgz#c786a6fd0e0b68941aaf624596fb987089195668" + integrity sha512-dZ6+mexnaTIbSBZWgou51U6OmzIhYM2VcNdtiTtI7qPNZm35Akpr0f6vtw3w1Kmn5PYo+tZVfh13WrhpS6oLqQ== dependencies: "@eslint-community/eslint-utils" "^4.2.0" "@eslint-community/regexpp" "^4.6.1" "@eslint/eslintrc" "^2.1.4" - "@eslint/js" "8.56.0" - "@humanwhocodes/config-array" "^0.11.13" + "@eslint/js" "8.57.0" + "@humanwhocodes/config-array" "^0.11.14" "@humanwhocodes/module-importer" "^1.0.1" "@nodelib/fs.walk" "^1.2.8" "@ungap/structured-clone" "^1.2.0" @@ -10276,10 +10324,10 @@ prettier-linter-helpers@^1.0.0: dependencies: fast-diff "^1.1.2" -prettier@^3.0.3: - version "3.0.3" - resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.0.3.tgz#432a51f7ba422d1469096c0fdc28e235db8f9643" - integrity sha512-L/4pUDMxcNa8R/EthV08Zt42WBO4h1rarVtK0K+QJG0X187OLo7l699jWw0GKuwzkPQ//jMFA/8Xm6Fh3J/DAg== +prettier@^3.2.5: + version "3.2.5" + resolved "https://registry.yarnpkg.com/prettier/-/prettier-3.2.5.tgz#e52bc3090586e824964a8813b09aba6233b28368" + integrity sha512-3/GWa9aOC0YeD7LUfvOG2NiDyhOWRvt1k+rcKhOuYnMY24iiCphgneUfJDyFXd6rZCAnuLBv6UeAULtrhT/F4A== pretty-format@^29.7.0: version "29.7.0" From 6ad9740a085574306cf46c7642e749d6ec9a4264 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 29 Feb 2024 16:03:49 +0100 Subject: [PATCH 20/34] chore: add eslint rule to restrict global fetch (#6500) * chore: add eslint rule to restrict global fetch * Add comment to eslint disable * Rephrase comment --- .eslintrc.js | 7 +++++++ packages/api/src/utils/client/fetch.ts | 2 ++ .../api/test/unit/client/httpClientFallback.test.ts | 10 +++++----- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index e38300c2caa9..fdfae1a1a00d 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -134,6 +134,13 @@ module.exports = { "no-console": "error", "no-loss-of-precision": "error", "no-prototype-builtins": 0, + "no-restricted-globals": [ + "error", + { + name: "fetch", + message: "Please use 'fetch' from '@lodestar/api' instead.", + }, + ], "no-restricted-imports": [ "error", { diff --git a/packages/api/src/utils/client/fetch.ts b/packages/api/src/utils/client/fetch.ts index 2cdd98a27af0..a338d82e521f 100644 --- a/packages/api/src/utils/client/fetch.ts +++ b/packages/api/src/utils/client/fetch.ts @@ -5,6 +5,8 @@ */ async function wrappedFetch(url: string | URL, init?: RequestInit): Promise { try { + // This function wraps global `fetch` which should only be directly called here + // eslint-disable-next-line no-restricted-globals return await fetch(url, init); } catch (e) { throw new FetchError(url, e); diff --git a/packages/api/test/unit/client/httpClientFallback.test.ts b/packages/api/test/unit/client/httpClientFallback.test.ts index ff02095b1cc6..e51119741d3c 100644 --- a/packages/api/test/unit/client/httpClientFallback.test.ts +++ b/packages/api/test/unit/client/httpClientFallback.test.ts @@ -1,5 +1,5 @@ import {describe, it, beforeEach, afterEach, expect, vi} from "vitest"; -import {HttpClient} from "../../../src/utils/client/index.js"; +import {HttpClient, fetch} from "../../../src/utils/client/index.js"; describe("httpClient fallback", () => { const testRoute = {url: "/test-route", method: "GET" as const}; @@ -7,7 +7,7 @@ describe("httpClient fallback", () => { // Using fetchSub instead of actually setting up servers because there are some strange // race conditions, where the server stub doesn't count the call in time before the test is over. - const fetchStub = vi.fn(); + const fetchStub = vi.fn, ReturnType>(); let httpClient: HttpClient; @@ -20,7 +20,7 @@ describe("httpClient fallback", () => { const serverErrors = new Map(); // With baseURLs above find the server index associated with that URL - function getServerIndex(url: URL): number { + function getServerIndex(url: URL | string): number { const i = baseUrls.findIndex((baseUrl) => url.toString().startsWith(baseUrl)); if (i < 0) { throw Error(`fetch called with unknown url ${url.toString()}`); @@ -33,7 +33,7 @@ describe("httpClient fallback", () => { httpClient = new HttpClient({ baseUrl: "", urls: baseUrls.map((baseUrl) => ({baseUrl})), - fetch: fetchStub as typeof fetch, + fetch: fetchStub, }); fetchStub.mockImplementation(async (url) => { @@ -57,7 +57,7 @@ describe("httpClient fallback", () => { const callCounts: number[] = []; for (let i = 0; i < serverCount; i++) callCounts[i] = 0; for (const call of fetchStub.mock.calls) { - callCounts[getServerIndex(call)]++; + callCounts[getServerIndex(call[0])]++; } expect(callCounts.join(",")).toBe(expectedCallCounts.join(",")); From 3ef43ef0fee9f9b7430cc84c5de73865dda57f59 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 1 Mar 2024 10:14:03 +0100 Subject: [PATCH 21/34] chore: restrict node version to 20.x (#6498) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 5b76d19a8760..720a021a6ed2 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "root", "private": true, "engines": { - "node": ">=18.17.0 <19 || >=20.1.0" + "node": ">=20.1.0 <21" }, "workspaces": [ "packages/*" From 1f18ec475b0062e0e1aa476c59a0ef104ab4ff30 Mon Sep 17 00:00:00 2001 From: Julien Date: Fri, 1 Mar 2024 02:53:01 -0800 Subject: [PATCH 22/34] chore: upgrade blst (#6492) --- packages/beacon-node/package.json | 2 +- packages/cli/package.json | 2 +- packages/state-transition/package.json | 3 +-- yarn.lock | 8 ++++---- 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 88951d1ab597..27ae5b1734ac 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -94,7 +94,7 @@ "dependencies": { "@chainsafe/as-sha256": "^0.4.1", "@chainsafe/bls": "7.1.3", - "@chainsafe/blst": "^0.2.9", + "@chainsafe/blst": "^0.2.10", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/libp2p-gossipsub": "^11.2.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 00cc5bd9bc1f..19aaf088d886 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -54,7 +54,7 @@ "@chainsafe/bls": "7.1.3", "@chainsafe/bls-keygen": "^0.4.0", "@chainsafe/bls-keystore": "^3.0.1", - "@chainsafe/blst": "^0.2.9", + "@chainsafe/blst": "^0.2.10", "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/persistent-merkle-tree": "^0.6.1", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index f861792e5c38..d7d65a2dab49 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -60,7 +60,7 @@ "dependencies": { "@chainsafe/as-sha256": "^0.4.1", "@chainsafe/bls": "7.1.3", - "@chainsafe/blst": "^0.2.9", + "@chainsafe/blst": "^0.2.10", "@chainsafe/persistent-merkle-tree": "^0.6.1", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.14.0", @@ -72,7 +72,6 @@ "buffer-xor": "^2.0.2" }, "devDependencies": { - "@chainsafe/blst": "^0.2.9", "@types/buffer-xor": "^2.0.0" }, "keywords": [ diff --git a/yarn.lock b/yarn.lock index b96e4de79135..8748b589bcc8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -301,10 +301,10 @@ "@chainsafe/bls-keygen" "^0.4.0" bls-eth-wasm "^0.4.8" -"@chainsafe/blst@^0.2.9": - version "0.2.9" - resolved "https://registry.yarnpkg.com/@chainsafe/blst/-/blst-0.2.9.tgz#b356b47759c3ce127677227fc24faa3ac6c72032" - integrity sha512-6MXBUy5Co6k6V9Bv0EC5YrHD7kwWIpzwBO4yCqurLw//Zm3cUmN6DohuYuEGcS4QMNEswa/cXqzZLf+LFBJPiw== +"@chainsafe/blst@^0.2.10": + version "0.2.10" + resolved "https://registry.yarnpkg.com/@chainsafe/blst/-/blst-0.2.10.tgz#77802e5b1ff2d98ec1d25dcd5f7d27b89d376a40" + integrity sha512-ofecTL5fWsNwnpS2oUh56dDXJRmCEcDKNNBFDb2ux+WtvdjrdSq6B+L/eNlg+sVBzXbzrCw1jq8Y8+cYiHg32w== dependencies: "@types/tar" "^6.1.4" node-fetch "^2.6.1" From b5712a60637f6b13b7f60c6dfcadc214a8c71bf7 Mon Sep 17 00:00:00 2001 From: Hiroyuki Naito <04hiroyuki28@gmail.com> Date: Sat, 2 Mar 2024 18:58:13 +0900 Subject: [PATCH 23/34] chore: remove lodash usage (#6501) * Delete lodash.pick package * Lint --------- Co-authored-by: Nico Flaig --- packages/beacon-node/package.json | 2 -- packages/beacon-node/test/unit/eth1/utils/eth1Data.test.ts | 3 +-- yarn.lock | 5 ----- 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 27ae5b1734ac..d715c1f77ed5 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -139,7 +139,6 @@ "interface-datastore": "^8.2.7", "it-all": "^3.0.4", "it-pipe": "^3.0.1", - "lodash": "^4.17.21", "jwt-simple": "0.5.6", "libp2p": "1.1.1", "multiformats": "^11.0.1", @@ -154,7 +153,6 @@ "devDependencies": { "@types/datastore-level": "^3.0.0", "@types/leveldown": "^4.0.3", - "@types/lodash": "^4.14.192", "@types/qs": "^6.9.7", "@types/tmp": "^0.2.3", "it-drain": "^3.0.3", diff --git a/packages/beacon-node/test/unit/eth1/utils/eth1Data.test.ts b/packages/beacon-node/test/unit/eth1/utils/eth1Data.test.ts index e5678b9f06d7..4b5bf74772b7 100644 --- a/packages/beacon-node/test/unit/eth1/utils/eth1Data.test.ts +++ b/packages/beacon-node/test/unit/eth1/utils/eth1Data.test.ts @@ -1,4 +1,3 @@ -import pick from "lodash/pick.js"; import {describe, it, expect} from "vitest"; import {Root, phase0, ssz} from "@lodestar/types"; import {toHex} from "@lodestar/utils"; @@ -107,7 +106,7 @@ describe("eth1 / util / getEth1DataForBlocks", function () { if (expectedEth1Data) { const eth1Datas = await eth1DatasPromise; - const eth1DatasPartial = eth1Datas.map((eth1Data) => pick(eth1Data, Object.keys(expectedEth1Data[0]))); + const eth1DatasPartial = eth1Datas.map(({blockNumber, depositCount}) => ({blockNumber, depositCount})); expect(eth1DatasPartial).toEqual(expectedEth1Data); } else if (error != null) { await expectRejectedWithLodestarError(eth1DatasPromise, error); diff --git a/yarn.lock b/yarn.lock index 8748b589bcc8..0f0c76c7d130 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2915,11 +2915,6 @@ "@types/abstract-leveldown" "*" "@types/node" "*" -"@types/lodash@^4.14.192": - version "4.14.202" - resolved "https://registry.yarnpkg.com/@types/lodash/-/lodash-4.14.202.tgz#f09dbd2fb082d507178b2f2a5c7e74bd72ff98f8" - integrity sha512-OvlIYQK9tNneDlS0VN54LLd5uiPCBOp7gS5Z0f1mjoJYBrtStzgmJBxONW3U6OZqdtNzZPmn9BS/7WI7BFFcFQ== - "@types/minimatch@^3.0.3": version "3.0.5" resolved "https://registry.npmjs.org/@types/minimatch/-/minimatch-3.0.5.tgz" From 3af0db1a7c96cbd3a26ce8df9921e33fffc7c733 Mon Sep 17 00:00:00 2001 From: Phil Ngo <58080811+philknows@users.noreply.github.com> Date: Sat, 2 Mar 2024 10:24:07 -0500 Subject: [PATCH 24/34] docs: update vc-configuration md doc (#6503) * update vc-configuration md doc * Grammar correction Co-authored-by: Nico Flaig * Show only one ref on text Co-authored-by: Nico Flaig * Put create keystore under setup validator --------- Co-authored-by: Nico Flaig --- .../validator-management/vc-configuration.md | 76 +++++++++---------- 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/docs/pages/validator-management/vc-configuration.md b/docs/pages/validator-management/vc-configuration.md index 4853bca5bfa6..52257dd53552 100644 --- a/docs/pages/validator-management/vc-configuration.md +++ b/docs/pages/validator-management/vc-configuration.md @@ -4,37 +4,24 @@ The following instructions are for stakers utilizing the Lodestar validator clie [TOC] -## Wallet configuration - -A wallet helps to manage many validators from a group of 12/24 words (also known as a "mnemonic" or "recovery phrase"). All validators and withdrawal keys can be re-generated from a backed-up mnemonic. - -The mnemonic is randomly generated during wallet creation and printed out to the terminal. It's important to make one or more backups of the mnemonic to ensure your ETH wallets are not lost in the case of data loss. - - -!!! danger - It is very important to keep your mnemonic private as it represents the ultimate control of your ETH wallets. - +## Setup your validator -### Create a wallet +Validators are represented by a BLS keypair. Use your generated mnemonic from one of the tools above to generate the keystore files required for validator duties on Ethereum using the Lodestar validator client. -Lodestar has removed its functionality to create wallets. +### Create a keystore -To create a wallet, we recommend using the official [`staking-deposit-cli`](https://github.com/ethereum/staking-deposit-cli/releases) from the Ethereum Foundation for users comfortable with command line interfaces. +To create a keystore, we recommend using the official [Staking Deposit CLI](https://github.com/ethereum/staking-deposit-cli/releases) from the Ethereum Foundation for users comfortable with command line interfaces. Alternatively, for a graphical user interface, you can use the [Stakehouse Wagyu Key Generator](https://wagyu.gg/) developed by members of the EthStaker community. -!!! info - These tools will generate files for staking validators as well as the important mnemonic. This mnemonic must be handled and stored securely. +!!! warning + These tools will generate keystore files for staking validators as well as the important mnemonic. This mnemonic must be handled and stored securely. -## Setup your validator - -Validators are represented by a BLS keypair. Use your generated mnemonic from one of the tools above to generate the keystore files required for validator duties on Lodestar. +### Import a validator keystore to Lodestar -### Import a validator keystore from your wallet to Lodestar - -To import a validator keystore that was created via one of the methods described above, you must locate the validator JSON keystores exported by those tools (ex. `keystore-m_12381_3600_0_0_0-1654128694.json`). +To import a validator JSON keystore that was created via one of the methods described above, you must locate the file for import (ex. `keystore-m_12381_3600_0_0_0-1654128694.json`). Inside the keystore JSON file, you should have an [EIP-2335 keystore file](https://github.com/ethereum/EIPs/blob/master/EIPS/eip-2335.md#json-schema). @@ -60,7 +47,7 @@ _Plaintext passphrase file import_ !!! info The interactive passphrase import method will prompt every keystore in the `validator_keys` folder for import and will ask for the individual password for each keystore. **This method will allow you to import multiple keystores with different passwords.** - The plaintext passphrase file import method will allow to import all keystores in the `validator_keys` folder with the same password contained in `password.txt` for efficiency. + The plaintext passphrase file import method will allow you to import all keystores in the `validator_keys` folder encrypted with the same password contained in `password.txt` for efficiency. Once imported with either method, these keystores will be automatically loaded when you start the validator. To list the imported keystores, use the `validator list` command. @@ -118,9 +105,19 @@ Example 3: Setting a `--builder.boostFactor=100` is the same as signaling `--bui ### Submit a validator deposit -Please use the official Ethereum Launchpad to perform your deposits +Please use the official Ethereum Launchpad to perform your deposits. Ensure your deposits are sent to the proper beacon chain deposit address on the correct network. + +#### Mainnet +- [Ethereum Mainnet Launchpad](https://launchpad.ethereum.org) +- [Beacon Chain Deposit Contract](https://etherscan.io/address/0x00000000219ab540356cbb839cbe05303d7705fa) `0x00000000219ab540356cBB839Cbe05303d7705Fa` + +#### Holesky Testnet +- [Ethereum Holesky Testnet Launchpad](https://holesky.launchpad.ethereum.org) +- [Holesky Beacon Chain Deposit Contract](https://holesky.etherscan.io/address/0x4242424242424242424242424242424242424242) `0x4242424242424242424242424242424242424242` -- Ethereum Foundation launchpad: +#### Ephemery Testnet +- [Ethereum Ephemery Testnet Launchpad](https://launchpad.ephemery.dev/) +- [Ephemeral Testnet Resources](https://ephemery.dev/) ## Run the validator @@ -133,19 +130,20 @@ To start a Lodestar validator run the command: You should see confirmation that modules have started. ```txt -Nov-29 10:47:13.647[] info: Lodestar network=sepolia, version=v1.2.2/f093b46, commit=f093b468ec3ab0dbbe8e2d2c8175f52ad88aa35f -Nov-29 10:47:13.649[] info: Connecting to LevelDB database path=/home/user/.local/share/lodestar/sepolia/validator-db -Nov-29 10:47:51.732[] info: 3 local keystores -Nov-29 10:47:51.735[] info: 0x800f6be579b31ea950a50be65f7de8f678b23b7466579c01ac26ebf9c19599fb2b446da40ad4fc92c6109fcd6793303f -Nov-29 10:47:51.735[] info: 0x81337ebe90d6942d8b61922ea880c4d28ebc745ddc10a1acc85b745a15c6c8754af1a73b1b3483b6a5024b783510b35c -Nov-29 10:47:51.757[] info: 0xb95fc0ec39596deee2c4363f57bb4786f5bb8dfb345c1e5b14e2927be482615971d0d81f9a88b3389fac7079b3cb2f46 -Nov-29 10:47:51.776[] info: Genesis fetched from the beacon node -Nov-29 10:47:51.781[] info: Verified connected beacon node and validator have same the config -Nov-29 10:47:51.837[] info: Verified connected beacon node and validator have the same genesisValidatorRoot -Nov-29 10:47:51.914[] info: Discovered new validators count=100 -Nov-29 10:48:00.197[] info: Published SyncCommitteeMessage slot=1165140, count=27 -Nov-29 10:48:02.296[] info: Published attestations slot=1165140, count=6 -Nov-29 10:48:08.122[] info: Published aggregateAndProofs slot=1165140, index=0, count=2 -Nov-29 10:48:12.102[] info: Published SyncCommitteeMessage slot=1165141, count=27 -Nov-29 10:48:14.236[] info: Published attestations slot=1165141, count=4 +Mar-01 03:06:35.048[] info: Lodestar network=holesky, version=v1.16.0/6ad9740, commit=6ad9740a085574306cf46c7642e749d6ec9a4264 +Mar-01 03:06:35.050[] info: Connecting to LevelDB database path=/keystoresDir/validator-db-holesky +Mar-01 03:06:35.697[] info: 100% of keystores imported. current=2 total=2 rate=1318.68keys/m +Mar-01 03:06:35.698[] info: 2 local keystores +Mar-01 03:06:35.698[] info: 0xa6fcfca12e1db6c7341d82327010cd57224dc239d1c5e4fb18286cc32edb877d813c5af1c870d474aef7b3ff7ab927ea +Mar-01 03:06:35.698[] info: 0x8f868e53bbe1451bcf6d42c9ab6d292cbd7fbfa09c59b6b99c1dd6a4977e2e7b4b752c328784ca2788dd6f63ffcbdb7e +Mar-01 03:06:35.732[] info: Beacon node urls=http://127.0.0.1:9596 +Mar-01 03:09:23.813[] info: Genesis fetched from the beacon node +Mar-01 03:09:23.816[] info: Verified connected beacon node and validator have same the config +Mar-01 03:09:23.818[] info: Verified connected beacon node and validator have the same genesisValidatorRoot +Mar-01 03:09:23.818[] info: Initializing validator useProduceBlockV3=deneb+, broadcastValidation=gossip, defaultBuilderSelection=executiononly, suggestedFeeRecipient=0xeeef273281fB83F56182eE960aA4bAfe7fE075DE, strictFeeRecipientCheck=false +Mar-01 03:09:23.830[] info: Validator seen on beacon chain validatorIndex=1234567, pubKey=0xa6fcfca12e1db6c7341d82327010cd57224dc239d1c5e4fb18286cc32edb877d813c5af1c870d474aef7b3ff7ab927ea +Mar-01 03:09:23.830[] info: Validator seen on beacon chain validatorIndex=1234568, pubKey=0x8f868e53bbe1451bcf6d42c9ab6d292cbd7fbfa09c59b6b99c1dd6a4977e2e7b4b752c328784ca2788dd6f63ffcbdb7e +Mar-01 03:09:23.830[] info: Validator statuses active=2, total=2 +Mar-01 03:15:50.191[] info: Published attestations slot=1113379, count=1 +Mar-01 03:16:02.728[] info: Published attestations slot=1113380, count=1 ``` From d10ed384d648ffd0dcc4d377d0d9565b8e0f7e9b Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Mon, 4 Mar 2024 08:21:05 +0700 Subject: [PATCH 25/34] feat: regen to consume state cache reload api (#6456) * feat: regen to consume state cache reload api * chore: address PR comments --- .../beacon-node/src/chain/regen/errors.ts | 4 +- packages/beacon-node/src/chain/regen/regen.ts | 94 ++++++++++++++----- .../src/metrics/metrics/lodestar.ts | 5 + 3 files changed, 76 insertions(+), 27 deletions(-) diff --git a/packages/beacon-node/src/chain/regen/errors.ts b/packages/beacon-node/src/chain/regen/errors.ts index 85d43d1a4fe8..7c1573b415f8 100644 --- a/packages/beacon-node/src/chain/regen/errors.ts +++ b/packages/beacon-node/src/chain/regen/errors.ts @@ -8,6 +8,7 @@ export enum RegenErrorCode { TOO_MANY_BLOCK_PROCESSED = "REGEN_ERROR_TOO_MANY_BLOCK_PROCESSED", BLOCK_NOT_IN_DB = "REGEN_ERROR_BLOCK_NOT_IN_DB", STATE_TRANSITION_ERROR = "REGEN_ERROR_STATE_TRANSITION_ERROR", + INVALID_STATE_ROOT = "REGEN_ERROR_INVALID_STATE_ROOT", } export type RegenErrorType = @@ -17,7 +18,8 @@ export type RegenErrorType = | {code: RegenErrorCode.NO_SEED_STATE} | {code: RegenErrorCode.TOO_MANY_BLOCK_PROCESSED; stateRoot: RootHex | Root} | {code: RegenErrorCode.BLOCK_NOT_IN_DB; blockRoot: RootHex | Root} - | {code: RegenErrorCode.STATE_TRANSITION_ERROR; error: Error}; + | {code: RegenErrorCode.STATE_TRANSITION_ERROR; error: Error} + | {code: RegenErrorCode.INVALID_STATE_ROOT; slot: Slot; expected: RootHex; actual: RootHex}; export class RegenError extends Error { type: RegenErrorType; diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 0d6bd89d8ce7..ab0e0b5f2dd7 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -10,29 +10,34 @@ import { stateTransition, } from "@lodestar/state-transition"; import {IForkChoice, ProtoBlock} from "@lodestar/fork-choice"; -import {sleep} from "@lodestar/utils"; +import {Logger, sleep} from "@lodestar/utils"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; import {Metrics} from "../../metrics/index.js"; import {IBeaconDb} from "../../db/index.js"; -import {CheckpointStateCache, StateContextCache} from "../stateCache/index.js"; import {getCheckpointFromState} from "../blocks/utils/checkpoint.js"; import {ChainEvent, ChainEventEmitter} from "../emitter.js"; +import {CheckpointStateCache, BlockStateCache} from "../stateCache/types.js"; import {IStateRegeneratorInternal, RegenCaller, StateCloneOpts} from "./interface.js"; import {RegenError, RegenErrorCode} from "./errors.js"; export type RegenModules = { db: IBeaconDb; forkChoice: IForkChoice; - stateCache: StateContextCache; + stateCache: BlockStateCache; checkpointStateCache: CheckpointStateCache; config: ChainForkConfig; emitter: ChainEventEmitter; + logger: Logger; metrics: Metrics | null; }; /** * Regenerates states that have already been processed by the fork choice + * Since Feb 2024, we support reloading checkpoint state from disk via allowDiskReload flag. Due to its performance impact + * this flag is only set to true in this case: + * - getPreState: this is for block processing, it's important to reload state in unfinality time + * - updateHeadState: rarely happen, but it's important to make sure we always can regen head state */ export class StateRegenerator implements IStateRegeneratorInternal { constructor(private readonly modules: RegenModules) {} @@ -41,11 +46,12 @@ export class StateRegenerator implements IStateRegeneratorInternal { * Get the state to run with `block`. May be: * - If parent is in same epoch -> Exact state at `block.parentRoot` * - If parent is in prev epoch -> State after `block.parentRoot` dialed forward through epoch transition + * - reload state if needed in this flow */ async getPreState( block: allForks.BeaconBlock, opts: StateCloneOpts, - rCaller: RegenCaller + regenCaller: RegenCaller ): Promise { const parentBlock = this.modules.forkChoice.getBlock(block.parentRoot); if (!parentBlock) { @@ -57,6 +63,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { const parentEpoch = computeEpochAtSlot(parentBlock.slot); const blockEpoch = computeEpochAtSlot(block.slot); + const allowDiskReload = true; // This may save us at least one epoch transition. // If the requested state crosses an epoch boundary @@ -64,11 +71,11 @@ export class StateRegenerator implements IStateRegeneratorInternal { // We may have the checkpoint state with parent root inside the checkpoint state cache // through gossip validation. if (parentEpoch < blockEpoch) { - return this.getCheckpointState({root: block.parentRoot, epoch: blockEpoch}, opts, rCaller); + return this.getCheckpointState({root: block.parentRoot, epoch: blockEpoch}, opts, regenCaller, allowDiskReload); } // Otherwise, get the state normally. - return this.getState(parentBlock.stateRoot, rCaller); + return this.getState(parentBlock.stateRoot, regenCaller, allowDiskReload); } /** @@ -77,20 +84,23 @@ export class StateRegenerator implements IStateRegeneratorInternal { async getCheckpointState( cp: phase0.Checkpoint, opts: StateCloneOpts, - rCaller: RegenCaller + regenCaller: RegenCaller, + allowDiskReload = false ): Promise { const checkpointStartSlot = computeStartSlotAtEpoch(cp.epoch); - return this.getBlockSlotState(toHexString(cp.root), checkpointStartSlot, opts, rCaller); + return this.getBlockSlotState(toHexString(cp.root), checkpointStartSlot, opts, regenCaller, allowDiskReload); } /** * Get state after block `blockRoot` dialed forward to `slot` + * - allowDiskReload should be used with care, as it will cause the state to be reloaded from disk */ async getBlockSlotState( blockRoot: RootHex, slot: Slot, opts: StateCloneOpts, - rCaller: RegenCaller + regenCaller: RegenCaller, + allowDiskReload = false ): Promise { const block = this.modules.forkChoice.getBlockHex(blockRoot); if (!block) { @@ -108,26 +118,35 @@ export class StateRegenerator implements IStateRegeneratorInternal { }); } - const latestCheckpointStateCtx = this.modules.checkpointStateCache.getLatest(blockRoot, computeEpochAtSlot(slot)); + const {checkpointStateCache} = this.modules; + const epoch = computeEpochAtSlot(slot); + const latestCheckpointStateCtx = allowDiskReload + ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch) + : checkpointStateCache.getLatest(blockRoot, epoch); // If a checkpoint state exists with the given checkpoint root, it either is in requested epoch // or needs to have empty slots processed until the requested epoch if (latestCheckpointStateCtx) { - return processSlotsByCheckpoint(this.modules, latestCheckpointStateCtx, slot, opts); + return processSlotsByCheckpoint(this.modules, latestCheckpointStateCtx, slot, regenCaller, opts); } // Otherwise, use the fork choice to get the stateRoot from block at the checkpoint root // regenerate that state, // then process empty slots until the requested epoch - const blockStateCtx = await this.getState(block.stateRoot, rCaller); - return processSlotsByCheckpoint(this.modules, blockStateCtx, slot, opts); + const blockStateCtx = await this.getState(block.stateRoot, regenCaller, allowDiskReload); + return processSlotsByCheckpoint(this.modules, blockStateCtx, slot, regenCaller, opts); } /** * Get state by exact root. If not in cache directly, requires finding the block that references the state from the * forkchoice and replaying blocks to get to it. + * - allowDiskReload should be used with care, as it will cause the state to be reloaded from disk */ - async getState(stateRoot: RootHex, _rCaller: RegenCaller): Promise { + async getState( + stateRoot: RootHex, + _rCaller: RegenCaller, + allowDiskReload = false + ): Promise { // Trivial case, state at stateRoot is already cached const cachedStateCtx = this.modules.stateCache.get(stateRoot); if (cachedStateCtx) { @@ -143,15 +162,17 @@ export class StateRegenerator implements IStateRegeneratorInternal { // gets reversed when replayed const blocksToReplay = [block]; let state: CachedBeaconStateAllForks | null = null; - for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.parentRoot)) { + const {checkpointStateCache} = this.modules; + // iterateAncestorBlocks only returns ancestor blocks, not the block itself + for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot)) { state = this.modules.stateCache.get(b.stateRoot); if (state) { break; } - state = this.modules.checkpointStateCache.getLatest( - b.blockRoot, - computeEpochAtSlot(blocksToReplay[blocksToReplay.length - 1].slot - 1) - ); + const epoch = computeEpochAtSlot(blocksToReplay[blocksToReplay.length - 1].slot - 1); + state = allowDiskReload + ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch) + : checkpointStateCache.getLatest(b.blockRoot, epoch); if (state) { break; } @@ -172,6 +193,8 @@ export class StateRegenerator implements IStateRegeneratorInternal { }); } + const replaySlots = blocksToReplay.map((b) => b.slot).join(","); + this.modules.logger.debug("Replaying blocks to get state", {stateRoot, replaySlots}); for (const b of blocksToReplay.reverse()) { const block = await this.modules.db.block.get(fromHexString(b.blockRoot)); if (!block) { @@ -195,11 +218,23 @@ export class StateRegenerator implements IStateRegeneratorInternal { verifyProposer: false, verifySignatures: false, }, - null + this.modules.metrics ); - // TODO: Persist states, note that regen could be triggered by old states. - // Should those take a place in the cache? + const stateRoot = toHexString(state.hashTreeRoot()); + if (b.stateRoot !== stateRoot) { + throw new RegenError({ + slot: b.slot, + code: RegenErrorCode.INVALID_STATE_ROOT, + actual: stateRoot, + expected: b.stateRoot, + }); + } + + if (allowDiskReload) { + // also with allowDiskReload flag, we "reload" it to the state cache too + this.modules.stateCache.add(state); + } // this avoids keeping our node busy processing blocks await sleep(0); @@ -210,13 +245,14 @@ export class StateRegenerator implements IStateRegeneratorInternal { }); } } + this.modules.logger.debug("Replayed blocks to get state", {stateRoot, replaySlots}); return state; } private findFirstStateBlock(stateRoot: RootHex): ProtoBlock { for (const block of this.modules.forkChoice.forwarditerateAncestorBlocks()) { - if (block !== undefined) { + if (block.stateRoot === stateRoot) { return block; } } @@ -237,9 +273,10 @@ async function processSlotsByCheckpoint( modules: {checkpointStateCache: CheckpointStateCache; metrics: Metrics | null; emitter: ChainEventEmitter}, preState: CachedBeaconStateAllForks, slot: Slot, + regenCaller: RegenCaller, opts: StateCloneOpts ): Promise { - let postState = await processSlotsToNearestCheckpoint(modules, preState, slot, opts); + let postState = await processSlotsToNearestCheckpoint(modules, preState, slot, regenCaller, opts); if (postState.slot < slot) { postState = processSlots(postState, slot, opts, modules.metrics); } @@ -257,6 +294,7 @@ async function processSlotsToNearestCheckpoint( modules: {checkpointStateCache: CheckpointStateCache; metrics: Metrics | null; emitter: ChainEventEmitter}, preState: CachedBeaconStateAllForks, slot: Slot, + regenCaller: RegenCaller, opts: StateCloneOpts ): Promise { const preSlot = preState.slot; @@ -272,12 +310,16 @@ async function processSlotsToNearestCheckpoint( ) { // processSlots calls .clone() before mutating postState = processSlots(postState, nextEpochSlot, opts, metrics); + modules.metrics?.epochTransitionByCaller.inc({caller: regenCaller}); - // Cache state to preserve epoch transition work + // this is usually added when we prepare for next slot or validate gossip block + // then when we process the 1st block of epoch, we don't have to do state transition again + // This adds Previous Root Checkpoint State to the checkpoint state cache + // This may becomes the "official" checkpoint state if the 1st block of epoch is skipped const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); checkpointStateCache.add(cp, checkpointState); - emitter.emit(ChainEvent.checkpoint, cp, checkpointState); + emitter.emit(ChainEvent.checkpoint, cp, checkpointState.clone()); // this avoids keeping our node busy processing blocks await sleep(0); diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index c42dc4a747b3..957a962de103 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -291,6 +291,11 @@ export function createLodestarMetrics( // Beacon state transition metrics + epochTransitionByCaller: register.gauge<{caller: RegenCaller}>({ + name: "lodestar_epoch_transition_by_caller_total", + help: "Total count of epoch transition by caller", + labelNames: ["caller"], + }), epochTransitionTime: register.histogram({ name: "lodestar_stfn_epoch_transition_seconds", help: "Time to process a single epoch transition in seconds", From 10c1b11447dd173b975d89b256799b6ef8dc5224 Mon Sep 17 00:00:00 2001 From: NC Date: Mon, 4 Mar 2024 13:55:56 +0800 Subject: [PATCH 26/34] feat: add endpoint for sync committee reward (#6260) * Add block rewards api * Add test * Add unit test * Lint * Address comment * Reduce code redundancy * Read reward cache first before calculate * Lint * Add endpoint definition for sync rewards * Add calculation logic * Lint * Follow convention from block rewards * Include getSyncCommitteeRewards in unit test * Update packages/beacon-node/src/chain/rewards/syncCommitteeRewards.ts Co-authored-by: Nico Flaig * Update packages/beacon-node/src/api/impl/beacon/rewards/index.ts Co-authored-by: Nico Flaig * Improve filtering logic * Early throw on empty preState in getBlockRewards * Add jsdoc * Address comment * Clarify comment * Address comment * Update packages/beacon-node/src/chain/rewards/syncCommitteeRewards.ts Co-authored-by: Nico Flaig * Improve naming of filters * Lint * Update packages/beacon-node/src/chain/rewards/syncCommitteeRewards.ts Co-authored-by: Nico Flaig * ids -> validatorIds --------- Co-authored-by: Nico Flaig --- .../api/src/beacon/routes/beacon/index.ts | 2 +- .../api/src/beacon/routes/beacon/rewards.ts | 47 +++++++++++++++ .../api/test/unit/beacon/oapiSpec.test.ts | 2 +- .../api/test/unit/beacon/testData/beacon.ts | 5 ++ .../src/api/impl/beacon/rewards/index.ts | 5 ++ packages/beacon-node/src/chain/chain.ts | 17 +++++- packages/beacon-node/src/chain/interface.ts | 5 ++ .../src/chain/rewards/syncCommitteeRewards.ts | 57 +++++++++++++++++++ 8 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 packages/beacon-node/src/chain/rewards/syncCommitteeRewards.ts diff --git a/packages/api/src/beacon/routes/beacon/index.ts b/packages/api/src/beacon/routes/beacon/index.ts index 92fcc2093188..af1fcdaecfe0 100644 --- a/packages/api/src/beacon/routes/beacon/index.ts +++ b/packages/api/src/beacon/routes/beacon/index.ts @@ -20,7 +20,7 @@ export * as rewards from "./rewards.js"; export {BroadcastValidation} from "./block.js"; export type {BlockId, BlockHeaderResponse} from "./block.js"; export type {AttestationFilters} from "./pool.js"; -export type {BlockRewards} from "./rewards.js"; +export type {BlockRewards, SyncCommitteeRewards} from "./rewards.js"; // TODO: Review if re-exporting all these types is necessary export type { StateId, diff --git a/packages/api/src/beacon/routes/beacon/rewards.ts b/packages/api/src/beacon/routes/beacon/rewards.ts index 42dced7d5c3f..926cb3033f06 100644 --- a/packages/api/src/beacon/routes/beacon/rewards.ts +++ b/packages/api/src/beacon/routes/beacon/rewards.ts @@ -7,10 +7,12 @@ import { Schema, ReqSerializers, ContainerDataExecutionOptimistic, + ArrayOf, } from "../../../utils/index.js"; import {HttpStatusCode} from "../../../utils/client/httpStatusCode.js"; import {ApiClientResponse} from "../../../interfaces.js"; import {BlockId} from "./block.js"; +import {ValidatorId} from "./state.js"; // See /packages/api/src/routes/index.ts for reasoning and instructions to add new routes @@ -38,6 +40,14 @@ export type BlockRewards = { attesterSlashings: number; }; +/** + * Rewards info for sync committee participation. Every reward value is in Gwei. + * Note: In the case that block proposer is present in `SyncCommitteeRewards`, the reward value only reflects rewards for + * participating in sync committee. Please refer to `BlockRewards.syncAggregate` for rewards of proposer including sync committee + * outputs into their block + */ +export type SyncCommitteeRewards = {validatorIndex: ValidatorIndex; reward: number}[]; + export type Api = { /** * Get block rewards @@ -54,6 +64,24 @@ export type Api = { HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND > >; + + /** + * Get sync committee rewards + * Returns participant reward value for each sync committee member at the given block. + * + * @param blockId Block identifier. + * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \, \. + * @param validatorIds List of validator indices or pubkeys to filter in + */ + getSyncCommitteeRewards( + blockId: BlockId, + validatorIds?: ValidatorId[] + ): Promise< + ApiClientResponse< + {[HttpStatusCode.OK]: {data: SyncCommitteeRewards; executionOptimistic: ExecutionOptimistic}}, + HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND + > + >; }; /** @@ -61,11 +89,13 @@ export type Api = { */ export const routesData: RoutesData = { getBlockRewards: {url: "/eth/v1/beacon/rewards/blocks/{block_id}", method: "GET"}, + getSyncCommitteeRewards: {url: "/eth/v1/beacon/rewards/sync_committee/{block_id}", method: "POST"}, }; export type ReqTypes = { /* eslint-disable @typescript-eslint/naming-convention */ getBlockRewards: {params: {block_id: string}}; + getSyncCommitteeRewards: {params: {block_id: string}; body: ValidatorId[]}; }; export function getReqSerializers(): ReqSerializers { @@ -75,6 +105,14 @@ export function getReqSerializers(): ReqSerializers { parseReq: ({params}) => [params.block_id], schema: {params: {block_id: Schema.StringRequired}}, }, + getSyncCommitteeRewards: { + writeReq: (block_id, validatorIds) => ({params: {block_id: String(block_id)}, body: validatorIds || []}), + parseReq: ({params, body}) => [params.block_id, body], + schema: { + params: {block_id: Schema.StringRequired}, + body: Schema.UintOrStringArray, + }, + }, }; } @@ -91,7 +129,16 @@ export function getReturnTypes(): ReturnTypes { {jsonCase: "eth2"} ); + const SyncCommitteeRewardsResponse = new ContainerType( + { + validatorIndex: ssz.ValidatorIndex, + reward: ssz.UintNum64, + }, + {jsonCase: "eth2"} + ); + return { getBlockRewards: ContainerDataExecutionOptimistic(BlockRewardsResponse), + getSyncCommitteeRewards: ContainerDataExecutionOptimistic(ArrayOf(SyncCommitteeRewardsResponse)), }; } diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index 7256d3f4ed63..f5d9a6c98c92 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -87,7 +87,6 @@ const testDatas = { const ignoredOperations = [ /* missing route */ /* https://github.com/ChainSafe/lodestar/issues/5694 */ - "getSyncCommitteeRewards", "getAttestationsRewards", /* https://github.com/ChainSafe/lodestar/issues/6058 */ "postStateValidators", @@ -126,6 +125,7 @@ const ignoredProperties: Record = { getBlockAttestations: {response: ["finalized"]}, getStateV2: {response: ["finalized"]}, getBlockRewards: {response: ["finalized"]}, + getSyncCommitteeRewards: {response: ["finalized"]}, /* https://github.com/ChainSafe/lodestar/issues/6168 diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index 9c39849906de..1c944a4d6db8 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -12,6 +12,7 @@ import {GenericServerTestCases} from "../../../utils/genericServerTest.js"; const root = new Uint8Array(32).fill(1); const randao = new Uint8Array(32).fill(1); const balance = 32e9; +const reward = 32e9; const pubkeyHex = toHexString(Buffer.alloc(48, 1)); const blockHeaderResponse: BlockHeaderResponse = { @@ -184,6 +185,10 @@ export const testData: GenericServerTestCases = { }, }, }, + getSyncCommitteeRewards: { + args: ["head", ["1300"]], + res: {executionOptimistic: true, data: [{validatorIndex: 1300, reward}]}, + }, // - diff --git a/packages/beacon-node/src/api/impl/beacon/rewards/index.ts b/packages/beacon-node/src/api/impl/beacon/rewards/index.ts index 03a182359d90..780068ebd518 100644 --- a/packages/beacon-node/src/api/impl/beacon/rewards/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/rewards/index.ts @@ -9,5 +9,10 @@ export function getBeaconRewardsApi({chain}: Pick): ServerA const data = await chain.getBlockRewards(block.message); return {data, executionOptimistic}; }, + async getSyncCommitteeRewards(blockId, validatorIds) { + const {block, executionOptimistic} = await resolveBlockId(chain, blockId); + const data = await chain.getSyncCommitteeRewards(block.message, validatorIds); + return {data, executionOptimistic}; + }, }; } diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 20a6ca343565..08743165cd05 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -81,6 +81,7 @@ import {ShufflingCache} from "./shufflingCache.js"; import {StateContextCache} from "./stateCache/stateContextCache.js"; import {SeenGossipBlockInput} from "./seenCache/index.js"; import {CheckpointStateCache} from "./stateCache/stateContextCheckpointsCache.js"; +import {SyncCommitteeRewards, computeSyncCommitteeRewards} from "./rewards/syncCommitteeRewards.js"; /** * Arbitrary constants, blobs and payloads should be consumed immediately in the same slot @@ -995,12 +996,26 @@ export class BeaconChain implements IBeaconChain { async getBlockRewards(block: allForks.FullOrBlindedBeaconBlock): Promise { const preState = this.regen.getPreStateSync(block); - const postState = this.regen.getStateSync(toHexString(block.stateRoot)) ?? undefined; if (preState === null) { throw Error(`Pre-state is unavailable given block's parent root ${toHexString(block.parentRoot)}`); } + const postState = this.regen.getStateSync(toHexString(block.stateRoot)) ?? undefined; + return computeBlockRewards(block, preState.clone(), postState?.clone()); } + + async getSyncCommitteeRewards( + block: allForks.FullOrBlindedBeaconBlock, + validatorIds?: (ValidatorIndex | string)[] + ): Promise { + const preState = this.regen.getPreStateSync(block); + + if (preState === null) { + throw Error(`Pre-state is unavailable given block's parent root ${toHexString(block.parentRoot)}`); + } + + return computeSyncCommitteeRewards(block, preState.clone(), validatorIds); + } } diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index 99c1b7ea0c4a..55f5ebf485a2 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -53,6 +53,7 @@ import {SeenAttestationDatas} from "./seenCache/seenAttestationData.js"; import {SeenGossipBlockInput} from "./seenCache/index.js"; import {ShufflingCache} from "./shufflingCache.js"; import {BlockRewards} from "./rewards/blockRewards.js"; +import {SyncCommitteeRewards} from "./rewards/syncCommitteeRewards.js"; export {BlockType, type AssembledBlockType}; export {type ProposerPreparationData}; @@ -201,6 +202,10 @@ export interface IBeaconChain { blsThreadPoolCanAcceptWork(): boolean; getBlockRewards(blockRef: allForks.FullOrBlindedBeaconBlock): Promise; + getSyncCommitteeRewards( + blockRef: allForks.FullOrBlindedBeaconBlock, + validatorIds?: (ValidatorIndex | string)[] + ): Promise; } export type SSZObjectType = diff --git a/packages/beacon-node/src/chain/rewards/syncCommitteeRewards.ts b/packages/beacon-node/src/chain/rewards/syncCommitteeRewards.ts new file mode 100644 index 000000000000..ba45d03adbab --- /dev/null +++ b/packages/beacon-node/src/chain/rewards/syncCommitteeRewards.ts @@ -0,0 +1,57 @@ +import {CachedBeaconStateAllForks, CachedBeaconStateAltair} from "@lodestar/state-transition"; +import {ValidatorIndex, allForks, altair} from "@lodestar/types"; +import {ForkName, SYNC_COMMITTEE_SIZE} from "@lodestar/params"; +import {routes} from "@lodestar/api"; + +export type SyncCommitteeRewards = routes.beacon.SyncCommitteeRewards; +type BalanceRecord = {val: number}; // Use val for convenient way to increment/decrement balance + +export async function computeSyncCommitteeRewards( + block: allForks.BeaconBlock, + preState: CachedBeaconStateAllForks, + validatorIds?: (ValidatorIndex | string)[] +): Promise { + const fork = preState.config.getForkName(block.slot); + if (fork === ForkName.phase0) { + throw Error("Cannot get sync rewards as phase0 block does not have sync committee"); + } + + const altairBlock = block as altair.BeaconBlock; + const preStateAltair = preState as CachedBeaconStateAltair; + const {index2pubkey} = preStateAltair.epochCtx; + + // Bound committeeIndices in case it goes beyond SYNC_COMMITTEE_SIZE just to be safe + const committeeIndices = preStateAltair.epochCtx.currentSyncCommitteeIndexed.validatorIndices.slice( + 0, + SYNC_COMMITTEE_SIZE + ); + const {syncParticipantReward} = preStateAltair.epochCtx; + const {syncCommitteeBits} = altairBlock.body.syncAggregate; + + // Use balance of each committee as starting point such that we cap the penalty to avoid balance dropping below 0 + const balances: Map = new Map( + committeeIndices.map((i) => [i, {val: preStateAltair.balances.get(i)}]) + ); + + for (const i of committeeIndices) { + const balanceRecord = balances.get(i) as BalanceRecord; + if (syncCommitteeBits.get(i)) { + // Positive rewards for participants + balanceRecord.val += syncParticipantReward; + } else { + // Negative rewards for non participants + balanceRecord.val = Math.max(0, balanceRecord.val - syncParticipantReward); + } + } + + const rewards = Array.from(balances, ([validatorIndex, v]) => ({validatorIndex, reward: v.val})); + + if (validatorIds !== undefined) { + const filtersSet = new Set(validatorIds); + return rewards.filter( + (reward) => filtersSet.has(reward.validatorIndex) || filtersSet.has(index2pubkey[reward.validatorIndex].toHex()) + ); + } else { + return rewards; + } +} From d66f607108e483c4b4519496ed53dfde8d751217 Mon Sep 17 00:00:00 2001 From: g11tech Date: Tue, 5 Mar 2024 19:52:59 +0530 Subject: [PATCH 27/34] feat: aggressively pull blobs as soon as we see the block (#6499) * feat: aggressively pull blobs as soon as we see the block * fixes * some logging improvement * improve log * fix tsc * improve comments * apply feedback * refactor to a separate event and handler for unknownblock input * fix e2e * reduce diff * add metric to track availability resolution * fix circular include issue --- .../beacon-node/src/chain/blocks/types.ts | 27 ++- .../blocks/verifyBlocksDataAvailability.ts | 11 +- .../chain/seenCache/seenGossipBlockInput.ts | 30 ++-- .../src/metrics/metrics/lodestar.ts | 8 +- packages/beacon-node/src/network/events.ts | 3 + .../src/network/processor/gossipHandlers.ts | 35 ++-- .../reqresp/beaconBlocksMaybeBlobsByRoot.ts | 53 +++++- packages/beacon-node/src/sync/interface.ts | 17 +- packages/beacon-node/src/sync/unknownBlock.ts | 164 ++++++++++++++++-- .../src/sync/utils/pendingBlocksTree.ts | 12 +- .../onWorker/dataSerialization.test.ts | 26 ++- .../seenCache/seenGossipBlockInput.test.ts | 28 +-- 12 files changed, 341 insertions(+), 73 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index 2fd16fa64705..63fe21df105a 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -29,7 +29,12 @@ export type BlockInputBlobs = {blobs: deneb.BlobSidecars; blobsBytes: (Uint8Arra export type BlockInput = {block: allForks.SignedBeaconBlock; source: BlockSource; blockBytes: Uint8Array | null} & ( | {type: BlockInputType.preDeneb} | ({type: BlockInputType.postDeneb} & BlockInputBlobs) - | {type: BlockInputType.blobsPromise; blobsCache: BlobsCache; availabilityPromise: Promise} + | { + type: BlockInputType.blobsPromise; + blobsCache: BlobsCache; + availabilityPromise: Promise; + resolveAvailability: (blobs: BlockInputBlobs) => void; + } ); export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean { @@ -85,7 +90,8 @@ export const getBlockInput = { source: BlockSource, blobsCache: BlobsCache, blockBytes: Uint8Array | null, - availabilityPromise: Promise + availabilityPromise: Promise, + resolveAvailability: (blobs: BlockInputBlobs) => void ): BlockInput { if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) { throw Error(`Pre Deneb block slot ${block.message.slot}`); @@ -97,10 +103,27 @@ export const getBlockInput = { blobsCache, blockBytes, availabilityPromise, + resolveAvailability, }; }, }; +export function getBlockInputBlobs(blobsCache: BlobsCache): BlockInputBlobs { + const blobs = []; + const blobsBytes = []; + + for (let index = 0; index < blobsCache.size; index++) { + const blobCache = blobsCache.get(index); + if (blobCache === undefined) { + throw Error(`Missing blobSidecar at index=${index}`); + } + const {blobSidecar, blobBytes} = blobCache; + blobs.push(blobSidecar); + blobsBytes.push(blobBytes); + } + return {blobs, blobsBytes}; +} + export enum AttestationImportOpt { Skip, Force, diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts index 9c45469d56dd..de7a9575ce06 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksDataAvailability.ts @@ -7,9 +7,9 @@ import {validateBlobSidecars} from "../validation/blobSidecar.js"; import {Metrics} from "../../metrics/metrics.js"; import {BlockInput, BlockInputType, ImportBlockOpts, BlobSidecarValidation} from "./types.js"; -// proposer boost is not available post 3 sec so try pulling using unknown block hash -// post 3 sec after throwing the availability error -const BLOB_AVAILABILITY_TIMEOUT = 3_000; +// we can now wait for full 12 seconds because unavailable block sync will try pulling +// the blobs from the network anyway after 500ms of seeing the block +const BLOB_AVAILABILITY_TIMEOUT = 12_000; /** * Verifies some early cheap sanity checks on the block before running the full state transition. @@ -59,7 +59,7 @@ export async function verifyBlocksDataAvailability( } async function maybeValidateBlobs( - chain: {config: ChainForkConfig; genesisTime: UintNum64}, + chain: {config: ChainForkConfig; genesisTime: UintNum64; logger: Logger}, blockInput: BlockInput, opts: ImportBlockOpts ): Promise { @@ -102,7 +102,7 @@ async function maybeValidateBlobs( * which may try unknownblock/blobs fill (by root). */ async function raceWithCutoff( - chain: {config: ChainForkConfig; genesisTime: UintNum64}, + chain: {config: ChainForkConfig; genesisTime: UintNum64; logger: Logger}, blockInput: BlockInput, availabilityPromise: Promise ): Promise { @@ -114,6 +114,7 @@ async function raceWithCutoff( 0 ); const cutoffTimeout = new Promise((_resolve, reject) => setTimeout(reject, cutoffTime)); + chain.logger.debug("Racing for blob availabilityPromise", {blockSlot, cutoffTime}); try { await Promise.race([availabilityPromise, cutoffTimeout]); diff --git a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts index 1f23503d3957..1f7e992ebada 100644 --- a/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts +++ b/packages/beacon-node/src/chain/seenCache/seenGossipBlockInput.ts @@ -11,7 +11,14 @@ import { BlockInputBlobs, BlobsCache, GossipedInputType, + getBlockInputBlobs, } from "../blocks/types.js"; +import {Metrics} from "../../metrics/index.js"; + +export enum BlockInputAvailabilitySource { + GOSSIP = "gossip", + UNKNOWN_SYNC = "unknown_sync", +} type GossipedBlockInput = | {type: GossipedInputType.block; signedBlock: allForks.SignedBeaconBlock; blockBytes: Uint8Array | null} @@ -52,7 +59,8 @@ export class SeenGossipBlockInput { getGossipBlockInput( config: ChainForkConfig, - gossipedInput: GossipedBlockInput + gossipedInput: GossipedBlockInput, + metrics: Metrics | null ): | { blockInput: BlockInput; @@ -113,6 +121,7 @@ export class SeenGossipBlockInput { if (blobKzgCommitments.length === blobsCache.size) { const allBlobs = getBlockInputBlobs(blobsCache); resolveAvailability(allBlobs); + metrics?.syncUnknownBlock.resolveAvailabilitySource.inc({source: BlockInputAvailabilitySource.GOSSIP}); const {blobs, blobsBytes} = allBlobs; return { blockInput: getBlockInput.postDeneb( @@ -133,7 +142,8 @@ export class SeenGossipBlockInput { BlockSource.gossip, blobsCache, blockBytes ?? null, - availabilityPromise + availabilityPromise, + resolveAvailability ), blockInputMeta: { pending: GossipedInputType.blob, @@ -165,19 +175,3 @@ function getEmptyBlockInputCacheEntry(): BlockInputCacheType { const blobsCache = new Map(); return {availabilityPromise, resolveAvailability, blobsCache}; } - -function getBlockInputBlobs(blobsCache: BlobsCache): BlockInputBlobs { - const blobs = []; - const blobsBytes = []; - - for (let index = 0; index < blobsCache.size; index++) { - const blobCache = blobsCache.get(index); - if (blobCache === undefined) { - throw Error(`Missing blobSidecar at index=${index}`); - } - const {blobSidecar, blobBytes} = blobCache; - blobs.push(blobSidecar); - blobsBytes.push(blobBytes); - } - return {blobs, blobsBytes}; -} diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 957a962de103..4f82969cff81 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -7,11 +7,12 @@ import {InsertOutcome} from "../../chain/opPools/types.js"; import {RegenCaller, RegenFnName} from "../../chain/regen/interface.js"; import {ReprocessStatus} from "../../chain/reprocess.js"; import {RejectReason} from "../../chain/seenCache/seenAttestationData.js"; +import {BlockInputAvailabilitySource} from "../../chain/seenCache/seenGossipBlockInput.js"; import {ExecutionPayloadStatus} from "../../execution/index.js"; import {GossipType} from "../../network/index.js"; import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js"; import {BackfillSyncMethod} from "../../sync/backfill/backfill.js"; -import {PendingBlockType} from "../../sync/interface.js"; +import {PendingBlockType} from "../../sync/index.js"; import {PeerSyncType, RangeSyncType} from "../../sync/utils/remoteSyncType.js"; import {LodestarMetadata} from "../options.js"; import {RegistryMetricCreator} from "../utils/registryMetricCreator.js"; @@ -592,6 +593,11 @@ export function createLodestarMetrics( help: "Time elapsed between block slot time and the time block received via unknown block sync", buckets: [0.5, 1, 2, 4, 6, 12], }), + resolveAvailabilitySource: register.gauge<{source: BlockInputAvailabilitySource}>({ + name: "lodestar_sync_blockinput_availability_source", + help: "Total number of blocks whose data availability was resolved", + labelNames: ["source"], + }), }, // Gossip sync committee diff --git a/packages/beacon-node/src/network/events.ts b/packages/beacon-node/src/network/events.ts index 67ea0a1dd0e0..65c5d56fb808 100644 --- a/packages/beacon-node/src/network/events.ts +++ b/packages/beacon-node/src/network/events.ts @@ -17,6 +17,7 @@ export enum NetworkEvent { // TODO remove this event, this is not a network-level concern, rather a chain / sync concern unknownBlockParent = "unknownBlockParent", unknownBlock = "unknownBlock", + unknownBlockInput = "unknownBlockInput", // Network processor events /** (Network -> App) A gossip message is ready for validation */ @@ -31,6 +32,7 @@ export type NetworkEventData = { [NetworkEvent.reqRespRequest]: {request: RequestTypedContainer; peer: PeerId}; [NetworkEvent.unknownBlockParent]: {blockInput: BlockInput; peer: PeerIdStr}; [NetworkEvent.unknownBlock]: {rootHex: RootHex; peer?: PeerIdStr}; + [NetworkEvent.unknownBlockInput]: {blockInput: BlockInput; peer?: PeerIdStr}; [NetworkEvent.pendingGossipsubMessage]: PendingGossipsubMessage; [NetworkEvent.gossipMessageValidationResult]: { msgId: string; @@ -45,6 +47,7 @@ export const networkEventDirection: Record = { [NetworkEvent.reqRespRequest]: EventDirection.none, // Only used internally in NetworkCore [NetworkEvent.unknownBlockParent]: EventDirection.workerToMain, [NetworkEvent.unknownBlock]: EventDirection.workerToMain, + [NetworkEvent.unknownBlockInput]: EventDirection.workerToMain, [NetworkEvent.pendingGossipsubMessage]: EventDirection.workerToMain, [NetworkEvent.gossipMessageValidationResult]: EventDirection.mainToWorker, }; diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 60186f8fb79f..d9efdd2b09a6 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -45,7 +45,7 @@ import {PeerAction} from "../peers/index.js"; import {validateLightClientFinalityUpdate} from "../../chain/validation/lightClientFinalityUpdate.js"; import {validateLightClientOptimisticUpdate} from "../../chain/validation/lightClientOptimisticUpdate.js"; import {validateGossipBlobSidecar} from "../../chain/validation/blobSidecar.js"; -import {BlockInput, GossipedInputType, BlobSidecarValidation} from "../../chain/blocks/types.js"; +import {BlockInput, GossipedInputType, BlobSidecarValidation, BlockInputType} from "../../chain/blocks/types.js"; import {sszDeserialize} from "../gossip/topic.js"; import {INetworkCore} from "../core/index.js"; import {INetwork} from "../interface.js"; @@ -118,11 +118,15 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler const recvToValLatency = Date.now() / 1000 - seenTimestampSec; // always set block to seen cache for all forks so that we don't need to download it - const blockInputRes = chain.seenGossipBlockInput.getGossipBlockInput(config, { - type: GossipedInputType.block, - signedBlock, - blockBytes, - }); + const blockInputRes = chain.seenGossipBlockInput.getGossipBlockInput( + config, + { + type: GossipedInputType.block, + signedBlock, + blockBytes, + }, + metrics + ); const blockInput = blockInputRes.blockInput; // blockInput can't be returned null, improve by enforcing via return types if (blockInput === null) { @@ -187,11 +191,15 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); const recvToValLatency = Date.now() / 1000 - seenTimestampSec; - const {blockInput, blockInputMeta} = chain.seenGossipBlockInput.getGossipBlockInput(config, { - type: GossipedInputType.blob, - blobSidecar, - blobBytes, - }); + const {blockInput, blockInputMeta} = chain.seenGossipBlockInput.getGossipBlockInput( + config, + { + type: GossipedInputType.blob, + blobSidecar, + blobBytes, + }, + metrics + ); try { await validateGossipBlobSidecar(chain, blobSidecar, gossipIndex); @@ -242,6 +250,10 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler // Handler - MUST NOT `await`, to allow validation result to be propagated metrics?.registerBeaconBlock(OpSource.gossip, seenTimestampSec, signedBlock.message); + // if blobs are not yet fully available start an aggressive blob pull + if (blockInput.type === BlockInputType.blobsPromise) { + events.emit(NetworkEvent.unknownBlockInput, {blockInput: blockInput, peer: peerIdStr}); + } chain .processBlock(blockInput, { @@ -276,7 +288,6 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler if (e instanceof BlockError) { switch (e.type.code) { case BlockErrorCode.DATA_UNAVAILABLE: { - // TODO: create a newevent unknownBlobs and only pull blobs const slot = signedBlock.message.slot; const forkTypes = config.getForkTypes(slot); const rootHex = toHexString(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message)); diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts index c85464a05b61..9562588d56db 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts @@ -1,9 +1,11 @@ import {ChainForkConfig} from "@lodestar/config"; import {phase0, deneb} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; -import {BlockInput, BlockSource} from "../../chain/blocks/types.js"; +import {BlockInput, BlockInputType, BlockSource, getBlockInputBlobs, getBlockInput} from "../../chain/blocks/types.js"; import {PeerIdStr} from "../../util/peerId.js"; import {INetwork} from "../interface.js"; +import {BlockInputAvailabilitySource} from "../../chain/seenCache/seenGossipBlockInput.js"; +import {Metrics} from "../../metrics/index.js"; import {matchBlockWithBlobs} from "./beaconBlocksMaybeBlobsByRange.js"; export async function beaconBlocksMaybeBlobsByRoot( @@ -39,3 +41,52 @@ export async function beaconBlocksMaybeBlobsByRoot( // and here it should be infinity since all bobs should match return matchBlockWithBlobs(config, allBlocks, allBlobSidecars, Infinity, BlockSource.byRoot); } + +export async function unavailableBeaconBlobsByRoot( + config: ChainForkConfig, + network: INetwork, + peerId: PeerIdStr, + unavailableBlockInput: BlockInput, + metrics: Metrics | null +): Promise { + if (unavailableBlockInput.type !== BlockInputType.blobsPromise) { + return unavailableBlockInput; + } + + const blobIdentifiers: deneb.BlobIdentifier[] = []; + const {block, blobsCache, resolveAvailability, blockBytes} = unavailableBlockInput; + + const slot = block.message.slot; + const blockRoot = config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block.message); + + const blobKzgCommitmentsLen = (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; + for (let index = 0; index < blobKzgCommitmentsLen; index++) { + if (blobsCache.has(index) === false) blobIdentifiers.push({blockRoot, index}); + } + + let allBlobSidecars: deneb.BlobSidecar[]; + if (blobIdentifiers.length > 0) { + allBlobSidecars = await network.sendBlobSidecarsByRoot(peerId, blobIdentifiers); + } else { + allBlobSidecars = []; + } + + // add them in cache so that its reflected in all the blockInputs that carry this + // for e.g. a blockInput that might be awaiting blobs promise fullfillment in + // verifyBlocksDataAvailability + for (const blobSidecar of allBlobSidecars) { + blobsCache.set(blobSidecar.index, {blobSidecar, blobBytes: null}); + } + + // check and see if all blobs are now available and in that case resolve availability + // if not this will error and the leftover blobs will be tried from another peer + const allBlobs = getBlockInputBlobs(blobsCache); + const {blobs, blobsBytes} = allBlobs; + if (blobs.length !== blobKzgCommitmentsLen) { + throw Error(`Not all blobs fetched missingBlobs=${blobKzgCommitmentsLen - blobs.length}`); + } + + resolveAvailability(allBlobs); + metrics?.syncUnknownBlock.resolveAvailabilitySource.inc({source: BlockInputAvailabilitySource.UNKNOWN_SYNC}); + return getBlockInput.postDeneb(config, block, BlockSource.byRoot, blobs, blockBytes, blobsBytes); +} diff --git a/packages/beacon-node/src/sync/interface.ts b/packages/beacon-node/src/sync/interface.ts index 4dd2fd96e21a..1c1ae1ceedf7 100644 --- a/packages/beacon-node/src/sync/interface.ts +++ b/packages/beacon-node/src/sync/interface.ts @@ -2,7 +2,7 @@ import {Logger} from "@lodestar/utils"; import {RootHex, Slot, phase0} from "@lodestar/types"; import {BeaconConfig} from "@lodestar/config"; import {routes} from "@lodestar/api"; -import {BlockInput} from "../chain/blocks/types.js"; +import {BlockInput, BlockInputType} from "../chain/blocks/types.js"; import {INetwork} from "../network/index.js"; import {IBeaconChain} from "../chain/index.js"; import {Metrics} from "../metrics/index.js"; @@ -56,7 +56,7 @@ export interface SyncModules { } export type UnknownAndAncestorBlocks = { - unknowns: UnknownBlock[]; + unknowns: (UnknownBlock | UnknownBlockInput)[]; ancestors: DownloadedBlock[]; }; @@ -66,7 +66,7 @@ export type UnknownAndAncestorBlocks = { * - store 1 record with known parentBlockRootHex & blockInput, blockRootHex as key, status downloaded * - store 1 record with undefined parentBlockRootHex & blockInput, parentBlockRootHex as key, status pending */ -export type PendingBlock = UnknownBlock | DownloadedBlock; +export type PendingBlock = UnknownBlock | UnknownBlockInput | DownloadedBlock; type PendingBlockCommon = { blockRootHex: RootHex; @@ -80,6 +80,15 @@ export type UnknownBlock = PendingBlockCommon & { blockInput: null; }; +/** + * either the blobs are unknown or in future some blobs and even the block is unknown + */ +export type UnknownBlockInput = PendingBlockCommon & { + status: PendingBlockStatus.pending | PendingBlockStatus.fetching; + parentBlockRootHex: null; + blockInput: BlockInput & {type: BlockInputType.blobsPromise}; +}; + export type DownloadedBlock = PendingBlockCommon & { status: PendingBlockStatus.downloaded | PendingBlockStatus.processing; parentBlockRootHex: RootHex; @@ -102,4 +111,6 @@ export enum PendingBlockType { * During gossip time, we may get a block but the parent root is unknown (not in forkchoice). */ UNKNOWN_PARENT = "unknown_parent", + + UNKNOWN_BLOCKINPUT = "unknown_blockinput", } diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index cefe2617900a..15a145eb5f84 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -1,18 +1,21 @@ import {fromHexString, toHexString} from "@chainsafe/ssz"; import {ChainForkConfig} from "@lodestar/config"; import {Logger, pruneSetToMax} from "@lodestar/utils"; -import {Root, RootHex} from "@lodestar/types"; +import {Root, RootHex, deneb} from "@lodestar/types"; import {INTERVALS_PER_SLOT} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; import {INetwork, NetworkEvent, NetworkEventData, PeerAction} from "../network/index.js"; import {PeerIdStr} from "../util/peerId.js"; import {IBeaconChain} from "../chain/index.js"; -import {BlockInput} from "../chain/blocks/types.js"; +import {BlockInput, BlockInputType} from "../chain/blocks/types.js"; import {Metrics} from "../metrics/index.js"; import {shuffle} from "../util/shuffle.js"; import {byteArrayEquals} from "../util/bytes.js"; import {BlockError, BlockErrorCode} from "../chain/errors/index.js"; -import {beaconBlocksMaybeBlobsByRoot} from "../network/reqresp/beaconBlocksMaybeBlobsByRoot.js"; +import { + beaconBlocksMaybeBlobsByRoot, + unavailableBeaconBlobsByRoot, +} from "../network/reqresp/beaconBlocksMaybeBlobsByRoot.js"; import {wrapError} from "../util/wrapError.js"; import {PendingBlock, PendingBlockStatus, PendingBlockType} from "./interface.js"; import {getDescendantBlocks, getAllDescendantBlocks, getUnknownAndAncestorBlocks} from "./utils/pendingBlocksTree.js"; @@ -59,6 +62,7 @@ export class UnknownBlockSync { if (!this.subscribedToNetworkEvents) { this.logger.verbose("UnknownBlockSync enabled."); this.network.events.on(NetworkEvent.unknownBlock, this.onUnknownBlock); + this.network.events.on(NetworkEvent.unknownBlockInput, this.onUnknownBlockInput); this.network.events.on(NetworkEvent.unknownBlockParent, this.onUnknownParent); this.network.events.on(NetworkEvent.peerConnected, this.triggerUnknownBlockSearch); this.subscribedToNetworkEvents = true; @@ -71,6 +75,7 @@ export class UnknownBlockSync { unsubscribeFromNetwork(): void { this.logger.verbose("UnknownBlockSync disabled."); this.network.events.off(NetworkEvent.unknownBlock, this.onUnknownBlock); + this.network.events.off(NetworkEvent.unknownBlockInput, this.onUnknownBlockInput); this.network.events.off(NetworkEvent.unknownBlockParent, this.onUnknownParent); this.network.events.off(NetworkEvent.peerConnected, this.triggerUnknownBlockSearch); this.subscribedToNetworkEvents = false; @@ -98,6 +103,19 @@ export class UnknownBlockSync { } }; + /** + * Process an unknownBlockInput event and register the block in `pendingBlocks` Map. + */ + private onUnknownBlockInput = (data: NetworkEventData[NetworkEvent.unknownBlockInput]): void => { + try { + this.addUnknownBlock(data.blockInput, data.peer); + this.triggerUnknownBlockSearch(); + this.metrics?.syncUnknownBlock.requests.inc({type: PendingBlockType.UNKNOWN_BLOCKINPUT}); + } catch (e) { + this.logger.debug("Error handling unknownBlockInput event", {}, e as Error); + } + }; + /** * Process an unknownBlockParent event and register the block in `pendingBlocks` Map. */ @@ -147,19 +165,42 @@ export class UnknownBlockSync { this.addUnknownBlock(parentBlockRootHex, peerIdStr); } - private addUnknownBlock(blockRootHex: RootHex, peerIdStr?: string): void { + private addUnknownBlock(blockInputOrRootHex: RootHex | BlockInput, peerIdStr?: string): void { + let blockRootHex; + let blockInput: BlockInput | null; + + if (typeof blockInputOrRootHex === "string") { + blockRootHex = blockInputOrRootHex; + blockInput = null; + } else { + const {block} = blockInputOrRootHex; + blockRootHex = toHexString(this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message)); + blockInput = blockInputOrRootHex; + } + let pendingBlock = this.pendingBlocks.get(blockRootHex); if (!pendingBlock) { pendingBlock = { blockRootHex, parentBlockRootHex: null, - blockInput: null, + blockInput: blockInput?.type === BlockInputType.blobsPromise ? blockInput : null, peerIdStrs: new Set(), status: PendingBlockStatus.pending, downloadAttempts: 0, - }; + } as PendingBlock; this.pendingBlocks.set(blockRootHex, pendingBlock); - this.logger.verbose("Added unknown block to pendingBlocks", {root: blockRootHex}); + + if (pendingBlock.blockInput?.type === BlockInputType.blobsPromise) { + this.logger.verbose("Added blockInput with unknown blobs to pendingBlocks", { + root: blockRootHex, + slot: blockInput?.block.message.slot ?? "unknown", + }); + } else { + this.logger.verbose("Added unknown block to pendingBlocks", { + root: blockRootHex, + slot: blockInput?.block.message.slot ?? "unknown", + }); + } } if (peerIdStr) { @@ -226,13 +267,29 @@ export class UnknownBlockSync { return; } - this.logger.verbose("Downloading unknown block", { - root: block.blockRootHex, - pendingBlocks: this.pendingBlocks.size, - }); + if (block.blockInput?.type === BlockInputType.blobsPromise) { + this.logger.verbose("Downloading unknown blockInput", { + root: block.blockRootHex, + pendingBlocks: this.pendingBlocks.size, + blockInputType: block.blockInput.type, + slot: block.blockInput?.block.message.slot ?? "unknown", + }); + } else { + this.logger.verbose("Downloading unknown block", { + root: block.blockRootHex, + pendingBlocks: this.pendingBlocks.size, + slot: block.blockInput?.block.message.slot ?? "unknown", + }); + } block.status = PendingBlockStatus.fetching; - const res = await wrapError(this.fetchUnknownBlockRoot(fromHexString(block.blockRootHex), connectedPeers)); + + let res; + if (block.blockInput === null) { + res = await wrapError(this.fetchUnknownBlockRoot(fromHexString(block.blockRootHex), connectedPeers)); + } else { + res = await wrapError(this.fetchUnavailableBlockInput(block.blockInput, connectedPeers)); + } if (res.err) this.metrics?.syncUnknownBlock.downloadedBlocksError.inc(); else this.metrics?.syncUnknownBlock.downloadedBlocksSuccess.inc(); @@ -252,11 +309,22 @@ export class UnknownBlockSync { this.metrics?.syncUnknownBlock.elapsedTimeTillReceived.observe(delaySec); const parentInForkchoice = this.chain.forkChoice.hasBlock(blockInput.block.message.parentRoot); - this.logger.verbose("Downloaded unknown block", { - root: block.blockRootHex, - pendingBlocks: this.pendingBlocks.size, - parentInForkchoice, - }); + + if (block.blockInput.type === BlockInputType.blobsPromise) { + this.logger.verbose("Downloaded unknown blobs", { + root: block.blockRootHex, + pendingBlocks: this.pendingBlocks.size, + parentInForkchoice, + blockInputType: blockInput.type, + }); + } else { + this.logger.verbose("Downloaded unknown block", { + root: block.blockRootHex, + pendingBlocks: this.pendingBlocks.size, + parentInForkchoice, + blockInputType: blockInput.type, + }); + } if (parentInForkchoice) { // Bingo! Process block. Add to pending blocks anyway for recycle the cache that prevents duplicate processing @@ -437,6 +505,68 @@ export class UnknownBlockSync { } } + /** + * Fetches missing blobs for the blockinput, in future can also pull block is thats also missing + * along with the blobs (i.e. only some blobs are available) + */ + private async fetchUnavailableBlockInput( + unavailableBlockInput: BlockInput, + connectedPeers: PeerIdStr[] + ): Promise<{blockInput: BlockInput; peerIdStr: string}> { + if (unavailableBlockInput.type !== BlockInputType.blobsPromise) { + return {blockInput: unavailableBlockInput, peerIdStr: ""}; + } + + const shuffledPeers = shuffle(connectedPeers); + const unavailableBlock = unavailableBlockInput.block; + const blockRoot = this.config + .getForkTypes(unavailableBlock.message.slot) + .BeaconBlock.hashTreeRoot(unavailableBlock.message); + const blockRootHex = toHexString(blockRoot); + + const blobKzgCommitmentsLen = (unavailableBlock.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; + const pendingBlobs = blobKzgCommitmentsLen - unavailableBlockInput.blobsCache.size; + + let lastError: Error | null = null; + for (let i = 0; i < MAX_ATTEMPTS_PER_BLOCK; i++) { + const peer = shuffledPeers[i % shuffledPeers.length]; + try { + const blockInput = await unavailableBeaconBlobsByRoot( + this.config, + this.network, + peer, + unavailableBlockInput, + this.metrics + ); + + // Peer does not have the block, try with next peer + if (blockInput === undefined) { + continue; + } + + // Verify block root is correct + const block = blockInput.block.message; + const receivedBlockRoot = this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block); + if (!byteArrayEquals(receivedBlockRoot, blockRoot)) { + throw Error(`Wrong block received by peer, got ${toHexString(receivedBlockRoot)} expected ${blockRootHex}`); + } + this.logger.debug("Fetched UnavailableBlockInput", {attempts: i, pendingBlobs, blobKzgCommitmentsLen}); + + return {blockInput, peerIdStr: peer}; + } catch (e) { + this.logger.debug("Error fetching UnavailableBlockInput", {attempt: i, blockRootHex, peer}, e as Error); + lastError = e as Error; + } + } + + if (lastError) { + lastError.message = `Error fetching UnavailableBlockInput after ${MAX_ATTEMPTS_PER_BLOCK} attempts: ${lastError.message}`; + throw lastError; + } else { + throw Error(`Error fetching UnavailableBlockInput after ${MAX_ATTEMPTS_PER_BLOCK}: unknown error`); + } + } + /** * Gets all descendant blocks of `block` recursively from `pendingBlocks`. * Assumes that if a parent block does not exist or is not processable, all descendant blocks are bad too. diff --git a/packages/beacon-node/src/sync/utils/pendingBlocksTree.ts b/packages/beacon-node/src/sync/utils/pendingBlocksTree.ts index cd3f606dff53..ca93fd0181af 100644 --- a/packages/beacon-node/src/sync/utils/pendingBlocksTree.ts +++ b/packages/beacon-node/src/sync/utils/pendingBlocksTree.ts @@ -5,8 +5,10 @@ import { PendingBlock, PendingBlockStatus, UnknownAndAncestorBlocks, + UnknownBlockInput, UnknownBlock, } from "../interface.js"; +import {BlockInputType} from "../../chain/blocks/types.js"; export function getAllDescendantBlocks(blockRootHex: RootHex, blocks: Map): PendingBlock[] { // Do one pass over all blocks to index by parent @@ -57,16 +59,20 @@ export function getDescendantBlocks(blockRootHex: RootHex, blocks: Map): UnknownAndAncestorBlocks { - const unknowns: UnknownBlock[] = []; + const unknowns: (UnknownBlock | UnknownBlockInput)[] = []; const ancestors: DownloadedBlock[] = []; for (const block of blocks.values()) { const parentHex = block.parentBlockRootHex; - if (block.status === PendingBlockStatus.pending && block.blockInput == null && parentHex == null) { + if ( + block.status === PendingBlockStatus.pending && + (block.blockInput == null || block.blockInput.type === BlockInputType.blobsPromise) && + parentHex == null + ) { unknowns.push(block); } - if (parentHex && !blocks.has(parentHex)) { + if (block.status === PendingBlockStatus.downloaded && parentHex && !blocks.has(parentHex)) { ancestors.push(block); } } diff --git a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts index a49c6923a9ea..976df5d2ae4d 100644 --- a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts +++ b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts @@ -15,7 +15,7 @@ import { ReqRespMethod, networkEventDirection, } from "../../../../src/network/index.js"; -import {BlockInputType, BlockSource} from "../../../../src/chain/blocks/types.js"; +import {BlockInputType, BlockSource, BlockInputBlobs, BlockInput} from "../../../../src/chain/blocks/types.js"; import {ZERO_HASH, ZERO_HASH_HEX} from "../../../../src/constants/constants.js"; import {IteratorEventType} from "../../../../src/util/asyncIterableToEvents.js"; import {NetworkWorkerApi} from "../../../../src/network/core/index.js"; @@ -92,6 +92,10 @@ describe.skip("data serialization through worker boundary", function () { rootHex: ZERO_HASH_HEX, peer, }, + [NetworkEvent.unknownBlockInput]: { + blockInput: getEmptyBlockInput(), + peer, + }, [NetworkEvent.pendingGossipsubMessage]: { topic: {type: GossipType.beacon_block, fork: ForkName.altair}, msg: { @@ -240,3 +244,23 @@ describe.skip("data serialization through worker boundary", function () { }); type Resolves> = T extends Promise ? (U extends void ? null : U) : never; + +function getEmptyBlockInput(): BlockInput { + let resolveAvailability: ((blobs: BlockInputBlobs) => void) | null = null; + const availabilityPromise = new Promise((resolveCB) => { + resolveAvailability = resolveCB; + }); + if (resolveAvailability === null) { + throw Error("Promise Constructor was not executed immediately"); + } + const blobsCache = new Map(); + return { + type: BlockInputType.blobsPromise, + block: ssz.deneb.SignedBeaconBlock.defaultValue(), + source: BlockSource.gossip, + blockBytes: ZERO_HASH, + blobsCache, + availabilityPromise, + resolveAvailability, + }; +} diff --git a/packages/beacon-node/test/unit/chain/seenCache/seenGossipBlockInput.test.ts b/packages/beacon-node/test/unit/chain/seenCache/seenGossipBlockInput.test.ts index 16af7b0df10e..ac7adb546663 100644 --- a/packages/beacon-node/test/unit/chain/seenCache/seenGossipBlockInput.test.ts +++ b/packages/beacon-node/test/unit/chain/seenCache/seenGossipBlockInput.test.ts @@ -105,11 +105,15 @@ describe("SeenGossipBlockInput", () => { try { if (eventType === GossipedInputType.block) { - const blockInputRes = seenGossipBlockInput.getGossipBlockInput(config, { - type: GossipedInputType.block, - signedBlock, - blockBytes: null, - }); + const blockInputRes = seenGossipBlockInput.getGossipBlockInput( + config, + { + type: GossipedInputType.block, + signedBlock, + blockBytes: null, + }, + null + ); if (expectedResponseType instanceof Error) { expect.fail(`expected to fail with error: ${expectedResponseType.message}`); @@ -123,11 +127,15 @@ describe("SeenGossipBlockInput", () => { const blobSidecar = blobSidecars[index]; expect(blobSidecar).not.toBeUndefined(); - const blobInputRes = seenGossipBlockInput.getGossipBlockInput(config, { - type: GossipedInputType.blob, - blobSidecar, - blobBytes: null, - }); + const blobInputRes = seenGossipBlockInput.getGossipBlockInput( + config, + { + type: GossipedInputType.blob, + blobSidecar, + blobBytes: null, + }, + null + ); if (expectedResponseType instanceof Error) { expect.fail(`expected to fail with error: ${expectedResponseType.message}`); From 918924eabad9ca615acf3d92ff50828fb40fedea Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Tue, 5 Mar 2024 21:54:24 +0700 Subject: [PATCH 28/34] feat: new getCheckpointStateOrBytes() api and clone states from cache (#6504) * feat: new getCheckpointStateOrBytes() api and clone states from cache * fix: clone states without transferring cache * fix: persistentCheckpointsCache.test.ts unit test * fix: remove .only in persistentCheckpointsCache.test.ts --- .../src/chain/archiver/archiveStates.ts | 26 ++++++++++++++----- .../src/chain/blocks/importBlock.ts | 2 +- .../beacon-node/src/chain/regen/interface.ts | 1 + .../beacon-node/src/chain/regen/queued.ts | 16 ++++++++++-- packages/beacon-node/src/chain/regen/regen.ts | 2 +- .../chain/stateCache/fifoBlockStateCache.ts | 2 +- .../stateCache/persistentCheckpointsCache.ts | 18 ++++++------- .../src/chain/stateCache/stateContextCache.ts | 2 +- .../stateContextCheckpointsCache.ts | 2 +- packages/beacon-node/src/util/multifork.ts | 8 ++++-- .../persistentCheckpointsCache.test.ts | 16 +++++++++--- 11 files changed, 67 insertions(+), 28 deletions(-) diff --git a/packages/beacon-node/src/chain/archiver/archiveStates.ts b/packages/beacon-node/src/chain/archiver/archiveStates.ts index e3ff48b02355..2231cd3ff513 100644 --- a/packages/beacon-node/src/chain/archiver/archiveStates.ts +++ b/packages/beacon-node/src/chain/archiver/archiveStates.ts @@ -5,6 +5,7 @@ import {computeEpochAtSlot, computeStartSlotAtEpoch} from "@lodestar/state-trans import {CheckpointWithHex} from "@lodestar/fork-choice"; import {IBeaconDb} from "../../db/index.js"; import {IStateRegenerator} from "../regen/interface.js"; +import {getStateSlotFromBytes} from "../../util/multifork.js"; /** * Minimum number of epochs between single temp archived states @@ -83,13 +84,26 @@ export class StatesArchiver { * Only the new finalized state is stored to disk */ async archiveState(finalized: CheckpointWithHex): Promise { - const finalizedState = this.regen.getCheckpointStateSync(finalized); - if (!finalizedState) { - throw Error("No state in cache for finalized checkpoint state epoch #" + finalized.epoch); + // starting from Mar 2024, the finalized state could be from disk or in memory + const finalizedStateOrBytes = await this.regen.getCheckpointStateOrBytes(finalized); + const {rootHex} = finalized; + if (!finalizedStateOrBytes) { + throw Error(`No state in cache for finalized checkpoint state epoch #${finalized.epoch} root ${rootHex}`); + } + if (finalizedStateOrBytes instanceof Uint8Array) { + const slot = getStateSlotFromBytes(finalizedStateOrBytes); + await this.db.stateArchive.putBinary(slot, finalizedStateOrBytes); + this.logger.verbose("Archived finalized state bytes", {epoch: finalized.epoch, slot, root: rootHex}); + } else { + // state + await this.db.stateArchive.put(finalizedStateOrBytes.slot, finalizedStateOrBytes); + // don't delete states before the finalized state, auto-prune will take care of it + this.logger.verbose("Archived finalized state", { + epoch: finalized.epoch, + slot: finalizedStateOrBytes.slot, + root: rootHex, + }); } - await this.db.stateArchive.put(finalizedState.slot, finalizedState); - // don't delete states before the finalized state, auto-prune will take care of it - this.logger.verbose("Archived finalized state", {finalizedEpoch: finalized.epoch}); } } diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index 0f1f2a7890d9..b755d2026f3c 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -360,7 +360,7 @@ export async function importBlock( const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); this.regen.addCheckpointState(cp, checkpointState); - this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState); + this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState.clone(true)); // Note: in-lined code from previos handler of ChainEvent.checkpoint this.logger.verbose("Checkpoint processed", toCheckpointHex(cp)); diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index b861378ff440..d72f83604e67 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -36,6 +36,7 @@ export interface IStateRegenerator extends IStateRegeneratorInternal { dumpCacheSummary(): routes.lodestar.StateCacheItem[]; getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null; getPreStateSync(block: allForks.BeaconBlock): CachedBeaconStateAllForks | null; + getCheckpointStateOrBytes(cp: CheckpointHex): Promise; getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null; getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null; pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void; diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index a65c462227f3..365a6d871a79 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -18,7 +18,6 @@ const REGEN_CAN_ACCEPT_WORK_THRESHOLD = 16; type QueuedStateRegeneratorModules = RegenModules & { signal: AbortSignal; - logger: Logger; }; type RegenRequestKey = keyof IStateRegeneratorInternal; @@ -54,6 +53,12 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.logger = modules.logger; } + async init(): Promise { + if (this.checkpointStateCache.init) { + return this.checkpointStateCache.init(); + } + } + canAcceptWork(): boolean { return this.jobQueue.jobLen < REGEN_CAN_ACCEPT_WORK_THRESHOLD; } @@ -105,6 +110,10 @@ export class QueuedStateRegenerator implements IStateRegenerator { return null; } + async getCheckpointStateOrBytes(cp: CheckpointHex): Promise { + return this.checkpointStateCache.getStateOrBytes(cp); + } + getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null { return this.checkpointStateCache.get(cp); } @@ -145,10 +154,13 @@ export class QueuedStateRegenerator implements IStateRegenerator { } else { // Trigger regen on head change if necessary this.logger.warn("Head state not available, triggering regen", {stateRoot: newHeadStateRoot}); + // it's important to reload state to regen head state here + const allowDiskReload = true; // head has changed, so the existing cached head state is no longer useful. Set strong reference to null to free // up memory for regen step below. During regen, node won't be functional but eventually head will be available + // for legacy StateContextCache only this.stateCache.setHeadState(null); - this.regen.getState(newHeadStateRoot, RegenCaller.processBlock).then( + this.regen.getState(newHeadStateRoot, RegenCaller.processBlock, allowDiskReload).then( (headStateRegen) => this.stateCache.setHeadState(headStateRegen), (e) => this.logger.error("Error on head state regen", {}, e) ); diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index ab0e0b5f2dd7..2e32b9ed2d71 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -319,7 +319,7 @@ async function processSlotsToNearestCheckpoint( const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); checkpointStateCache.add(cp, checkpointState); - emitter.emit(ChainEvent.checkpoint, cp, checkpointState.clone()); + emitter.emit(ChainEvent.checkpoint, cp, checkpointState.clone(true)); // this avoids keeping our node busy processing blocks await sleep(0); diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index 854983101c04..7cce1566baf3 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -80,7 +80,7 @@ export class FIFOBlockStateCache implements BlockStateCache { this.metrics?.hits.inc(); this.metrics?.stateClonedCount.observe(item.clonedCount); - return item; + return item.clone(true); } /** diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index b57e51f8f2d6..9580ef1701c7 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -187,7 +187,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { async getOrReload(cp: CheckpointHex): Promise { const stateOrStateBytesData = await this.getStateOrLoadDb(cp); if (stateOrStateBytesData === null || isCachedBeaconState(stateOrStateBytesData)) { - return stateOrStateBytesData; + return stateOrStateBytesData?.clone(true) ?? null; } const {persistedKey, stateBytes} = stateOrStateBytesData; const logMeta = {persistedKey: toHexString(persistedKey)}; @@ -242,7 +242,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.cache.set(cpKey, {type: CacheItemType.inMemory, state: newCachedState, persistedKey}); this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex); // don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch - return newCachedState; + return newCachedState.clone(true); } catch (e) { this.logger.debug("Reload: error loading cached state", logMeta, e as Error); return null; @@ -312,7 +312,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { if (isInMemoryCacheItem(cacheItem)) { const {state} = cacheItem; this.metrics?.stateClonedCount.observe(state.clonedCount); - return state; + return state.clone(true); } return null; @@ -352,9 +352,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { .filter((e) => e <= maxEpoch); for (const epoch of epochs) { if (this.epochIndex.get(epoch)?.has(rootHex)) { - const inMemoryState = this.get({rootHex, epoch}); - if (inMemoryState) { - return inMemoryState; + const inMemoryClonedState = this.get({rootHex, epoch}); + if (inMemoryClonedState) { + return inMemoryClonedState; } } } @@ -376,9 +376,9 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { for (const epoch of epochs) { if (this.epochIndex.get(epoch)?.has(rootHex)) { try { - const state = await this.getOrReload({rootHex, epoch}); - if (state) { - return state; + const clonedState = await this.getOrReload({rootHex, epoch}); + if (clonedState) { + return clonedState; } } catch (e) { this.logger.debug("Error get or reload state", {epoch, rootHex}, e as Error); diff --git a/packages/beacon-node/src/chain/stateCache/stateContextCache.ts b/packages/beacon-node/src/chain/stateCache/stateContextCache.ts index 3a04c4f4a258..a11398f2d962 100644 --- a/packages/beacon-node/src/chain/stateCache/stateContextCache.ts +++ b/packages/beacon-node/src/chain/stateCache/stateContextCache.ts @@ -48,7 +48,7 @@ export class StateContextCache implements BlockStateCache { this.metrics?.hits.inc(); this.metrics?.stateClonedCount.observe(item.clonedCount); - return item; + return item.clone(true); } add(item: CachedBeaconStateAllForks): void { diff --git a/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts index a177db9b7c87..a7e907a97014 100644 --- a/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts @@ -72,7 +72,7 @@ export class CheckpointStateCache implements CheckpointStateCacheInterface { this.metrics?.stateClonedCount.observe(item.clonedCount); - return item; + return item.clone(true); } add(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void { diff --git a/packages/beacon-node/src/util/multifork.ts b/packages/beacon-node/src/util/multifork.ts index 81b4921a0a4a..4abeacd2e566 100644 --- a/packages/beacon-node/src/util/multifork.ts +++ b/packages/beacon-node/src/util/multifork.ts @@ -1,5 +1,5 @@ import {ChainForkConfig} from "@lodestar/config"; -import {allForks} from "@lodestar/types"; +import {Slot, allForks} from "@lodestar/types"; import {bytesToInt} from "@lodestar/utils"; import {getSlotFromSignedBeaconBlockSerialized} from "./sszBytes.js"; @@ -36,10 +36,14 @@ export function getStateTypeFromBytes( config: ChainForkConfig, bytes: Buffer | Uint8Array ): allForks.AllForksSSZTypes["BeaconState"] { - const slot = bytesToInt(bytes.subarray(SLOT_BYTES_POSITION_IN_STATE, SLOT_BYTES_POSITION_IN_STATE + SLOT_BYTE_COUNT)); + const slot = getStateSlotFromBytes(bytes); return config.getForkTypes(slot).BeaconState; } +export function getStateSlotFromBytes(bytes: Uint8Array): Slot { + return bytesToInt(bytes.subarray(SLOT_BYTES_POSITION_IN_STATE, SLOT_BYTES_POSITION_IN_STATE + SLOT_BYTE_COUNT)); +} + /** * First field in update is beacon, first field in beacon is slot * diff --git a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts index 9c37b863623d..af7c118f5e99 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/persistentCheckpointsCache.test.ts @@ -132,7 +132,9 @@ describe("PersistentCheckpointStateCache", function () { it("pruneFinalized and getStateOrBytes", async function () { cache.add(cp2, states["cp2"]); - expect(await cache.getStateOrBytes(cp0bHex)).toEqual(states["cp0b"]); + expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( + states["cp0b"].hashTreeRoot() + ); expect(await cache.processState(toHexString(cp2.root), states["cp2"])).toEqual(1); // cp0 is persisted expect(fileApisBuffer.size).toEqual(1); @@ -484,7 +486,9 @@ describe("PersistentCheckpointStateCache", function () { // regen needs to reload cp0b cache.add(cp0b, states["cp0b"]); - expect(await cache.getStateOrBytes(cp0bHex)).toEqual(states["cp0b"]); + expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( + states["cp0b"].hashTreeRoot() + ); // regen generates cp1b const cp1b = {epoch: 21, root: root0b}; @@ -670,7 +674,9 @@ describe("PersistentCheckpointStateCache", function () { // simulate regen cache.add(cp0b, states["cp0b"]); - expect(await cache.getStateOrBytes(cp0bHex)).toEqual(states["cp0b"]); + expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( + states["cp0b"].hashTreeRoot() + ); // root2, regen cp0b const cp1bState = states["cp0b"].clone(); cp1bState.slot = 21 * SLOTS_PER_EPOCH; @@ -847,7 +853,9 @@ describe("PersistentCheckpointStateCache", function () { // simulate reload cp1b cache.add(cp0b, states["cp0b"]); - expect(await cache.getStateOrBytes(cp0bHex)).toEqual(states["cp0b"]); + expect(((await cache.getStateOrBytes(cp0bHex)) as CachedBeaconStateAllForks).hashTreeRoot()).toEqual( + states["cp0b"].hashTreeRoot() + ); const root1b = Buffer.alloc(32, 101); const state1b = states["cp0b"].clone(); state1b.slot = state1a.slot + 1; From 52ac1555c7359b3505592fe4009bcf45a3b23890 Mon Sep 17 00:00:00 2001 From: tuyennhv Date: Wed, 6 Mar 2024 20:47:28 +0700 Subject: [PATCH 29/34] feat: add clone option to state caches (#6512) * feat: clone option for state caches * fix: compilation error --- .../src/chain/blocks/importBlock.ts | 1 + .../src/chain/blocks/verifyBlock.ts | 1 + .../beacon-node/src/chain/prepareNextSlot.ts | 1 + .../beacon-node/src/chain/regen/interface.ts | 2 +- .../beacon-node/src/chain/regen/queued.ts | 53 ++++++++++++++----- packages/beacon-node/src/chain/regen/regen.ts | 19 ++++--- .../chain/stateCache/fifoBlockStateCache.ts | 5 +- .../stateCache/persistentCheckpointsCache.ts | 35 +++++++----- .../src/chain/stateCache/stateContextCache.ts | 5 +- .../stateContextCheckpointsCache.ts | 24 +++++---- .../beacon-node/src/chain/stateCache/types.ts | 15 ++++-- 11 files changed, 108 insertions(+), 53 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index b755d2026f3c..d82448a4e932 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -360,6 +360,7 @@ export async function importBlock( const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); this.regen.addCheckpointState(cp, checkpointState); + // consumers should not mutate or get the transfered cache this.emitter.emit(ChainEvent.checkpoint, cp, checkpointState.clone(true)); // Note: in-lined code from previos handler of ChainEvent.checkpoint diff --git a/packages/beacon-node/src/chain/blocks/verifyBlock.ts b/packages/beacon-node/src/chain/blocks/verifyBlock.ts index 94a42a39a6ae..658ac05d3908 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlock.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlock.ts @@ -65,6 +65,7 @@ export async function verifyBlocksInEpoch( // TODO: Skip in process chain segment // Retrieve preState from cache (regen) const preState0 = await this.regen + // transfer cache to process faster, postState will be in block state cache .getPreState(block0.message, {dontTransferCache: false}, RegenCaller.processBlocksInEpoch) .catch((e) => { throw new BlockError(block0, {code: BlockErrorCode.PRESTATE_MISSING, error: e as Error}); diff --git a/packages/beacon-node/src/chain/prepareNextSlot.ts b/packages/beacon-node/src/chain/prepareNextSlot.ts index 60658b69ca98..c155c3198269 100644 --- a/packages/beacon-node/src/chain/prepareNextSlot.ts +++ b/packages/beacon-node/src/chain/prepareNextSlot.ts @@ -107,6 +107,7 @@ export class PrepareNextSlotScheduler { headRoot, prepareSlot, // the slot 0 of next epoch will likely use this Previous Root Checkpoint state for state transition so we transfer cache here + // the resulting state with cache will be cached in Checkpoint State Cache which is used for the upcoming block processing // for other slots dontTransferCached=true because we don't run state transition on this state {dontTransferCache: !isEpochTransition}, RegenCaller.precomputeEpoch diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index d72f83604e67..650d92143a8e 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -84,5 +84,5 @@ export interface IStateRegeneratorInternal { /** * Return the exact state with `stateRoot` */ - getState(stateRoot: RootHex, rCaller: RegenCaller): Promise; + getState(stateRoot: RootHex, rCaller: RegenCaller, opts?: StateCloneOpts): Promise; } diff --git a/packages/beacon-node/src/chain/regen/queued.ts b/packages/beacon-node/src/chain/regen/queued.ts index 365a6d871a79..1b2456d2b325 100644 --- a/packages/beacon-node/src/chain/regen/queued.ts +++ b/packages/beacon-node/src/chain/regen/queued.ts @@ -72,11 +72,23 @@ export class QueuedStateRegenerator implements IStateRegenerator { return [...this.stateCache.dumpSummary(), ...this.checkpointStateCache.dumpSummary()]; } + /** + * Get a state from block state cache. + * This is not for block processing so don't transfer cache + */ getStateSync(stateRoot: RootHex): CachedBeaconStateAllForks | null { - return this.stateCache.get(stateRoot); + return this.stateCache.get(stateRoot, {dontTransferCache: true}); } - getPreStateSync(block: allForks.BeaconBlock): CachedBeaconStateAllForks | null { + /** + * Get state for block processing. + * By default, do not transfer cache except for the block at clock slot + * which is usually the gossip block. + */ + getPreStateSync( + block: allForks.BeaconBlock, + opts: StateCloneOpts = {dontTransferCache: true} + ): CachedBeaconStateAllForks | null { const parentRoot = toHexString(block.parentRoot); const parentBlock = this.forkChoice.getBlockHex(parentRoot); if (!parentBlock) { @@ -91,7 +103,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { // Check the checkpoint cache (if the pre-state is a checkpoint state) if (parentEpoch < blockEpoch) { - const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch); + const checkpointState = this.checkpointStateCache.getLatest(parentRoot, blockEpoch, opts); if (checkpointState && computeEpochAtSlot(checkpointState.slot) === blockEpoch) { return checkpointState; } @@ -101,7 +113,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { // Otherwise the state transition may not be cached and wasted. Queue for regen since the // work required will still be significant. if (parentEpoch === blockEpoch) { - const state = this.stateCache.get(parentBlock.stateRoot); + const state = this.stateCache.get(parentBlock.stateRoot, opts); if (state) { return state; } @@ -114,12 +126,21 @@ export class QueuedStateRegenerator implements IStateRegenerator { return this.checkpointStateCache.getStateOrBytes(cp); } + /** + * Get checkpoint state from cache, this function is not for block processing so don't transfer cache + */ getCheckpointStateSync(cp: CheckpointHex): CachedBeaconStateAllForks | null { - return this.checkpointStateCache.get(cp); + return this.checkpointStateCache.get(cp, {dontTransferCache: true}); } + /** + * Get state closest to head, this function is not for block processing so don't transfer cache + */ getClosestHeadState(head: ProtoBlock): CachedBeaconStateAllForks | null { - return this.checkpointStateCache.getLatest(head.blockRoot, Infinity) || this.stateCache.get(head.stateRoot); + const opts = {dontTransferCache: true}; + return ( + this.checkpointStateCache.getLatest(head.blockRoot, Infinity, opts) || this.stateCache.get(head.stateRoot, opts) + ); } pruneOnCheckpoint(finalizedEpoch: Epoch, justifiedEpoch: Epoch, headStateRoot: RootHex): void { @@ -144,10 +165,12 @@ export class QueuedStateRegenerator implements IStateRegenerator { } updateHeadState(newHeadStateRoot: RootHex, maybeHeadState: CachedBeaconStateAllForks): void { + // the resulting state will be added to block state cache so we transfer the cache in this flow + const cloneOpts = {dontTransferCache: true}; const headState = newHeadStateRoot === toHexString(maybeHeadState.hashTreeRoot()) ? maybeHeadState - : this.stateCache.get(newHeadStateRoot); + : this.stateCache.get(newHeadStateRoot, cloneOpts); if (headState) { this.stateCache.setHeadState(headState); @@ -160,7 +183,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { // up memory for regen step below. During regen, node won't be functional but eventually head will be available // for legacy StateContextCache only this.stateCache.setHeadState(null); - this.regen.getState(newHeadStateRoot, RegenCaller.processBlock, allowDiskReload).then( + this.regen.getState(newHeadStateRoot, RegenCaller.processBlock, cloneOpts, allowDiskReload).then( (headStateRegen) => this.stateCache.setHeadState(headStateRegen), (e) => this.logger.error("Error on head state regen", {}, e) ); @@ -183,7 +206,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getPreState}); // First attempt to fetch the state from caches before queueing - const cachedState = this.getPreStateSync(block); + const cachedState = this.getPreStateSync(block, opts); if (cachedState !== null) { return cachedState; @@ -202,7 +225,7 @@ export class QueuedStateRegenerator implements IStateRegenerator { this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getCheckpointState}); // First attempt to fetch the state from cache before queueing - const checkpointState = this.checkpointStateCache.get(toCheckpointHex(cp)); + const checkpointState = this.checkpointStateCache.get(toCheckpointHex(cp), opts); if (checkpointState) { return checkpointState; } @@ -230,18 +253,22 @@ export class QueuedStateRegenerator implements IStateRegenerator { return this.jobQueue.push({key: "getBlockSlotState", args: [blockRoot, slot, opts, rCaller]}); } - async getState(stateRoot: RootHex, rCaller: RegenCaller): Promise { + async getState( + stateRoot: RootHex, + rCaller: RegenCaller, + opts: StateCloneOpts = {dontTransferCache: true} + ): Promise { this.metrics?.regenFnCallTotal.inc({caller: rCaller, entrypoint: RegenFnName.getState}); // First attempt to fetch the state from cache before queueing - const state = this.stateCache.get(stateRoot); + const state = this.stateCache.get(stateRoot, opts); if (state) { return state; } // The state is not immediately available in the cache, enqueue the job this.metrics?.regenFnQueuedTotal.inc({caller: rCaller, entrypoint: RegenFnName.getState}); - return this.jobQueue.push({key: "getState", args: [stateRoot, rCaller]}); + return this.jobQueue.push({key: "getState", args: [stateRoot, rCaller, opts]}); } private jobQueueProcessor = async (regenRequest: RegenRequest): Promise => { diff --git a/packages/beacon-node/src/chain/regen/regen.ts b/packages/beacon-node/src/chain/regen/regen.ts index 2e32b9ed2d71..ccfbd44a8f7b 100644 --- a/packages/beacon-node/src/chain/regen/regen.ts +++ b/packages/beacon-node/src/chain/regen/regen.ts @@ -75,7 +75,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { } // Otherwise, get the state normally. - return this.getState(parentBlock.stateRoot, regenCaller, allowDiskReload); + return this.getState(parentBlock.stateRoot, regenCaller, opts, allowDiskReload); } /** @@ -121,8 +121,8 @@ export class StateRegenerator implements IStateRegeneratorInternal { const {checkpointStateCache} = this.modules; const epoch = computeEpochAtSlot(slot); const latestCheckpointStateCtx = allowDiskReload - ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch) - : checkpointStateCache.getLatest(blockRoot, epoch); + ? await checkpointStateCache.getOrReloadLatest(blockRoot, epoch, opts) + : checkpointStateCache.getLatest(blockRoot, epoch, opts); // If a checkpoint state exists with the given checkpoint root, it either is in requested epoch // or needs to have empty slots processed until the requested epoch @@ -133,7 +133,7 @@ export class StateRegenerator implements IStateRegeneratorInternal { // Otherwise, use the fork choice to get the stateRoot from block at the checkpoint root // regenerate that state, // then process empty slots until the requested epoch - const blockStateCtx = await this.getState(block.stateRoot, regenCaller, allowDiskReload); + const blockStateCtx = await this.getState(block.stateRoot, regenCaller, opts, allowDiskReload); return processSlotsByCheckpoint(this.modules, blockStateCtx, slot, regenCaller, opts); } @@ -145,10 +145,12 @@ export class StateRegenerator implements IStateRegeneratorInternal { async getState( stateRoot: RootHex, _rCaller: RegenCaller, + opts?: StateCloneOpts, + // internal option, don't want to expose to external caller allowDiskReload = false ): Promise { // Trivial case, state at stateRoot is already cached - const cachedStateCtx = this.modules.stateCache.get(stateRoot); + const cachedStateCtx = this.modules.stateCache.get(stateRoot, opts); if (cachedStateCtx) { return cachedStateCtx; } @@ -165,14 +167,14 @@ export class StateRegenerator implements IStateRegeneratorInternal { const {checkpointStateCache} = this.modules; // iterateAncestorBlocks only returns ancestor blocks, not the block itself for (const b of this.modules.forkChoice.iterateAncestorBlocks(block.blockRoot)) { - state = this.modules.stateCache.get(b.stateRoot); + state = this.modules.stateCache.get(b.stateRoot, opts); if (state) { break; } const epoch = computeEpochAtSlot(blocksToReplay[blocksToReplay.length - 1].slot - 1); state = allowDiskReload - ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch) - : checkpointStateCache.getLatest(b.blockRoot, epoch); + ? await checkpointStateCache.getOrReloadLatest(b.blockRoot, epoch, opts) + : checkpointStateCache.getLatest(b.blockRoot, epoch, opts); if (state) { break; } @@ -319,6 +321,7 @@ async function processSlotsToNearestCheckpoint( const checkpointState = postState; const cp = getCheckpointFromState(checkpointState); checkpointStateCache.add(cp, checkpointState); + // consumers should not mutate or get the transfered cache emitter.emit(ChainEvent.checkpoint, cp, checkpointState.clone(true)); // this avoids keeping our node busy processing blocks diff --git a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts index 7cce1566baf3..942825581c4a 100644 --- a/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts +++ b/packages/beacon-node/src/chain/stateCache/fifoBlockStateCache.ts @@ -4,6 +4,7 @@ import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {routes} from "@lodestar/api"; import {Metrics} from "../../metrics/index.js"; import {LinkedList} from "../../util/array.js"; +import {StateCloneOpts} from "../regen/interface.js"; import {MapTracker} from "./mapMetrics.js"; import {BlockStateCache} from "./types.js"; @@ -70,7 +71,7 @@ export class FIFOBlockStateCache implements BlockStateCache { /** * Get a state from this cache given a state root hex. */ - get(rootHex: RootHex): CachedBeaconStateAllForks | null { + get(rootHex: RootHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { this.metrics?.lookups.inc(); const item = this.cache.get(rootHex); if (!item) { @@ -80,7 +81,7 @@ export class FIFOBlockStateCache implements BlockStateCache { this.metrics?.hits.inc(); this.metrics?.stateClonedCount.observe(item.clonedCount); - return item.clone(true); + return item.clone(opts?.dontTransferCache); } /** diff --git a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts index 9580ef1701c7..4aea5ad53a6a 100644 --- a/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/persistentCheckpointsCache.ts @@ -9,6 +9,7 @@ import {Metrics} from "../../metrics/index.js"; import {IClock} from "../../util/clock.js"; import {ShufflingCache} from "../shufflingCache.js"; import {BufferPool, BufferWithKey} from "../../util/bufferPool.js"; +import {StateCloneOpts} from "../regen/interface.js"; import {MapTracker} from "./mapMetrics.js"; import {CPStateDatastore, DatastoreKey, datastoreKeyToCheckpoint} from "./datastore/index.js"; import {CheckpointHex, CacheItemType, CheckpointStateCache} from "./types.js"; @@ -184,10 +185,10 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - Get block for processing * - Regen head state */ - async getOrReload(cp: CheckpointHex): Promise { - const stateOrStateBytesData = await this.getStateOrLoadDb(cp); + async getOrReload(cp: CheckpointHex, opts?: StateCloneOpts): Promise { + const stateOrStateBytesData = await this.getStateOrLoadDb(cp, opts); if (stateOrStateBytesData === null || isCachedBeaconState(stateOrStateBytesData)) { - return stateOrStateBytesData?.clone(true) ?? null; + return stateOrStateBytesData?.clone(opts?.dontTransferCache) ?? null; } const {persistedKey, stateBytes} = stateOrStateBytesData; const logMeta = {persistedKey: toHexString(persistedKey)}; @@ -242,7 +243,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { this.cache.set(cpKey, {type: CacheItemType.inMemory, state: newCachedState, persistedKey}); this.epochIndex.getOrDefault(cp.epoch).add(cp.rootHex); // don't prune from memory here, call it at the last 1/3 of slot 0 of an epoch - return newCachedState.clone(true); + return newCachedState.clone(opts?.dontTransferCache); } catch (e) { this.logger.debug("Reload: error loading cached state", logMeta, e as Error); return null; @@ -253,7 +254,8 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * Return either state or state bytes loaded from db. */ async getStateOrBytes(cp: CheckpointHex): Promise { - const stateOrLoadedState = await this.getStateOrLoadDb(cp); + // don't have to transfer cache for this specific api + const stateOrLoadedState = await this.getStateOrLoadDb(cp, {dontTransferCache: true}); if (stateOrLoadedState === null || isCachedBeaconState(stateOrLoadedState)) { return stateOrLoadedState; } @@ -263,9 +265,12 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Return either state or state bytes with persisted key loaded from db. */ - async getStateOrLoadDb(cp: CheckpointHex): Promise { + async getStateOrLoadDb( + cp: CheckpointHex, + opts?: StateCloneOpts + ): Promise { const cpKey = toCacheKey(cp); - const inMemoryState = this.get(cpKey); + const inMemoryState = this.get(cpKey, opts); if (inMemoryState) { return inMemoryState; } @@ -294,7 +299,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Similar to get() api without reloading from disk */ - get(cpOrKey: CheckpointHex | string): CachedBeaconStateAllForks | null { + get(cpOrKey: CheckpointHex | string, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { this.metrics?.lookups.inc(); const cpKey = typeof cpOrKey === "string" ? cpOrKey : toCacheKey(cpOrKey); const cacheItem = this.cache.get(cpKey); @@ -312,7 +317,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { if (isInMemoryCacheItem(cacheItem)) { const {state} = cacheItem; this.metrics?.stateClonedCount.observe(state.clonedCount); - return state.clone(true); + return state.clone(opts?.dontTransferCache); } return null; @@ -345,14 +350,14 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { /** * Searches in-memory state for the latest cached state with a `root` without reload, starting with `epoch` and descending */ - getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null { + getLatest(rootHex: RootHex, maxEpoch: Epoch, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) .sort((a, b) => b - a) .filter((e) => e <= maxEpoch); for (const epoch of epochs) { if (this.epochIndex.get(epoch)?.has(rootHex)) { - const inMemoryClonedState = this.get({rootHex, epoch}); + const inMemoryClonedState = this.get({rootHex, epoch}, opts); if (inMemoryClonedState) { return inMemoryClonedState; } @@ -368,7 +373,11 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { * - Get block for processing * - Regen head state */ - async getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise { + async getOrReloadLatest( + rootHex: RootHex, + maxEpoch: Epoch, + opts?: StateCloneOpts + ): Promise { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) .sort((a, b) => b - a) @@ -376,7 +385,7 @@ export class PersistentCheckpointStateCache implements CheckpointStateCache { for (const epoch of epochs) { if (this.epochIndex.get(epoch)?.has(rootHex)) { try { - const clonedState = await this.getOrReload({rootHex, epoch}); + const clonedState = await this.getOrReload({rootHex, epoch}, opts); if (clonedState) { return clonedState; } diff --git a/packages/beacon-node/src/chain/stateCache/stateContextCache.ts b/packages/beacon-node/src/chain/stateCache/stateContextCache.ts index a11398f2d962..1b1067a3cec7 100644 --- a/packages/beacon-node/src/chain/stateCache/stateContextCache.ts +++ b/packages/beacon-node/src/chain/stateCache/stateContextCache.ts @@ -3,6 +3,7 @@ import {Epoch, RootHex} from "@lodestar/types"; import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {routes} from "@lodestar/api"; import {Metrics} from "../../metrics/index.js"; +import {StateCloneOpts} from "../regen/interface.js"; import {MapTracker} from "./mapMetrics.js"; import {BlockStateCache} from "./types.js"; @@ -38,7 +39,7 @@ export class StateContextCache implements BlockStateCache { } } - get(rootHex: RootHex): CachedBeaconStateAllForks | null { + get(rootHex: RootHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { this.metrics?.lookups.inc(); const item = this.head?.stateRoot === rootHex ? this.head.state : this.cache.get(rootHex); if (!item) { @@ -48,7 +49,7 @@ export class StateContextCache implements BlockStateCache { this.metrics?.hits.inc(); this.metrics?.stateClonedCount.observe(item.clonedCount); - return item.clone(true); + return item.clone(opts?.dontTransferCache); } add(item: CachedBeaconStateAllForks): void { diff --git a/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts b/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts index a7e907a97014..873e7ac9e465 100644 --- a/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts +++ b/packages/beacon-node/src/chain/stateCache/stateContextCheckpointsCache.ts @@ -4,6 +4,7 @@ import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {MapDef} from "@lodestar/utils"; import {routes} from "@lodestar/api"; import {Metrics} from "../../metrics/index.js"; +import {StateCloneOpts} from "../regen/interface.js"; import {MapTracker} from "./mapMetrics.js"; import {CheckpointStateCache as CheckpointStateCacheInterface, CacheItemType} from "./types.js"; @@ -38,16 +39,21 @@ export class CheckpointStateCache implements CheckpointStateCacheInterface { } } - async getOrReload(cp: CheckpointHex): Promise { - return this.get(cp); + async getOrReload(cp: CheckpointHex, opts?: StateCloneOpts): Promise { + return this.get(cp, opts); } async getStateOrBytes(cp: CheckpointHex): Promise { - return this.get(cp); + // no need to transfer cache for this api + return this.get(cp, {dontTransferCache: true}); } - async getOrReloadLatest(rootHex: string, maxEpoch: number): Promise { - return this.getLatest(rootHex, maxEpoch); + async getOrReloadLatest( + rootHex: string, + maxEpoch: number, + opts?: StateCloneOpts + ): Promise { + return this.getLatest(rootHex, maxEpoch, opts); } async processState(): Promise { @@ -55,7 +61,7 @@ export class CheckpointStateCache implements CheckpointStateCacheInterface { return 0; } - get(cp: CheckpointHex): CachedBeaconStateAllForks | null { + get(cp: CheckpointHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { this.metrics?.lookups.inc(); const cpKey = toCheckpointKey(cp); const item = this.cache.get(cpKey); @@ -72,7 +78,7 @@ export class CheckpointStateCache implements CheckpointStateCacheInterface { this.metrics?.stateClonedCount.observe(item.clonedCount); - return item.clone(true); + return item.clone(opts?.dontTransferCache); } add(cp: phase0.Checkpoint, item: CachedBeaconStateAllForks): void { @@ -89,14 +95,14 @@ export class CheckpointStateCache implements CheckpointStateCacheInterface { /** * Searches for the latest cached state with a `root`, starting with `epoch` and descending */ - getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null { + getLatest(rootHex: RootHex, maxEpoch: Epoch, opts?: StateCloneOpts): CachedBeaconStateAllForks | null { // sort epochs in descending order, only consider epochs lte `epoch` const epochs = Array.from(this.epochIndex.keys()) .sort((a, b) => b - a) .filter((e) => e <= maxEpoch); for (const epoch of epochs) { if (this.epochIndex.get(epoch)?.has(rootHex)) { - return this.get({rootHex, epoch}); + return this.get({rootHex, epoch}, opts); } } return null; diff --git a/packages/beacon-node/src/chain/stateCache/types.ts b/packages/beacon-node/src/chain/stateCache/types.ts index 5867d7d356c1..4a86a0527889 100644 --- a/packages/beacon-node/src/chain/stateCache/types.ts +++ b/packages/beacon-node/src/chain/stateCache/types.ts @@ -1,6 +1,7 @@ import {CachedBeaconStateAllForks} from "@lodestar/state-transition"; import {Epoch, RootHex, phase0} from "@lodestar/types"; import {routes} from "@lodestar/api"; +import {StateCloneOpts} from "../regen/interface.js"; export type CheckpointHex = {epoch: Epoch; rootHex: RootHex}; @@ -20,7 +21,7 @@ export type CheckpointHex = {epoch: Epoch; rootHex: RootHex}; * The cache key is state root */ export interface BlockStateCache { - get(rootHex: RootHex): CachedBeaconStateAllForks | null; + get(rootHex: RootHex, opts?: StateCloneOpts): CachedBeaconStateAllForks | null; add(item: CachedBeaconStateAllForks): void; setHeadState(item: CachedBeaconStateAllForks | null): void; clear(): void; @@ -53,12 +54,16 @@ export interface BlockStateCache { */ export interface CheckpointStateCache { init?: () => Promise; - getOrReload(cp: CheckpointHex): Promise; + getOrReload(cp: CheckpointHex, opts?: StateCloneOpts): Promise; getStateOrBytes(cp: CheckpointHex): Promise; - get(cpOrKey: CheckpointHex | string): CachedBeaconStateAllForks | null; + get(cpOrKey: CheckpointHex | string, opts?: StateCloneOpts): CachedBeaconStateAllForks | null; add(cp: phase0.Checkpoint, state: CachedBeaconStateAllForks): void; - getLatest(rootHex: RootHex, maxEpoch: Epoch): CachedBeaconStateAllForks | null; - getOrReloadLatest(rootHex: RootHex, maxEpoch: Epoch): Promise; + getLatest(rootHex: RootHex, maxEpoch: Epoch, opts?: StateCloneOpts): CachedBeaconStateAllForks | null; + getOrReloadLatest( + rootHex: RootHex, + maxEpoch: Epoch, + opts?: StateCloneOpts + ): Promise; updatePreComputedCheckpoint(rootHex: RootHex, epoch: Epoch): number | null; prune(finalizedEpoch: Epoch, justifiedEpoch: Epoch): void; pruneFinalized(finalizedEpoch: Epoch): void; From 8ad2cb0ad1856fe66fd0998a0c1daecc3f0e4598 Mon Sep 17 00:00:00 2001 From: Julien Date: Wed, 6 Mar 2024 06:41:43 -0800 Subject: [PATCH 30/34] chore: simplify LightClient usage (#6506) * chore: simplify LightClient usage * chore: lint * chore: lint --- .../test/e2e/chain/lightclient.test.ts | 2 +- packages/cli/src/cmds/lightclient/handler.ts | 2 +- packages/light-client/README.md | 54 +++---------------- packages/light-client/src/index.ts | 19 +++++-- packages/light-client/src/utils/utils.ts | 20 ++++++- .../light-client/test/unit/sync.node.test.ts | 2 +- .../src/proof_provider/proof_provider.ts | 14 ++--- 7 files changed, 49 insertions(+), 64 deletions(-) diff --git a/packages/beacon-node/test/e2e/chain/lightclient.test.ts b/packages/beacon-node/test/e2e/chain/lightclient.test.ts index 6857ff8824ec..dc440d5982ec 100644 --- a/packages/beacon-node/test/e2e/chain/lightclient.test.ts +++ b/packages/beacon-node/test/e2e/chain/lightclient.test.ts @@ -131,7 +131,7 @@ describe("chain / lightclient", function () { }); loggerLC.info("Initialized lightclient", {headSlot: lightclient.getHead().beacon.slot}); - lightclient.start(); + void lightclient.start(); return new Promise((resolve, reject) => { bn.chain.emitter.on(routes.events.EventType.head, async (head) => { diff --git a/packages/cli/src/cmds/lightclient/handler.ts b/packages/cli/src/cmds/lightclient/handler.ts index 02bb98cb4d1d..04c833af92d5 100644 --- a/packages/cli/src/cmds/lightclient/handler.ts +++ b/packages/cli/src/cmds/lightclient/handler.ts @@ -33,5 +33,5 @@ export async function lightclientHandler(args: ILightClientArgs & GlobalArgs): P transport: new LightClientRestTransport(api), }); - client.start(); + void client.start(); } diff --git a/packages/light-client/README.md b/packages/light-client/README.md index 759576334489..00ebcf180874 100644 --- a/packages/light-client/README.md +++ b/packages/light-client/README.md @@ -51,51 +51,23 @@ lodestar lightclient \ For this example we will assume there is a running beacon node at `https://beacon-node.your-domain.com` ```ts -import type {Api} from "@lodestar/api/beacon"; -import {ApiError} from "@lodestar/api"; -import type {Bytes32} from "@lodestar/types"; +import {getClient} from "@lodestar/api"; import {createChainForkConfig} from "@lodestar/config"; import {networksChainConfig} from "@lodestar/config/networks"; -import { - type GenesisData, - Lightclient, - LightclientEvent, - RunStatusCode -} from "@lodestar/light-client"; -import {getClient} from "@lodestar/api"; +import {Lightclient, LightclientEvent} from "@lodestar/light-client"; import {LightClientRestTransport} from "@lodestar/light-client/transport"; -import {getLcLoggerConsole} from "@lodestar/light-client/utils"; - -async function getGenesisData(api: Pick): Promise { - const res = await api.beacon.getGenesis(); - ApiError.assert(res); - - return { - genesisTime: Number(res.response.data.genesisTime), - genesisValidatorsRoot: res.response.data.genesisValidatorsRoot, - }; -} - -async function getSyncCheckpoint(api: Pick): Promise { - const res = await api.beacon.getStateFinalityCheckpoints("head"); - ApiError.assert(res); - return res.response.data.finalized.root; -} +import {getFinalizedSyncCheckpoint, getGenesisData, getLcLoggerConsole} from "@lodestar/light-client/utils"; const config = createChainForkConfig(networksChainConfig.mainnet); - const logger = getLcLoggerConsole({logDebug: Boolean(process.env.DEBUG)}); - const api = getClient({urls: ["https://beacon-node.your-domain.com"]}, {config}); -const transport = new LightClientRestTransport(api); - const lightclient = await Lightclient.initializeFromCheckpointRoot({ config, logger, - transport, + transport: new LightClientRestTransport(api), genesisData: await getGenesisData(api), - checkpointRoot: await getSyncCheckpoint(api), + checkpointRoot: await getFinalizedSyncCheckpoint(api), opts: { allowForcedUpdates: true, updateHeadersOnForcedUpdate: true, @@ -103,26 +75,16 @@ const lightclient = await Lightclient.initializeFromCheckpointRoot({ }); // Wait for the lightclient to start -await new Promise((resolve) => { - const lightclientStarted = (status: RunStatusCode): void => { - if (status === RunStatusCode.started) { - lightclient?.emitter.off(LightclientEvent.statusChange, lightclientStarted); - resolve(); - } - }; - lightclient?.emitter.on(LightclientEvent.statusChange, lightclientStarted); - logger.info("Initiating lightclient"); - lightclient?.start(); -}); +await lightclient.start(); logger.info("Lightclient synced"); lightclient.emitter.on(LightclientEvent.lightClientFinalityHeader, async (finalityUpdate) => { - console.log(finalityUpdate); + logger.info(finalityUpdate); }); lightclient.emitter.on(LightclientEvent.lightClientOptimisticHeader, async (optimisticUpdate) => { - console.log(optimisticUpdate); + logger.info(optimisticUpdate); }); ``` diff --git a/packages/light-client/src/index.ts b/packages/light-client/src/index.ts index deac9f66f4d9..4df6f80607be 100644 --- a/packages/light-client/src/index.ts +++ b/packages/light-client/src/index.ts @@ -167,10 +167,23 @@ export class Lightclient { return new Lightclient({...args, bootstrap}); } - start(): void { - this.runLoop().catch((e) => { - this.logger.error("Error on runLoop", {}, e as Error); + /** + * @returns a `Promise` that will resolve once `LightclientEvent.statusChange` with `RunStatusCode.started` value is emitted + */ + start(): Promise { + const startPromise = new Promise((resolve) => { + const lightclientStarted = (status: RunStatusCode): void => { + if (status === RunStatusCode.started) { + this.emitter.off(LightclientEvent.statusChange, lightclientStarted); + resolve(); + } + }; + this.emitter.on(LightclientEvent.statusChange, lightclientStarted); }); + + void this.runLoop(); + + return startPromise; } stop(): void { diff --git a/packages/light-client/src/utils/utils.ts b/packages/light-client/src/utils/utils.ts index 9960921eee90..b7c2c29319e3 100644 --- a/packages/light-client/src/utils/utils.ts +++ b/packages/light-client/src/utils/utils.ts @@ -1,8 +1,10 @@ import bls from "@chainsafe/bls"; import type {PublicKey} from "@chainsafe/bls/types"; import {BitArray} from "@chainsafe/ssz"; -import {altair, Root, ssz} from "@lodestar/types"; +import {Api, ApiError} from "@lodestar/api"; +import {altair, Bytes32, Root, ssz} from "@lodestar/types"; import {BeaconBlockHeader} from "@lodestar/types/phase0"; +import {GenesisData} from "../index.js"; import {SyncCommitteeFast} from "../types.js"; export function sumBits(bits: BitArray): number { @@ -78,3 +80,19 @@ export function isEmptyHeader(header: BeaconBlockHeader): boolean { // Thanks https://github.com/iliakan/detect-node/blob/master/index.esm.js export const isNode = Object.prototype.toString.call(typeof process !== "undefined" ? process : 0) === "[object process]"; + +export async function getGenesisData(api: Pick): Promise { + const res = await api.beacon.getGenesis(); + ApiError.assert(res); + + return { + genesisTime: res.response.data.genesisTime, + genesisValidatorsRoot: res.response.data.genesisValidatorsRoot, + }; +} + +export async function getFinalizedSyncCheckpoint(api: Pick): Promise { + const res = await api.beacon.getStateFinalityCheckpoints("head"); + ApiError.assert(res); + return res.response.data.finalized.root; +} diff --git a/packages/light-client/test/unit/sync.node.test.ts b/packages/light-client/test/unit/sync.node.test.ts index 75073c80070b..4d4212d626cf 100644 --- a/packages/light-client/test/unit/sync.node.test.ts +++ b/packages/light-client/test/unit/sync.node.test.ts @@ -104,7 +104,7 @@ describe("sync", () => { resolve(); } }); - lightclient.start(); + void lightclient.start(); }); // Wait for lightclient to subscribe to header updates diff --git a/packages/prover/src/proof_provider/proof_provider.ts b/packages/prover/src/proof_provider/proof_provider.ts index d888f4269840..f88063e4f02f 100644 --- a/packages/prover/src/proof_provider/proof_provider.ts +++ b/packages/prover/src/proof_provider/proof_provider.ts @@ -96,18 +96,10 @@ export class ProofProvider { }); assertLightClient(this.lightClient); + + this.logger.info("Initiating lightclient"); // Wait for the lightclient to start - await new Promise((resolve) => { - const lightClientStarted = (status: RunStatusCode): void => { - if (status === RunStatusCode.started) { - this.lightClient?.emitter.off(LightclientEvent.statusChange, lightClientStarted); - resolve(); - } - }; - this.lightClient?.emitter.on(LightclientEvent.statusChange, lightClientStarted); - this.logger.info("Initiating lightclient"); - this.lightClient?.start(); - }); + await this.lightClient?.start(); this.logger.info("Lightclient synced", this.getStatus()); this.registerEvents(); From 36f50cf08337e70b0f8078949a23162256ec03a2 Mon Sep 17 00:00:00 2001 From: Cayman Date: Thu, 7 Mar 2024 08:56:44 -0600 Subject: [PATCH 31/34] feat: use Uint32Array for shuffling/committees (#6475) * feat: use Uint32Array for shuffling/committees * chore: address pr comments --- packages/api/package.json | 2 +- .../api/src/beacon/routes/beacon/state.ts | 2 +- packages/beacon-node/package.json | 2 +- .../opPools/aggregatedAttestationPool.ts | 10 +++++----- .../chain/seenCache/seenAttestationData.ts | 2 +- .../src/chain/validation/aggregateAndProof.ts | 6 +++--- .../src/chain/validation/attestation.ts | 4 ++-- packages/beacon-node/test/fixtures/phase0.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 4 ++-- .../stateCache/fifoBlockStateCache.test.ts | 4 ++-- .../stateCache/stateContextCache.test.ts | 4 ++-- packages/cli/package.json | 2 +- packages/config/package.json | 2 +- packages/db/package.json | 2 +- packages/fork-choice/package.json | 2 +- packages/light-client/package.json | 2 +- packages/state-transition/package.json | 2 +- .../state-transition/src/cache/epochCache.ts | 4 ++-- .../src/util/epochShuffling.ts | 19 ++++++++++--------- packages/state-transition/src/util/seed.ts | 6 +++--- packages/state-transition/src/util/shuffle.ts | 14 ++++++++++---- .../src/util/syncCommittee.ts | 2 +- .../test/perf/shuffle/shuffle.test.ts | 4 ++-- packages/types/package.json | 2 +- packages/validator/package.json | 2 +- yarn.lock | 8 ++++---- 26 files changed, 61 insertions(+), 54 deletions(-) diff --git a/packages/api/package.json b/packages/api/package.json index db84e9618ad6..73d31b5ec4d1 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -68,7 +68,7 @@ }, "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.6.1", - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@lodestar/config": "^1.16.0", "@lodestar/params": "^1.16.0", "@lodestar/types": "^1.16.0", diff --git a/packages/api/src/beacon/routes/beacon/state.ts b/packages/api/src/beacon/routes/beacon/state.ts index 7047eaa42e77..322d7ba5e796 100644 --- a/packages/api/src/beacon/routes/beacon/state.ts +++ b/packages/api/src/beacon/routes/beacon/state.ts @@ -66,7 +66,7 @@ export type ValidatorBalance = { export type EpochCommitteeResponse = { index: CommitteeIndex; slot: Slot; - validators: ValidatorIndex[]; + validators: ArrayLike; }; export type EpochSyncCommitteeResponse = { diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index d715c1f77ed5..d9518f0ebb64 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -102,7 +102,7 @@ "@chainsafe/libp2p-noise": "^14.1.0", "@chainsafe/persistent-merkle-tree": "^0.6.1", "@chainsafe/prometheus-gc-stats": "^1.0.0", - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@chainsafe/threads": "^1.11.1", "@ethersproject/abi": "^5.7.0", "@fastify/bearer-auth": "^9.0.0", diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index db7922758732..03cf447b44f4 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -26,7 +26,7 @@ type AttestationWithScore = {attestation: phase0.Attestation; score: number}; * This function returns not seen participation for a given epoch and committee. * Return null if all validators are seen or no info to check. */ -type GetNotSeenValidatorsFn = (epoch: Epoch, committee: number[]) => Set | null; +type GetNotSeenValidatorsFn = (epoch: Epoch, committee: Uint32Array) => Set | null; type ValidateAttestationDataFn = (attData: phase0.AttestationData) => boolean; @@ -79,7 +79,7 @@ export class AggregatedAttestationPool { attestation: phase0.Attestation, dataRootHex: RootHex, attestingIndicesCount: number, - committee: ValidatorIndex[] + committee: Uint32Array ): InsertOutcome { const slot = attestation.data.slot; const lowestPermissibleSlot = this.lowestPermissibleSlot; @@ -271,7 +271,7 @@ export class MatchingDataAttestationGroup { constructor( // TODO: no need committee here - readonly committee: ValidatorIndex[], + readonly committee: Uint32Array, readonly data: phase0.AttestationData ) {} @@ -398,7 +398,7 @@ export function getNotSeenValidatorsFn(state: CachedBeaconStateAllForks): GetNot state ); - return (epoch: Epoch, committee: number[]) => { + return (epoch: Epoch, committee: Uint32Array) => { const participants = epoch === stateEpoch ? currentEpochParticipants : epoch === stateEpoch - 1 ? previousEpochParticipants : null; if (participants === null) { @@ -426,7 +426,7 @@ export function getNotSeenValidatorsFn(state: CachedBeaconStateAllForks): GetNot const currentParticipation = altairState.currentEpochParticipation.getAll(); const stateEpoch = computeEpochAtSlot(state.slot); - return (epoch: Epoch, committee: number[]) => { + return (epoch: Epoch, committee: Uint32Array) => { const participationStatus = epoch === stateEpoch ? currentParticipation : epoch === stateEpoch - 1 ? previousParticipation : null; diff --git a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts index a19476497e9f..9312f3b517a7 100644 --- a/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts +++ b/packages/beacon-node/src/chain/seenCache/seenAttestationData.ts @@ -6,7 +6,7 @@ import {InsertOutcome} from "../opPools/types.js"; export type AttestationDataCacheEntry = { // part of shuffling data, so this does not take memory - committeeIndices: number[]; + committeeIndices: Uint32Array; // IndexedAttestationData signing root, 32 bytes signingRoot: Uint8Array; // to be consumed by forkchoice and oppool diff --git a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts index 5c6a308808ce..170e7421024e 100644 --- a/packages/beacon-node/src/chain/validation/aggregateAndProof.ts +++ b/packages/beacon-node/src/chain/validation/aggregateAndProof.ts @@ -1,6 +1,6 @@ import {toHexString} from "@chainsafe/ssz"; import {ForkName} from "@lodestar/params"; -import {phase0, RootHex, ssz, ValidatorIndex} from "@lodestar/types"; +import {phase0, RootHex, ssz} from "@lodestar/types"; import { computeEpochAtSlot, isAggregatorFromCommitteeLength, @@ -21,7 +21,7 @@ import { export type AggregateAndProofValidationResult = { indexedAttestation: phase0.IndexedAttestation; - committeeIndices: ValidatorIndex[]; + committeeIndices: Uint32Array; attDataRootHex: RootHex; }; @@ -148,7 +148,7 @@ async function validateAggregateAndProof( RegenCaller.validateGossipAttestation ); - const committeeIndices: number[] = cachedAttData + const committeeIndices = cachedAttData ? cachedAttData.committeeIndices : getCommitteeIndices(shuffling, attSlot, attIndex); diff --git a/packages/beacon-node/src/chain/validation/attestation.ts b/packages/beacon-node/src/chain/validation/attestation.ts index eae171631025..fc39534b45e6 100644 --- a/packages/beacon-node/src/chain/validation/attestation.ts +++ b/packages/beacon-node/src/chain/validation/attestation.ts @@ -319,7 +319,7 @@ async function validateGossipAttestationNoSignatureCheck( }); } - let committeeIndices: number[]; + let committeeIndices: Uint32Array; let getSigningRoot: () => Uint8Array; let expectedSubnet: number; if (attestationOrCache.cache) { @@ -702,7 +702,7 @@ export function getCommitteeIndices( shuffling: EpochShuffling, attestationSlot: Slot, attestationIndex: number -): number[] { +): Uint32Array { const {committees} = shuffling; const slotCommittees = committees[attestationSlot % SLOTS_PER_EPOCH]; diff --git a/packages/beacon-node/test/fixtures/phase0.ts b/packages/beacon-node/test/fixtures/phase0.ts index a273f55e967d..56b419a824e7 100644 --- a/packages/beacon-node/test/fixtures/phase0.ts +++ b/packages/beacon-node/test/fixtures/phase0.ts @@ -20,7 +20,7 @@ export function generateIndexedAttestations( for (let committeeIndex = 0; committeeIndex < committeeCount; committeeIndex++) { result.push({ - attestingIndices: state.epochCtx.getBeaconCommittee(slot, committeeIndex), + attestingIndices: Array.from(state.epochCtx.getBeaconCommittee(slot, committeeIndex)), data: { slot: slot, index: committeeIndex, diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index 00aa8a40168b..8d7718a69ebd 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -181,7 +181,7 @@ describe("MatchingDataAttestationGroup.add()", () => { ]; const attestationData = ssz.phase0.AttestationData.defaultValue(); - const committee = linspace(0, 7); + const committee = Uint32Array.from(linspace(0, 7)); for (const {id, attestationsToAdd} of testCases) { it(id, () => { @@ -251,7 +251,7 @@ describe("MatchingDataAttestationGroup.getAttestationsForBlock", () => { ]; const attestationData = ssz.phase0.AttestationData.defaultValue(); - const committee = linspace(0, 7); + const committee = Uint32Array.from(linspace(0, 7)); for (const {id, notSeenAttestingBits, attestationsToAdd} of testCases) { it(id, () => { diff --git a/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts index 4628b1b07220..994cf3f7c085 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/fifoBlockStateCache.test.ts @@ -10,8 +10,8 @@ describe("FIFOBlockStateCache", function () { let cache: FIFOBlockStateCache; const shuffling: EpochShuffling = { epoch: 0, - activeIndices: [], - shuffling: [], + activeIndices: new Uint32Array(), + shuffling: new Uint32Array(), committees: [], committeesPerSlot: 1, }; diff --git a/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts b/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts index 5a18346ff929..cca4d7ea7734 100644 --- a/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts +++ b/packages/beacon-node/test/unit/chain/stateCache/stateContextCache.test.ts @@ -12,8 +12,8 @@ describe("StateContextCache", function () { let key1: Root, key2: Root; const shuffling: EpochShuffling = { epoch: 0, - activeIndices: [], - shuffling: [], + activeIndices: new Uint32Array(), + shuffling: new Uint32Array(), committees: [], committeesPerSlot: 1, }; diff --git a/packages/cli/package.json b/packages/cli/package.json index 19aaf088d886..73cdf19ffe01 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -58,7 +58,7 @@ "@chainsafe/discv5": "^9.0.0", "@chainsafe/enr": "^3.0.0", "@chainsafe/persistent-merkle-tree": "^0.6.1", - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@chainsafe/threads": "^1.11.1", "@libp2p/crypto": "^3.0.4", "@libp2p/peer-id": "^4.0.4", diff --git a/packages/config/package.json b/packages/config/package.json index 669eb6f6e07c..7f0cbefdcf6f 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -64,7 +64,7 @@ "blockchain" ], "dependencies": { - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@lodestar/params": "^1.16.0", "@lodestar/types": "^1.16.0" } diff --git a/packages/db/package.json b/packages/db/package.json index 631bbbc37e7c..3f2cc739f24d 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -35,7 +35,7 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@lodestar/config": "^1.16.0", "@lodestar/utils": "^1.16.0", "it-all": "^3.0.4", diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index f60551531589..65cf401ae569 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -36,7 +36,7 @@ "check-readme": "typescript-docs-verifier" }, "dependencies": { - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@lodestar/config": "^1.16.0", "@lodestar/params": "^1.16.0", "@lodestar/state-transition": "^1.16.0", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 2f392b169f07..6e96ecd43ddf 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -67,7 +67,7 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/persistent-merkle-tree": "^0.6.1", - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@lodestar/api": "^1.16.0", "@lodestar/config": "^1.16.0", "@lodestar/params": "^1.16.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index d7d65a2dab49..b3a55938b1a6 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -63,7 +63,7 @@ "@chainsafe/blst": "^0.2.10", "@chainsafe/persistent-merkle-tree": "^0.6.1", "@chainsafe/persistent-ts": "^0.19.1", - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@lodestar/config": "^1.16.0", "@lodestar/params": "^1.16.0", "@lodestar/types": "^1.16.0", diff --git a/packages/state-transition/src/cache/epochCache.ts b/packages/state-transition/src/cache/epochCache.ts index 78cccacf1d00..9565898eb09d 100644 --- a/packages/state-transition/src/cache/epochCache.ts +++ b/packages/state-transition/src/cache/epochCache.ts @@ -584,7 +584,7 @@ export class EpochCache { /** * Return the beacon committee at slot for index. */ - getBeaconCommittee(slot: Slot, index: CommitteeIndex): ValidatorIndex[] { + getBeaconCommittee(slot: Slot, index: CommitteeIndex): Uint32Array { const slotCommittees = this.getShufflingAtSlot(slot).committees[slot % SLOTS_PER_EPOCH]; if (index >= slotCommittees.length) { throw new EpochCacheError({ @@ -745,7 +745,7 @@ export class EpochCache { const committee = this.getBeaconCommittee(slot, i); if (committee.includes(validatorIndex)) { return { - validators: committee, + validators: Array.from(committee), committeeIndex: i, slot, }; diff --git a/packages/state-transition/src/util/epochShuffling.ts b/packages/state-transition/src/util/epochShuffling.ts index f9172126250f..12f270d29792 100644 --- a/packages/state-transition/src/util/epochShuffling.ts +++ b/packages/state-transition/src/util/epochShuffling.ts @@ -30,12 +30,12 @@ export type EpochShuffling = { /** * Non-shuffled active validator indices */ - activeIndices: ValidatorIndex[]; + activeIndices: Uint32Array; /** * The active validator indices, shuffled into their committee */ - shuffling: ValidatorIndex[]; + shuffling: Uint32Array; /** * List of list of committees Committees @@ -45,7 +45,7 @@ export type EpochShuffling = { * Note: With a high amount of shards, or low amount of validators, * some shards may not have a committee this epoch */ - committees: ValidatorIndex[][][]; + committees: Uint32Array[][]; /** * Committees per slot, for fast attestation verification @@ -61,13 +61,14 @@ export function computeCommitteeCount(activeValidatorCount: number): number { export function computeEpochShuffling( state: BeaconStateAllForks, - activeIndices: ValidatorIndex[], + activeIndices: ArrayLike, epoch: Epoch ): EpochShuffling { const seed = getSeed(state, epoch, DOMAIN_BEACON_ATTESTER); // copy - const shuffling = activeIndices.slice(); + const _activeIndices = new Uint32Array(activeIndices); + const shuffling = _activeIndices.slice(); unshuffleList(shuffling, seed); const activeValidatorCount = activeIndices.length; @@ -75,9 +76,9 @@ export function computeEpochShuffling( const committeeCount = committeesPerSlot * SLOTS_PER_EPOCH; - const committees: ValidatorIndex[][][] = []; + const committees: Uint32Array[][] = []; for (let slot = 0; slot < SLOTS_PER_EPOCH; slot++) { - const slotCommittees: ValidatorIndex[][] = []; + const slotCommittees: Uint32Array[] = []; for (let committeeIndex = 0; committeeIndex < committeesPerSlot; committeeIndex++) { const index = slot * committeesPerSlot + committeeIndex; const startOffset = Math.floor((activeValidatorCount * index) / committeeCount); @@ -85,14 +86,14 @@ export function computeEpochShuffling( if (!(startOffset <= endOffset)) { throw new Error(`Invalid offsets: start ${startOffset} must be less than or equal end ${endOffset}`); } - slotCommittees.push(shuffling.slice(startOffset, endOffset)); + slotCommittees.push(shuffling.subarray(startOffset, endOffset)); } committees.push(slotCommittees); } return { epoch, - activeIndices, + activeIndices: _activeIndices, shuffling, committees, committeesPerSlot, diff --git a/packages/state-transition/src/util/seed.ts b/packages/state-transition/src/util/seed.ts index b73851badaf5..cf48fda8bec4 100644 --- a/packages/state-transition/src/util/seed.ts +++ b/packages/state-transition/src/util/seed.ts @@ -21,7 +21,7 @@ import {computeEpochAtSlot} from "./epoch.js"; */ export function computeProposers( epochSeed: Uint8Array, - shuffling: {epoch: Epoch; activeIndices: ValidatorIndex[]}, + shuffling: {epoch: Epoch; activeIndices: ArrayLike}, effectiveBalanceIncrements: EffectiveBalanceIncrements ): number[] { const startSlot = computeStartSlotAtEpoch(shuffling.epoch); @@ -45,7 +45,7 @@ export function computeProposers( */ export function computeProposerIndex( effectiveBalanceIncrements: EffectiveBalanceIncrements, - indices: ValidatorIndex[], + indices: ArrayLike, seed: Uint8Array ): ValidatorIndex { if (indices.length === 0) { @@ -91,7 +91,7 @@ export function computeProposerIndex( */ export function getNextSyncCommitteeIndices( state: BeaconStateAllForks, - activeValidatorIndices: ValidatorIndex[], + activeValidatorIndices: ArrayLike, effectiveBalanceIncrements: EffectiveBalanceIncrements ): ValidatorIndex[] { // TODO: Bechmark if it's necessary to inline outside of this function diff --git a/packages/state-transition/src/util/shuffle.ts b/packages/state-transition/src/util/shuffle.ts index 1915b08938c2..07457c5975d0 100644 --- a/packages/state-transition/src/util/shuffle.ts +++ b/packages/state-transition/src/util/shuffle.ts @@ -1,15 +1,21 @@ import {digest} from "@chainsafe/as-sha256"; import {SHUFFLE_ROUND_COUNT} from "@lodestar/params"; -import {ValidatorIndex, Bytes32} from "@lodestar/types"; +import {Bytes32} from "@lodestar/types"; import {assert, bytesToBigInt} from "@lodestar/utils"; +// ArrayLike but with settable indices +type Shuffleable = { + readonly length: number; + [index: number]: number; +}; + // ShuffleList shuffles a list, using the given seed for randomness. Mutates the input list. -export function shuffleList(input: ValidatorIndex[], seed: Bytes32): void { +export function shuffleList(input: Shuffleable, seed: Bytes32): void { innerShuffleList(input, seed, true); } // UnshuffleList undoes a list shuffling using the seed of the shuffling. Mutates the input list. -export function unshuffleList(input: ValidatorIndex[], seed: Bytes32): void { +export function unshuffleList(input: Shuffleable, seed: Bytes32): void { innerShuffleList(input, seed, false); } @@ -70,7 +76,7 @@ function setPositionUint32(value: number, buf: Buffer): void { } // Shuffles or unshuffles, depending on the `dir` (true for shuffling, false for unshuffling -function innerShuffleList(input: ValidatorIndex[], seed: Bytes32, dir: boolean): void { +function innerShuffleList(input: Shuffleable, seed: Bytes32, dir: boolean): void { if (input.length <= 1) { // nothing to (un)shuffle return; diff --git a/packages/state-transition/src/util/syncCommittee.ts b/packages/state-transition/src/util/syncCommittee.ts index 2dcc46ae8ae3..89c476b69c04 100644 --- a/packages/state-transition/src/util/syncCommittee.ts +++ b/packages/state-transition/src/util/syncCommittee.ts @@ -20,7 +20,7 @@ import {getNextSyncCommitteeIndices} from "./seed.js"; */ export function getNextSyncCommittee( state: BeaconStateAllForks, - activeValidatorIndices: ValidatorIndex[], + activeValidatorIndices: ArrayLike, effectiveBalanceIncrements: EffectiveBalanceIncrements ): {indices: ValidatorIndex[]; syncCommittee: altair.SyncCommittee} { const indices = getNextSyncCommitteeIndices(state, activeValidatorIndices, effectiveBalanceIncrements); diff --git a/packages/state-transition/test/perf/shuffle/shuffle.test.ts b/packages/state-transition/test/perf/shuffle/shuffle.test.ts index 9822c5291792..ea1a9d606184 100644 --- a/packages/state-transition/test/perf/shuffle/shuffle.test.ts +++ b/packages/state-transition/test/perf/shuffle/shuffle.test.ts @@ -14,12 +14,12 @@ describe("shuffle list", () => { // Don't run 4_000_000 since it's very slow and not testnet has gotten there yet // 4e6, ]) { - itBench({ + itBench({ id: `shuffle list - ${listSize} els`, before: () => { const input: number[] = []; for (let i = 0; i < listSize; i++) input[i] = i; - return input; + return new Uint32Array(input); }, beforeEach: (input) => input, fn: (input) => unshuffleList(input, seed), diff --git a/packages/types/package.json b/packages/types/package.json index c492eed43014..286d4b99e5cd 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -73,7 +73,7 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@lodestar/params": "^1.16.0", "ethereum-cryptography": "^2.0.0" }, diff --git a/packages/validator/package.json b/packages/validator/package.json index 1eebe5706b9a..b5dba33d5d32 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -46,7 +46,7 @@ ], "dependencies": { "@chainsafe/bls": "7.1.3", - "@chainsafe/ssz": "^0.14.0", + "@chainsafe/ssz": "^0.14.3", "@lodestar/api": "^1.16.0", "@lodestar/config": "^1.16.0", "@lodestar/db": "^1.16.0", diff --git a/yarn.lock b/yarn.lock index 0f0c76c7d130..c744a9b4ede6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -465,10 +465,10 @@ "@chainsafe/as-sha256" "^0.4.1" "@chainsafe/persistent-merkle-tree" "^0.6.1" -"@chainsafe/ssz@^0.14.0": - version "0.14.0" - resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.14.0.tgz#fe9e4fd3cf673013bd57f77c3ab0fdc5ebc5d916" - integrity sha512-KTc33pWu7ItXlzMAz5/1osOHsvhx25kpM3j7Ez+PNZLyyhIoNzAhhozvxy+ul0fCDfHbvaCRp3lJQnzsb5Iv0A== +"@chainsafe/ssz@^0.14.3": + version "0.14.3" + resolved "https://registry.yarnpkg.com/@chainsafe/ssz/-/ssz-0.14.3.tgz#caae48ae2670b2f8b6febed22b0e0619a636f316" + integrity sha512-ldOx4Rk9OC8YMvFdwvHKtRc7KpFRLcXlb9ATCdQ5fHtLT438LRQyxdWFufC9+M8jFHSZcgq31h2BJsSva6sZ0w== dependencies: "@chainsafe/as-sha256" "^0.4.1" "@chainsafe/persistent-merkle-tree" "^0.6.1" From adc0534782436ee45614968c090915f0724121e1 Mon Sep 17 00:00:00 2001 From: Julien Date: Thu, 7 Mar 2024 23:00:41 -0800 Subject: [PATCH 32/34] chore: remove state-transition dependency (#6519) --- packages/light-client/package.json | 1 - packages/light-client/src/spec/utils.ts | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 6e96ecd43ddf..def5f45bbcf5 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -71,7 +71,6 @@ "@lodestar/api": "^1.16.0", "@lodestar/config": "^1.16.0", "@lodestar/params": "^1.16.0", - "@lodestar/state-transition": "^1.16.0", "@lodestar/types": "^1.16.0", "@lodestar/utils": "^1.16.0", "mitt": "^3.0.0", diff --git a/packages/light-client/src/spec/utils.ts b/packages/light-client/src/spec/utils.ts index cd0cfc28d4f2..2a5720a1f637 100644 --- a/packages/light-client/src/spec/utils.ts +++ b/packages/light-client/src/spec/utils.ts @@ -10,9 +10,8 @@ import { } from "@lodestar/params"; import {altair, phase0, ssz, allForks, capella, deneb, Slot} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; -import {computeEpochAtSlot} from "@lodestar/state-transition"; -import {isValidMerkleBranch, computeSyncPeriodAtSlot} from "../utils/index.js"; +import {isValidMerkleBranch, computeEpochAtSlot, computeSyncPeriodAtSlot} from "../utils/index.js"; import {LightClientStore} from "./store.js"; export const GENESIS_SLOT = 0; From cae26be359ae6115079ad592cc5539920b89616d Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Fri, 8 Mar 2024 13:51:06 +0100 Subject: [PATCH 33/34] fix: increase max attestation inclusion slot for post deneb blocks (#6522) --- .../chain/opPools/aggregatedAttestationPool.ts | 16 +++++++++++++--- .../src/chain/produceBlock/produceBlockBody.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 2 +- .../opPools/aggregatedAttestationPool.test.ts | 11 ++++++----- 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts index 03cf447b44f4..c94e5d81e823 100644 --- a/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts +++ b/packages/beacon-node/src/chain/opPools/aggregatedAttestationPool.ts @@ -1,6 +1,6 @@ import bls from "@chainsafe/bls"; import {toHexString} from "@chainsafe/ssz"; -import {ForkName, MAX_ATTESTATIONS, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {ForkName, ForkSeq, MAX_ATTESTATIONS, MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; import {phase0, Epoch, Slot, ssz, ValidatorIndex, RootHex} from "@lodestar/types"; import { CachedBeaconStateAllForks, @@ -117,7 +117,11 @@ export class AggregatedAttestationPool { /** * Get attestations to be included in a block. Returns $MAX_ATTESTATIONS items */ - getAttestationsForBlock(forkChoice: IForkChoice, state: CachedBeaconStateAllForks): phase0.Attestation[] { + getAttestationsForBlock( + fork: ForkName, + forkChoice: IForkChoice, + state: CachedBeaconStateAllForks + ): phase0.Attestation[] { const stateSlot = state.slot; const stateEpoch = state.epochCtx.epoch; const statePrevEpoch = stateEpoch - 1; @@ -144,7 +148,13 @@ export class AggregatedAttestationPool { continue; // Invalid attestations } // validateAttestation condition: Attestation slot not within inclusion window - if (!(slot + MIN_ATTESTATION_INCLUSION_DELAY <= stateSlot && stateSlot <= slot + SLOTS_PER_EPOCH)) { + if ( + !( + slot + MIN_ATTESTATION_INCLUSION_DELAY <= stateSlot && + // Post deneb, attestations are valid for current and previous epoch + (ForkSeq[fork] >= ForkSeq.deneb || stateSlot <= slot + SLOTS_PER_EPOCH) + ) + ) { continue; // Invalid attestations } diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index d598ddcb9688..7697e89807ea 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -606,7 +606,7 @@ export async function produceCommonBlockBody( this.opPool.getSlashingsAndExits(currentState, blockType, this.metrics); const endAttestations = stepsMetrics?.startTimer(); - const attestations = this.aggregatedAttestationPool.getAttestationsForBlock(this.forkChoice, currentState); + const attestations = this.aggregatedAttestationPool.getAttestationsForBlock(fork, this.forkChoice, currentState); endAttestations?.({ step: BlockProductionStep.attestations, }); diff --git a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts index 4ed8215ac85b..a9ce54e9d3f4 100644 --- a/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/perf/chain/opPools/aggregatedAttestationPool.test.ts @@ -159,7 +159,7 @@ describe(`getAttestationsForBlock vc=${vc}`, () => { return {state, pool}; }, fn: ({state, pool}) => { - pool.getAttestationsForBlock(forkchoice, state); + pool.getAttestationsForBlock(state.config.getForkName(state.slot), forkchoice, state); }, }); } diff --git a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts index 8d7718a69ebd..3c248ad4d194 100644 --- a/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts +++ b/packages/beacon-node/test/unit/chain/opPools/aggregatedAttestationPool.test.ts @@ -3,7 +3,7 @@ import bls from "@chainsafe/bls"; import {BitArray, fromHexString, toHexString} from "@chainsafe/ssz"; import {describe, it, expect, beforeEach, beforeAll, afterEach, vi} from "vitest"; import {CachedBeaconStateAllForks, newFilledArray} from "@lodestar/state-transition"; -import {FAR_FUTURE_EPOCH, MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH} from "@lodestar/params"; +import {FAR_FUTURE_EPOCH, ForkName, MAX_EFFECTIVE_BALANCE, SLOTS_PER_EPOCH} from "@lodestar/params"; import {ssz, phase0} from "@lodestar/types"; import {CachedBeaconStateAltair} from "@lodestar/state-transition/src/types.js"; import {MockedForkChoice, getMockedForkChoice} from "../../../mocks/mockedBeaconChain.js"; @@ -28,6 +28,7 @@ const validSignature = fromHexString( describe("AggregatedAttestationPool", function () { let pool: AggregatedAttestationPool; + const fork = ForkName.altair; const altairForkEpoch = 2020; const currentEpoch = altairForkEpoch + 10; const currentSlot = SLOTS_PER_EPOCH * currentEpoch; @@ -115,9 +116,9 @@ describe("AggregatedAttestationPool", function () { forkchoiceStub.getBlockHex.mockReturnValue(generateProtoBlock()); forkchoiceStub.getDependentRoot.mockReturnValue(ZERO_HASH_HEX); if (isReturned) { - expect(pool.getAttestationsForBlock(forkchoiceStub, altairState).length).toBeGreaterThan(0); + expect(pool.getAttestationsForBlock(fork, forkchoiceStub, altairState).length).toBeGreaterThan(0); } else { - expect(pool.getAttestationsForBlock(forkchoiceStub, altairState).length).toEqual(0); + expect(pool.getAttestationsForBlock(fork, forkchoiceStub, altairState).length).toEqual(0); } // "forkchoice should be called to check pivot block" expect(forkchoiceStub.getDependentRoot).toHaveBeenCalledTimes(1); @@ -129,7 +130,7 @@ describe("AggregatedAttestationPool", function () { // all attesters are not seen const attestingIndices = [2, 3]; pool.add(attestation, attDataRootHex, attestingIndices.length, committee); - expect(pool.getAttestationsForBlock(forkchoiceStub, altairState)).toEqual([]); + expect(pool.getAttestationsForBlock(fork, forkchoiceStub, altairState)).toEqual([]); // "forkchoice should not be called" expect(forkchoiceStub.iterateAncestorBlocks).not.toHaveBeenCalledTimes(1); }); @@ -140,7 +141,7 @@ describe("AggregatedAttestationPool", function () { pool.add(attestation, attDataRootHex, attestingIndices.length, committee); forkchoiceStub.getBlockHex.mockReturnValue(generateProtoBlock()); forkchoiceStub.getDependentRoot.mockReturnValue("0xWeird"); - expect(pool.getAttestationsForBlock(forkchoiceStub, altairState)).toEqual([]); + expect(pool.getAttestationsForBlock(fork, forkchoiceStub, altairState)).toEqual([]); // "forkchoice should be called to check pivot block" expect(forkchoiceStub.getDependentRoot).toHaveBeenCalledTimes(1); }); From 88b2564e4f82df7156593ac8a7920505fe294cc4 Mon Sep 17 00:00:00 2001 From: Phil Ngo Date: Fri, 8 Mar 2024 10:39:20 -0500 Subject: [PATCH 34/34] v1.17.0 --- lerna.json | 2 +- packages/api/package.json | 10 +++++----- packages/beacon-node/package.json | 26 +++++++++++++------------- packages/cli/package.json | 26 +++++++++++++------------- packages/config/package.json | 6 +++--- packages/db/package.json | 8 ++++---- packages/flare/package.json | 14 +++++++------- packages/fork-choice/package.json | 12 ++++++------ packages/light-client/package.json | 12 ++++++------ packages/logger/package.json | 6 +++--- packages/params/package.json | 2 +- packages/prover/package.json | 18 +++++++++--------- packages/reqresp/package.json | 12 ++++++------ packages/spec-test-util/package.json | 4 ++-- packages/state-transition/package.json | 10 +++++----- packages/test-utils/package.json | 6 +++--- packages/types/package.json | 4 ++-- packages/utils/package.json | 2 +- packages/validator/package.json | 18 +++++++++--------- 19 files changed, 99 insertions(+), 99 deletions(-) diff --git a/lerna.json b/lerna.json index a79413a21c37..2103cc613c5c 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useNx": true, - "version": "1.16.0", + "version": "1.17.0", "stream": true, "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index 73d31b5ec4d1..8407d3705a6c 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": { ".": { @@ -69,10 +69,10 @@ "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.6.1", "@chainsafe/ssz": "^0.14.3", - "@lodestar/config": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/types": "^1.16.0", - "@lodestar/utils": "^1.16.0", + "@lodestar/config": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/types": "^1.17.0", + "@lodestar/utils": "^1.17.0", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index d9518f0ebb64..609cc5c9f04f 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": { ".": { @@ -118,18 +118,18 @@ "@libp2p/peer-id-factory": "^4.0.3", "@libp2p/prometheus-metrics": "^3.0.10", "@libp2p/tcp": "9.0.10", - "@lodestar/api": "^1.16.0", - "@lodestar/config": "^1.16.0", - "@lodestar/db": "^1.16.0", - "@lodestar/fork-choice": "^1.16.0", - "@lodestar/light-client": "^1.16.0", - "@lodestar/logger": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/reqresp": "^1.16.0", - "@lodestar/state-transition": "^1.16.0", - "@lodestar/types": "^1.16.0", - "@lodestar/utils": "^1.16.0", - "@lodestar/validator": "^1.16.0", + "@lodestar/api": "^1.17.0", + "@lodestar/config": "^1.17.0", + "@lodestar/db": "^1.17.0", + "@lodestar/fork-choice": "^1.17.0", + "@lodestar/light-client": "^1.17.0", + "@lodestar/logger": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/reqresp": "^1.17.0", + "@lodestar/state-transition": "^1.17.0", + "@lodestar/types": "^1.17.0", + "@lodestar/utils": "^1.17.0", + "@lodestar/validator": "^1.17.0", "@multiformats/multiaddr": "^12.1.3", "c-kzg": "^2.1.2", "datastore-core": "^9.1.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 73cdf19ffe01..2bf6bc6dc708 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.16.0", + "version": "1.17.0", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -63,17 +63,17 @@ "@libp2p/crypto": "^3.0.4", "@libp2p/peer-id": "^4.0.4", "@libp2p/peer-id-factory": "^4.0.3", - "@lodestar/api": "^1.16.0", - "@lodestar/beacon-node": "^1.16.0", - "@lodestar/config": "^1.16.0", - "@lodestar/db": "^1.16.0", - "@lodestar/light-client": "^1.16.0", - "@lodestar/logger": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/state-transition": "^1.16.0", - "@lodestar/types": "^1.16.0", - "@lodestar/utils": "^1.16.0", - "@lodestar/validator": "^1.16.0", + "@lodestar/api": "^1.17.0", + "@lodestar/beacon-node": "^1.17.0", + "@lodestar/config": "^1.17.0", + "@lodestar/db": "^1.17.0", + "@lodestar/light-client": "^1.17.0", + "@lodestar/logger": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/state-transition": "^1.17.0", + "@lodestar/types": "^1.17.0", + "@lodestar/utils": "^1.17.0", + "@lodestar/validator": "^1.17.0", "@multiformats/multiaddr": "^12.1.3", "deepmerge": "^4.3.1", "ethers": "^6.7.0", @@ -89,7 +89,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.16.0", + "@lodestar/test-utils": "^1.17.0", "@types/debug": "^4.1.7", "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", diff --git a/packages/config/package.json b/packages/config/package.json index 7f0cbefdcf6f..60f8484ee928 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.16.0", + "version": "1.17.0", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -65,7 +65,7 @@ ], "dependencies": { "@chainsafe/ssz": "^0.14.3", - "@lodestar/params": "^1.16.0", - "@lodestar/types": "^1.16.0" + "@lodestar/params": "^1.17.0", + "@lodestar/types": "^1.17.0" } } diff --git a/packages/db/package.json b/packages/db/package.json index 3f2cc739f24d..dcb0b66b920f 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.16.0", + "version": "1.17.0", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -36,12 +36,12 @@ }, "dependencies": { "@chainsafe/ssz": "^0.14.3", - "@lodestar/config": "^1.16.0", - "@lodestar/utils": "^1.16.0", + "@lodestar/config": "^1.17.0", + "@lodestar/utils": "^1.17.0", "it-all": "^3.0.4", "level": "^8.0.0" }, "devDependencies": { - "@lodestar/logger": "^1.16.0" + "@lodestar/logger": "^1.17.0" } } diff --git a/packages/flare/package.json b/packages/flare/package.json index 6281ea0f81f1..0d4241d96efa 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.16.0", + "version": "1.17.0", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -60,12 +60,12 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/bls-keygen": "^0.4.0", - "@lodestar/api": "^1.16.0", - "@lodestar/config": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/state-transition": "^1.16.0", - "@lodestar/types": "^1.16.0", - "@lodestar/utils": "^1.16.0", + "@lodestar/api": "^1.17.0", + "@lodestar/config": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/state-transition": "^1.17.0", + "@lodestar/types": "^1.17.0", + "@lodestar/utils": "^1.17.0", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 65cf401ae569..a8c708ec7525 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": "./lib/index.js", "types": "./lib/index.d.ts", @@ -37,11 +37,11 @@ }, "dependencies": { "@chainsafe/ssz": "^0.14.3", - "@lodestar/config": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/state-transition": "^1.16.0", - "@lodestar/types": "^1.16.0", - "@lodestar/utils": "^1.16.0" + "@lodestar/config": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/state-transition": "^1.17.0", + "@lodestar/types": "^1.17.0", + "@lodestar/utils": "^1.17.0" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index def5f45bbcf5..7bc3cfd36434 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": { ".": { @@ -68,11 +68,11 @@ "@chainsafe/bls": "7.1.3", "@chainsafe/persistent-merkle-tree": "^0.6.1", "@chainsafe/ssz": "^0.14.3", - "@lodestar/api": "^1.16.0", - "@lodestar/config": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/types": "^1.16.0", - "@lodestar/utils": "^1.16.0", + "@lodestar/api": "^1.17.0", + "@lodestar/config": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/types": "^1.17.0", + "@lodestar/utils": "^1.17.0", "mitt": "^3.0.0", "strict-event-emitter-types": "^2.0.0" }, diff --git a/packages/logger/package.json b/packages/logger/package.json index e0d773e8fe25..b249c8c50ec7 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": { ".": { @@ -66,14 +66,14 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@lodestar/utils": "^1.16.0", + "@lodestar/utils": "^1.17.0", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1", "winston-transport": "^4.5.0" }, "devDependencies": { "@chainsafe/threads": "^1.11.1", - "@lodestar/test-utils": "^1.16.0", + "@lodestar/test-utils": "^1.17.0", "@types/triple-beam": "^1.3.2", "triple-beam": "^1.3.0" }, diff --git a/packages/params/package.json b/packages/params/package.json index 8902b2bceb8e..a9242ab3528e 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.16.0", + "version": "1.17.0", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index a0b105bef18e..92e6c4fd679c 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": { ".": { @@ -69,13 +69,13 @@ "@ethereumjs/tx": "^4.1.2", "@ethereumjs/util": "^8.0.6", "@ethereumjs/vm": "^6.4.2", - "@lodestar/api": "^1.16.0", - "@lodestar/config": "^1.16.0", - "@lodestar/light-client": "^1.16.0", - "@lodestar/logger": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/types": "^1.16.0", - "@lodestar/utils": "^1.16.0", + "@lodestar/api": "^1.17.0", + "@lodestar/config": "^1.17.0", + "@lodestar/light-client": "^1.17.0", + "@lodestar/logger": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/types": "^1.17.0", + "@lodestar/utils": "^1.17.0", "ethereum-cryptography": "^2.0.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", @@ -84,7 +84,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.16.0", + "@lodestar/test-utils": "^1.17.0", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 20627f4de384..413eb737295a 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": { ".": { @@ -54,9 +54,9 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface": "^1.1.1", - "@lodestar/config": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/utils": "^1.16.0", + "@lodestar/config": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/utils": "^1.17.0", "it-all": "^3.0.4", "it-pipe": "^3.0.1", "snappy": "^7.2.2", @@ -65,8 +65,8 @@ "uint8arraylist": "^2.4.7" }, "devDependencies": { - "@lodestar/logger": "^1.16.0", - "@lodestar/types": "^1.16.0", + "@lodestar/logger": "^1.17.0", + "@lodestar/types": "^1.17.0", "libp2p": "1.1.1" }, "peerDependencies": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 8b4ab28da0fd..484172b02937 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.16.0", + "version": "1.17.0", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -62,7 +62,7 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.16.0", + "@lodestar/utils": "^1.17.0", "axios": "^1.3.4", "rimraf": "^4.4.1", "snappyjs": "^0.7.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index b3a55938b1a6..270cfa441357 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": { ".": { @@ -64,10 +64,10 @@ "@chainsafe/persistent-merkle-tree": "^0.6.1", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.14.3", - "@lodestar/config": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/types": "^1.16.0", - "@lodestar/utils": "^1.16.0", + "@lodestar/config": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/types": "^1.17.0", + "@lodestar/utils": "^1.17.0", "bigint-buffer": "^1.1.5", "buffer-xor": "^2.0.2" }, diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index b46e56303e08..284f88e378a9 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.16.0", + "version": "1.17.0", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -59,8 +59,8 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/bls-keystore": "^3.0.1", - "@lodestar/params": "^1.16.0", - "@lodestar/utils": "^1.16.0", + "@lodestar/params": "^1.17.0", + "@lodestar/utils": "^1.17.0", "axios": "^1.3.4", "testcontainers": "^10.2.1", "tmp": "^0.2.1", diff --git a/packages/types/package.json b/packages/types/package.json index 286d4b99e5cd..376778e6e8d3 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": { ".": { @@ -74,7 +74,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/ssz": "^0.14.3", - "@lodestar/params": "^1.16.0", + "@lodestar/params": "^1.17.0", "ethereum-cryptography": "^2.0.0" }, "keywords": [ diff --git a/packages/utils/package.json b/packages/utils/package.json index 6c3f372c0a1b..7ea0af4d738b 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.16.0", + "version": "1.17.0", "type": "module", "exports": "./lib/index.js", "files": [ diff --git a/packages/validator/package.json b/packages/validator/package.json index b5dba33d5d32..31ec656d0918 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.16.0", + "version": "1.17.0", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -47,17 +47,17 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/ssz": "^0.14.3", - "@lodestar/api": "^1.16.0", - "@lodestar/config": "^1.16.0", - "@lodestar/db": "^1.16.0", - "@lodestar/params": "^1.16.0", - "@lodestar/state-transition": "^1.16.0", - "@lodestar/types": "^1.16.0", - "@lodestar/utils": "^1.16.0", + "@lodestar/api": "^1.17.0", + "@lodestar/config": "^1.17.0", + "@lodestar/db": "^1.17.0", + "@lodestar/params": "^1.17.0", + "@lodestar/state-transition": "^1.17.0", + "@lodestar/types": "^1.17.0", + "@lodestar/utils": "^1.17.0", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { - "@lodestar/test-utils": "^1.16.0", + "@lodestar/test-utils": "^1.17.0", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" }