From ab09a001c4c8a4f0e093ee55222b12ef38060046 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 18 Aug 2023 15:03:19 +0200 Subject: [PATCH 1/5] fix api --- packages/core/src/api.ts | 15 ++++++----- packages/core/src/blockchain/block.ts | 7 +++++- packages/core/src/blockchain/index.ts | 6 +++++ packages/core/src/blockchain/storage-layer.ts | 6 ++--- packages/core/src/setup.test.ts | 8 ++++++ packages/core/src/setup.ts | 25 ++++++++++++++----- packages/e2e/src/helper.ts | 10 +++++++- 7 files changed, 60 insertions(+), 17 deletions(-) create mode 100644 packages/core/src/setup.test.ts diff --git a/packages/core/src/api.ts b/packages/core/src/api.ts index ba0912c7..56935806 100644 --- a/packages/core/src/api.ts +++ b/packages/core/src/api.ts @@ -87,30 +87,33 @@ export class Api { } async getMetadata(hash?: string) { - return this.#provider.send('state_getMetadata', hash ? [hash] : []) + return this.#provider.send('state_getMetadata', hash ? [hash] : []) } async getBlockHash(blockNumber?: number) { - return this.#provider.send('chain_getBlockHash', Number.isInteger(blockNumber) ? [blockNumber] : []) + return this.#provider.send( + 'chain_getBlockHash', + Number.isInteger(blockNumber) ? [blockNumber] : [], + ) } async getHeader(hash?: string) { - return this.#provider.send
('chain_getHeader', hash ? [hash] : []) + return this.#provider.send
('chain_getHeader', hash ? [hash] : []) } async getBlock(hash?: string) { - return this.#provider.send('chain_getBlock', hash ? [hash] : []) + return this.#provider.send('chain_getBlock', hash ? [hash] : []) } async getStorage(key: string, hash?: string) { const params = [key] if (hash) params.push(hash) - return this.#provider.send('state_getStorage', params) + return this.#provider.send('state_getStorage', params) } async getKeysPaged(prefix: string, pageSize: number, startKey: string, hash?: string) { const params = [prefix, pageSize, startKey] if (hash) params.push(hash) - return this.#provider.send('state_getKeysPaged', params) + return this.#provider.send('state_getKeysPaged', params) } } diff --git a/packages/core/src/blockchain/block.ts b/packages/core/src/blockchain/block.ts index 24262042..22df6b6b 100644 --- a/packages/core/src/blockchain/block.ts +++ b/packages/core/src/blockchain/block.ts @@ -86,7 +86,12 @@ export class Block { get extrinsics(): HexString[] | Promise { if (!this.#extrinsics) { - this.#extrinsics = this.#chain.api.getBlock(this.hash).then((b) => b.block.extrinsics) + this.#extrinsics = this.#chain.api.getBlock(this.hash).then((b) => { + if (!b) { + throw new Error(`Block ${this.hash} not found`) + } + return b.block.extrinsics + }) } return this.#extrinsics } diff --git a/packages/core/src/blockchain/index.ts b/packages/core/src/blockchain/index.ts index ecdf110f..e0c74fbe 100644 --- a/packages/core/src/blockchain/index.ts +++ b/packages/core/src/blockchain/index.ts @@ -163,6 +163,9 @@ export class Blockchain { return blockFromDB } const hash = await this.api.getBlockHash(number) + if (!hash) { + return undefined + } const block = new Block(this, number, hash) this.#registerBlock(block) } @@ -184,6 +187,9 @@ export class Blockchain { const blockFromDB = await this.loadBlockFromDB(hash) if (!blockFromDB) { const header = await this.api.getHeader(hash) + if (!header) { + throw new Error(`Block ${hash} not found`) + } const block = new Block(this, Number(header.number), hash) this.#registerBlock(block) } diff --git a/packages/core/src/blockchain/storage-layer.ts b/packages/core/src/blockchain/storage-layer.ts index 841e18b5..879bd712 100644 --- a/packages/core/src/blockchain/storage-layer.ts +++ b/packages/core/src/blockchain/storage-layer.ts @@ -46,7 +46,7 @@ export class RemoteStorageLayer implements StorageLayerProvider { } } logger.trace({ at: this.#at, key }, 'RemoteStorageLayer get') - const data = await this.#api.getStorage(key, this.#at) + const data = await this.#api.getStorage(key, this.#at).then((data) => data ?? undefined) keyValuePair?.upsert({ key, blockHash: this.#at, value: data }, ['key', 'blockHash']) return data } @@ -61,7 +61,7 @@ export class RemoteStorageLayer implements StorageLayerProvider { logger.trace({ at: this.#at, prefix, pageSize, startKey }, 'RemoteStorageLayer getKeysPaged') // can't handle keyCache without prefix if (prefix.length < 66) { - return this.#api.getKeysPaged(prefix, pageSize, startKey, this.#at) + return this.#api.getKeysPaged(prefix, pageSize, startKey, this.#at).then((keys) => keys ?? []) } let batchComplete = false @@ -79,7 +79,7 @@ export class RemoteStorageLayer implements StorageLayerProvider { } // fetch a batch of keys - const batch = await this.#api.getKeysPaged(prefix, BATCH_SIZE, startKey, this.#at) + const batch = await this.#api.getKeysPaged(prefix, BATCH_SIZE, startKey, this.#at).then((keys) => keys ?? []) batchComplete = batch.length < BATCH_SIZE // feed the key cache diff --git a/packages/core/src/setup.test.ts b/packages/core/src/setup.test.ts new file mode 100644 index 00000000..12f9a4ca --- /dev/null +++ b/packages/core/src/setup.test.ts @@ -0,0 +1,8 @@ +import { test, expect } from "vitest"; +import { setup } from "./setup"; + +test("handle invalid block ", async () => { + await expect(setup({ endpoint: "wss://acala-rpc-0.aca-api.network", block: "0x" })).rejects.toThrow('invalid length') + await expect(setup({ endpoint: "wss://acala-rpc-0.aca-api.network", block: 999999999 })).rejects.toThrow('Cannot find block hash for 999999999') + await expect(setup({ endpoint: "wss://acala-rpc-0.aca-api.network", block: '0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d' })).rejects.toThrow('Cannot find header for 0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d') +}) diff --git a/packages/core/src/setup.ts b/packages/core/src/setup.ts index fcbd5001..fe2d6ef2 100644 --- a/packages/core/src/setup.ts +++ b/packages/core/src/setup.ts @@ -51,14 +51,22 @@ export const setup = async (options: Options) => { let blockHash: string if (options.block == null) { - blockHash = await api.getBlockHash() + blockHash = await api.getBlockHash().then((hash) => { + if (!hash) { + // should not happen, but just in case + throw new Error('Cannot find block hash') + } + return hash + }) } else if (typeof options.block === 'string' && options.block.startsWith('0x')) { blockHash = options.block as string } else if (Number.isInteger(+options.block)) { - blockHash = await api.getBlockHash(Number(options.block)) - if (!blockHash) { - throw new Error(`Cannot find block hash for ${options.block}`) - } + blockHash = await api.getBlockHash(Number(options.block)).then((hash) => { + if (!hash) { + throw new Error(`Cannot find block hash for ${options.block}`) + } + return hash + }) } else { throw new Error(`Invalid block number or hash: ${options.block}`) } @@ -70,7 +78,12 @@ export const setup = async (options: Options) => { db = await openDb(options.db) } - const header = await api.getHeader(blockHash) + const header = await api.getHeader(blockHash).then((header) => { + if (!header) { + throw new Error(`Cannot find header for ${blockHash}`) + } + return header + }) const inherents = new InherentProviders(new SetTimestamp(), [ new SetValidationData(), diff --git a/packages/e2e/src/helper.ts b/packages/e2e/src/helper.ts index b7ea1b7f..24c573dd 100644 --- a/packages/e2e/src/helper.ts +++ b/packages/e2e/src/helper.ts @@ -60,6 +60,9 @@ export const setupAll = async ({ await api.isReady const header = await api.getHeader(blockHash) + if (!header) { + throw new Error(`Cannot find header for ${blockHash}`) + } return { async setup() { @@ -70,12 +73,17 @@ export const setupAll = async ({ new SetBabeRandomness(), ]) + blockHash ??= await api.getBlockHash().then((hash) => hash ?? undefined) + if (!blockHash) { + throw new Error('Cannot find block hash') + } + const chain = new Blockchain({ api, buildBlockMode: BuildBlockMode.Manual, inherentProvider: inherents, header: { - hash: blockHash || (await api.getBlockHash()), + hash: blockHash, number: Number(header.number), }, mockSignatureHost, From 8890779f9d6ea3fbee23ceac753c5b38ba307c19 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 18 Aug 2023 15:10:54 +0200 Subject: [PATCH 2/5] test api --- packages/core/src/api.test.ts | 16 ++++++++++++++++ packages/core/src/setup.test.ts | 19 +++++++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) create mode 100644 packages/core/src/api.test.ts diff --git a/packages/core/src/api.test.ts b/packages/core/src/api.test.ts new file mode 100644 index 00000000..a74df902 --- /dev/null +++ b/packages/core/src/api.test.ts @@ -0,0 +1,16 @@ +import { Api } from './api' +import { WsProvider } from '@polkadot/api' +import { expect, test } from 'vitest' + +test('handle invalid block hash', async () => { + const api = new Api(new WsProvider('wss://acala-rpc-0.aca-api.network')) + await api.isReady + + await expect(api.getHeader('0x')).rejects.toThrow('invalid length') + expect(await api.getHeader('0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d')).toBeNull() + expect(await api.getBlock('0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d')).toBeNull() + expect(await api.getBlockHash(999999999)).toBeNull() + expect(await api.getBlockHash()).toBeTruthy() + + await api.disconnect() +}) diff --git a/packages/core/src/setup.test.ts b/packages/core/src/setup.test.ts index 12f9a4ca..42fd2cfc 100644 --- a/packages/core/src/setup.test.ts +++ b/packages/core/src/setup.test.ts @@ -1,8 +1,15 @@ -import { test, expect } from "vitest"; -import { setup } from "./setup"; +import { expect, test } from 'vitest' +import { setup } from './setup' -test("handle invalid block ", async () => { - await expect(setup({ endpoint: "wss://acala-rpc-0.aca-api.network", block: "0x" })).rejects.toThrow('invalid length') - await expect(setup({ endpoint: "wss://acala-rpc-0.aca-api.network", block: 999999999 })).rejects.toThrow('Cannot find block hash for 999999999') - await expect(setup({ endpoint: "wss://acala-rpc-0.aca-api.network", block: '0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d' })).rejects.toThrow('Cannot find header for 0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d') +test('handle invalid block ', async () => { + await expect(setup({ endpoint: 'wss://acala-rpc-0.aca-api.network', block: '0x' })).rejects.toThrow('invalid length') + await expect(setup({ endpoint: 'wss://acala-rpc-0.aca-api.network', block: 999999999 })).rejects.toThrow( + 'Cannot find block hash for 999999999', + ) + await expect( + setup({ + endpoint: 'wss://acala-rpc-0.aca-api.network', + block: '0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d', + }), + ).rejects.toThrow('Cannot find header for 0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d') }) From 6c504b46fadfa8818253df9fcbb616f0b809ec4f Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 18 Aug 2023 15:42:05 +0200 Subject: [PATCH 3/5] cleanup --- packages/core/src/api.test.ts | 4 ++++ packages/core/src/api.ts | 6 +----- packages/core/src/blockchain/index.ts | 3 ++- packages/core/src/blockchain/storage-layer.ts | 4 ++-- packages/core/src/db/entities.ts | 6 +++--- packages/core/src/genesis-provider.ts | 16 +--------------- 6 files changed, 13 insertions(+), 26 deletions(-) diff --git a/packages/core/src/api.test.ts b/packages/core/src/api.test.ts index a74df902..42bd0900 100644 --- a/packages/core/src/api.test.ts +++ b/packages/core/src/api.test.ts @@ -11,6 +11,10 @@ test('handle invalid block hash', async () => { expect(await api.getBlock('0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d')).toBeNull() expect(await api.getBlockHash(999999999)).toBeNull() expect(await api.getBlockHash()).toBeTruthy() + expect(await api.getStorage('0x0001')).toBeNull() + await expect( + api.getKeysPaged('0x', 1, '0x', '0xc87ae632b2cc4583a37659785f5098947acfdc6a36dbb07abcfa6ad694f97c5d'), + ).rejects.toThrow('Header was not found') await api.disconnect() }) diff --git a/packages/core/src/api.ts b/packages/core/src/api.ts index 56935806..92eab26a 100644 --- a/packages/core/src/api.ts +++ b/packages/core/src/api.ts @@ -86,10 +86,6 @@ export class Api { return this.#provider.send('system_chain', []) } - async getMetadata(hash?: string) { - return this.#provider.send('state_getMetadata', hash ? [hash] : []) - } - async getBlockHash(blockNumber?: number) { return this.#provider.send( 'chain_getBlockHash', @@ -114,6 +110,6 @@ export class Api { async getKeysPaged(prefix: string, pageSize: number, startKey: string, hash?: string) { const params = [prefix, pageSize, startKey] if (hash) params.push(hash) - return this.#provider.send('state_getKeysPaged', params) + return this.#provider.send('state_getKeysPaged', params) } } diff --git a/packages/core/src/blockchain/index.ts b/packages/core/src/blockchain/index.ts index e0c74fbe..1dfa4b7b 100644 --- a/packages/core/src/blockchain/index.ts +++ b/packages/core/src/blockchain/index.ts @@ -140,8 +140,9 @@ export class Blockchain { .getRepository(BlockEntity) .findOne({ where: { [typeof key === 'number' ? 'number' : 'hash']: key } }) if (blockData) { - const { hash, number, header, extrinsics, parentHash, storageDiff } = blockData + const { hash, number, header, extrinsics, parentHash } = blockData const parentBlock = parentHash ? this.#blocksByHash[parentHash] : undefined + const storageDiff = blockData.storageDiff ?? undefined const block = new Block(this, number, hash, parentBlock, { header, extrinsics, storageDiff }) this.#registerBlock(block) return block diff --git a/packages/core/src/blockchain/storage-layer.ts b/packages/core/src/blockchain/storage-layer.ts index 879bd712..64b73f09 100644 --- a/packages/core/src/blockchain/storage-layer.ts +++ b/packages/core/src/blockchain/storage-layer.ts @@ -42,7 +42,7 @@ export class RemoteStorageLayer implements StorageLayerProvider { if (this.#db) { const res = await keyValuePair?.findOne({ where: { key, blockHash: this.#at } }) if (res) { - return res.value + return res.value ?? undefined } } logger.trace({ at: this.#at, key }, 'RemoteStorageLayer get') @@ -79,7 +79,7 @@ export class RemoteStorageLayer implements StorageLayerProvider { } // fetch a batch of keys - const batch = await this.#api.getKeysPaged(prefix, BATCH_SIZE, startKey, this.#at).then((keys) => keys ?? []) + const batch = await this.#api.getKeysPaged(prefix, BATCH_SIZE, startKey, this.#at) batchComplete = batch.length < BATCH_SIZE // feed the key cache diff --git a/packages/core/src/db/entities.ts b/packages/core/src/db/entities.ts index 1ecced6e..a07762f6 100644 --- a/packages/core/src/db/entities.ts +++ b/packages/core/src/db/entities.ts @@ -5,7 +5,7 @@ import { HexString } from '@polkadot/util/types' export const KeyValuePair = new EntitySchema<{ blockHash: string key: string - value?: string + value: string | null }>({ name: 'KeyValuePair', columns: { @@ -30,9 +30,9 @@ export const BlockEntity = new EntitySchema<{ hash: HexString number: number header: Header - parentHash?: HexString + parentHash: HexString | null extrinsics: HexString[] - storageDiff?: Record + storageDiff: Record | null }>({ name: 'Block', columns: { diff --git a/packages/core/src/genesis-provider.ts b/packages/core/src/genesis-provider.ts index 2541a6ec..fb2acd75 100644 --- a/packages/core/src/genesis-provider.ts +++ b/packages/core/src/genesis-provider.ts @@ -7,11 +7,10 @@ import { ProviderInterfaceEmitted, ProviderStats, } from '@polkadot/rpc-provider/types' -import { stringToHex } from '@polkadot/util' import axios from 'axios' import { Genesis, genesisSchema } from './schema' -import { JsCallback, calculateStateRoot, emptyTaskHandler, runTask } from './executor' +import { JsCallback, calculateStateRoot, emptyTaskHandler } from './executor' import { isUrl } from './utils' export class GenesisProvider implements ProviderInterface { @@ -152,19 +151,6 @@ export class GenesisProvider implements ProviderInterface { return this.#genesis.id case 'system_name': return this.#genesis.name - case 'state_getMetadata': { - const code = this.#genesis.genesis.raw.top[stringToHex(':code')] as HexString - return runTask( - { - wasm: code, - calls: [['Metadata_metadata', []]], - mockSignatureHost: false, - allowUnresolvedImports: true, - runtimeLogLevel: 0, - }, - this._jsCallback, - ) - } case 'chain_getHeader': return this.getHeader() case 'chain_getBlock': From 8318899b768bd88627c765ab2e8d9fff15eb2f77 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 18 Aug 2023 16:19:31 +0200 Subject: [PATCH 4/5] cleanup --- packages/core/src/blockchain/storage-layer.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/core/src/blockchain/storage-layer.ts b/packages/core/src/blockchain/storage-layer.ts index 64b73f09..7a7b28f2 100644 --- a/packages/core/src/blockchain/storage-layer.ts +++ b/packages/core/src/blockchain/storage-layer.ts @@ -46,9 +46,9 @@ export class RemoteStorageLayer implements StorageLayerProvider { } } logger.trace({ at: this.#at, key }, 'RemoteStorageLayer get') - const data = await this.#api.getStorage(key, this.#at).then((data) => data ?? undefined) + const data = await this.#api.getStorage(key, this.#at) keyValuePair?.upsert({ key, blockHash: this.#at, value: data }, ['key', 'blockHash']) - return data + return data ?? undefined } async foldInto(_into: StorageLayer): Promise { @@ -61,7 +61,7 @@ export class RemoteStorageLayer implements StorageLayerProvider { logger.trace({ at: this.#at, prefix, pageSize, startKey }, 'RemoteStorageLayer getKeysPaged') // can't handle keyCache without prefix if (prefix.length < 66) { - return this.#api.getKeysPaged(prefix, pageSize, startKey, this.#at).then((keys) => keys ?? []) + return this.#api.getKeysPaged(prefix, pageSize, startKey, this.#at) } let batchComplete = false From 47f83fb9028f3bf0cbec57ef710460d5b6d801a5 Mon Sep 17 00:00:00 2001 From: Ermal Kaleci Date: Fri, 18 Aug 2023 16:21:48 +0200 Subject: [PATCH 5/5] cleanup --- packages/core/src/setup.ts | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/packages/core/src/setup.ts b/packages/core/src/setup.ts index fe2d6ef2..a202b623 100644 --- a/packages/core/src/setup.ts +++ b/packages/core/src/setup.ts @@ -78,12 +78,10 @@ export const setup = async (options: Options) => { db = await openDb(options.db) } - const header = await api.getHeader(blockHash).then((header) => { - if (!header) { - throw new Error(`Cannot find header for ${blockHash}`) - } - return header - }) + const header = await api.getHeader(blockHash) + if (!header) { + throw new Error(`Cannot find header for ${blockHash}`) + } const inherents = new InherentProviders(new SetTimestamp(), [ new SetValidationData(),