run block rpc (#476)

* run block rpc

* include logs

* improve

* no need to set head

* to u8a

* fix

* fix

packages/chopsticks/src/plugins/dry-run/dry-run-preimage.ts

@@ -81,7 +81,7 @@ export const dryRunPreimage = async (argv: Config) => {
     taskHandler(block),
   )
 
-  if (result.Error) {
+  if ('Error' in result) {
     throw new Error(result.Error)
   }
 

packages/chopsticks/src/plugins/run-block/index.ts

@@ -1,14 +1,18 @@
-import type yargs from 'yargs'
-
+import { GenericExtrinsic } from '@polkadot/types'
+import { Header } from '@polkadot/types/interfaces'
 import { HexString } from '@polkadot/util/types'
-import { defaultOptions, mockOptions } from '../../cli-options'
 import { writeFileSync } from 'node:fs'
+import { z } from 'zod'
+import _ from 'lodash'
+import type yargs from 'yargs'
+
+import { Block, Context, decodeKeyValue, runTask, taskHandler } from '@acala-network/chopsticks-core'
 
 import { Config } from '../../schema'
 import { defaultLogger } from '../../logger'
+import { defaultOptions, mockOptions } from '../../cli-options'
 import { generateHtmlDiffPreviewFile } from '../../utils/generate-html-diff'
 import { openHtml } from '../../utils/open-html'
-import { runTask, taskHandler } from '@acala-network/chopsticks-core'
 import { setupContext } from '../../context'
 
 export const cli = (y: yargs.Argv) => {
@@ -58,7 +62,7 @@ export const cli = (y: yargs.Argv) => {
         taskHandler(parent),
       )
 
-      if (result.Error) {
+      if ('Error' in result) {
         throw new Error(result.Error)
       }
 
@@ -82,3 +86,228 @@ export const cli = (y: yargs.Argv) => {
     },
   )
 }
+
+const zHex = z.custom<HexString>((val: any) => /^0x\w+$/.test(val))
+const zHash = z.string().length(66).and(zHex)
+
+const schema = z.object({
+  includeRaw: z.boolean().optional(),
+  includeParsed: z.boolean().optional(),
+  includeBlockDetails: z.boolean().optional(),
+  parent: zHash.optional(),
+  block: z.object({
+    header: z.any(),
+    extrinsics: z.array(zHex),
+  }),
+})
+
+type Params = z.infer<typeof schema>
+
+export interface RunBlockParams {
+  /**
+   * Include raw storage diff. Default to true
+   */
+  includeRaw: Params['includeRaw']
+  /**
+   * Include parsed storage diff in json format
+   */
+  includeParsed: Params['includeParsed']
+  /**
+   * Include block details such as parsed extrinsics in json format
+   */
+  includeBlockDetails: Params['includeBlockDetails']
+  /**
+   * The parent block hash to run on top of. Deafult to chain head.
+   */
+  parent: Params['parent']
+  /**
+   * Block to run
+   */
+  block: Params['block']
+}
+
+/**
+ * The phase of an execution.
+ * `number` means the phase is ApplyExtrinsic and the value is the extrinsic index.
+ */
+export type Phase = 'Initialization' | 'Finalization' | number // extrinsic index
+
+export interface RunBlockResponse {
+  /**
+   * The storage diff of each phase.
+   */
+  phases: {
+    /**
+     * The phase of the execution. See {@link Phase}.
+     */
+    phase: Phase
+    /**
+     * Parsed storage diff. Only available when `includeParsed` is true.
+     */
+    parsed?: Record<string, Record<string, any>>
+    /**
+     * Raw storage diff. Only available when `includeRaw` is true.
+     */
+    raw?: [HexString, HexString | null][]
+    /**
+     * Runtime logs.
+     */
+    logs?: string[]
+  }[]
+  /**
+   * Block details. Only available when `includeBlockDetails` is true.
+   */
+  blockDetails?: {
+    /**
+     * Block timestamp in ms
+     */
+    timestamp?: string
+    /**
+     * Parsed events in this block.
+     */
+    events?: { phase: Phase; section: string; method: string; args: any[] }[]
+    /**
+     * Parsed extrinsics in this block.
+     */
+    extrinsics: {
+      section: string
+      method: string
+      args: any[]
+      success: boolean
+    }[]
+  }
+}
+
+export const name = 'runBlock'
+
+/**
+ * Run a set of extrinsics on top of a block and get the storage diff
+ * and optionally the parsed storage diff and block details.
+ * NOTE: The extrinsics should include inherents or tranasctions may have unexpected results.
+ *
+ * This function is a dev rpc handler. Use `dev_runBlock` as the method name when calling it.
+ */
+export const rpc = async ({ chain }: Context, [params]: [RunBlockParams]): Promise<RunBlockResponse> => {
+  const { includeRaw, includeParsed, includeBlockDetails, parent, block } = schema.parse(params)
+
+  const includeRawStorage = includeRaw ?? true
+
+  const parentBlock = await chain.getBlock(parent)
+  if (!parentBlock) {
+    throw Error(`Invalid block hash ${parent}`)
+  }
+
+  const registry = await parentBlock.registry
+  const header = registry.createType<Header>('Header', block.header)
+
+  const wasm = await parentBlock.wasm
+
+  const blockNumber = parentBlock.number + 1
+  const hash: HexString = `0x${Math.round(Math.random() * 100000000)
+    .toString(16)
+    .padEnd(64, '0')}`
+
+  const newBlock = new Block(chain, blockNumber, hash, parentBlock, {
+    header,
+    extrinsics: [],
+    storage: parentBlock.storage,
+  })
+
+  const resp = {
+    phases: [],
+  } as RunBlockResponse
+
+  const run = async (fn: string, args: HexString[]) => {
+    const result = await runTask(
+      {
+        wasm,
+        calls: [[fn, args]],
+        mockSignatureHost: false,
+        allowUnresolvedImports: false,
+        runtimeLogLevel: 5,
+      },
+      taskHandler(newBlock),
+    )
+
+    if ('Error' in result) {
+      throw new Error(result.Error)
+    }
+
+    const resp = {} as any
+    const raw = result.Call.storageDiff
+
+    newBlock.pushStorageLayer().setAll(raw)
+
+    if (includeRawStorage) {
+      resp.raw = raw
+    }
+
+    if (includeParsed) {
+      const meta = await newBlock.meta
+      const parsed = {}
+      for (const [key, value] of raw) {
+        _.merge(parsed, decodeKeyValue(meta, newBlock, key, value, false))
+      }
+
+      // clear events because it can be stupidly large and redudant
+      if (parsed['system']?.['events']) {
+        delete parsed['system']['events']
+      }
+
+      resp.parsed = parsed
+    }
+
+    resp.logs = result.Call.runtimeLogs
+
+    return resp
+  }
+
+  const resInit = await run('Core_initialize_block', [header.toHex()])
+  resp.phases.push({ phase: 'Initialization', ...resInit })
+
+  for (const extrinsic of block.extrinsics) {
+    const res = await run('BlockBuilder_apply_extrinsic', [extrinsic])
+    resp.phases.push({ phase: resp.phases.length - 1, ...res })
+  }
+
+  const resFinalize = await run('BlockBuilder_finalize_block', [])
+  resp.phases.push({ phase: 'Finalization', ...resFinalize })
+
+  if (includeBlockDetails) {
+    const meta = await newBlock.meta
+    const registry = await newBlock.registry
+    const timestamp = await newBlock.read('u64', meta.query.timestamp.now)
+    const events = await newBlock.read('Vec<EventRecord>', meta.query.system.events)
+    const parsedEvents = events?.map((event) => ({
+      phase: event.phase.isApplyExtrinsic ? event.phase.asApplyExtrinsic.toNumber() : (event.phase.toString() as Phase),
+      section: event.event.section,
+      method: event.event.method,
+      args: event.event.data.map((arg) => arg.toJSON()),
+    }))
+    const extrinsics = block.extrinsics.map((extrinsic, idx) => {
+      const parsed = registry.createType<GenericExtrinsic>('GenericExtrinsic', extrinsic)
+      const resultEvent = events?.find(
+        ({ event, phase }) =>
+          event.section === 'system' &&
+          (event.method === 'ExtrinsicSuccess' || event.method === 'ExtrinsicFailed') &&
+          phase.isApplyExtrinsic &&
+          phase.asApplyExtrinsic.eq(idx),
+      )
+
+      return {
+        section: parsed.method.section,
+        method: parsed.method.method,
+        args: parsed.method.args.map((arg) => arg.toJSON()),
+        success: resultEvent?.event.method === 'ExtrinsicSuccess',
+      }
+    })
+
+    resp.blockDetails = {
+      timestamp: timestamp?.toString(),
+      events: parsedEvents,
+      extrinsics,
+    }
+  }
+
+  return resp
+}

packages/core/src/blockchain/block.ts

@@ -1,6 +1,7 @@
 import { ChainProperties, Header } from '@polkadot/types/interfaces'
 import { DecoratedMeta } from '@polkadot/types/metadata/decorate/types'
 import { Metadata, TypeRegistry } from '@polkadot/types'
+import { StorageEntry } from '@polkadot/types/primitive/types'
 import { expandMetadata } from '@polkadot/types/metadata'
 import { getSpecExtensions, getSpecHasher, getSpecTypes } from '@polkadot/types-known/util'
 import { hexToU8a, objectSpread, stringToHex } from '@polkadot/util'
@@ -164,6 +165,17 @@ export class Block {
     }
   }
 
+  async read<T extends string>(type: T, query: StorageEntry, ...args: any[]) {
+    const key = compactHex(query(...args))
+    const value = await this.get(key)
+    if (!value) {
+      return undefined
+    }
+
+    const registry = await this.registry
+    return registry.createType(type, hexToU8a(value))
+  }
+
   /**
    * Get paged storage keys.
    */
@@ -309,7 +321,7 @@ export class Block {
       },
       taskHandler(this),
     )
-    if (response.Call) {
+    if ('Call' in response) {
       for (const log of response.Call.runtimeLogs) {
         defaultLogger.info(`RuntimeLogs:\n${log}`)
       }

packages/core/src/blockchain/inherent/index.ts

@@ -20,7 +20,7 @@ export class SetTimestamp implements InherentProvider {
     const meta = await parent.meta
     const slotDuration = await getSlotDuration(parent.chain)
     const currentTimestamp = await getCurrentTimestamp(parent.chain)
-    return [new GenericExtrinsic(meta.registry, meta.tx.timestamp.set(currentTimestamp + slotDuration)).toHex()]
+    return [new GenericExtrinsic(meta.registry, meta.tx.timestamp.set(currentTimestamp + BigInt(slotDuration))).toHex()]
   }
 }
 

packages/core/src/utils/decoder.ts

@@ -48,7 +48,13 @@ export const decodeKey = (
   return {}
 }
 
-export const decodeKeyValue = (meta: DecoratedMeta, block: Block, key: HexString, value?: HexString | null) => {
+export const decodeKeyValue = (
+  meta: DecoratedMeta,
+  block: Block,
+  key: HexString,
+  value?: HexString | null,
+  toHuman = true,
+) => {
   const { storage, decodedKey } = decodeKey(meta, block, key)
 
   if (!storage || !decodedKey) {
@@ -60,7 +66,7 @@ export const decodeKeyValue = (meta: DecoratedMeta, block: Block, key: HexString
     if (storage.section === 'substrate' && storage.method === 'code') {
       return `:code blake2_256 ${blake2AsHex(value, 256)} (${hexToU8a(value).length} bytes)`
     }
-    return meta.registry.createType(decodedKey.outputType, hexToU8a(value)).toHuman()
+    return meta.registry.createType(decodedKey.outputType, hexToU8a(value))[toHuman ? 'toHuman' : 'toJSON']()
   }
 
   switch (decodedKey.args.length) {

packages/core/src/utils/index.ts

@@ -1,6 +1,6 @@
 import { HexString } from '@polkadot/util/types'
 import { StorageKey } from '@polkadot/types'
-import { compactStripLength, hexToU8a, u8aToHex } from '@polkadot/util'
+import { compactStripLength, u8aToHex } from '@polkadot/util'
 import { hexAddPrefix, hexStripPrefix } from '@polkadot/util/hex'
 
 import { Blockchain } from '../blockchain'
@@ -46,9 +46,11 @@ export const compactHex = (value: Uint8Array): HexString => {
 
 export const getParaId = async (chain: Blockchain) => {
   const meta = await chain.head.meta
-  const raw = await chain.head.get(compactHex(meta.query.parachainInfo.parachainId()))
-  if (!raw) throw new Error('Cannot find parachain id')
-  return meta.registry.createType('u32', hexToU8a(raw))
+  const id = await chain.head.read('u32', meta.query.parachainInfo.parachainId)
+  if (!id) {
+    throw new Error('Cannot find parachain id')
+  }
+  return id
 }
 
 export const isUrl = (url: string) => {

packages/core/src/utils/time-travel.ts

@@ -19,8 +19,8 @@ export const getCurrentSlot = async (chain: Blockchain) => {
 
 export const getCurrentTimestamp = async (chain: Blockchain) => {
   const meta = await chain.head.meta
-  const currentTimestampRaw = (await chain.head.get(compactHex(meta.query.timestamp.now()))) || '0x'
-  return meta.registry.createType('u64', hexToU8a(currentTimestampRaw)).toNumber()
+  const timestamp = await chain.head.read('u64', meta.query.timestamp.now)
+  return timestamp?.toBigInt() ?? 0n
 }
 
 export const getSlotDuration = async (chain: Blockchain) => {