Skip to content

Commit

Permalink
improve load rpc plugin (#514)
Browse files Browse the repository at this point in the history
* improve load plugin

* restore

* add load rpc plugin

* fix

* add cli plugin lazy load

* fix

* fix DISABLE_PLUGINS

* refactor

* update

* dev container

* keep extend plugin

* fix lint

* update read rpc methods

* revert executor build

* fix lint
  • Loading branch information
qiweiii committed Nov 10, 2023
1 parent c4144c7 commit 3488064
Show file tree
Hide file tree
Showing 27 changed files with 783 additions and 760 deletions.
3 changes: 3 additions & 0 deletions .devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
{
"postCreateCommand": "git submodule update --init"
}
8 changes: 8 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -158,3 +158,11 @@ External documentation on Chopsticks can be found at the following links:

- [Chopsticks types documentation](https://acalanetwork.github.io/chopsticks/docs)
- [Moonbeam documentation site](https://docs.moonbeam.network/builders/build/substrate-api/chopsticks/)

## Plugins

Chopsticks is designed to be extensible. You can write your own plugin to extend Chopsticks' functionality.
There are 2 types of plugins: `cli` and `rpc`. `cli` plugins are used to extend Chopsticks' CLI, while `rpc` plugins are used to extend Chopsticks' RPC.
To create a new plugin, you could check out the [run-block plugin](packages/chopsticks/src/plugins/run-block/) as an example.
2 changes: 1 addition & 1 deletion executor/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"url": "https://github.com/AcalaNetwork/chopsticks"
},
"scripts": {
"clean": "rm -rf web node",
"clean": "rm -rf browser node",
"build": "wasm-pack build --target web --out-dir browser; wasm-pack build --target nodejs --out-dir node; scripts/pack-wasm.cjs"
},
"dependencies": {
Expand Down
5 changes: 0 additions & 5 deletions packages/chopsticks/src/context.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ import { BlockEntry, GenesisProvider, defaultLogger, isUrl, setup, timeTravel }
import { Config } from './schema/index.js'
import { HexString } from '@polkadot/util/types'
import { SqliteDatabase } from '@acala-network/chopsticks-db'
import { loadRPCPlugins } from './plugins/index.js'
import { overrideStorage, overrideWasm } from './utils/override.js'
import axios from 'axios'

Expand Down Expand Up @@ -88,9 +87,5 @@ export const setupContext = async (argv: Config, overrideParent = false) => {
await overrideWasm(chain, argv['wasm-override'], at)
await overrideStorage(chain, argv['import-storage'], at)

if (!process.env.DISABLE_PLUGINS) {
await loadRPCPlugins()
}

return { chain }
}
39 changes: 39 additions & 0 deletions packages/chopsticks/src/plugins/decode-key/cli.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
import { Config } from '../../schema/index.js'
import { HexString } from '@polkadot/util/types'
import { decodeKey } from '@acala-network/chopsticks-core'
import { defaultOptions } from '../../cli-options.js'
import { setupContext } from '../../context.js'
import type { Argv } from 'yargs'

export const cli = (y: Argv) => {
y.command(
'decode-key <key>',
'Deocde a key',
(yargs) =>
yargs
.positional('key', {
desc: 'Key to decode',
type: 'string',
})
.options({
...defaultOptions,
}),
async (argv) => {
const context = await setupContext(argv as Config)
const { storage, decodedKey } = decodeKey(
await context.chain.head.meta,
context.chain.head,
argv.key as HexString,
)
if (storage && decodedKey) {
console.log(
`${storage.section}.${storage.method}`,
decodedKey.args.map((x) => JSON.stringify(x.toHuman())).join(', '),
)
} else {
console.log('Unknown')
}
process.exit(0)
},
)
}
40 changes: 1 addition & 39 deletions packages/chopsticks/src/plugins/decode-key/index.ts
Original file line number Diff line number Diff line change
@@ -1,39 +1 @@
import { Config } from '../../schema/index.js'
import { HexString } from '@polkadot/util/types'
import { decodeKey } from '@acala-network/chopsticks-core'
import { defaultOptions } from '../../cli-options.js'
import { setupContext } from '../../context.js'
import type { Argv } from 'yargs'

export const cli = (y: Argv) => {
y.command(
'decode-key <key>',
'Deocde a key',
(yargs) =>
yargs
.positional('key', {
desc: 'Key to decode',
type: 'string',
})
.options({
...defaultOptions,
}),
async (argv) => {
const context = await setupContext(argv as Config)
const { storage, decodedKey } = decodeKey(
await context.chain.head.meta,
context.chain.head,
argv.key as HexString,
)
if (storage && decodedKey) {
console.log(
`${storage.section}.${storage.method}`,
decodedKey.args.map((x) => JSON.stringify(x.toHuman())).join(', '),
)
} else {
console.log('Unknown')
}
process.exit(0)
},
)
}
export * from './cli.js'
3 changes: 0 additions & 3 deletions packages/chopsticks/src/plugins/dry-run/rpc.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,6 @@ const schema = z.object({
at: zHash.optional(),
})

// custom rpc name (optional). e.g. dryRun will be called as dev_dryRun
export const name = 'dryRun'

type Params = z.infer<typeof schema>

export interface DryRunParams {
Expand Down
91 changes: 91 additions & 0 deletions packages/chopsticks/src/plugins/follow-chain/cli.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
import { Block, defaultLogger, runTask, taskHandler } from '@acala-network/chopsticks-core'
import { Header } from '@polkadot/types/interfaces'
import { HexString } from '@polkadot/util/types'
import _ from 'lodash'
import type { Argv } from 'yargs'

import { createServer } from '../../server.js'
import { defaultOptions } from '../../cli-options.js'
import { handler } from '../../rpc/index.js'
import { setupContext } from '../../context.js'
import type { Config } from '../../schema/index.js'

const logger = defaultLogger.child({ name: 'follow-chain' })
const options = _.pick(defaultOptions, ['endpoint', 'wasm-override', 'runtime-log-level', 'offchain-worker'])

export const cli = (y: Argv) => {
y.command(
'follow-chain',
'Always follow the latest block on upstream',
(yargs) =>
yargs.options({
...options,
port: {
desc: 'Port to listen on',
number: true,
},
'head-mode': {
desc: 'Head mode',
choices: ['latest', 'finalized'],
default: 'finalized',
},
}),
async (argv) => {
const port = argv.port ?? 8000
const endpoint = argv.endpoint as string
if (/^(https|http):\/\//.test(endpoint || '')) {
throw Error('http provider is not supported')
}

const context = await setupContext(argv as Config, true)
const { close, port: listenPort } = await createServer(handler(context), port)
logger.info(`${await context.chain.api.getSystemChain()} RPC listening on port ${listenPort}`)

const chain = context.chain

chain.api[argv.headMode === 'latest' ? 'subscribeRemoteNewHeads' : 'subscribeRemoteFinalizedHeads'](
async (error, data) => {
try {
if (error) throw error
logger.info({ header: data }, `Follow ${argv.headMode} head from upstream`)
const parent = await chain.getBlock(data.parentHash)
if (!parent) throw Error(`Cannot find parent', ${data.parentHash}`)
const registry = await parent.registry
const header = registry.createType<Header>('Header', data)
const wasm = await parent.wasm

const block = new Block(chain, header.number.toNumber(), header.hash.toHex(), parent)
await chain.setHead(block)

const calls: [string, HexString[]][] = [['Core_initialize_block', [header.toHex()]]]

for (const extrinsic of await block.extrinsics) {
calls.push(['BlockBuilder_apply_extrinsic', [extrinsic]])
}

calls.push(['BlockBuilder_finalize_block', []])

const result = await runTask(
{
wasm,
calls,
mockSignatureHost: false,
allowUnresolvedImports: false,
runtimeLogLevel: (argv.runtimeLogLevel as number) || 0,
},
taskHandler(parent),
)

if ('Error' in result) {
throw new Error(result.Error)
}
} catch (e) {
logger.error(e, 'Error when processing new head')
await close()
process.exit(1)
}
},
)
},
)
}
92 changes: 1 addition & 91 deletions packages/chopsticks/src/plugins/follow-chain/index.ts
Original file line number Diff line number Diff line change
@@ -1,91 +1 @@
import { Block, defaultLogger, runTask, taskHandler } from '@acala-network/chopsticks-core'
import { Header } from '@polkadot/types/interfaces'
import { HexString } from '@polkadot/util/types'
import _ from 'lodash'
import type { Argv } from 'yargs'

import { createServer } from '../../server.js'
import { defaultOptions } from '../../cli-options.js'
import { handler } from '../../rpc/index.js'
import { setupContext } from '../../context.js'
import type { Config } from '../../schema/index.js'

const logger = defaultLogger.child({ name: 'follow-chain' })
const options = _.pick(defaultOptions, ['endpoint', 'wasm-override', 'runtime-log-level', 'offchain-worker'])

export const cli = (y: Argv) => {
y.command(
'follow-chain',
'Always follow the latest block on upstream',
(yargs) =>
yargs.options({
...options,
port: {
desc: 'Port to listen on',
number: true,
},
'head-mode': {
desc: 'Head mode',
choices: ['latest', 'finalized'],
default: 'finalized',
},
}),
async (argv) => {
const port = argv.port ?? 8000
const endpoint = argv.endpoint as string
if (/^(https|http):\/\//.test(endpoint || '')) {
throw Error('http provider is not supported')
}

const context = await setupContext(argv as Config, true)
const { close, port: listenPort } = await createServer(handler(context), port)
logger.info(`${await context.chain.api.getSystemChain()} RPC listening on port ${listenPort}`)

const chain = context.chain

chain.api[argv.headMode === 'latest' ? 'subscribeRemoteNewHeads' : 'subscribeRemoteFinalizedHeads'](
async (error, data) => {
try {
if (error) throw error
logger.info({ header: data }, `Follow ${argv.headMode} head from upstream`)
const parent = await chain.getBlock(data.parentHash)
if (!parent) throw Error(`Cannot find parent', ${data.parentHash}`)
const registry = await parent.registry
const header = registry.createType<Header>('Header', data)
const wasm = await parent.wasm

const block = new Block(chain, header.number.toNumber(), header.hash.toHex(), parent)
await chain.setHead(block)

const calls: [string, HexString[]][] = [['Core_initialize_block', [header.toHex()]]]

for (const extrinsic of await block.extrinsics) {
calls.push(['BlockBuilder_apply_extrinsic', [extrinsic]])
}

calls.push(['BlockBuilder_finalize_block', []])

const result = await runTask(
{
wasm,
calls,
mockSignatureHost: false,
allowUnresolvedImports: false,
runtimeLogLevel: (argv.runtimeLogLevel as number) || 0,
},
taskHandler(parent),
)

if ('Error' in result) {
throw new Error(result.Error)
}
} catch (e) {
logger.error(e, 'Error when processing new head')
await close()
process.exit(1)
}
},
)
},
)
}
export * from './cli.js'
33 changes: 23 additions & 10 deletions packages/chopsticks/src/plugins/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,23 +7,36 @@ import { defaultLogger } from '../logger.js'

const logger = defaultLogger.child({ name: 'plugin' })

export const pluginHandlers: Handlers = {}
export const rpcPluginHandlers: Handlers = {}

// list of plugins directory
const plugins = readdirSync(new URL('.', import.meta.url)).filter((file) =>
lstatSync(new URL(file, import.meta.url)).isDirectory(),
)

export const loadRPCPlugins = async () => {
for (const plugin of plugins) {
const location = new URL(`${plugin}/index.js`, import.meta.url)
const { rpc, name } = await import(location.pathname)
if (rpc) {
const methodName = name || _.camelCase(plugin)
pluginHandlers[`dev_${methodName}`] = rpc
logger.debug(`Registered plugin RPC: ${`dev_${methodName}`}`)
}
// find all rpc methods
export const rpcPluginMethods = plugins
.filter((name) => readdirSync(new URL(name, import.meta.url)).some((file) => file.startsWith('rpc')))
.map((name) => `dev_${_.camelCase(name)}`)

export const loadRpcPlugin = async (method: string) => {
if (process.env.DISABLE_PLUGINS) {
return undefined
}
if (rpcPluginHandlers[method]) return rpcPluginHandlers[method]

const plugin = _.snakeCase(method.split('dev_')[1]).replaceAll('_', '-')
if (!plugin) return undefined

const location = new URL(`${plugin}/index.js`, import.meta.url)

const { rpc } = await import(location.pathname)
if (!rpc) return undefined

rpcPluginHandlers[method] = rpc
logger.debug(`Registered plugin ${plugin} RPC`)

return rpc
}

export const pluginExtendCli = async (y: Argv) => {
Expand Down
Loading

0 comments on commit 3488064

Please sign in to comment.