Skip to content

Commit

Permalink
support well known keys (#507)
Browse files Browse the repository at this point in the history
* support well known keys

* no need to toString

* fix decode

* code is handled

* fix

* fmt

* Update packages/chopsticks/src/plugins/run-block/index.ts

Co-authored-by: Qiwei Yang <yangqiwei97@gmail.com>

---------

Co-authored-by: Qiwei Yang <yangqiwei97@gmail.com>
  • Loading branch information
xlc and qiweiii authored Nov 2, 2023
1 parent 08fccbc commit 6e22206
Show file tree
Hide file tree
Showing 5 changed files with 206 additions and 31 deletions.
11 changes: 9 additions & 2 deletions packages/chopsticks/src/plugins/run-block/index.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import { GenericExtrinsic } from '@polkadot/types'
import { Header } from '@polkadot/types/interfaces'
import { HexString } from '@polkadot/util/types'
import { u8aToHex } from '@polkadot/util'
import { writeFileSync } from 'node:fs'
import { z } from 'zod'
import _ from 'lodash'
Expand Down Expand Up @@ -200,6 +201,7 @@ export const name = 'runBlock'
* Run a set of extrinsics on top of a block and get the storage diff
* and optionally the parsed storage diff and block details.
* NOTE: The extrinsics should include inherents or tranasctions may have unexpected results.
* NOTE: system.events and system.extrinsicData are excluded from storage diff to reduce size.
*
* This function is a dev rpc handler. Use `dev_runBlock` as the method name when calling it.
*/
Expand Down Expand Up @@ -236,6 +238,8 @@ export const rpc = async ({ chain }: Context, [params]: [RunBlockParams]): Promi

// exclude system events because it can be stupidly large and redudant
const systemEventsKey = compactHex(meta.query.system.events())
// large and not really useful
const systemExtrinsicDataKey = u8aToHex(meta.query.system.extrinsicData.keyPrefix())

const run = async (fn: string, args: HexString[]) => {
const result = await runTask(
Expand All @@ -262,6 +266,9 @@ export const rpc = async ({ chain }: Context, [params]: [RunBlockParams]): Promi
if (key === systemEventsKey) {
continue
}
if (key.startsWith(systemExtrinsicDataKey)) {
continue
}

const obj = {} as (typeof resp)['storageDiff'][number]
if (includeRawStorage) {
Expand All @@ -273,8 +280,8 @@ export const rpc = async ({ chain }: Context, [params]: [RunBlockParams]): Promi
obj.parsed = {
section: decoded.section,
method: decoded.method,
key: decoded.key?.map((x) => x.toString()),
value: decoded.value?.toString(),
key: decoded.key,
value: decoded.value,
}
}
}
Expand Down
16 changes: 12 additions & 4 deletions packages/core/src/utils/decoder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,11 @@ import { DecoratedMeta } from '@polkadot/types/metadata/decorate/types'
import { HexString } from '@polkadot/util/types'
import { StorageEntry } from '@polkadot/types/primitive/types'
import { StorageKey } from '@polkadot/types'
import { blake2AsHex } from '@polkadot/util-crypto'
import { hexToU8a, u8aToHex } from '@polkadot/util'
import _ from 'lodash'

import { decodeWellKnownKey } from './well-known-keys'

const _CACHE: Record<string, Map<HexString, StorageEntry>> = {}

const getCache = (uid: string): Map<HexString, StorageEntry> => {
Expand Down Expand Up @@ -55,6 +56,16 @@ export const decodeKeyValue = (
value?: HexString | null,
toHuman = true,
) => {
const res = decodeWellKnownKey(meta.registry, key, value)
if (res) {
return {
section: 'substrate',
method: res.name,
key: res.key,
value: res.value,
}
}

const { storage, decodedKey } = decodeKey(meta, block, key)

if (!storage || !decodedKey) {
Expand All @@ -63,9 +74,6 @@ export const decodeKeyValue = (

const decodeValue = () => {
if (!value) return null
if (storage.section === 'substrate' && storage.method === 'code') {
return `:code blake2_256 ${blake2AsHex(value, 256)} (${hexToU8a(value).length} bytes)`
}
return meta.registry.createType(decodedKey.outputType, hexToU8a(value))[toHuman ? 'toHuman' : 'toJSON']()
}

Expand Down
75 changes: 75 additions & 0 deletions packages/core/src/utils/well-known-keys.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
import { HexString } from '@polkadot/util/types'
import { Registry } from '@polkadot/types-codec/types'
import { blake2AsHex } from '@polkadot/util-crypto'
import { hexToU8a, stringToHex } from '@polkadot/util'

const decodeValue = (type: string) => (registry: Registry, value: HexString) => {
return registry.createType(type, hexToU8a(value)).toJSON()
}

// https://github.com/paritytech/polkadot-sdk/issues/2126
const wellKnownKeys = [
{
name: 'code',
key: ':code',
decodeValue: (_registry: Registry, value: HexString) => {
return `<:code blake2_256 ${blake2AsHex(value, 256)} (${value.length / 2 - 1} bytes)>`
},
},
{
name: 'heapPages',
key: ':heappages',
type: 'u64',
},
{
name: 'extrinsicIndex',
key: ':extrinsic_index',
type: 'u32',
},
{
name: 'intrablockEntropy',
key: ':intrablock_entropy',
type: '[u8; 32]',
},
{
name: 'transactionLevel',
key: ':transaction_level:',
type: 'u32',
},
{
name: 'grandpaAuthorities',
key: ':grandpa_authorities',
type: '(u8, AuthorityList)',
},
{
name: 'relayDispatchQueueRemainingCapacity',
prefix: ':relay_dispatch_queue_remaining_capacity',
decodeKey: (registry: Registry, key: HexString) => {
return [registry.createType('u32', hexToU8a(key)).toJSON()]
},
type: '(u32, u32)',
},
].map((def) => {
const prefix = stringToHex(def.prefix || def.key)
return {
name: def.name,
prefix,
decodeKey: def.decodeKey || ((_registry: Registry, key: HexString) => [key]),
decodeValue: def.decodeValue || decodeValue(def.type),
}
})

export const decodeWellKnownKey = (registry: Registry, key: HexString, value?: HexString | null) => {
for (const defs of wellKnownKeys) {
if (key.startsWith(defs.prefix)) {
const remaining = key.slice(defs.prefix.length)
const decodedKey = remaining ? defs.decodeKey(registry, `0x${remaining}`) : undefined
const decodedValue = value ? defs.decodeValue(registry, value) : undefined
return {
name: defs.name,
key: decodedKey ?? [],
value: decodedValue,
}
}
}
}
73 changes: 69 additions & 4 deletions packages/e2e/src/__snapshots__/decoder.test.ts.snap
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html

exports[`decoder > decode key-value 1`] = `
exports[`decoder > with acala > decode key-value 1`] = `
{
"key": [
"25fqepuLngYL2DK9ApTejNzqPadUUZ9ALYyKWX2jyvEiuZLa",
Expand All @@ -22,7 +22,7 @@ exports[`decoder > decode key-value 1`] = `
}
`;

exports[`decoder > decode key-value 2`] = `
exports[`decoder > with acala > decode key-value 2`] = `
{
"system": {
"account": {
Expand All @@ -43,7 +43,7 @@ exports[`decoder > decode key-value 2`] = `
}
`;

exports[`decoder > decode key-value 3`] = `
exports[`decoder > with acala > decode key-value 3`] = `
{
"key": [
"25fqepuLngYL2DK9ApTejNzqPadUUZ9ALYyKWX2jyvEiuZLa",
Expand All @@ -61,7 +61,7 @@ exports[`decoder > decode key-value 3`] = `
}
`;

exports[`decoder > decode key-value 4`] = `
exports[`decoder > with acala > decode key-value 4`] = `
{
"tokens": {
"accounts": {
Expand All @@ -77,6 +77,71 @@ exports[`decoder > decode key-value 4`] = `
}
`;

exports[`decoder > with acala > decode key-value 5`] = `
{
"key": [],
"method": "now",
"section": "timestamp",
"value": "64,188,750,128,742,400",
}
`;

exports[`decoder > with acala > decode key-value 6`] = `
{
"timestamp": {
"now": "64,188,750,128,742,400",
},
}
`;

exports[`decoder > with acala > decode keys 1`] = `
{
"decodedKey": "0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d",
"storage": [Function],
}
`;

exports[`decoder > with acala > works with well known keys 1`] = `
{
"key": [],
"method": "code",
"section": "substrate",
"value": "<:code blake2_256 0x97e86e044a53385b642a902fd8ed05534d7590412a608f43dbb70e1f0e3664c7 (4 bytes)>",
}
`;

exports[`decoder > with acala > works with well known keys 2`] = `
{
"key": [
3340,
],
"method": "relayDispatchQueueRemainingCapacity",
"section": "substrate",
"value": [
174762,
1048576,
],
}
`;

exports[`decoder > with acala > works with well known keys 3`] = `
{
"key": [],
"method": "transactionLevel",
"section": "substrate",
"value": undefined,
}
`;

exports[`decoder > with acala > works with well known keys 4`] = `
{
"key": [],
"method": "extrinsicIndex",
"section": "substrate",
"value": 2,
}
`;

exports[`decoder > works with multiple chains 1`] = `
{
"key": [
Expand Down
62 changes: 41 additions & 21 deletions packages/e2e/src/decoder.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,33 +7,53 @@ const SYSTEM_ACCOUNT =
'0x26aa394eea5630e07c48ae0c9558cef7b99d880ec681799c0cf30e8886371da9de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d'
const TOKENS_ACCOUNTS =
'0x99971b5749ac43e0235e41b0d37869188ee7418a6531173d60d1f6a82d8f4d51de1e86a9a8c739864cf3cc5ec2bea59fd43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d01a12dfa1fa4ab9a0000'
const TIMESTAMPE_NOW = '0xf0c365c3cf59d671eb72da0e7a4113c49f1f0515f462cdcf84e0f1d6045dfcbb'

describe('decoder', async () => {
const { chain, teardown } = await networks.acala()
describe('with acala', async () => {
const { chain, teardown } = await networks.acala()

afterAll(async () => {
await teardown()
})
afterAll(async () => {
await teardown()
})

it('decode keys', async () => {
const { storage, decodedKey } = decodeKey(await chain.head.meta, chain.head, SYSTEM_ACCOUNT)
expect(storage?.section).eq('system')
expect(storage?.method).eq('account')
expect(decodedKey?.args.map((x) => x.toHuman())).contains('25fqepuLngYL2DK9ApTejNzqPadUUZ9ALYyKWX2jyvEiuZLa')
})
it('decode keys', async () => {
expect(decodeKey(await chain.head.meta, chain.head, SYSTEM_ACCOUNT)).toMatchSnapshot()
})

it('decode key-value', async () => {
const meta = await chain.head.meta
const data = { data: { free: 10000000000 } }
const value = meta.registry.createType('AccountInfo', data)
const decoded = decodeKeyValue(meta, chain.head, SYSTEM_ACCOUNT, value.toHex())
expect(decoded).toMatchSnapshot()
expect(toStorageObject(decoded)).toMatchSnapshot()
it('decode key-value', async () => {
const meta = await chain.head.meta
const data = { data: { free: 10000000000 } }
const value = meta.registry.createType('AccountInfo', data)
const decoded = decodeKeyValue(meta, chain.head, SYSTEM_ACCOUNT, value.toHex())
expect(decoded).toMatchSnapshot()
expect(toStorageObject(decoded)).toMatchSnapshot()

const ormlAccountData = meta.registry.createType('AccountData', data.data)
const decoded2 = decodeKeyValue(meta, chain.head, TOKENS_ACCOUNTS, ormlAccountData.toHex())
expect(decoded2).toMatchSnapshot()
expect(toStorageObject(decoded2)).toMatchSnapshot()

const timestampNow = meta.registry.createType('Moment', data.data)
const decoded3 = decodeKeyValue(meta, chain.head, TIMESTAMPE_NOW, timestampNow.toHex())
expect(decoded3).toMatchSnapshot()
expect(toStorageObject(decoded3)).toMatchSnapshot()
})

const ormlAccountData = meta.registry.createType('AccountData', data.data)
const decoded2 = decodeKeyValue(meta, chain.head, TOKENS_ACCOUNTS, ormlAccountData.toHex())
expect(decoded2).toMatchSnapshot()
expect(toStorageObject(decoded2)).toMatchSnapshot()
it('works with well known keys', async () => {
const meta = await chain.head.meta
expect(decodeKeyValue(meta, chain.head, '0x3a636f6465', '0x12345678')).toMatchSnapshot()
expect(
decodeKeyValue(
meta,
chain.head,
'0x3a72656c61795f64697370617463685f71756575655f72656d61696e696e675f63617061636974790c0d0000',
'0xaaaa020000001000',
),
).toMatchSnapshot()
expect(decodeKeyValue(meta, chain.head, '0x3a7472616e73616374696f6e5f6c6576656c3a')).toMatchSnapshot()
expect(decodeKeyValue(meta, chain.head, '0x3a65787472696e7369635f696e646578', '0x02000000')).toMatchSnapshot()
})
})

it('works with multiple chains', async () => {
Expand Down

0 comments on commit 6e22206

Please sign in to comment.