diff --git a/compose.test.yml b/compose.test.yml index 7dc6e5b0..8f62cb9a 100644 --- a/compose.test.yml +++ b/compose.test.yml @@ -29,7 +29,7 @@ services: POSTGRES_USER: test POSTGRES_PASSWORD: test healthcheck: - test: ["CMD-SHELL", "pg_isready -U test"] + test: ['CMD-SHELL', 'pg_isready -U test'] interval: 1s timeout: 3s retries: 5 @@ -37,7 +37,7 @@ services: redis: image: redis:7-alpine healthcheck: - test: ["CMD-SHELL", "redis-cli ping"] + test: ['CMD-SHELL', 'redis-cli ping'] interval: 1s timeout: 3s retries: 5 diff --git a/docs/api.md b/docs/api.md index 39b2b5fc..fca37da8 100644 --- a/docs/api.md +++ b/docs/api.md @@ -30,6 +30,7 @@ This returns the status of the indexer. "lastStakingBlockHeightExported": string | null "lastWasmBlockHeightExported": string | null "lastBankBlockHeightExported": string | null + "lastGovBlockHeightExported": string | null } ``` diff --git a/src/core/env.ts b/src/core/env.ts index e1056249..6ec1dbf7 100644 --- a/src/core/env.ts +++ b/src/core/env.ts @@ -3,6 +3,7 @@ import { Op, Sequelize } from 'sequelize' import { BankStateEvent, Contract, + GovStateEvent, StakingSlashEvent, WasmStateEvent, WasmStateEventTransformation, @@ -26,6 +27,8 @@ import { FormulaMapGetter, FormulaPrefetch, FormulaPrefetchTransformations, + FormulaProposalGetter, + FormulaProposalsGetter, FormulaSlashEventsGetter, FormulaTransformationDateGetter, FormulaTransformationMapGetter, @@ -1090,6 +1093,122 @@ export const getEnv = ({ ) } + const getProposal: FormulaProposalGetter = async (proposalId) => { + const dependentKey = getDependentKey( + GovStateEvent.dependentKeyNamespace, + proposalId + ) + dependentKeys?.push({ + key: dependentKey, + prefix: false, + }) + + // Check cache. + const cachedEvent = cache.events[dependentKey] + const event = + // If undefined, we haven't tried to fetch it yet. If not undefined, + // either it exists or it doesn't (null). + cachedEvent !== undefined + ? cachedEvent?.[0] + : await GovStateEvent.findOne({ + where: { + proposalId, + blockHeight: blockHeightFilter, + }, + order: [['blockHeight', 'DESC']], + }) + + // Type-check. Should never happen assuming dependent key namespaces are + // unique across different event types. + if (event && !(event instanceof GovStateEvent)) { + throw new Error('Incorrect event type.') + } + + // Cache event, null if nonexistent. + if (cachedEvent === undefined) { + cache.events[dependentKey] = event ? [event] : null + } + + // If no event found, return undefined. + if (!event) { + return + } + + // Call hook. + await onFetch?.([event]) + + return event.value + } + + const getProposals: FormulaProposalsGetter = async () => { + const dependentKey = + getDependentKey(GovStateEvent.dependentKeyNamespace) + ':' + dependentKeys?.push({ + key: dependentKey, + prefix: true, + }) + + // Check cache. + const cachedEvents = cache.events[dependentKey] + + const events = + // If undefined, we haven't tried to fetch them yet. If not undefined, + // either they exist or they don't (null). + cachedEvents !== undefined + ? ((cachedEvents ?? []) as GovStateEvent[]) + : await GovStateEvent.findAll({ + attributes: [ + // DISTINCT ON is not directly supported by Sequelize, so we need + // to cast to unknown and back to string to insert this at the + // beginning of the query. This ensures we use the most recent + // version of each denom. + Sequelize.literal( + 'DISTINCT ON("proposalId") \'\'' + ) as unknown as string, + 'proposalId', + 'blockHeight', + 'blockTimeUnixMs', + 'value', + ], + where: { + blockHeight: blockHeightFilter, + }, + order: [ + // Needs to be first so we can use DISTINCT ON. + ['proposalId', 'ASC'], + ['blockHeight', 'DESC'], + ], + }) + + // Type-check. Should never happen assuming dependent key namespaces are + // unique across different event types. + if (events.some((event) => !(event instanceof GovStateEvent))) { + throw new Error('Incorrect event type.') + } + + // Cache events, null if nonexistent. + if (cachedEvents === undefined) { + cache.events[dependentKey] = events.length ? events : null + } + + // If no events found, return undefined. + if (!events.length) { + return + } + + // Call hook. + await onFetch?.(events) + + // Create denom balance map. + return events.reduce( + (acc, { proposalId, value }) => ({ + ...acc, + [proposalId]: value, + }), + {} as Record> + ) + } + return { chainId, block, @@ -1119,5 +1238,8 @@ export const getEnv = ({ getBalance, getBalances, + + getProposal, + getProposals, } } diff --git a/src/core/types.ts b/src/core/types.ts index cfebb924..e95cceae 100644 --- a/src/core/types.ts +++ b/src/core/types.ts @@ -243,6 +243,14 @@ export type FormulaBalancesGetter = ( address: string ) => Promise | undefined> +export type FormulaProposalGetter = ( + proposalId: string +) => Promise | undefined> + +export type FormulaProposalsGetter = () => Promise< + Record> | undefined +> + export type Env = {}> = { chainId: string block: Block @@ -271,6 +279,8 @@ export type Env = {}> = { getTxEvents: FormulaTxEventsGetter getBalance: FormulaBalanceGetter getBalances: FormulaBalancesGetter + getProposal: FormulaProposalGetter + getProposals: FormulaProposalsGetter } export interface EnvOptions { @@ -466,6 +476,14 @@ export type ParsedBankStateEvent = { balance: string } +export type ParsedGovStateEvent = { + proposalId: string + blockHeight: string + blockTimeUnixMs: string + blockTimestamp: Date + value: Record +} + type RequireAtLeastOne = Pick< T, Exclude diff --git a/src/core/utils/block.ts b/src/core/utils/block.ts index 2661a0e9..62a4211c 100644 --- a/src/core/utils/block.ts +++ b/src/core/utils/block.ts @@ -1,13 +1,13 @@ import { Op } from 'sequelize' -import { BankStateEvent, WasmStateEvent } from '@/db' +import { BankStateEvent, GovStateEvent, WasmStateEvent } from '@/db' import { Block } from '../types' export const getBlockForTime = async ( blockTimeUnixMs: bigint ): Promise => { - const [wasmEvent, bankEvent] = await Promise.all([ + const [wasmEvent, bankEvent, govEvent] = await Promise.all([ await WasmStateEvent.findOne({ where: { blockTimeUnixMs: { @@ -26,17 +26,27 @@ export const getBlockForTime = async ( }, order: [['blockTimeUnixMs', 'DESC']], }), + await GovStateEvent.findOne({ + where: { + blockTimeUnixMs: { + [Op.gt]: 0, + [Op.lte]: blockTimeUnixMs, + }, + }, + order: [['blockTimeUnixMs', 'DESC']], + }), ]) // Choose latest block. return [ ...(wasmEvent ? [wasmEvent.block] : []), ...(bankEvent ? [bankEvent.block] : []), + ...(govEvent ? [govEvent.block] : []), ].sort((a, b) => Number(b.height - a.height))[0] } export const getFirstBlock = async (): Promise => { - const [wasmEvent, bankEvent] = await Promise.all([ + const [wasmEvent, bankEvent, govEvent] = await Promise.all([ await WasmStateEvent.findOne({ where: { blockTimeUnixMs: { @@ -53,11 +63,20 @@ export const getFirstBlock = async (): Promise => { }, order: [['blockTimeUnixMs', 'ASC']], }), + await GovStateEvent.findOne({ + where: { + blockTimeUnixMs: { + [Op.gt]: 0, + }, + }, + order: [['blockTimeUnixMs', 'ASC']], + }), ]) // Choose latest block. return [ ...(wasmEvent ? [wasmEvent.block] : []), ...(bankEvent ? [bankEvent.block] : []), + ...(govEvent ? [govEvent.block] : []), ].sort((a, b) => Number(b.height - a.height))[0] } diff --git a/src/core/utils/index.ts b/src/core/utils/index.ts index 322d738a..ff46c3b5 100644 --- a/src/core/utils/index.ts +++ b/src/core/utils/index.ts @@ -92,7 +92,7 @@ export const getDependentKey = ( namespace: DependentKeyNamespace, // If empty/undefined, wildcard used. ...keys: (string | undefined)[] -) => `${namespace}:${keys.map((key) => key || '*').join(':')}` +) => `${[namespace, ...keys].map((key) => key || '*').join(':')}` export const validateBlockString = (block: string, subject: string): Block => { let parsedBlock diff --git a/src/data/formulas/generic/gov.ts b/src/data/formulas/generic/gov.ts new file mode 100644 index 00000000..95f11305 --- /dev/null +++ b/src/data/formulas/generic/gov.ts @@ -0,0 +1,88 @@ +import { GenericFormula } from '@/core' + +export const proposal: GenericFormula< + Record | undefined, + { proposalId: string } +> = { + compute: async ({ getProposal, args: { proposalId } }) => { + if (!proposalId) { + throw new Error('missing `proposalId`') + } + + return await getProposal(proposalId) + }, +} + +export const proposals: GenericFormula< + { + proposals: Record[] + total: number + }, + { + offset?: string + limit?: string + } +> = { + compute: async ({ getProposals, args: { offset, limit } }) => { + const offsetNum = offset ? Math.max(0, Number(offset)) : 0 + const limitNum = limit ? Math.max(0, Number(limit)) : Infinity + + if (isNaN(offsetNum)) { + throw new Error('invalid `offset`') + } + if (isNaN(limitNum)) { + throw new Error('invalid `limit`') + } + + const proposals = (await getProposals()) || {} + + // Sort ascending. + const proposalIds = Object.keys(proposals).sort((a, b) => + Number(BigInt(a) - BigInt(b)) + ) + + return { + proposals: proposalIds + .slice(offsetNum, offsetNum + limitNum) + .map((proposalId) => proposals[proposalId]), + total: proposalIds.length, + } + }, +} + +export const reverseProposals: GenericFormula< + { + proposals: Record[] + total: number + }, + { + offset?: string + limit?: string + } +> = { + compute: async ({ getProposals, args: { offset, limit } }) => { + const offsetNum = offset ? Math.max(0, Number(offset)) : 0 + const limitNum = limit ? Math.max(0, Number(limit)) : Infinity + + if (isNaN(offsetNum)) { + throw new Error('invalid `offset`') + } + if (isNaN(limitNum)) { + throw new Error('invalid `limit`') + } + + const proposals = (await getProposals()) || {} + + // Sort descending. + const proposalIds = Object.keys(proposals).sort((a, b) => + Number(BigInt(b) - BigInt(a)) + ) + + return { + proposals: proposalIds + .slice(offsetNum, offsetNum + limitNum) + .map((proposalId) => proposals[proposalId]), + total: proposalIds.length, + } + }, +} diff --git a/src/data/formulas/generic/index.ts b/src/data/formulas/generic/index.ts index b4da0d97..137d4b32 100644 --- a/src/data/formulas/generic/index.ts +++ b/src/data/formulas/generic/index.ts @@ -1,2 +1,3 @@ export * from './priority_featured_daos' +export * as gov from './gov' export * as token from './token' diff --git a/src/db/connection.ts b/src/db/connection.ts index 2ff3e0d1..67e9ef62 100644 --- a/src/db/connection.ts +++ b/src/db/connection.ts @@ -16,6 +16,7 @@ import { Computation, ComputationDependency, Contract, + GovStateEvent, StakingSlashEvent, State, Validator, @@ -40,6 +41,7 @@ const getModelsForType = (type: DbType): SequelizeOptions['models'] => Computation, ComputationDependency, Contract, + GovStateEvent, StakingSlashEvent, State, Validator, diff --git a/src/db/models/GovStateEvent.ts b/src/db/models/GovStateEvent.ts new file mode 100644 index 00000000..05d9e50d --- /dev/null +++ b/src/db/models/GovStateEvent.ts @@ -0,0 +1,116 @@ +import { Op, WhereOptions } from 'sequelize' +import { AllowNull, Column, DataType, Table } from 'sequelize-typescript' + +import { Block, ComputationDependentKey, getDependentKey } from '@/core' + +import { DependendableEventModel, DependentKeyNamespace } from '../types' + +@Table({ + timestamps: true, + indexes: [ + // Only one event can happen to a proposal ID at a given block height. This + // ensures events are not duplicated if they attempt exporting multiple + // times. + { + unique: true, + fields: ['blockHeight', 'proposalId'], + }, + { + // Speeds up queries finding first newer dependent key to validate a + // computation. + fields: ['proposalId'], + }, + { + // Speed up ordering queries. + fields: ['blockHeight'], + }, + { + // Speed up ordering queries. + fields: ['blockTimeUnixMs'], + }, + ], +}) +export class GovStateEvent extends DependendableEventModel { + @AllowNull(false) + @Column(DataType.BIGINT) + proposalId!: string + + @AllowNull(false) + @Column(DataType.BIGINT) + blockHeight!: string + + @AllowNull(false) + @Column(DataType.BIGINT) + blockTimeUnixMs!: string + + @AllowNull(false) + @Column(DataType.DATE) + blockTimestamp!: Date + + @AllowNull + @Column(DataType.JSONB) + value!: any + + get block(): Block { + return { + height: BigInt(this.blockHeight), + timeUnixMs: BigInt(this.blockTimeUnixMs), + } + } + + get dependentKey(): string { + return getDependentKey(GovStateEvent.dependentKeyNamespace, this.proposalId) + } + + // Get the previous event for this proposalId. If this is the first event for + // this proposalId, return null. Cache the result so it can be reused since + // this shouldn't change. + previousEvent?: GovStateEvent | null + async getPreviousEvent(cache = true): Promise { + if (this.previousEvent === undefined || !cache) { + this.previousEvent = await GovStateEvent.findOne({ + where: { + proposalId: this.proposalId, + blockHeight: { + [Op.lt]: this.blockHeight, + }, + }, + order: [['blockHeight', 'DESC']], + }) + } + + return this.previousEvent + } + + static dependentKeyNamespace = DependentKeyNamespace.GovStateEvent + static blockHeightKey: string = 'blockHeight' + + // Returns a where clause that will match all events that are described by the + // dependent keys. + static getWhereClauseForDependentKeys( + dependentKeys: ComputationDependentKey[] + ): WhereOptions { + // If any dependent keys are prefixed or contain wildcards, just look for + // any proposal, since you can't wildcard search a bigint (and it would make + // no sense to do so). A formula will only ever need a specific proposal or + // all proposals. + if ( + dependentKeys.some(({ key, prefix }) => prefix || key.includes('*')) || + !dependentKeys.length + ) { + return {} + } + + const exactKeys = dependentKeys + .filter(({ key, prefix }) => !prefix && !key.includes('*')) + .map(({ key }) => + key.replace(new RegExp(`^${this.dependentKeyNamespace}:`), '') + ) + + return { + // Related logic in `makeComputationDependencyWhere` in + // `src/db/utils.ts`. + proposalId: exactKeys, + } + } +} diff --git a/src/db/models/State.ts b/src/db/models/State.ts index 1e4a26e6..485ab180 100644 --- a/src/db/models/State.ts +++ b/src/db/models/State.ts @@ -48,6 +48,10 @@ export class State extends Model { @Column(DataType.BIGINT) lastBankBlockHeightExported!: string | null + @AllowNull + @Column(DataType.BIGINT) + lastGovBlockHeightExported!: string | null + get latestBlock(): Block { return { height: BigInt(this.latestBlockHeight), @@ -77,6 +81,7 @@ export class State extends Model { lastStakingBlockHeightExported: 0n, lastWasmBlockHeightExported: 0n, lastBankBlockHeightExported: 0n, + lastGovBlockHeightExported: 0n, }) } diff --git a/src/db/models/index.ts b/src/db/models/index.ts index 1d3a17b0..e5b6160b 100644 --- a/src/db/models/index.ts +++ b/src/db/models/index.ts @@ -10,6 +10,7 @@ export * from './BankStateEvent' export * from './Computation' export * from './ComputationDependency' export * from './Contract' +export * from './GovStateEvent' export * from './StakingSlashEvent' export * from './State' export * from './Validator' diff --git a/src/db/types.ts b/src/db/types.ts index da8b3349..2556b027 100644 --- a/src/db/types.ts +++ b/src/db/types.ts @@ -13,6 +13,7 @@ export enum DependentKeyNamespace { WasmTxEvent = 'wasm_tx', StakingSlash = 'staking_slash', BankStateEvent = 'bank_state', + GovStateEvent = 'gov_state', } // Interface that event models must implement to be depended on by computations. diff --git a/src/db/utils.ts b/src/db/utils.ts index 59193ccf..69761ab4 100644 --- a/src/db/utils.ts +++ b/src/db/utils.ts @@ -8,6 +8,7 @@ import { BankStateEvent, Computation, ComputationDependency, + GovStateEvent, StakingSlashEvent, WasmStateEvent, WasmStateEventTransformation, @@ -261,6 +262,7 @@ export const getDependableEventModels = WasmTxEvent, StakingSlashEvent, BankStateEvent, + GovStateEvent, ] // Get the dependable event model for a given key based on its namespace. diff --git a/src/scripts/export/handlers/gov.ts b/src/scripts/export/handlers/gov.ts new file mode 100644 index 00000000..9f2a5487 --- /dev/null +++ b/src/scripts/export/handlers/gov.ts @@ -0,0 +1,141 @@ +import { fromBase64 } from '@cosmjs/encoding' +import retry from 'async-await-retry' +import { Proposal as ProposalV1 } from 'cosmjs-types/cosmos/gov/v1/gov' +import { Proposal as ProposalV1Beta1 } from 'cosmjs-types/cosmos/gov/v1beta1/gov' +import { Sequelize } from 'sequelize' + +import { ParsedGovStateEvent } from '@/core' +import { GovStateEvent, State } from '@/db' + +import { Handler, HandlerMaker } from '../types' + +const STORE_NAME = 'gov' + +export const gov: HandlerMaker = async () => { + const match: Handler['match'] = (trace) => { + // ProposalsKeyPrefix = 0x00 + // gov keys are formatted as: + // ProposalsKeyPrefix || proposalIdBytes + + // Not sure why a proposal would ever be deleted... + if (trace.operation === 'delete') { + return + } + + const keyData = fromBase64(trace.key) + if (keyData[0] !== 0x00 || keyData.length !== 10) { + return + } + + let proposalId + try { + proposalId = Buffer.from(keyData.slice(2)).readBigUInt64BE().toString() + } catch { + // Ignore decoding errors. + return + } + + // Get code ID and block timestamp from chain. + const blockHeight = BigInt(trace.metadata.blockHeight).toString() + + const blockTimeUnixMs = BigInt(trace.blockTimeUnixMs).toString() + const blockTimestamp = new Date(trace.blockTimeUnixMs) + + // Convert base64 value to utf-8 string, if present. + let valueData + try { + valueData = trace.value && fromBase64(trace.value) + } catch { + return + } + + // If failed to parse value, skip. + if (!valueData) { + return + } + + // Attempt v1 decoding, falling back to v1beta1. If both fail, ignore. + let value + try { + value = ProposalV1.decode(valueData) + } catch { + try { + value = ProposalV1Beta1.decode(valueData) + } catch { + return + } + } + + return { + id: [blockHeight, proposalId].join(':'), + proposalId, + blockHeight, + blockTimeUnixMs, + blockTimestamp, + value, + } + } + + const process: Handler['process'] = async (events) => { + const exportEvents = async () => + // Unique index on [blockHeight, proposalId] ensures that we don't insert + // duplicate events. If we encounter a duplicate, we update the `value` + // and `valueJson` fields in case event processing for a block was batched + // separately. + events.length > 0 + ? await GovStateEvent.bulkCreate(events, { + updateOnDuplicate: ['value', 'valueJson'], + }) + : [] + + // Retry 3 times with exponential backoff starting at 100ms delay. + const exportedEvents = (await retry(exportEvents, [], { + retriesMax: 3, + exponential: true, + interval: 100, + })) as GovStateEvent[] + + // TODO(computations): Re-enable computations when they are invalidated in the background. + // if (updateComputations) { + // await updateComputationValidityDependentOnChanges(exportedEvents) + // } + + // Store last block height exported, and update latest block + // height/time if the last export is newer. + const lastBlockHeightExported = + exportedEvents[exportedEvents.length - 1].blockHeight + const lastBlockTimeUnixMsExported = + exportedEvents[exportedEvents.length - 1].blockTimeUnixMs + await State.update( + { + lastGovBlockHeightExported: Sequelize.fn( + 'GREATEST', + Sequelize.col('lastGovBlockHeightExported'), + lastBlockHeightExported + ), + + latestBlockHeight: Sequelize.fn( + 'GREATEST', + Sequelize.col('latestBlockHeight'), + lastBlockHeightExported + ), + latestBlockTimeUnixMs: Sequelize.fn( + 'GREATEST', + Sequelize.col('latestBlockTimeUnixMs'), + lastBlockTimeUnixMsExported + ), + }, + { + where: { + singleton: true, + }, + } + ) + } + + return { + storeName: STORE_NAME, + match, + process, + } +} diff --git a/src/scripts/export/handlers/index.ts b/src/scripts/export/handlers/index.ts index c5227ee4..dfec816e 100644 --- a/src/scripts/export/handlers/index.ts +++ b/src/scripts/export/handlers/index.ts @@ -1,8 +1,10 @@ import { HandlerMaker } from '../types' import { bank } from './bank' +import { gov } from './gov' import { wasm } from './wasm' export const handlerMakers: Record> = { bank, + gov, wasm, } diff --git a/src/server/routes/indexer/getStatus.ts b/src/server/routes/indexer/getStatus.ts index af1d15cd..7c8f580d 100644 --- a/src/server/routes/indexer/getStatus.ts +++ b/src/server/routes/indexer/getStatus.ts @@ -11,6 +11,7 @@ type GetStatusResponse = lastStakingBlockHeightExported: string | null lastWasmBlockHeightExported: string | null lastBankBlockHeightExported: string | null + lastGovBlockHeightExported: string | null } | { error: string @@ -39,5 +40,7 @@ export const getStatus: Router.Middleware< state.lastWasmBlockHeightExported?.toString() || null, lastBankBlockHeightExported: state.lastBankBlockHeightExported?.toString() || null, + lastGovBlockHeightExported: + state.lastBankBlockHeightExported?.toString() || null, } } diff --git a/src/server/test/indexer/computer/formulas/gov.ts b/src/server/test/indexer/computer/formulas/gov.ts new file mode 100644 index 00000000..9de87f64 --- /dev/null +++ b/src/server/test/indexer/computer/formulas/gov.ts @@ -0,0 +1,401 @@ +import request from 'supertest' + +import { GovStateEvent, State } from '@/db' + +import { app } from '../../app' +import { ComputerTestOptions } from '../types' + +export const loadGovTests = (options: ComputerTestOptions) => { + describe('gov', () => { + describe('basic', () => { + beforeEach(async () => { + // Set up gov. + const blockTimestamp = new Date() + await GovStateEvent.bulkCreate([ + { + proposalId: '1', + blockHeight: 1, + blockTimeUnixMs: 1, + blockTimestamp, + value: { + proposal: '1-1', + }, + }, + { + proposalId: '1', + blockHeight: 2, + blockTimeUnixMs: 2, + blockTimestamp, + value: { + proposal: '1-2', + }, + }, + { + proposalId: '2', + blockHeight: 3, + blockTimeUnixMs: 3, + blockTimestamp, + value: { + proposal: '2-3', + }, + }, + { + proposalId: '3', + blockHeight: 4, + blockTimeUnixMs: 4, + blockTimestamp, + value: { + proposal: '3-4', + }, + }, + ]) + + await (await State.getSingleton())!.update({ + latestBlockHeight: 4, + latestBlockTimeUnixMs: 4, + lastGovBlockHeightExported: 4, + }) + }) + + it('returns correct proposal response for a single block', async () => { + await request(app.callback()) + .get('/generic/_/gov/proposal?block=1:1&proposalId=1') + .set('x-api-key', options.apiKey) + .expect(200) + .expect({ + proposal: '1-1', + }) + + await request(app.callback()) + .get('/generic/_/gov/proposal?block=3:3&proposalId=1') + .set('x-api-key', options.apiKey) + .expect(200) + .expect({ + proposal: '1-2', + }) + + // Returns latest if no block. + await request(app.callback()) + .get('/generic/_/gov/proposal?proposalId=1') + .set('x-api-key', options.apiKey) + .expect(200) + .expect({ + proposal: '1-2', + }) + }) + + it('returns correct proposal response for multiple blocks', async () => { + await request(app.callback()) + .get('/generic/_/gov/proposal?blocks=1:1..3:3&proposalId=1') + .set('x-api-key', options.apiKey) + .expect(200) + .expect([ + { + value: { + proposal: '1-1', + }, + blockHeight: 1, + blockTimeUnixMs: 1, + }, + { + value: { + proposal: '1-2', + }, + blockHeight: 2, + blockTimeUnixMs: 2, + }, + ]) + + await request(app.callback()) + .get( + '/generic/_/gov/proposal?blocks=1:1..3:3&blockStep=2&proposalId=1' + ) + .set('x-api-key', options.apiKey) + .expect(200) + .expect([ + { + at: '1', + value: { + proposal: '1-1', + }, + blockHeight: 1, + blockTimeUnixMs: 1, + }, + { + at: '3', + value: { + proposal: '1-2', + }, + blockHeight: 2, + blockTimeUnixMs: 2, + }, + ]) + }) + + it('returns correct proposal response for multiple times', async () => { + await request(app.callback()) + .get('/generic/_/gov/proposal?times=1..3&proposalId=1') + .set('x-api-key', options.apiKey) + .expect(200) + .expect([ + { + value: { + proposal: '1-1', + }, + blockHeight: 1, + blockTimeUnixMs: 1, + }, + { + value: { + proposal: '1-2', + }, + blockHeight: 2, + blockTimeUnixMs: 2, + }, + ]) + + await request(app.callback()) + .get('/generic/_/gov/proposal?times=1..3&timeStep=2&proposalId=1') + .set('x-api-key', options.apiKey) + .expect(200) + .expect([ + { + at: '1', + value: { + proposal: '1-1', + }, + blockHeight: 1, + blockTimeUnixMs: 1, + }, + { + at: '3', + value: { + proposal: '1-2', + }, + blockHeight: 2, + blockTimeUnixMs: 2, + }, + ]) + }) + + it('returns correct proposals response for a single block', async () => { + await request(app.callback()) + .get('/generic/_/gov/proposals?block=1:1') + .set('x-api-key', options.apiKey) + .expect(200) + .expect({ + proposals: [ + { + proposal: '1-1', + }, + ], + total: 1, + }) + + await request(app.callback()) + .get('/generic/_/gov/proposals?block=3:3') + .set('x-api-key', options.apiKey) + .expect(200) + .expect({ + proposals: [ + { + proposal: '1-2', + }, + { + proposal: '2-3', + }, + ], + total: 2, + }) + + // Returns latest if no block. + await request(app.callback()) + .get('/generic/_/gov/proposals') + .set('x-api-key', options.apiKey) + .expect(200) + .expect({ + proposals: [ + { + proposal: '1-2', + }, + { + proposal: '2-3', + }, + { + proposal: '3-4', + }, + ], + total: 3, + }) + }) + + it('returns correct proposals response for multiple blocks', async () => { + await request(app.callback()) + .get('/generic/_/gov/proposals?blocks=1:1..3:3') + .set('x-api-key', options.apiKey) + .expect(200) + .expect([ + { + value: { + proposals: [ + { + proposal: '1-1', + }, + ], + total: 1, + }, + blockHeight: 1, + blockTimeUnixMs: 1, + }, + { + value: { + proposals: [ + { + proposal: '1-2', + }, + ], + total: 1, + }, + blockHeight: 2, + blockTimeUnixMs: 2, + }, + { + value: { + proposals: [ + { + proposal: '1-2', + }, + { + proposal: '2-3', + }, + ], + total: 2, + }, + blockHeight: 3, + blockTimeUnixMs: 3, + }, + ]) + + await request(app.callback()) + .get('/generic/_/gov/proposals?blocks=1:1..3:3&blockStep=2') + .set('x-api-key', options.apiKey) + .expect(200) + .expect([ + { + at: '1', + value: { + proposals: [ + { + proposal: '1-1', + }, + ], + total: 1, + }, + blockHeight: 1, + blockTimeUnixMs: 1, + }, + { + at: '3', + value: { + proposals: [ + { + proposal: '1-2', + }, + { + proposal: '2-3', + }, + ], + total: 2, + }, + blockHeight: 3, + blockTimeUnixMs: 3, + }, + ]) + }) + + it('returns correct proposals response for multiple times', async () => { + await request(app.callback()) + .get('/generic/_/gov/proposals?times=1..3') + .set('x-api-key', options.apiKey) + .expect(200) + .expect([ + { + value: { + proposals: [ + { + proposal: '1-1', + }, + ], + total: 1, + }, + blockHeight: 1, + blockTimeUnixMs: 1, + }, + { + value: { + proposals: [ + { + proposal: '1-2', + }, + ], + total: 1, + }, + blockHeight: 2, + blockTimeUnixMs: 2, + }, + { + value: { + proposals: [ + { + proposal: '1-2', + }, + { + proposal: '2-3', + }, + ], + total: 2, + }, + blockHeight: 3, + blockTimeUnixMs: 3, + }, + ]) + + await request(app.callback()) + .get('/generic/_/gov/proposals?times=1..3&timeStep=2') + .set('x-api-key', options.apiKey) + .expect(200) + .expect([ + { + at: '1', + value: { + proposals: [ + { + proposal: '1-1', + }, + ], + total: 1, + }, + blockHeight: 1, + blockTimeUnixMs: 1, + }, + { + at: '3', + value: { + proposals: [ + { + proposal: '1-2', + }, + { + proposal: '2-3', + }, + ], + total: 2, + }, + blockHeight: 3, + blockTimeUnixMs: 3, + }, + ]) + }) + }) + }) +} diff --git a/src/server/test/indexer/computer/formulas/index.ts b/src/server/test/indexer/computer/formulas/index.ts index c323b27e..b7092410 100644 --- a/src/server/test/indexer/computer/formulas/index.ts +++ b/src/server/test/indexer/computer/formulas/index.ts @@ -6,6 +6,7 @@ import { Contract } from '@/db' import { app } from '../../app' import { ComputerTestOptions } from '../types' import { loadBankTests } from './bank' +import { loadGovTests } from './gov' import { loadStakingTests } from './staking' import { loadWasmTests } from './wasm' @@ -16,6 +17,7 @@ export const loadFormulasTests = (options: ComputerTestOptions) => { }) loadBankTests(options) + loadGovTests(options) loadStakingTests(options) loadWasmTests(options) diff --git a/src/server/test/indexer/getStatus.test.ts b/src/server/test/indexer/getStatus.test.ts index 28163dc9..c51c7588 100644 --- a/src/server/test/indexer/getStatus.test.ts +++ b/src/server/test/indexer/getStatus.test.ts @@ -22,6 +22,8 @@ describe('GET /status', () => { state!.lastWasmBlockHeightExported?.toString() || null, lastBankBlockHeightExported: state!.lastBankBlockHeightExported?.toString() || null, + lastGovBlockHeightExported: + state!.lastGovBlockHeightExported?.toString() || null, }) }) }) diff --git a/state-dump/dump.go b/state-dump/dump.go index fdbedcac..747e39bf 100644 --- a/state-dump/dump.go +++ b/state-dump/dump.go @@ -36,9 +36,13 @@ type ( ) var ( - BalancesPrefix = []byte{0x02} + // bank + BalancesPrefix = []byte{0x02} + // wasm ContractKeyPrefix = []byte{0x02} ContractStorePrefix = []byte{0x03} + // gov + ProposalsKeyPrefix = []byte{0x00} ) func main() { @@ -97,9 +101,20 @@ func main() { count := 0 for ; iter.Valid(); iter.Next() { key := iter.Key() - // Only write contract keys. - if key[0] != ContractKeyPrefix[0] && key[0] != ContractStorePrefix[0] { - continue + + // Only write relevant keys. + if storeName == "wasm" { + if key[0] != ContractKeyPrefix[0] && key[0] != ContractStorePrefix[0] { + continue + } + } else if storeName == "bank" { + if key[0] != BalancesPrefix[0] { + continue + } + } else if storeName == "gov" { + if key[0] != ProposalsKeyPrefix[0] { + continue + } } // Make sure key is for the given address. Different stores have the address