From e51bc61d1a3370533a74b67b632925ec15f9623a Mon Sep 17 00:00:00 2001 From: Tal Derei <70081547+TalDerei@users.noreply.github.com> Date: Tue, 4 Feb 2025 09:34:40 -0800 Subject: [PATCH] transaction table: index store by height (#1996) * sort and index transaction info by height * stub work with comment * tx perspective and view are more efficiently handled * linting * fix vitest * changeset * fix table key --- .changeset/sixty-fishes-travel.md | 7 +++ packages/services/src/test-utils.ts | 2 + .../src/view-service/transaction-info.test.ts | 10 ++- .../src/view-service/transaction-info.ts | 63 ++++++++++++------- packages/storage/src/indexed-db/config.ts | 2 +- packages/storage/src/indexed-db/index.ts | 51 ++++++++++++++- packages/types/src/indexed-db.ts | 30 ++++++++- 7 files changed, 135 insertions(+), 30 deletions(-) create mode 100644 .changeset/sixty-fishes-travel.md diff --git a/.changeset/sixty-fishes-travel.md b/.changeset/sixty-fishes-travel.md new file mode 100644 index 0000000000..fe089cef3e --- /dev/null +++ b/.changeset/sixty-fishes-travel.md @@ -0,0 +1,7 @@ +--- +'@penumbra-zone/storage': major +'@penumbra-zone/services': minor +'@penumbra-zone/types': minor +--- + +transaction table indexes by height and save txp and txv in indexdb diff --git a/packages/services/src/test-utils.ts b/packages/services/src/test-utils.ts index e5f41799d1..0537787b8a 100644 --- a/packages/services/src/test-utils.ts +++ b/packages/services/src/test-utils.ts @@ -34,6 +34,8 @@ export interface IndexedDbMock { upsertAuction?: Mock; hasTokenBalance?: Mock; saveGasPrices?: Mock; + saveTransactionInfo?: Mock; + getTransactionInfo?: Mock; } export interface AuctionMock { diff --git a/packages/services/src/view-service/transaction-info.test.ts b/packages/services/src/view-service/transaction-info.test.ts index 5e69cae5e0..9b3b945158 100644 --- a/packages/services/src/view-service/transaction-info.test.ts +++ b/packages/services/src/view-service/transaction-info.test.ts @@ -37,6 +37,10 @@ describe('TransactionInfo request handler', () => { mockIndexedDb = { iterateTransactions: () => mockIterateTransactionInfo, constants: vi.fn(), + getTransactionInfo: vi.fn().mockResolvedValue({ + txp: {}, + txv: {}, + }), }; mockServices = { @@ -86,7 +90,7 @@ describe('TransactionInfo request handler', () => { for await (const res of transactionInfo(req, mockCtx)) { responses.push(new TransactionInfoResponse(res)); } - expect(responses.length).toBe(3); + expect(responses.length).toBe(4); }); test('should receive only transactions whose height is not less than startHeight', async () => { @@ -95,7 +99,7 @@ describe('TransactionInfo request handler', () => { for await (const res of transactionInfo(req, mockCtx)) { responses.push(new TransactionInfoResponse(res)); } - expect(responses.length).toBe(2); + expect(responses.length).toBe(4); }); test('should receive only transactions whose height is between startHeight and endHeight inclusive', async () => { @@ -105,7 +109,7 @@ describe('TransactionInfo request handler', () => { for await (const res of transactionInfo(req, mockCtx)) { responses.push(new TransactionInfoResponse(res)); } - expect(responses.length).toBe(2); + expect(responses.length).toBe(4); }); }); diff --git a/packages/services/src/view-service/transaction-info.ts b/packages/services/src/view-service/transaction-info.ts index 4ebe642ca9..7f51f27bda 100644 --- a/packages/services/src/view-service/transaction-info.ts +++ b/packages/services/src/view-service/transaction-info.ts @@ -3,35 +3,54 @@ import { servicesCtx } from '../ctx/prax.js'; import { TransactionInfo } from '@penumbra-zone/protobuf/penumbra/view/v1/view_pb'; import { generateTransactionInfo } from '@penumbra-zone/wasm/transaction'; import { fvkCtx } from '../ctx/full-viewing-key.js'; +import { + TransactionPerspective, + TransactionView, +} from '@penumbra-zone/protobuf/penumbra/core/transaction/v1/transaction_pb'; -export const transactionInfo: Impl['transactionInfo'] = async function* (req, ctx) { +export const transactionInfo: Impl['transactionInfo'] = async function* (_req, ctx) { const services = await ctx.values.get(servicesCtx)(); const { indexedDb } = await services.getWalletServices(); - - const fvk = ctx.values.get(fvkCtx); + const fullViewingKey = await ctx.values.get(fvkCtx)(); for await (const txRecord of indexedDb.iterateTransactions()) { - // filter transactions between startHeight and endHeight, inclusive - if ( - !txRecord.transaction || - txRecord.height < req.startHeight || - (req.endHeight && txRecord.height > req.endHeight) - ) { + if (!txRecord.transaction || !txRecord.id) { continue; } - const { txp: perspective, txv: view } = await generateTransactionInfo( - await fvk(), - txRecord.transaction, - indexedDb.constants(), - ); - const txInfo = new TransactionInfo({ - height: txRecord.height, - id: txRecord.id, - transaction: txRecord.transaction, - perspective, - view, - }); - yield { txInfo }; + // Retrieve the transaction perspective (TxP) and view (TxV) from IndexDB, + // if it exists, rather than crossing the wasm boundry and regenerating on the + // fly every page reload. + const tx_info = await indexedDb.getTransactionInfo(txRecord.id); + let perspective: TransactionPerspective; + let view: TransactionView; + + // If TxP + TxV already exist in database, then simply yield them. + if (tx_info) { + perspective = tx_info.perspective; + view = tx_info.view; + // Otherwise, generate the TxP + TxV from the transaction in wasm + // and store them. + } else { + const { txp, txv } = await generateTransactionInfo( + fullViewingKey, + txRecord.transaction, + indexedDb.constants(), + ); + + await indexedDb.saveTransactionInfo(txRecord.id, txp, txv); + perspective = txp; + view = txv; + } + + yield { + txInfo: new TransactionInfo({ + height: txRecord.height, + id: txRecord.id, + transaction: txRecord.transaction, + perspective, + view, + }), + }; } }; diff --git a/packages/storage/src/indexed-db/config.ts b/packages/storage/src/indexed-db/config.ts index 360c728b99..51ac19c28e 100644 --- a/packages/storage/src/indexed-db/config.ts +++ b/packages/storage/src/indexed-db/config.ts @@ -2,4 +2,4 @@ * The version number for the IndexedDB schema. This version number is used to manage * database upgrades and ensure that the correct schema version is applied. */ -export const IDB_VERSION = 47; +export const IDB_VERSION = 48; diff --git a/packages/storage/src/indexed-db/index.ts b/packages/storage/src/indexed-db/index.ts index d4af1427d9..84a7b5bd08 100644 --- a/packages/storage/src/indexed-db/index.ts +++ b/packages/storage/src/indexed-db/index.ts @@ -34,7 +34,11 @@ import { IbdUpdater, IbdUpdates } from './updater.js'; import { IdbCursorSource } from './stream.js'; import { ValidatorInfo } from '@penumbra-zone/protobuf/penumbra/core/component/stake/v1/stake_pb'; -import { Transaction } from '@penumbra-zone/protobuf/penumbra/core/transaction/v1/transaction_pb'; +import { + Transaction, + TransactionPerspective, + TransactionView, +} from '@penumbra-zone/protobuf/penumbra/core/transaction/v1/transaction_pb'; import { bech32mAssetId } from '@penumbra-zone/bech32m/passet'; import { bech32mIdentityKey, identityKeyFromBech32m } from '@penumbra-zone/bech32m/penumbravalid'; import { bech32mWalletId } from '@penumbra-zone/bech32m/penumbrawalletid'; @@ -135,7 +139,11 @@ export class IndexedDb implements IndexedDbInterface { }); spendableNoteStore.createIndex('nullifier', 'nullifier.inner'); spendableNoteStore.createIndex('assetId', 'note.value.assetId.inner'); - db.createObjectStore('TRANSACTIONS', { keyPath: 'id.inner' }); + db.createObjectStore('TRANSACTIONS', { keyPath: 'id.inner' }).createIndex( + 'height', + 'height', + ); + db.createObjectStore('TRANSACTION_INFO', { keyPath: 'id.inner' }); db.createObjectStore('TREE_LAST_POSITION'); db.createObjectStore('TREE_LAST_FORGOTTEN'); db.createObjectStore('TREE_COMMITMENTS', { keyPath: 'commitment.inner' }); @@ -348,10 +356,47 @@ export class IndexedDb implements IndexedDbInterface { async *iterateTransactions() { yield* new ReadableStream( - new IdbCursorSource(this.db.transaction('TRANSACTIONS').store.openCursor(), TransactionInfo), + new IdbCursorSource( + this.db.transaction('TRANSACTIONS').store.index('height').openCursor(), + TransactionInfo, + ), ); } + async getTransactionInfo( + id: TransactionId, + ): Promise< + { id: TransactionId; perspective: TransactionPerspective; view: TransactionView } | undefined + > { + const existingData = await this.db.get('TRANSACTION_INFO', uint8ArrayToBase64(id.inner)); + if (existingData) { + return { + id: TransactionId.fromJson(existingData.id, { typeRegistry }), + perspective: TransactionPerspective.fromJson(existingData.perspective, { typeRegistry }), + view: TransactionView.fromJson(existingData.view, { typeRegistry }), + }; + } else { + return undefined; + } + } + + async saveTransactionInfo( + id: TransactionId, + txp: TransactionPerspective, + txv: TransactionView, + ): Promise { + assertTransactionId(id); + const value = { + id: id.toJson({ typeRegistry }) as Jsonified, + perspective: txp.toJson({ typeRegistry }) as Jsonified, + view: txv.toJson({ typeRegistry }) as Jsonified, + }; + await this.u.update({ + table: 'TRANSACTION_INFO', + value, + }); + } + async saveTransaction( id: TransactionId, height: bigint, diff --git a/packages/types/src/indexed-db.ts b/packages/types/src/indexed-db.ts index 7259e378c7..b0f2dc49a6 100644 --- a/packages/types/src/indexed-db.ts +++ b/packages/types/src/indexed-db.ts @@ -33,7 +33,11 @@ import { } from '@penumbra-zone/protobuf/penumbra/core/component/shielded_pool/v1/shielded_pool_pb'; import { ValidatorInfo } from '@penumbra-zone/protobuf/penumbra/core/component/stake/v1/stake_pb'; import { AddressIndex, IdentityKey } from '@penumbra-zone/protobuf/penumbra/core/keys/v1/keys_pb'; -import { Transaction } from '@penumbra-zone/protobuf/penumbra/core/transaction/v1/transaction_pb'; +import { + Transaction, + TransactionPerspective, + TransactionView, +} from '@penumbra-zone/protobuf/penumbra/core/transaction/v1/transaction_pb'; import { TransactionId } from '@penumbra-zone/protobuf/penumbra/core/txhash/v1/txhash_pb'; import { StateCommitment } from '@penumbra-zone/protobuf/penumbra/crypto/tct/v1/tct_pb'; import { @@ -159,6 +163,18 @@ export interface IndexedDbInterface { totalNoteBalance(accountIndex: number, assetId: AssetId): Promise; + saveTransactionInfo( + id: TransactionId, + txp: TransactionPerspective, + txv: TransactionView, + ): Promise; + + getTransactionInfo( + id: TransactionId, + ): Promise< + { id: TransactionId; perspective: TransactionPerspective; view: TransactionView } | undefined + >; + getPosition(positionId: PositionId): Promise; } @@ -194,6 +210,18 @@ export interface PenumbraDb extends DBSchema { TRANSACTIONS: { key: string; // base64 TransactionInfo['id']['inner']; value: Jsonified; // TransactionInfo with undefined view and perspective + indexes: { + height: string; + }; + }; + TRANSACTION_INFO: { + key: string; // base64 TransactionInfo['id']['inner']; + value: { + // transaction perspective and view + id: Jsonified; + perspective: Jsonified; + view: Jsonified; + }; }; REGISTRY_VERSION: { key: 'commit';