From be2c2f30b412a45785904d9e6ec6c3cc0577d025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=A5=E5=9B=BD=E5=AE=87?= <841185308@qq.com> Date: Wed, 20 Sep 2023 10:51:40 +0800 Subject: [PATCH] fix: Fix miss saving some inputs and outputs with optimize input and output storage (#2841) --- .../src/block-sync-renderer/index.ts | 2 + .../sync/indexer-cache-service.ts | 52 ++-- .../chain/entities/indexer-tx-hash-cache.ts | 21 ++ .../src/database/chain/entities/sync-info.ts | 4 +- .../src/database/chain/entities/tx-lock.ts | 12 +- .../migrations/1694746034975-TxLockAddArgs.ts | 15 ++ .../src/database/chain/ormconfig.ts | 2 + .../neuron-wallet/src/services/addresses.ts | 37 +-- .../src/services/tx/transaction-persistor.ts | 93 +++++++- .../neuron-wallet/src/services/wallets.ts | 10 +- packages/neuron-wallet/src/utils/queue.ts | 6 +- .../index/resetSyncTask.test.ts | 1 + .../tests/services/address.test.ts | 122 +++++----- .../services/tx/transaction-persistor.test.ts | 223 ++++++++++++++++-- .../tests/services/wallets.test.ts | 52 ++-- 15 files changed, 484 insertions(+), 168 deletions(-) create mode 100644 packages/neuron-wallet/src/database/chain/migrations/1694746034975-TxLockAddArgs.ts diff --git a/packages/neuron-wallet/src/block-sync-renderer/index.ts b/packages/neuron-wallet/src/block-sync-renderer/index.ts index d06a6d4a42..929ba1e7bf 100644 --- a/packages/neuron-wallet/src/block-sync-renderer/index.ts +++ b/packages/neuron-wallet/src/block-sync-renderer/index.ts @@ -19,6 +19,7 @@ import MultisigConfigDbChangedSubject from '../models/subjects/multisig-config-d import Multisig from '../services/multisig' import { SyncAddressType } from '../database/chain/entities/sync-progress' import { debounceTime } from 'rxjs/operators' +import { TransactionPersistor } from '../services/tx' let network: Network | null let child: ChildProcess | null = null @@ -63,6 +64,7 @@ export const resetSyncTask = async (startTask = true) => { if (startTask) { await WalletService.getInstance().maintainAddressesIfNecessary() + await TransactionPersistor.checkTxLock() await CommonUtils.sleep(3000) await createBlockSyncTask() } diff --git a/packages/neuron-wallet/src/block-sync-renderer/sync/indexer-cache-service.ts b/packages/neuron-wallet/src/block-sync-renderer/sync/indexer-cache-service.ts index 012889fe55..0bd628201f 100644 --- a/packages/neuron-wallet/src/block-sync-renderer/sync/indexer-cache-service.ts +++ b/packages/neuron-wallet/src/block-sync-renderer/sync/indexer-cache-service.ts @@ -12,7 +12,7 @@ export default class IndexerCacheService { private rpcService: RpcService private walletId: string private indexer: CkbIndexer - #cacheBlockNumberEntity?: SyncInfoEntity + #cacheBlockNumberEntityMap: Map = new Map() constructor(walletId: string, addressMetas: AddressMeta[], rpcService: RpcService, indexer: CkbIndexer) { for (const addressMeta of addressMetas) { @@ -71,10 +71,10 @@ export default class IndexerCacheService { } private async fetchTxMapping(): Promise>> { - const lastCacheBlockNumber = await this.getCachedBlockNumber() const currentHeaderBlockNumber = await this.rpcService.getTipBlockNumber() - const mappingsByTxHash = new Map() + const mappingsByTxHash = new Map>() for (const addressMeta of this.addressMetas) { + const lastCacheBlockNumber = await this.getCachedBlockNumber(addressMeta.blake160) const lockScripts = [ addressMeta.generateDefaultLockScript(), addressMeta.generateACPLockScript(), @@ -147,25 +147,31 @@ export default class IndexerCacheService { return mappingsByTxHash } - private async getCachedBlockNumber() { - if (!this.#cacheBlockNumberEntity) { - this.#cacheBlockNumberEntity = + private async getCachedBlockNumber(blake160: string) { + let cacheBlockNumberEntity = this.#cacheBlockNumberEntityMap.get(blake160) + if (!cacheBlockNumberEntity) { + cacheBlockNumberEntity = (await getConnection() .getRepository(SyncInfoEntity) - .findOne({ name: SyncInfoEntity.getLastCachedKey(this.walletId) })) ?? + .findOne({ name: SyncInfoEntity.getLastCachedKey(blake160) })) ?? SyncInfoEntity.fromObject({ - name: SyncInfoEntity.getLastCachedKey(this.walletId), + name: SyncInfoEntity.getLastCachedKey(blake160), value: '0x0', }) + this.#cacheBlockNumberEntityMap.set(blake160, cacheBlockNumberEntity) } - return this.#cacheBlockNumberEntity + return cacheBlockNumberEntity } private async saveCacheBlockNumber(cacheBlockNumber: string) { - let cacheBlockNumberEntity = await this.getCachedBlockNumber() - cacheBlockNumberEntity.value = cacheBlockNumber - await getConnection().manager.save(cacheBlockNumberEntity) + const entities = this.addressMetas.map(v => + SyncInfoEntity.fromObject({ + name: SyncInfoEntity.getLastCachedKey(v.blake160), + value: cacheBlockNumber, + }) + ) + await getConnection().manager.save(entities, { chunk: 100 }) } public async upsertTxHashes(): Promise { @@ -210,28 +216,30 @@ export default class IndexerCacheService { fetchBlockDetailsQueue.drain(resolve) }) + const indexerCaches: IndexerTxHashCache[] = [] for (const txWithStatus of txsWithStatus) { const { transaction, txStatus } = txWithStatus - const mappings = mappingsByTxHash.get(transaction.hash!)! + const mappings = mappingsByTxHash.get(transaction.hash!) + if (!mappings) { + continue + } for (const { lockHash, address } of mappings) { - await getConnection() - .createQueryBuilder() - .insert() - .into(IndexerTxHashCache) - .values({ - txHash: transaction.hash, + indexerCaches.push( + IndexerTxHashCache.fromObject({ + txHash: transaction.hash!, blockNumber: parseInt(transaction.blockNumber!), blockHash: txStatus.blockHash!, - blockTimestamp: transaction.timestamp, + blockTimestamp: transaction.timestamp!, lockHash, address, walletId: this.walletId, - isProcessed: false, }) - .execute() + ) } } + indexerCaches.sort((a, b) => a.blockNumber - b.blockNumber) + await getConnection().manager.save(indexerCaches, { chunk: 100 }) await this.saveCacheBlockNumber(tipBlockNumber) return newTxHashes diff --git a/packages/neuron-wallet/src/database/chain/entities/indexer-tx-hash-cache.ts b/packages/neuron-wallet/src/database/chain/entities/indexer-tx-hash-cache.ts index 3aaf054b5b..0a278e44c4 100644 --- a/packages/neuron-wallet/src/database/chain/entities/indexer-tx-hash-cache.ts +++ b/packages/neuron-wallet/src/database/chain/entities/indexer-tx-hash-cache.ts @@ -66,4 +66,25 @@ export default class IndexerTxHashCache extends BaseEntity { onUpdate: 'CURRENT_TIMESTAMP', }) updatedAt!: Date + + static fromObject(obj: { + txHash: string + blockNumber: number + blockHash: string + blockTimestamp: string + lockHash: string + address: string + walletId: string + }) { + const result = new IndexerTxHashCache() + result.txHash = obj.txHash + result.blockNumber = obj.blockNumber + result.blockHash = obj.blockHash + result.blockTimestamp = obj.blockTimestamp + result.lockHash = obj.lockHash + result.address = obj.address + result.walletId = obj.walletId + result.isProcessed = false + return result + } } diff --git a/packages/neuron-wallet/src/database/chain/entities/sync-info.ts b/packages/neuron-wallet/src/database/chain/entities/sync-info.ts index c64813204e..facc70fdbb 100644 --- a/packages/neuron-wallet/src/database/chain/entities/sync-info.ts +++ b/packages/neuron-wallet/src/database/chain/entities/sync-info.ts @@ -21,7 +21,7 @@ export default class SyncInfo { return res } - static getLastCachedKey(walletId: string) { - return `lastCachedBlockNumber_${walletId}` + static getLastCachedKey(blake160: string) { + return `lastCachedBlockNumber_${blake160}` } } diff --git a/packages/neuron-wallet/src/database/chain/entities/tx-lock.ts b/packages/neuron-wallet/src/database/chain/entities/tx-lock.ts index db700983b0..bc5014c4fe 100644 --- a/packages/neuron-wallet/src/database/chain/entities/tx-lock.ts +++ b/packages/neuron-wallet/src/database/chain/entities/tx-lock.ts @@ -1,4 +1,4 @@ -import { BaseEntity, Entity, PrimaryColumn } from 'typeorm' +import { BaseEntity, Column, Entity, Index, PrimaryColumn } from 'typeorm' @Entity() export default class TxLock extends BaseEntity { @@ -12,10 +12,18 @@ export default class TxLock extends BaseEntity { }) lockHash!: string - static fromObject(obj: { txHash: string; lockHash: string }) { + @Column({ + type: 'varchar', + }) + @Index() + // check whether saving wallet blake160 + lockArgs!: string + + static fromObject(obj: { txHash: string; lockHash: string; lockArgs: string }) { const res = new TxLock() res.transactionHash = obj.txHash res.lockHash = obj.lockHash + res.lockArgs = obj.lockArgs return res } } diff --git a/packages/neuron-wallet/src/database/chain/migrations/1694746034975-TxLockAddArgs.ts b/packages/neuron-wallet/src/database/chain/migrations/1694746034975-TxLockAddArgs.ts new file mode 100644 index 0000000000..7384b229fe --- /dev/null +++ b/packages/neuron-wallet/src/database/chain/migrations/1694746034975-TxLockAddArgs.ts @@ -0,0 +1,15 @@ +import {MigrationInterface, QueryRunner, TableIndex} from "typeorm"; + +export class TxLockAddArgs1694746034975 implements MigrationInterface { + name = 'TxLockAddArgs1694746034975' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "tx_lock" ADD COLUMN "lockArgs" varchar;`); + await queryRunner.createIndex("tx_lock", new TableIndex({ columnNames: ["lockArgs"] })) + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE "tx_lock" DROP COLUMN "lockArgs" varchar;`); + } + +} diff --git a/packages/neuron-wallet/src/database/chain/ormconfig.ts b/packages/neuron-wallet/src/database/chain/ormconfig.ts index 78f0502a68..5a2a9440aa 100644 --- a/packages/neuron-wallet/src/database/chain/ormconfig.ts +++ b/packages/neuron-wallet/src/database/chain/ormconfig.ts @@ -54,6 +54,7 @@ import { AddSyncProgress1676441837373 } from './migrations/1676441837373-AddSync import { AddTypeSyncProgress1681360188494 } from './migrations/1681360188494-AddTypeSyncProgress' import { TxLock1684488676083 } from './migrations/1684488676083-TxLock' import { ResetSyncProgressPrimaryKey1690361215400 } from './migrations/1690361215400-ResetSyncProgressPrimaryKey' +import { TxLockAddArgs1694746034975 } from './migrations/1694746034975-TxLockAddArgs' export const CONNECTION_NOT_FOUND_NAME = 'ConnectionNotFoundError' @@ -124,6 +125,7 @@ const connectOptions = async (genesisBlockHash: string): Promise v.walletId)) if (walletIds.size !== 1) { @@ -74,8 +74,7 @@ export default class AddressService { ) const generatedAddresses: AddressInterface[] = [...addresses.receiving, ...addresses.change] - await AddressService.createQueue.asyncPush({ addresses: generatedAddresses }) - + await AddressService.create({ addresses: generatedAddresses }) return generatedAddresses } @@ -141,23 +140,29 @@ export default class AddressService { return allGeneratedAddresses } - public static async generateAndSaveForExtendedKey( - walletId: string, - extendedKey: AccountExtendedPublicKey, - isImporting: boolean | undefined, - receivingAddressCount: number = DefaultAddressNumber.Receiving, - changeAddressCount: number = DefaultAddressNumber.Change - ): Promise { - const generatedAddresses = await this.recursiveGenerateAndSave( + public static async generateAndSaveForExtendedKey({ + walletId, + extendedKey, + isImporting, + receivingAddressCount, + changeAddressCount, + }: { + walletId: string + extendedKey: AccountExtendedPublicKey + isImporting?: boolean + receivingAddressCount?: number + changeAddressCount?: number + }) { + const generatedAddresses = await AddressService.recursiveGenerateAndSave( walletId, extendedKey, isImporting, - receivingAddressCount, - changeAddressCount + receivingAddressCount ?? DefaultAddressNumber.Receiving, + changeAddressCount ?? DefaultAddressNumber.Change ) if (generatedAddresses) { - this.notifyAddressCreated(generatedAddresses, isImporting) + AddressService.notifyAddressCreated(generatedAddresses, isImporting) } return generatedAddresses @@ -201,7 +206,7 @@ export default class AddressService { await getConnection().manager.save(publicKeyInfo) const addressMeta = AddressMeta.fromHdPublicKeyInfoModel(publicKeyInfo.toModel()) - this.notifyAddressCreated([addressMeta], undefined) + AddressService.notifyAddressCreated([addressMeta], undefined) } // Generate both receiving and change addresses. diff --git a/packages/neuron-wallet/src/services/tx/transaction-persistor.ts b/packages/neuron-wallet/src/services/tx/transaction-persistor.ts index db36988007..9a8b1dc88e 100644 --- a/packages/neuron-wallet/src/services/tx/transaction-persistor.ts +++ b/packages/neuron-wallet/src/services/tx/transaction-persistor.ts @@ -1,4 +1,4 @@ -import { getConnection, QueryRunner } from 'typeorm' +import { getConnection, In, QueryRunner } from 'typeorm' import InputEntity from '../../database/chain/entities/input' import OutputEntity from '../../database/chain/entities/output' import TransactionEntity from '../../database/chain/entities/transaction' @@ -11,6 +11,7 @@ import Transaction, { TransactionStatus } from '../../models/chain/transaction' import Input from '../../models/chain/input' import TxLockEntity from '../../database/chain/entities/tx-lock' import { CHEQUE_ARGS_LENGTH, DEFAULT_ARGS_LENGTH } from '../../utils/const' +import IndexerTxHashCache from '../../database/chain/entities/indexer-tx-hash-cache' export enum TxSaveType { Sent = 'sent', @@ -269,6 +270,8 @@ export class TransactionPersistor { } const outputsData = transaction.outputsData! + const useTxInputs = await connection.getRepository(InputEntity).find({ outPointTxHash: tx.hash }) + const useTxIndices = new Set(useTxInputs.map(v => v.outPointIndex)) const outputs: OutputEntity[] = transaction.outputs.map((o, index) => { const output = new OutputEntity() output.outPointTxHash = transaction.hash || transaction.computeHash() @@ -280,6 +283,9 @@ export class TransactionPersistor { output.lockHash = o.lockHash! output.transaction = tx output.status = outputStatus + if (useTxIndices.has(index.toString())) { + output.status = OutputStatus.Dead + } output.multiSignBlake160 = o.multiSignBlake160 || null if (o.type) { output.typeCodeHash = o.type.codeHash @@ -309,12 +315,11 @@ export class TransactionPersistor { if (lockArgsSetNeedsDetail?.size) { willSaveDetailInputs = inputs.filter(v => this.shouldSaveDetail(v, lockArgsSetNeedsDetail)) willSaveDetailOutputs = outputs.filter(v => this.shouldSaveDetail(v, lockArgsSetNeedsDetail)) - txLocks = [ - ...new Set([ - ...inputs.filter(v => v.lockHash && !this.shouldSaveDetail(v, lockArgsSetNeedsDetail)).map(v => v.lockHash!), - ...outputs.filter(v => !this.shouldSaveDetail(v, lockArgsSetNeedsDetail)).map(v => v.lockHash), - ]), - ].map(v => TxLockEntity.fromObject({ txHash: tx.hash, lockHash: v })) + txLocks = await TransactionPersistor.findAndCreateTxLocks( + [...inputs, ...outputs], + lockArgsSetNeedsDetail, + tx.hash + ) } const chunk = 100 @@ -403,6 +408,80 @@ export class TransactionPersistor { ))) ) } + + public static async checkTxLock() { + const resetTxLocks = await getConnection() + .getRepository(TxLockEntity) + .createQueryBuilder() + .select('transactionHash') + .where('lockArgs IN (SELECT publicKeyInBlake160 from hd_public_key_info)') + .getRawMany<{ transactionHash: string }>() + if (!resetTxLocks?.length) { + return + } + const resetTxHashes = resetTxLocks.map(v => v.transactionHash) + const queryRunner = getConnection().createQueryRunner() + await TransactionPersistor.waitUntilTransactionFinished(queryRunner) + await queryRunner.startTransaction() + try { + await queryRunner.manager + .createQueryBuilder() + .update(IndexerTxHashCache) + .set({ isProcessed: false }) + .where({ txHash: In(resetTxHashes) }) + .execute() + await queryRunner.manager + .createQueryBuilder() + .delete() + .from(TransactionEntity) + .where({ hash: In(resetTxHashes) }) + .execute() + await queryRunner.manager + .createQueryBuilder() + .delete() + .from(InputEntity) + .where({ transactionHash: In(resetTxHashes) }) + .execute() + await queryRunner.manager + .createQueryBuilder() + .delete() + .from(OutputEntity) + .where({ outPointTxHash: In(resetTxHashes) }) + .execute() + await queryRunner.manager + .createQueryBuilder() + .delete() + .from(TxLockEntity) + .where({ transactionHash: In(resetTxHashes) }) + .execute() + await queryRunner.commitTransaction() + } catch (err) { + logger.error('Database:\tReset tx entity error:', err) + await queryRunner.rollbackTransaction() + throw err + } finally { + await queryRunner.release() + } + } + + private static findAndCreateTxLocks( + cells: (InputEntity | OutputEntity)[], + lockArgsSetNeedsDetail: Set, + txHash: string + ) { + const lockHashArgs: Record = {} + const lockHashSet = new Set( + cells + .filter(v => v.lockHash && !this.shouldSaveDetail(v, lockArgsSetNeedsDetail)) + .map(v => { + if (v.lockHash) { + lockHashArgs[v.lockHash] = v.lockArgs! + } + return v.lockHash! + }) + ) + return [...lockHashSet].map(v => TxLockEntity.fromObject({ txHash, lockHash: v, lockArgs: lockHashArgs[v] })) + } } export default TransactionPersistor diff --git a/packages/neuron-wallet/src/services/wallets.ts b/packages/neuron-wallet/src/services/wallets.ts index c3964a0e40..ac993ce4d0 100644 --- a/packages/neuron-wallet/src/services/wallets.ts +++ b/packages/neuron-wallet/src/services/wallets.ts @@ -183,13 +183,13 @@ export class FileKeystoreWallet extends Wallet { receivingAddressCount: number = DefaultAddressNumber.Receiving, changeAddressCount: number = DefaultAddressNumber.Change ): Promise => { - return await AddressService.generateAndSaveForExtendedKey( - this.id, - this.accountExtendedPublicKey(), + return await AddressService.generateAndSaveForExtendedKeyQueue.asyncPush({ + walletId: this.id, + extendedKey: this.accountExtendedPublicKey(), isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) } public getNextAddress = async (): Promise => { diff --git a/packages/neuron-wallet/src/utils/queue.ts b/packages/neuron-wallet/src/utils/queue.ts index 5868adca2d..4bd08a9956 100644 --- a/packages/neuron-wallet/src/utils/queue.ts +++ b/packages/neuron-wallet/src/utils/queue.ts @@ -1,10 +1,10 @@ -import { AsyncQueue, AsyncResultIterator, queue } from 'async' +import { QueueObject, AsyncResultIterator, queue } from 'async' export default function queueWrapper( fn: (item: T) => Promise, concurrency?: number, ignoreSameItem?: boolean -): AsyncQueue & { asyncPush: (item: T) => Promise } { +): QueueObject & { asyncPush: (item: T) => Promise } { const itemList: T[] = [] const promiseList: Promise[] = [] const queueFn: AsyncResultIterator = (item: T, callback: (err?: E | null, res?: R) => void) => { @@ -45,5 +45,5 @@ export default function queueWrapper( return promiseList[promiseList.length - 1] }, writable: false, - }) as AsyncQueue & { asyncPush: (item: T) => Promise } + }) as QueueObject & { asyncPush: (item: T) => Promise } } diff --git a/packages/neuron-wallet/tests/block-sync-renderer/index/resetSyncTask.test.ts b/packages/neuron-wallet/tests/block-sync-renderer/index/resetSyncTask.test.ts index ac88946a9e..8cf8258bab 100644 --- a/packages/neuron-wallet/tests/block-sync-renderer/index/resetSyncTask.test.ts +++ b/packages/neuron-wallet/tests/block-sync-renderer/index/resetSyncTask.test.ts @@ -6,6 +6,7 @@ describe(`Reset sync task`, () => { getInstance: () => ({ maintainAddressesIfNecessary: stubbedmaintainAddressesIfNecessary }), })) jest.doMock('utils/common', () => ({ sleep: stubbedSleep, timeout: stubbedTimeout })) + jest.doMock('services/tx', () => ({ TransactionPersistor: { checkTxLock: jest.fn() } })) const blockSyncRenderer = require('block-sync-renderer') const spyCreateBlockSyncTask = jest diff --git a/packages/neuron-wallet/tests/services/address.test.ts b/packages/neuron-wallet/tests/services/address.test.ts index deb7124db1..5467f85fa3 100644 --- a/packages/neuron-wallet/tests/services/address.test.ts +++ b/packages/neuron-wallet/tests/services/address.test.ts @@ -142,13 +142,13 @@ describe('integration tests for AddressService', () => { describe('when the newly created public keys have not been used', () => { beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) }) @@ -179,13 +179,13 @@ describe('integration tests for AddressService', () => { ) await linkNewCell([receivingAddresses[0].blake160, changeAddresses[0].blake160]) - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) }) it('generates new addresses for both receiving and change', () => { @@ -202,13 +202,13 @@ describe('integration tests for AddressService', () => { }) describe('when none of public keys are used', () => { beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) }) it('should not generate new addresses', () => { @@ -223,13 +223,13 @@ describe('integration tests for AddressService', () => { ) await linkNewCell([receivingAddresses[0].blake160]) - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) }) it('generates addresses for receiving', () => { @@ -253,13 +253,13 @@ describe('integration tests for AddressService', () => { ) await linkNewCell([changeAddresses[0].blake160]) - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) }) it('generates addresses for change', () => { @@ -280,13 +280,13 @@ describe('integration tests for AddressService', () => { beforeEach(async () => { await linkNewCell(preloadedPublicKeys.filter((k: any) => k.addressIndex < 4).map((k: any) => k.publicKeyHash)) - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) }) it('recursively generates both receiving and change addresses', () => { @@ -357,20 +357,20 @@ describe('integration tests for AddressService', () => { const changeAddressCount = 4 let allAddresses: Address[] = [] beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( - '1', + await AddressService.generateAndSaveForExtendedKey({ + walletId: '1', extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) - await AddressService.generateAndSaveForExtendedKey( - '2', + changeAddressCount, + }) + await AddressService.generateAndSaveForExtendedKey({ + walletId: '2', extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) allAddresses = await AddressService.getAddressesByAllWallets() }) it('returns all addresses', () => { @@ -387,13 +387,13 @@ describe('integration tests for AddressService', () => { let address: Address const walletId = '1' beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) address = await AddressService.getFirstAddressByWalletId(walletId) }) it('returns the first addresses by by wallet id', () => { @@ -410,16 +410,16 @@ describe('integration tests for AddressService', () => { describe('#nextUnusedAddress', () => { const receivingAddressCount = 2 const changeAddressCount = 2 - let publicKeysToUse = [] + let publicKeysToUse: string[] = [] describe('when there are unused receiving addresses', () => { beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) publicKeysToUse = [generatedAddresses[0].blake160] await linkNewCell(publicKeysToUse) @@ -432,13 +432,13 @@ describe('integration tests for AddressService', () => { }) describe('when there are unused change addresses but no receiving addresses', () => { beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) publicKeysToUse = generatedAddresses .filter((addr: Address) => addr.addressType === AddressType.Receiving) @@ -454,13 +454,13 @@ describe('integration tests for AddressService', () => { }) describe('when there is no receiving or change unused address', () => { beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) publicKeysToUse = generatedAddresses.map((addr: Address) => addr.blake160) @@ -474,18 +474,18 @@ describe('integration tests for AddressService', () => { }) describe('#allUnusedReceivingAddresses', () => { - let publicKeysToUse = [] + let publicKeysToUse: string[] = [] const receivingAddressCount = 4 const changeAddressCount = 4 describe('when there are unused receiving addresses', () => { beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) publicKeysToUse = [generatedAddresses[0].blake160] await linkNewCell(publicKeysToUse) @@ -500,13 +500,13 @@ describe('integration tests for AddressService', () => { }) describe('when there is no unused receiving address', () => { beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) publicKeysToUse = generatedAddresses.map((addr: Address) => addr.blake160) await linkNewCell(publicKeysToUse) @@ -519,17 +519,17 @@ describe('integration tests for AddressService', () => { }) describe('#nextUnusedChangeAddress', () => { - let publicKeysToUse = [] + let publicKeysToUse: string[] = [] const receivingAddressCount = 4 const changeAddressCount = 4 beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) }) describe('when there are unused change addresses', () => { beforeEach(async () => { @@ -563,13 +563,13 @@ describe('integration tests for AddressService', () => { const receivingAddressCount = 1 const changeAddressCount = 1 beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( + await AddressService.generateAndSaveForExtendedKey({ walletId, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) generatedAddresses = await AddressService.getAddressesByWalletId(walletId) publicKeysToUse = generatedAddresses.map((addr: Address) => addr.blake160) @@ -607,20 +607,20 @@ describe('integration tests for AddressService', () => { describe('when saved description', () => { beforeEach(async () => { - await AddressService.generateAndSaveForExtendedKey( - walletId1, + await AddressService.generateAndSaveForExtendedKey({ + walletId: walletId1, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) - await AddressService.generateAndSaveForExtendedKey( - walletId2, + changeAddressCount, + }) + await AddressService.generateAndSaveForExtendedKey({ + walletId: walletId2, extendedKey, isImporting, receivingAddressCount, - changeAddressCount - ) + changeAddressCount, + }) const generatedAddresses1 = await AddressService.getAddressesByWalletId(walletId1) addressToUpdate = generatedAddresses1[0] diff --git a/packages/neuron-wallet/tests/services/tx/transaction-persistor.test.ts b/packages/neuron-wallet/tests/services/tx/transaction-persistor.test.ts index 9d236e5b11..0dab3e8f9b 100644 --- a/packages/neuron-wallet/tests/services/tx/transaction-persistor.test.ts +++ b/packages/neuron-wallet/tests/services/tx/transaction-persistor.test.ts @@ -1,4 +1,4 @@ -import Transaction from '../../../src/models/chain/transaction' +import Transaction, { TransactionStatus } from '../../../src/models/chain/transaction' import { TransactionPersistor, TxSaveType } from '../../../src/services/tx' import initConnection from '../../../src/database/chain/ormconfig' import TransactionEntity from '../../../src/database/chain/entities/transaction' @@ -8,6 +8,10 @@ import AssetAccountInfo from '../../../src/models/asset-account-info' import SystemScriptInfo from '../../../src/models/system-script-info' import Multisig from '../../../src/models/multisig' import TxLockEntity from '../../../src/database/chain/entities/tx-lock' +import { OutputStatus } from '../../../src/models/chain/output' +import OutputEntity from '../../../src/database/chain/entities/output' +import HdPublicKeyInfo from '../../../src/database/chain/entities/hd-public-key-info' +import InputEntity from '../../../src/database/chain/entities/input' const [tx, tx2] = transactions @@ -88,14 +92,14 @@ describe('TransactionPersistor', () => { expect(loadedTx?.inputs.length).toBe(0) expect(loadedTx?.outputs.length).toBe(1) expect(loadedTx?.outputs[0].lockArgs).toBe(tx.outputs[0].lock.args) - const txLocks = await getConnection() - .getRepository(TxLockEntity) - .find({ transactionHash: tx.hash }) + const txLocks = await getConnection().getRepository(TxLockEntity).find({ transactionHash: tx.hash }) expect(txLocks.length).toBe(1) expect(txLocks[0].lockHash).toBe(tx.inputs[0].lock?.computeHash()) }) it('all args is current', async () => { - const args = [...tx.inputs.map(v => v.lock?.args), ...tx.outputs.map(v => v.lock.args)].filter((v): v is string => !!v) + const args = [...tx.inputs.map(v => v.lock?.args), ...tx.outputs.map(v => v.lock.args)].filter( + (v): v is string => !!v + ) await TransactionPersistor.convertTransactionAndSave(tx, TxSaveType.Fetch, new Set(args)) const loadedTx = await getConnection() .getRepository(TransactionEntity) @@ -106,22 +110,26 @@ describe('TransactionPersistor', () => { .getOne() expect(loadedTx?.inputs.length).toBe(1) expect(loadedTx?.outputs.length).toBe(2) - const txLocks = await getConnection() - .getRepository(TxLockEntity) - .find({ transactionHash: tx.hash }) + const txLocks = await getConnection().getRepository(TxLockEntity).find({ transactionHash: tx.hash }) expect(txLocks.length).toBe(0) }) it('filter with receive cheque and send cheque', async () => { const assetAccountInfo = new AssetAccountInfo() const txWithCheque = Transaction.fromObject(tx) - const outputReceiveChequeLock = assetAccountInfo.generateChequeScript(txWithCheque.outputs[0].lockHash, `0x${'0'.repeat(42)}`) - const outputSendChequeLock = assetAccountInfo.generateChequeScript(`0x${'0'.repeat(42)}`, txWithCheque.outputs[0].lockHash) + const outputReceiveChequeLock = assetAccountInfo.generateChequeScript( + txWithCheque.outputs[0].lockHash, + `0x${'0'.repeat(42)}` + ) + const outputSendChequeLock = assetAccountInfo.generateChequeScript( + `0x${'0'.repeat(42)}`, + txWithCheque.outputs[0].lockHash + ) txWithCheque.outputs[0].setLock(outputReceiveChequeLock) txWithCheque.outputs[1].setLock(outputSendChequeLock) - const args = [...tx.inputs.map(v => v.lock?.args), ...tx.outputs.map(v => v.lock.args)].filter((v): v is string => !!v).map(v => [ - v, - SystemScriptInfo.generateSecpScript(v).computeHash().slice(0, 42), - ]).flat() + const args = [...tx.inputs.map(v => v.lock?.args), ...tx.outputs.map(v => v.lock.args)] + .filter((v): v is string => !!v) + .map(v => [v, SystemScriptInfo.generateSecpScript(v).computeHash().slice(0, 42)]) + .flat() await TransactionPersistor.convertTransactionAndSave(txWithCheque, TxSaveType.Fetch, new Set(args)) const loadedTx = await getConnection() .getRepository(TransactionEntity) @@ -132,19 +140,19 @@ describe('TransactionPersistor', () => { .getOne() expect(loadedTx?.inputs.length).toBe(1) expect(loadedTx?.outputs.length).toBe(2) - const txLocks = await getConnection() - .getRepository(TxLockEntity) - .find({ transactionHash: tx.hash }) + const txLocks = await getConnection().getRepository(TxLockEntity).find({ transactionHash: tx.hash }) expect(txLocks.length).toBe(0) }) it('filter with multi lock time', async () => { const txWithCheque = Transaction.fromObject(tx) - const multisigLockTimeLock = SystemScriptInfo.generateMultiSignScript(Multisig.hash([txWithCheque.outputs[0].lock.args])) + const multisigLockTimeLock = SystemScriptInfo.generateMultiSignScript( + Multisig.hash([txWithCheque.outputs[0].lock.args]) + ) txWithCheque.outputs[0].setLock(multisigLockTimeLock) - const args = [...tx.inputs.map(v => v.lock?.args), ...tx.outputs.map(v => v.lock.args)].filter((v): v is string => !!v).map(v => [ - v, - Multisig.hash([v]), - ]).flat() + const args = [...tx.inputs.map(v => v.lock?.args), ...tx.outputs.map(v => v.lock.args)] + .filter((v): v is string => !!v) + .map(v => [v, Multisig.hash([v])]) + .flat() await TransactionPersistor.convertTransactionAndSave(txWithCheque, TxSaveType.Fetch, new Set(args)) const loadedTx = await getConnection() .getRepository(TransactionEntity) @@ -155,10 +163,175 @@ describe('TransactionPersistor', () => { .getOne() expect(loadedTx?.inputs.length).toBe(1) expect(loadedTx?.outputs.length).toBe(2) - const txLocks = await getConnection() - .getRepository(TxLockEntity) - .find({ transactionHash: tx.hash }) + const txLocks = await getConnection().getRepository(TxLockEntity).find({ transactionHash: tx.hash }) expect(txLocks.length).toBe(0) }) }) + + describe('#saveWithSent', () => { + const createMock = jest.fn() + const originalCreate = TransactionPersistor.create + beforeAll(() => { + TransactionPersistor.create = createMock + }) + afterAll(() => { + TransactionPersistor.create = originalCreate + }) + afterEach(async () => { + await getConnection().synchronize(true) + createMock.mockReset() + }) + + it('there is no transaction', async () => { + //@ts-ignore private method + await TransactionPersistor.saveWithSent(tx) + expect(createMock).toBeCalledWith(tx, OutputStatus.Sent, OutputStatus.Pending) + }) + it('there is a transaction but status is failed', async () => { + const entity = new TransactionEntity() + entity.hash = tx.hash || tx.computeHash() + entity.version = tx.version + entity.headerDeps = tx.headerDeps + entity.cellDeps = tx.cellDeps + entity.timestamp = tx.timestamp! + entity.blockHash = tx.blockHash! + entity.blockNumber = tx.blockNumber! + entity.witnesses = tx.witnessesAsString() + entity.description = '' // tx desc is saved in leveldb as wallet property + entity.status = TransactionStatus.Failed + entity.inputs = [] + entity.outputs = [] + await getConnection().manager.save(entity) + //@ts-ignore private method + await TransactionPersistor.saveWithSent(tx) + expect(createMock).toBeCalledWith(tx, OutputStatus.Sent, OutputStatus.Pending) + }) + it('there is a transaction but status is not failed', async () => { + await originalCreate(tx, OutputStatus.Dead, OutputStatus.Dead) + //@ts-ignore private method + await TransactionPersistor.saveWithSent(tx) + expect(createMock).toBeCalledTimes(0) + }) + }) + + describe('#saveWithFetch', () => { + const createMock = jest.fn() + const originalCreate = TransactionPersistor.create + beforeAll(() => { + TransactionPersistor.create = createMock + }) + afterAll(() => { + TransactionPersistor.create = originalCreate + }) + afterEach(async () => { + await getConnection().synchronize(true) + createMock.mockReset() + }) + + it('there is no transaction', async () => { + //@ts-ignore private method + await TransactionPersistor.saveWithFetch(tx) + expect(createMock).toBeCalledWith(tx, OutputStatus.Live, OutputStatus.Dead, undefined) + }) + it('update sent output to live', async () => { + const entity = await originalCreate(tx, OutputStatus.Sent, OutputStatus.Pending) + expect(entity.outputs[0]?.status).toBe(OutputStatus.Sent) + //@ts-ignore private method + await TransactionPersistor.saveWithFetch(tx) + expect(createMock).toBeCalledTimes(0) + const output = await getConnection().getRepository(OutputEntity).findOne({ outPointTxHash: entity.hash }) + expect(output?.status).toBe(OutputStatus.Live) + }) + it('update live output to dead because refer to input', async () => { + const entity = await originalCreate(tx, OutputStatus.Live, OutputStatus.Dead) + expect(entity.outputs[0]?.status).toBe(OutputStatus.Live) + await originalCreate(tx2, OutputStatus.Sent, OutputStatus.Pending) + //@ts-ignore private method + await TransactionPersistor.saveWithFetch(tx2) + const output = await getConnection().getRepository(OutputEntity).findOne({ + outPointTxHash: tx2.inputs[0].previousOutput?.txHash, + outPointIndex: tx2.inputs[0].previousOutput?.index, + }) + expect(output?.status).toBe(OutputStatus.Dead) + }) + }) + + describe('#create', () => { + afterEach(async () => { + await getConnection().synchronize(true) + }) + it('set output to dead if it is in input', async () => { + await TransactionPersistor.convertTransactionAndSave(tx2, TxSaveType.Sent) + await TransactionPersistor.convertTransactionAndSave(tx, TxSaveType.Sent) + const output = await getConnection().getRepository(OutputEntity).findOne({ + outPointTxHash: tx2.inputs[0].previousOutput?.txHash, + outPointIndex: tx2.inputs[0].previousOutput?.index, + }) + expect(output?.status).toBe(OutputStatus.Dead) + }) + }) + + describe('#checkTxLock', () => { + afterEach(async () => { + await getConnection().synchronize(true) + }) + + it('no tx lock saving wrong', async () => { + const mock = jest.fn() + jest.doMock('typeorm', () => ({ + getConnection() { + return { createQueryRunner: mock } + }, + })) + await TransactionPersistor.checkTxLock() + expect(mock).toBeCalledTimes(0) + }) + + it('some tx lock saving wrong', async () => { + const txLocks = TxLockEntity.fromObject({ + txHash: tx.hash!, + lockHash: tx.inputs[0].lockHash!, + lockArgs: tx.inputs[0].lock!.args!, + }) + const hdPublicKeyInfo = HdPublicKeyInfo.fromObject({ + walletId: 'w1', + addressType: 0, + addressIndex: 0, + publicKeyInBlake160: tx.inputs[0].lock!.args!, + }) + await getConnection().manager.save([txLocks, hdPublicKeyInfo]) + await TransactionPersistor.convertTransactionAndSave(tx, TxSaveType.Sent) + let output = await getConnection().getRepository(OutputEntity).findOne({ outPointTxHash: tx.hash }) + let input = await getConnection().getRepository(InputEntity).findOne({ transactionHash: tx.hash }) + expect(output).toBeDefined() + expect(input).toBeDefined() + await TransactionPersistor.checkTxLock() + output = await getConnection().getRepository(OutputEntity).findOne({ outPointTxHash: tx.hash }) + input = await getConnection().getRepository(InputEntity).findOne({ transactionHash: tx.hash }) + expect(output).toBeUndefined() + expect(input).toBeUndefined() + }) + }) + + describe('#findAndCreateTxLocks', () => { + let inputEntities: InputEntity[] = [] + let outputEntities: OutputEntity[] = [] + beforeAll(async () => { + await TransactionPersistor.convertTransactionAndSave(tx, TxSaveType.Sent) + inputEntities = await getConnection().getRepository(InputEntity).find({ transactionHash: tx.hash }) + outputEntities = await getConnection().getRepository(OutputEntity).find({ outPointTxHash: tx.hash }) + }) + it('no filter', () => { + const cells = [...inputEntities, ...outputEntities] + //@ts-ignore private method + const result = TransactionPersistor.findAndCreateTxLocks(cells, new Set(), tx.hash!) + expect(result).toHaveLength(new Set(cells.map(v => v.lockHash!)).size) + }) + it('filter with lock args', () => { + const cells = [...inputEntities, ...outputEntities] + //@ts-ignore private method + const result = TransactionPersistor.findAndCreateTxLocks(cells, new Set([tx.inputs[0].lock?.args]), tx.hash!) + expect(result).toHaveLength(1) + }) + }) }) diff --git a/packages/neuron-wallet/tests/services/wallets.test.ts b/packages/neuron-wallet/tests/services/wallets.test.ts index f23029ae7e..663602234f 100644 --- a/packages/neuron-wallet/tests/services/wallets.test.ts +++ b/packages/neuron-wallet/tests/services/wallets.test.ts @@ -7,7 +7,7 @@ import { AddressType } from '../../src/models/keys/address' import { Manufacturer } from '../../src/services/hardware/common' const stubbedDeletedByWalletIdFn = jest.fn() -const stubbedGenerateAndSaveForExtendedKeyFn = jest.fn() +const stubbedGenerateAndSaveForExtendedKeyQueue = jest.fn() const stubbedGenerateAndSaveForPublicKeyQueueAsyncPush = jest.fn() const stubbedGetNextUnusedAddressByWalletIdFn = jest.fn() const stubbedGetNextUnusedChangeAddressByWalletIdFn = jest.fn() @@ -20,10 +20,12 @@ const stubbedDeleteByWalletId = jest.fn() jest.doMock('../../src/services/addresses', () => { return { deleteByWalletId: stubbedDeleteByWalletId, - generateAndSaveForExtendedKey: stubbedGenerateAndSaveForExtendedKeyFn, generateAndSaveForPublicKeyQueue: { asyncPush: stubbedGenerateAndSaveForPublicKeyQueueAsyncPush, }, + generateAndSaveForExtendedKeyQueue: { + asyncPush: stubbedGenerateAndSaveForExtendedKeyQueue, + }, getNextUnusedAddressByWalletId: stubbedGetNextUnusedAddressByWalletIdFn, getNextUnusedChangeAddressByWalletId: stubbedGetNextUnusedChangeAddressByWalletIdFn, getUnusedReceivingAddressesByWalletId: stubbedGetUnusedReceivingAddressesByWalletIdFn, @@ -37,7 +39,7 @@ import HdPublicKeyInfo from '../../src/database/chain/entities/hd-public-key-inf const resetMocks = () => { stubbedDeletedByWalletIdFn.mockReset() - stubbedGenerateAndSaveForExtendedKeyFn.mockReset() + stubbedGenerateAndSaveForExtendedKeyQueue.mockReset() stubbedGenerateAndSaveForPublicKeyQueueAsyncPush.mockReset() stubbedGetNextUnusedAddressByWalletIdFn.mockReset() stubbedGetNextUnusedChangeAddressByWalletIdFn.mockReset() @@ -201,16 +203,16 @@ describe('wallet service', () => { await wallet.checkAndGenerateAddresses() }) it('calls AddressService.accountExtendedPublicKey', async () => { - expect(stubbedGenerateAndSaveForExtendedKeyFn).toHaveBeenCalledWith( - createdWallet.id, - expect.objectContaining({ + expect(stubbedGenerateAndSaveForExtendedKeyQueue).toHaveBeenCalledWith({ + walletId: createdWallet.id, + extendedKey: expect.objectContaining({ chainCode: createdWallet.accountExtendedPublicKey().chainCode, publicKey: createdWallet.accountExtendedPublicKey().publicKey, }), - false, - 20, - 10 - ) + isImporting: false, + receivingAddressCount: 20, + changeAddressCount: 10, + }) }) }) describe('#getNextAddressByWalletId', () => { @@ -411,23 +413,23 @@ describe('wallet service', () => { await walletService.maintainAddressesIfNecessary() }) it('should not generate addresses for wallets already having addresses', () => { - expect(stubbedGenerateAndSaveForExtendedKeyFn).not.toHaveBeenCalledWith(createdWallet1.id) + expect(stubbedGenerateAndSaveForExtendedKeyQueue).not.toHaveBeenCalledWith(createdWallet1.id) }) it('generates addresses for wallets not having addresses', () => { - expect(stubbedGenerateAndSaveForExtendedKeyFn).toHaveBeenCalledWith( - createdWallet2.id, - expect.objectContaining({ publicKey: 'a' }), - false, - 20, - 10 - ) - expect(stubbedGenerateAndSaveForExtendedKeyFn).toHaveBeenCalledWith( - createdWallet3.id, - expect.objectContaining({ publicKey: 'a' }), - false, - 20, - 10 - ) + expect(stubbedGenerateAndSaveForExtendedKeyQueue).toHaveBeenCalledWith({ + walletId: createdWallet2.id, + extendedKey: expect.objectContaining({ publicKey: 'a' }), + isImporting: false, + receivingAddressCount: 20, + changeAddressCount: 10, + }) + expect(stubbedGenerateAndSaveForExtendedKeyQueue).toHaveBeenCalledWith({ + walletId: createdWallet3.id, + extendedKey: expect.objectContaining({ publicKey: 'a' }), + isImporting: false, + receivingAddressCount: 20, + changeAddressCount: 10, + }) }) describe('when having invalid wallet ids in key info', () => { const deletedWalletId = 'wallet4'