fix: Fix miss saving some inputs and outputs with optimize input and …

…output storage (#2841)

packages/neuron-wallet/src/block-sync-renderer/index.ts

@@ -19,6 +19,7 @@ import MultisigConfigDbChangedSubject from '../models/subjects/multisig-config-d
 import Multisig from '../services/multisig'
 import { SyncAddressType } from '../database/chain/entities/sync-progress'
 import { debounceTime } from 'rxjs/operators'
+import { TransactionPersistor } from '../services/tx'
 
 let network: Network | null
 let child: ChildProcess | null = null
@@ -63,6 +64,7 @@ export const resetSyncTask = async (startTask = true) => {
 
   if (startTask) {
     await WalletService.getInstance().maintainAddressesIfNecessary()
+    await TransactionPersistor.checkTxLock()
     await CommonUtils.sleep(3000)
     await createBlockSyncTask()
   }

packages/neuron-wallet/src/block-sync-renderer/sync/indexer-cache-service.ts

@@ -12,7 +12,7 @@ export default class IndexerCacheService {
   private rpcService: RpcService
   private walletId: string
   private indexer: CkbIndexer
-  #cacheBlockNumberEntity?: SyncInfoEntity
+  #cacheBlockNumberEntityMap: Map<string, SyncInfoEntity> = new Map()
 
   constructor(walletId: string, addressMetas: AddressMeta[], rpcService: RpcService, indexer: CkbIndexer) {
     for (const addressMeta of addressMetas) {
@@ -71,10 +71,10 @@ export default class IndexerCacheService {
   }
 
   private async fetchTxMapping(): Promise<Map<string, Array<{ address: string; lockHash: string }>>> {
-    const lastCacheBlockNumber = await this.getCachedBlockNumber()
     const currentHeaderBlockNumber = await this.rpcService.getTipBlockNumber()
-    const mappingsByTxHash = new Map()
+    const mappingsByTxHash = new Map<string, Array<{ address: string; lockHash: string }>>()
     for (const addressMeta of this.addressMetas) {
+      const lastCacheBlockNumber = await this.getCachedBlockNumber(addressMeta.blake160)
       const lockScripts = [
         addressMeta.generateDefaultLockScript(),
         addressMeta.generateACPLockScript(),
@@ -147,25 +147,31 @@ export default class IndexerCacheService {
     return mappingsByTxHash
   }
 
-  private async getCachedBlockNumber() {
-    if (!this.#cacheBlockNumberEntity) {
-      this.#cacheBlockNumberEntity =
+  private async getCachedBlockNumber(blake160: string) {
+    let cacheBlockNumberEntity = this.#cacheBlockNumberEntityMap.get(blake160)
+    if (!cacheBlockNumberEntity) {
+      cacheBlockNumberEntity =
         (await getConnection()
           .getRepository(SyncInfoEntity)
-          .findOne({ name: SyncInfoEntity.getLastCachedKey(this.walletId) })) ??
+          .findOne({ name: SyncInfoEntity.getLastCachedKey(blake160) })) ??
         SyncInfoEntity.fromObject({
-          name: SyncInfoEntity.getLastCachedKey(this.walletId),
+          name: SyncInfoEntity.getLastCachedKey(blake160),
           value: '0x0',
         })
+      this.#cacheBlockNumberEntityMap.set(blake160, cacheBlockNumberEntity)
     }
 
-    return this.#cacheBlockNumberEntity
+    return cacheBlockNumberEntity
   }
 
   private async saveCacheBlockNumber(cacheBlockNumber: string) {
-    let cacheBlockNumberEntity = await this.getCachedBlockNumber()
-    cacheBlockNumberEntity.value = cacheBlockNumber
-    await getConnection().manager.save(cacheBlockNumberEntity)
+    const entities = this.addressMetas.map(v =>
+      SyncInfoEntity.fromObject({
+        name: SyncInfoEntity.getLastCachedKey(v.blake160),
+        value: cacheBlockNumber,
+      })
+    )
+    await getConnection().manager.save(entities, { chunk: 100 })
   }
 
   public async upsertTxHashes(): Promise<string[]> {
@@ -210,28 +216,30 @@ export default class IndexerCacheService {
       fetchBlockDetailsQueue.drain(resolve)
     })
 
+    const indexerCaches: IndexerTxHashCache[] = []
     for (const txWithStatus of txsWithStatus) {
       const { transaction, txStatus } = txWithStatus
-      const mappings = mappingsByTxHash.get(transaction.hash!)!
+      const mappings = mappingsByTxHash.get(transaction.hash!)
+      if (!mappings) {
+        continue
+      }
 
       for (const { lockHash, address } of mappings) {
-        await getConnection()
-          .createQueryBuilder()
-          .insert()
-          .into(IndexerTxHashCache)
-          .values({
-            txHash: transaction.hash,
+        indexerCaches.push(
+          IndexerTxHashCache.fromObject({
+            txHash: transaction.hash!,
             blockNumber: parseInt(transaction.blockNumber!),
             blockHash: txStatus.blockHash!,
-            blockTimestamp: transaction.timestamp,
+            blockTimestamp: transaction.timestamp!,
             lockHash,
             address,
             walletId: this.walletId,
-            isProcessed: false,
           })
-          .execute()
+        )
       }
     }
+    indexerCaches.sort((a, b) => a.blockNumber - b.blockNumber)
+    await getConnection().manager.save(indexerCaches, { chunk: 100 })
 
     await this.saveCacheBlockNumber(tipBlockNumber)
     return newTxHashes

packages/neuron-wallet/src/database/chain/entities/indexer-tx-hash-cache.ts

@@ -66,4 +66,25 @@ export default class IndexerTxHashCache extends BaseEntity {
     onUpdate: 'CURRENT_TIMESTAMP',
   })
   updatedAt!: Date
+
+  static fromObject(obj: {
+    txHash: string
+    blockNumber: number
+    blockHash: string
+    blockTimestamp: string
+    lockHash: string
+    address: string
+    walletId: string
+  }) {
+    const result = new IndexerTxHashCache()
+    result.txHash = obj.txHash
+    result.blockNumber = obj.blockNumber
+    result.blockHash = obj.blockHash
+    result.blockTimestamp = obj.blockTimestamp
+    result.lockHash = obj.lockHash
+    result.address = obj.address
+    result.walletId = obj.walletId
+    result.isProcessed = false
+    return result
+  }
 }

packages/neuron-wallet/src/database/chain/entities/sync-info.ts

@@ -21,7 +21,7 @@ export default class SyncInfo {
     return res
   }
 
-  static getLastCachedKey(walletId: string) {
-    return `lastCachedBlockNumber_${walletId}`
+  static getLastCachedKey(blake160: string) {
+    return `lastCachedBlockNumber_${blake160}`
   }
 }

packages/neuron-wallet/src/database/chain/entities/tx-lock.ts

@@ -1,4 +1,4 @@
-import { BaseEntity, Entity, PrimaryColumn } from 'typeorm'
+import { BaseEntity, Column, Entity, Index, PrimaryColumn } from 'typeorm'
 
 @Entity()
 export default class TxLock extends BaseEntity {
@@ -12,10 +12,18 @@ export default class TxLock extends BaseEntity {
   })
   lockHash!: string
 
-  static fromObject(obj: { txHash: string; lockHash: string }) {
+  @Column({
+    type: 'varchar',
+  })
+  @Index()
+  // check whether saving wallet blake160
+  lockArgs!: string
+
+  static fromObject(obj: { txHash: string; lockHash: string; lockArgs: string }) {
     const res = new TxLock()
     res.transactionHash = obj.txHash
     res.lockHash = obj.lockHash
+    res.lockArgs = obj.lockArgs
     return res
   }
 }

packages/neuron-wallet/src/database/chain/migrations/1694746034975-TxLockAddArgs.ts

@@ -0,0 +1,15 @@
+import {MigrationInterface, QueryRunner, TableIndex} from "typeorm";
+
+export class TxLockAddArgs1694746034975 implements MigrationInterface {
+    name = 'TxLockAddArgs1694746034975'
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "tx_lock" ADD COLUMN "lockArgs" varchar;`);
+        await queryRunner.createIndex("tx_lock", new TableIndex({ columnNames: ["lockArgs"] }))
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+        await queryRunner.query(`ALTER TABLE "tx_lock" DROP COLUMN "lockArgs" varchar;`);
+    }
+
+}

packages/neuron-wallet/src/database/chain/ormconfig.ts

@@ -54,6 +54,7 @@ import { AddSyncProgress1676441837373 } from './migrations/1676441837373-AddSync
 import { AddTypeSyncProgress1681360188494 } from './migrations/1681360188494-AddTypeSyncProgress'
 import { TxLock1684488676083 } from './migrations/1684488676083-TxLock'
 import { ResetSyncProgressPrimaryKey1690361215400 } from './migrations/1690361215400-ResetSyncProgressPrimaryKey'
+import { TxLockAddArgs1694746034975 } from './migrations/1694746034975-TxLockAddArgs'
 
 export const CONNECTION_NOT_FOUND_NAME = 'ConnectionNotFoundError'
 
@@ -124,6 +125,7 @@ const connectOptions = async (genesisBlockHash: string): Promise<SqliteConnectio
       AddTypeSyncProgress1681360188494,
       TxLock1684488676083,
       ResetSyncProgressPrimaryKey1690361215400,
+      TxLockAddArgs1694746034975,
     ],
     logger: 'simple-console',
     logging,

packages/neuron-wallet/src/services/addresses.ts

@@ -27,10 +27,10 @@ export interface AddressMetaInfo {
 export default class AddressService {
   private static minUnusedAddressCount: number = 3
 
-  private static createQueue = queueWrapper(AddressService.create)
-
   public static generateAndSaveForPublicKeyQueue = queueWrapper(AddressService.generateAndSaveForPublicKey)
 
+  public static generateAndSaveForExtendedKeyQueue = queueWrapper(AddressService.generateAndSaveForExtendedKey)
+
   private static async create({ addresses }: { addresses: AddressInterface[] }) {
     const walletIds = new Set(addresses.map(v => v.walletId))
     if (walletIds.size !== 1) {
@@ -74,8 +74,7 @@ export default class AddressService {
     )
 
     const generatedAddresses: AddressInterface[] = [...addresses.receiving, ...addresses.change]
-    await AddressService.createQueue.asyncPush({ addresses: generatedAddresses })
-
+    await AddressService.create({ addresses: generatedAddresses })
     return generatedAddresses
   }
 
@@ -141,23 +140,29 @@ export default class AddressService {
     return allGeneratedAddresses
   }
 
-  public static async generateAndSaveForExtendedKey(
-    walletId: string,
-    extendedKey: AccountExtendedPublicKey,
-    isImporting: boolean | undefined,
-    receivingAddressCount: number = DefaultAddressNumber.Receiving,
-    changeAddressCount: number = DefaultAddressNumber.Change
-  ): Promise<AddressInterface[] | undefined> {
-    const generatedAddresses = await this.recursiveGenerateAndSave(
+  public static async generateAndSaveForExtendedKey({
+    walletId,
+    extendedKey,
+    isImporting,
+    receivingAddressCount,
+    changeAddressCount,
+  }: {
+    walletId: string
+    extendedKey: AccountExtendedPublicKey
+    isImporting?: boolean
+    receivingAddressCount?: number
+    changeAddressCount?: number
+  }) {
+    const generatedAddresses = await AddressService.recursiveGenerateAndSave(
       walletId,
       extendedKey,
       isImporting,
-      receivingAddressCount,
-      changeAddressCount
+      receivingAddressCount ?? DefaultAddressNumber.Receiving,
+      changeAddressCount ?? DefaultAddressNumber.Change
     )
 
     if (generatedAddresses) {
-      this.notifyAddressCreated(generatedAddresses, isImporting)
+      AddressService.notifyAddressCreated(generatedAddresses, isImporting)
     }
 
     return generatedAddresses
@@ -201,7 +206,7 @@ export default class AddressService {
     await getConnection().manager.save(publicKeyInfo)
 
     const addressMeta = AddressMeta.fromHdPublicKeyInfoModel(publicKeyInfo.toModel())
-    this.notifyAddressCreated([addressMeta], undefined)
+    AddressService.notifyAddressCreated([addressMeta], undefined)
   }
 
   // Generate both receiving and change addresses.