refactor: Optimize input and output storage (#2672)

1. Storage others' cells tx and locks into tx-lock for search others' addresses.
2. Get transaction detail by rpc.

packages/neuron-wallet/src/block-sync-renderer/sync/indexer-cache-service.ts

@@ -4,13 +4,15 @@ import AddressMeta from '../../database/address/meta'
 import IndexerTxHashCache from '../../database/chain/entities/indexer-tx-hash-cache'
 import RpcService from '../../services/rpc-service'
 import TransactionWithStatus from '../../models/chain/transaction-with-status'
+import SyncInfoEntity from '../../database/chain/entities/sync-info'
 import { TransactionCollector, CellCollector, Indexer as CkbIndexer } from '@ckb-lumos/ckb-indexer'
 
 export default class IndexerCacheService {
   private addressMetas: AddressMeta[]
   private rpcService: RpcService
   private walletId: string
   private indexer: CkbIndexer
+  #cacheBlockNumberEntity?: SyncInfoEntity
 
   constructor(walletId: string, addressMetas: AddressMeta[], rpcService: RpcService, indexer: CkbIndexer) {
     for (const addressMeta of addressMetas) {
@@ -25,16 +27,6 @@ export default class IndexerCacheService {
     this.indexer = indexer
   }
 
-  private async countTxHashes(): Promise<number> {
-    return getConnection()
-      .getRepository(IndexerTxHashCache)
-      .createQueryBuilder()
-      .where({
-        walletId: this.walletId,
-      })
-      .getCount()
-  }
-
   private async getTxHashes(): Promise<IndexerTxHashCache[]> {
     return getConnection()
       .getRepository(IndexerTxHashCache)
@@ -79,6 +71,8 @@ export default class IndexerCacheService {
   }
 
   private async fetchTxMapping(): Promise<Map<string, Array<{ address: string; lockHash: string }>>> {
+    const lastCacheBlockNumber = await this.getCachedBlockNumber()
+    const currentHeaderBlockNumber = await this.rpcService.getTipBlockNumber()
     const mappingsByTxHash = new Map()
     for (const addressMeta of this.addressMetas) {
       const lockScripts = [
@@ -88,9 +82,18 @@ export default class IndexerCacheService {
       ]
 
       for (const lockScript of lockScripts) {
-        const transactionCollector = new TransactionCollector(this.indexer, { lock: lockScript }, this.rpcService.url, {
-          includeStatus: false,
-        })
+        const transactionCollector = new TransactionCollector(
+          this.indexer,
+          {
+            lock: lockScript,
+            fromBlock: lastCacheBlockNumber.value,
+            toBlock: currentHeaderBlockNumber,
+          },
+          this.rpcService.url,
+          {
+            includeStatus: false,
+          }
+        )
 
         const fetchedTxHashes = await transactionCollector.getTransactionHashes()
         if (!fetchedTxHashes.length) {
@@ -126,6 +129,8 @@ export default class IndexerCacheService {
             args: lockScript.args.slice(0, 42),
           },
           argsLen,
+          fromBlock: lastCacheBlockNumber.value,
+          toBlock: currentHeaderBlockNumber,
         })
 
         for await (const cell of cellCollector.collect()) {
@@ -142,17 +147,33 @@ export default class IndexerCacheService {
     return mappingsByTxHash
   }
 
+  private async getCachedBlockNumber() {
+    if (!this.#cacheBlockNumberEntity) {
+      this.#cacheBlockNumberEntity = (await getConnection().getRepository(SyncInfoEntity).findOne({ name: SyncInfoEntity.getLastCachedKey(this.walletId) })) ?? 
+        SyncInfoEntity.fromObject({
+          name: SyncInfoEntity.getLastCachedKey(this.walletId),
+          value: '0x0'
+        })
+    }
+
+    return this.#cacheBlockNumberEntity
+  }
+
+  private async saveCacheBlockNumber(cacheBlockNumber: string) {
+    let cacheBlockNumberEntity = await this.getCachedBlockNumber()
+    cacheBlockNumberEntity.value = cacheBlockNumber
+    await getConnection().manager.save(cacheBlockNumberEntity)
+  }
+
   public async upsertTxHashes(): Promise<string[]> {
+    const tipBlockNumber = await this.rpcService.getTipBlockNumber()
     const mappingsByTxHash = await this.fetchTxMapping()
 
     const fetchedTxHashes = [...mappingsByTxHash.keys()]
-    const fetchedTxHashCount = fetchedTxHashes.reduce((sum, txHash) => sum + mappingsByTxHash.get(txHash)!.length, 0)
-
-    const txCount = await this.countTxHashes()
-    if (fetchedTxHashCount === txCount) {
+    if (!fetchedTxHashes.length) {
+      await this.saveCacheBlockNumber(tipBlockNumber)
       return []
     }
-
     const txMetasCaches = await this.getTxHashes()
     const cachedTxHashes = txMetasCaches.map(meta => meta.txHash.toString())
 
@@ -161,6 +182,7 @@ export default class IndexerCacheService {
     const newTxHashes = fetchedTxHashes.filter(hash => !cachedTxHashesSet.has(hash))
 
     if (!newTxHashes.length) {
+      await this.saveCacheBlockNumber(tipBlockNumber)
       return []
     }
 
@@ -208,6 +230,7 @@ export default class IndexerCacheService {
       }
     }
 
+    await this.saveCacheBlockNumber(tipBlockNumber)
     return newTxHashes
   }
 

packages/neuron-wallet/src/block-sync-renderer/sync/queue.ts

@@ -30,6 +30,7 @@ export default class Queue {
   #rpcService: RpcService
   #indexerConnector: Connector | undefined
   #checkAndSaveQueue: QueueObject<{ txHashes: CKBComponents.Hash[]; params: unknown }> | undefined
+  #lockArgsSet: Set<string> = new Set()
 
   #multiSignBlake160s: string[]
   #anyoneCanPayLockHashes: string[]
@@ -44,6 +45,11 @@ export default class Queue {
     this.#lockHashes = AddressParser.batchToLockHash(this.#addresses.map(meta => meta.address))
 
     const blake160s = this.#addresses.map(meta => meta.blake160)
+    this.#lockArgsSet = new Set(blake160s.map(blake160 => [
+      blake160,
+      Multisig.hash([blake160]),
+      SystemScriptInfo.generateSecpScript(blake160).computeHash().slice(0, 42)
+    ]).flat())
     this.#multiSignBlake160s = blake160s.map(blake160 => Multisig.hash([blake160]))
     this.#anyoneCanPayLockHashes = blake160s.map(b =>
       this.#assetAccountInfo.generateAnyoneCanPayScript(b).computeHash()
@@ -216,7 +222,7 @@ export default class Queue {
           }
         }
       }
-      await TransactionPersistor.saveFetchTx(tx)
+      await TransactionPersistor.saveFetchTx(tx, this.#lockArgsSet)
       for (const info of anyoneCanPayInfos) {
         await AssetAccountService.checkAndSaveAssetAccountWhenSync(info.tokenID, info.blake160)
       }

packages/neuron-wallet/src/database/chain/entities/sync-info.ts

@@ -13,4 +13,18 @@ export default class SyncInfo {
     type: 'varchar',
   })
   value!: string
+
+  static fromObject(params: {
+    name: string,
+    value: string
+  }) {
+    const res = new SyncInfo()
+    res.name = params.name
+    res.value = params.value
+    return res
+  }
+
+  static getLastCachedKey(walletId: string) {
+    return `lastCachedBlockNumber_${walletId}`
+  }
 }

packages/neuron-wallet/src/database/chain/entities/tx-lock.ts

@@ -0,0 +1,24 @@
+import { BaseEntity, Entity, PrimaryColumn } from 'typeorm'
+
+@Entity()
+export default class TxLock extends BaseEntity {
+  @PrimaryColumn({
+    type: 'varchar'
+  })
+  transactionHash!: string
+
+  @PrimaryColumn({
+    type: 'varchar',
+  })
+  lockHash!: string
+
+  static fromObject(obj: {
+    txHash: string
+    lockHash: string
+  }) {
+    const res = new TxLock()
+    res.transactionHash = obj.txHash
+    res.lockHash = obj.lockHash
+    return res
+  }
+}

packages/neuron-wallet/src/database/chain/index.ts

@@ -8,14 +8,17 @@ import SyncInfoEntity from './entities/sync-info'
 import IndexerTxHashCache from './entities/indexer-tx-hash-cache'
 import MultisigOutput from './entities/multisig-output'
 import SyncProgress from './entities/sync-progress'
+import TxLock from './entities/tx-lock'
 
 /*
  * Clean local sqlite storage
  */
 export const clean = async (clearAllLightClientData?: boolean) => {
   await Promise.all([
-    ...[InputEntity, OutputEntity, TransactionEntity, IndexerTxHashCache, MultisigOutput].map(entity => {
-      return getConnection().getRepository(entity).clear()
+    ...[InputEntity, OutputEntity, TransactionEntity, IndexerTxHashCache, MultisigOutput, TxLock].map(entity => {
+      return getConnection()
+        .getRepository(entity)
+        .clear()
     }),
     clearAllLightClientData
       ? getConnection().getRepository(SyncProgress).clear()

packages/neuron-wallet/src/database/chain/migrations/1684488676083-TxLock.ts

@@ -0,0 +1,15 @@
+import {MigrationInterface, QueryRunner, TableIndex} from "typeorm";
+
+export class TxLock1684488676083 implements MigrationInterface {
+
+    public async up(queryRunner: QueryRunner): Promise<void> {
+      await queryRunner.query(`CREATE TABLE "tx_lock" ("lockHash" varchar NOT NULL, "transactionHash" varchar NOT NULL, PRIMARY KEY ("transactionHash", "lockHash"))`)
+      await queryRunner.createIndex("tx_lock", new TableIndex({ columnNames: ['lockHash'] }))
+      await queryRunner.createIndex("tx_lock", new TableIndex({ columnNames: ['transactionHash'] }))
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<void> {
+      await queryRunner.query(`DROP TABLE "tx_lock"`)
+    }
+
+}

packages/neuron-wallet/src/database/chain/ormconfig.ts

@@ -18,6 +18,7 @@ import AddressDescription from './entities/address-description'
 import MultisigConfig from './entities/multisig-config'
 import MultisigOuput from './entities/multisig-output'
 import SyncProgress from './entities/sync-progress'
+import TxLock from './entities/tx-lock'
 
 import { InitMigration1566959757554 } from './migrations/1566959757554-InitMigration'
 import { AddTypeAndHasData1567144517514 } from './migrations/1567144517514-AddTypeAndHasData'
@@ -51,6 +52,7 @@ import { UpdateOutputChequeLockHash1652945662504 } from './migrations/1652945662
 import { RemoveAddressesMultisigConfig1651820157100 } from './migrations/1651820157100-RemoveAddressesMultisigConfig'
 import { AddSyncProgress1676441837373 } from './migrations/1676441837373-AddSyncProgress'
 import { AddTypeSyncProgress1681360188494 } from './migrations/1681360188494-AddTypeSyncProgress'
+import { TxLock1684488676083 } from './migrations/1684488676083-TxLock'
 
 export const CONNECTION_NOT_FOUND_NAME = 'ConnectionNotFoundError'
 
@@ -84,6 +86,7 @@ const connectOptions = async (genesisBlockHash: string): Promise<SqliteConnectio
       MultisigConfig,
       MultisigOuput,
       SyncProgress,
+      TxLock,
     ],
     migrations: [
       InitMigration1566959757554,
@@ -118,6 +121,7 @@ const connectOptions = async (genesisBlockHash: string): Promise<SqliteConnectio
       RemoveAddressesMultisigConfig1651820157100,
       AddSyncProgress1676441837373,
       AddTypeSyncProgress1681360188494,
+      TxLock1684488676083,
     ],
     logger: 'simple-console',
     logging,