From 9b43f5a5aca50efcbe626905fe26a67831f1d8ad Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Wed, 8 Nov 2023 14:32:25 +0900 Subject: [PATCH 1/5] blockchain, mempool, wallet: update verifyudata to include cache bool On verification, there's an option for the accumulator to cache the proof and the hashes so that they can be proven at a later date. The change here exposes that option to the callers of verifyudata. --- blockchain/utreexoviewpoint.go | 8 ++++++-- mempool/mempool.go | 8 +++++--- wallet/watchonly.go | 2 +- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/blockchain/utreexoviewpoint.go b/blockchain/utreexoviewpoint.go index 15a1e726..58fbe8db 100644 --- a/blockchain/utreexoviewpoint.go +++ b/blockchain/utreexoviewpoint.go @@ -826,7 +826,7 @@ func (b *BlockChain) IsUtreexoViewActive() bool { // // This function does not modify the underlying UtreexoViewpoint. // This function is safe for concurrent access. -func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn) error { +func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn, remember bool) error { // Nothing to prove. if len(txIns) == 0 { return nil @@ -909,7 +909,7 @@ func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn) error { // VerifyBatchProof checks that the utreexo proofs are valid without // mutating the accumulator. - err := b.utreexoView.accumulator.Verify(delHashes, ud.AccProof, false) + err := b.utreexoView.accumulator.Verify(delHashes, ud.AccProof, remember) if err != nil { str := "Verify fail. All txIns-leaf datas:\n" for i, txIn := range txIns { @@ -921,6 +921,10 @@ func (b *BlockChain) VerifyUData(ud *wire.UData, txIns []*wire.TxIn) error { return fmt.Errorf(str) } + if remember { + log.Debugf("cached hashes: %v", delHashes) + } + return nil } diff --git a/mempool/mempool.go b/mempool/mempool.go index be82a36c..95c164fd 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -97,7 +97,7 @@ type Config struct { // VerifyUData defines the function to use to verify the utreexo // data. This is only used when the node is run with the UtreexoView // activated. - VerifyUData func(ud *wire.UData, txIns []*wire.TxIn) error + VerifyUData func(ud *wire.UData, txIns []*wire.TxIn, remember bool) error // SigCache defines a signature cache to use. SigCache *txscript.SigCache @@ -1083,9 +1083,11 @@ func (mp *TxPool) maybeAcceptTransaction(tx *btcutil.Tx, isNew, rateLimit, rejec // First verify the proof to ensure that the proof the peer has // sent was over valid. - err := mp.cfg.VerifyUData(ud, tx.MsgTx().TxIn) + err = mp.cfg.VerifyUData(ud, tx.MsgTx().TxIn, false) if err != nil { - return nil, nil, err + str := fmt.Sprintf("transaction %v failed the utreexo data verification.", + txHash) + return nil, nil, txRuleError(wire.RejectInvalid, str) } log.Debugf("VerifyUData passed for tx %s", txHash.String()) diff --git a/wallet/watchonly.go b/wallet/watchonly.go index 349fbd3f..bc9cbe34 100644 --- a/wallet/watchonly.go +++ b/wallet/watchonly.go @@ -864,7 +864,7 @@ func (wm *WatchOnlyWalletManager) ProveTx(tx *btcutil.Tx) (*wire.UData, error) { } // Verify that the generated proof passes verification. - err = wm.config.Chain.VerifyUData(&ud, tx.MsgTx().TxIn) + err = wm.config.Chain.VerifyUData(&ud, tx.MsgTx().TxIn, false) if err != nil { return nil, fmt.Errorf("Couldn't prove tx %s. Generated proof "+ "fails verification. Error: %v", tx.Hash(), err) From 2ee165f86f447825a2417ddc5b8111f3730bf350 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Wed, 8 Nov 2023 14:47:17 +0900 Subject: [PATCH 2/5] wire: add IsCompact method to LeafData IsCompact returns whether or not the leafdata is in the compact form or not. Useful for determining if the correct hash can be genereated from just the leaf data or if additional data is needed. --- wire/leaf.go | 7 ++++++ wire/leaf_test.go | 60 +++++++++++++++++++++++++++++++++++++++++++++++ wire/udata.go | 8 +++++++ 3 files changed, 75 insertions(+) diff --git a/wire/leaf.go b/wire/leaf.go index 06ef8991..e92292ef 100644 --- a/wire/leaf.go +++ b/wire/leaf.go @@ -149,6 +149,13 @@ func (l *LeafData) SetUnconfirmed() { l.Height = -1 } +// IsCompact returns if the leaf data is in the compact state. +func (l *LeafData) IsCompact() bool { + return l.BlockHash == empty && + l.OutPoint.Hash == empty && + l.OutPoint.Index == 0 +} + // ----------------------------------------------------------------------------- // LeafData serialization includes all the data needed for generating the hash // commitment of the LeafData. diff --git a/wire/leaf_test.go b/wire/leaf_test.go index 12df4e5f..c093dc45 100644 --- a/wire/leaf_test.go +++ b/wire/leaf_test.go @@ -570,3 +570,63 @@ func TestLeafDataJsonMarshal(t *testing.T) { } } } + +func TestIsCompact(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ld LeafData + }{ + { + name: "Testnet3 tx 061bb0bf... from block 1600000", + ld: LeafData{ + BlockHash: *newHashFromStr("00000000000172ff8a4e14441512072bacaf8d38b995a3fcd2f8435efc61717d"), + OutPoint: OutPoint{ + Hash: *newHashFromStr("061bb0bf3a1b9df13773da06bf92920394887a9c2b8b8772ac06be4e077df5eb"), + Index: 10, + }, + Amount: 200000, + PkScript: hexToBytes("a914e8d74935cfa223f9750a32b18d609cba17a5c3fe87"), + Height: 1599255, + IsCoinBase: false, + }, + }, + { + name: "Mainnet coinbase tx fa201b65... from block 573123", + ld: LeafData{ + BlockHash: *newHashFromStr("000000000000000000278eb9386b4e70b850a4ec21907af3a27f50330b7325aa"), + OutPoint: OutPoint{ + Hash: *newHashFromStr("fa201b650eef761f5701afbb610e4a211b86985da4745aec3ac0f4b7a8e2c8d2"), + Index: 0, + }, + Amount: 1315080370, + PkScript: hexToBytes("76a9142cc2b87a28c8a097f48fcc1d468ced6e7d39958d88ac"), + Height: 573123, + IsCoinBase: true, + }, + }, + } + + for _, test := range tests { + if test.ld.IsCompact() { + t.Fatalf("leafdata %v is not compact but IsCompact returned %v", test.ld, test.ld.IsCompact()) + } + + var w bytes.Buffer + err := test.ld.SerializeCompact(&w, false) + if err != nil { + t.Fatal(err) + } + + compact := NewLeafData() + err = compact.DeserializeCompact(&w, false) + if err != nil { + t.Fatal(err) + } + + if !compact.IsCompact() { + t.Fatalf("leafdata %v is compact but IsCompact returned %v", test.ld, compact.IsCompact()) + } + } +} diff --git a/wire/udata.go b/wire/udata.go index ab7457ed..01d9ed4d 100644 --- a/wire/udata.go +++ b/wire/udata.go @@ -446,6 +446,14 @@ func GenerateUData(txIns []LeafData, pollard utreexo.Utreexo) ( unconfirmedCount++ continue } + + // We can't calculate the correct hash if the leaf data is in + // the compact state. + if ld.IsCompact() { + return nil, fmt.Errorf("leafdata is compact. Unable " + + "to generate a leafhash") + } + delHashes = append(delHashes, ld.LeafHash()) } From 6bf7025e476c4219e76af43f4cc422a1818d70f1 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Wed, 8 Nov 2023 14:56:53 +0900 Subject: [PATCH 3/5] blockchain: add PruneFromAccumulator method to blockchain PruneFromAccumulator can be used for csn nodes to prune away the given hashes from the accumulator. --- blockchain/utreexoviewpoint.go | 36 ++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/blockchain/utreexoviewpoint.go b/blockchain/utreexoviewpoint.go index 58fbe8db..345772ee 100644 --- a/blockchain/utreexoviewpoint.go +++ b/blockchain/utreexoviewpoint.go @@ -943,6 +943,42 @@ func (b *BlockChain) GenerateUData(dels []wire.LeafData) (*wire.UData, error) { return ud, nil } +// PruneFromAccumulator uncaches the given hashes from the accumulator. No action is taken +// if the hashes are not already cached. +func (b *BlockChain) PruneFromAccumulator(leaves []wire.LeafData) error { + if b.utreexoView == nil { + return fmt.Errorf("This blockchain instance doesn't have an " + + "accumulator. Cannot prune leaves") + } + + b.chainLock.Lock() + defer b.chainLock.Unlock() + + hashes := make([]utreexo.Hash, 0, len(leaves)) + for i := range leaves { + // Unconfirmed leaves aren't present in the accumulator. + if leaves[i].IsUnconfirmed() { + continue + } + + if leaves[i].IsCompact() { + return fmt.Errorf("Cannot generate hash as " + + "the leafdata is compact") + } + + hashes = append(hashes, leaves[i].LeafHash()) + } + + log.Debugf("uncaching hashes: %v", hashes) + + err := b.utreexoView.accumulator.Prune(hashes) + if err != nil { + return err + } + + return nil +} + // ChainTipProof represents all the information that is needed to prove that a // utxo exists in the chain tip with utreexo accumulator proof. type ChainTipProof struct { From 5604fb9f328949ba3f6ffec60f5c9d61876d86a7 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Wed, 8 Nov 2023 16:03:06 +0900 Subject: [PATCH 4/5] mempool, main: cache tx proofs for CSNs For utreexo CSN to CSN proof serving, each node needs to keep track of the proof for the given mempool tx. The change made here introduces a pool of leaf datas that are stored in the mempool and when adding transactions to the pool, the accumulator proof for that is cached in the accumulator as well. This code is mostly for the p2p to eventually support receiving and serving partial proofs for utreexo peers. --- mempool/mempool.go | 62 +++++++++++++++++++++++++++++++++++++++++++--- server.go | 46 +++++++++++++++++++++++++++------- 2 files changed, 96 insertions(+), 12 deletions(-) diff --git a/mempool/mempool.go b/mempool/mempool.go index 95c164fd..a19dda56 100644 --- a/mempool/mempool.go +++ b/mempool/mempool.go @@ -99,6 +99,9 @@ type Config struct { // activated. VerifyUData func(ud *wire.UData, txIns []*wire.TxIn, remember bool) error + // PruneFromAccumulator uncaches the given hashes from the accumulator. + PruneFromAccumulator func(hashes []wire.LeafData) error + // SigCache defines a signature cache to use. SigCache *txscript.SigCache @@ -189,6 +192,7 @@ type TxPool struct { mtx sync.RWMutex cfg Config pool map[chainhash.Hash]*TxDesc + poolLeaves map[chainhash.Hash][]wire.LeafData orphans map[chainhash.Hash]*orphanTx orphansByPrev map[wire.OutPoint]map[chainhash.Hash]*btcutil.Tx outpoints map[wire.OutPoint]*btcutil.Tx @@ -496,6 +500,25 @@ func (mp *TxPool) removeTransaction(tx *btcutil.Tx, removeRedeemers bool) { mp.cfg.AddrIndex.RemoveUnconfirmedTx(txHash) } + // If the utreexo view is active, then remove the cached hashes from the + // accumulator. + if mp.cfg.IsUtreexoViewActive != nil && mp.cfg.IsUtreexoViewActive() { + leaves, found := mp.poolLeaves[*txHash] + if !found { + log.Infof("missing the leaf hashes for tx %s from while "+ + "removing it from the pool", + tx.MsgTx().TxHash().String()) + } else { + delete(mp.poolLeaves, *txHash) + + err := mp.cfg.PruneFromAccumulator(leaves) + if err != nil { + log.Infof("err while pruning proof for inputs of tx %s: ", + err, tx.MsgTx().TxHash().String()) + } + } + } + // Mark the referenced outpoints as unspent by the pool. for _, txIn := range txDesc.Tx.MsgTx().TxIn { delete(mp.outpoints, txIn.PreviousOutPoint) @@ -543,7 +566,21 @@ func (mp *TxPool) RemoveDoubleSpends(tx *btcutil.Tx) { // helper for maybeAcceptTransaction. // // This function MUST be called with the mempool lock held (for writes). -func (mp *TxPool) addTransaction(utxoView *blockchain.UtxoViewpoint, tx *btcutil.Tx, height int32, fee int64) *TxDesc { +func (mp *TxPool) addTransaction(utxoView *blockchain.UtxoViewpoint, tx *btcutil.Tx, height int32, fee int64) (*TxDesc, error) { + if mp.cfg.IsUtreexoViewActive != nil && mp.cfg.IsUtreexoViewActive() { + // Ingest the proof. Shouldn't error out with the proof being invalid + // here since we've already verified it above. + err := mp.cfg.VerifyUData(tx.MsgTx().UData, tx.MsgTx().TxIn, true) + if err != nil { + return nil, fmt.Errorf("error while ingesting proof. %v", err) + } + + mp.poolLeaves[*tx.Hash()] = tx.MsgTx().UData.LeafDatas + } + + // Nil out uneeded udata for the mempool. + tx.MsgTx().UData = nil + // Add the transaction to the pool and mark the referenced outpoints // as spent by the pool. txD := &TxDesc{ @@ -574,7 +611,7 @@ func (mp *TxPool) addTransaction(utxoView *blockchain.UtxoViewpoint, tx *btcutil mp.cfg.FeeEstimator.ObserveTransaction(txD) } - return txD + return txD, nil } // checkPoolDoubleSpend checks whether or not the passed transaction is @@ -880,6 +917,21 @@ func (mp *TxPool) FetchTransaction(txHash *chainhash.Hash) (*btcutil.Tx, error) return nil, fmt.Errorf("transaction is not in the pool") } +// FetchLeafDatas returns the leafdatas for the given tx. Returns an error if +// the leaves for the given tx is not in the pool. +func (mp *TxPool) FetchLeafDatas(txHash *chainhash.Hash) ([]wire.LeafData, error) { + // Protect concurrent access. + mp.mtx.RLock() + leaves, exists := mp.poolLeaves[*txHash] + mp.mtx.RUnlock() + + if exists { + return leaves, nil + } + + return nil, fmt.Errorf("leafdata for the transaction is not in the pool") +} + // validateReplacement determines whether a transaction is deemed as a valid // replacement of all of its conflicts according to the RBF policy. If it is // valid, no error is returned. Otherwise, an error is returned indicating what @@ -1305,7 +1357,10 @@ func (mp *TxPool) maybeAcceptTransaction(tx *btcutil.Tx, isNew, rateLimit, rejec // this call as they'll be removed eventually. mp.removeTransaction(conflict, false) } - txD := mp.addTransaction(utxoView, tx, bestHeight, txFee) + txD, err := mp.addTransaction(utxoView, tx, bestHeight, txFee) + if err != nil { + return nil, txD, err + } log.Debugf("Accepted transaction %v (pool size: %v)", txHash, len(mp.pool)) @@ -1626,6 +1681,7 @@ func New(cfg *Config) *TxPool { return &TxPool{ cfg: *cfg, pool: make(map[chainhash.Hash]*TxDesc), + poolLeaves: make(map[chainhash.Hash][]wire.LeafData), orphans: make(map[chainhash.Hash]*orphanTx), orphansByPrev: make(map[wire.OutPoint]map[chainhash.Hash]*btcutil.Tx), nextExpireScan: time.Now().Add(orphanExpireScanInterval), diff --git a/server.go b/server.go index d487ad7c..f85ea457 100644 --- a/server.go +++ b/server.go @@ -1529,8 +1529,35 @@ func (s *server) pushTxMsg(sp *serverPeer, hash *chainhash.Hash, doneChan chan<- return err } - // We already checked that at least one is active. Pick one and - // generate the UData. + // For compact state nodes. + if cfg.Utreexo { + // Fetch the necessary leafdatas to create the utreexo data. + leafDatas, err := s.txMemPool.FetchLeafDatas(tx.Hash()) + if err != nil { + chanLog.Errorf(err.Error()) + if doneChan != nil { + doneChan <- struct{}{} + } + return err + } + + btcdLog.Debugf("fetched %v for tx %s", leafDatas, tx.Hash()) + + // This creates the accumulator proof and also puts the leaf datas + // in the utreexo data. + ud, err := s.chain.GenerateUData(leafDatas) + if err != nil { + chanLog.Errorf(err.Error()) + if doneChan != nil { + doneChan <- struct{}{} + } + return err + } + + tx.MsgTx().UData = ud + } + + // For bridge nodes. if s.utreexoProofIndex != nil { leafDatas, err := blockchain.TxToDelLeaves(tx, s.chain) if err != nil { @@ -3160,13 +3187,14 @@ func newServer(listenAddrs, agentBlacklist, agentWhitelist []string, CalcSequenceLock: func(tx *btcutil.Tx, view *blockchain.UtxoViewpoint) (*blockchain.SequenceLock, error) { return s.chain.CalcSequenceLock(tx, view, true) }, - IsDeploymentActive: s.chain.IsDeploymentActive, - IsUtreexoViewActive: s.chain.IsUtreexoViewActive, - VerifyUData: s.chain.VerifyUData, - SigCache: s.sigCache, - HashCache: s.hashCache, - AddrIndex: s.addrIndex, - FeeEstimator: s.feeEstimator, + IsDeploymentActive: s.chain.IsDeploymentActive, + IsUtreexoViewActive: s.chain.IsUtreexoViewActive, + VerifyUData: s.chain.VerifyUData, + PruneFromAccumulator: s.chain.PruneFromAccumulator, + SigCache: s.sigCache, + HashCache: s.hashCache, + AddrIndex: s.addrIndex, + FeeEstimator: s.feeEstimator, } s.txMemPool = mempool.New(&txC) From d6edbc6b18a898af4f089b588d087cfde30803f0 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Wed, 8 Nov 2023 16:28:55 +0900 Subject: [PATCH 5/5] main: only update utreexo data stats for sending if it's not nil --- server.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/server.go b/server.go index f85ea457..d3b33697 100644 --- a/server.go +++ b/server.go @@ -2602,8 +2602,10 @@ func (s *server) UpdateProofBytesWritten(msgTx *wire.MsgTx) error { s.addAccBytesSent(uint64(accSize)) } else if s.chain.IsUtreexoViewActive() { - s.addProofBytesSent(uint64(msgTx.UData.SerializeSizeCompact(true))) - s.addAccBytesSent(uint64(msgTx.UData.SerializeAccSizeCompact())) + if msgTx.UData != nil { + s.addProofBytesSent(uint64(msgTx.UData.SerializeSizeCompact(true))) + s.addAccBytesSent(uint64(msgTx.UData.SerializeAccSizeCompact())) + } } return nil