[WIP] feat: Assume-Valid SwiftSync with accumulator building

Cargo.toml

@@ -92,3 +92,6 @@ opt-level = 3     # maximum optimization level
 strip = true      # strip symbol tables and metadata
 lto = true        # enable link-time optimizations
 codegen-units = 1 # compile a single codegen unit
+
+[patch.crates-io]
+rustreexo = { git = "https://github.com/Davidson-Souza/rustreexo.git", branch = "swift-sync" }

bin/florestad/src/main.rs

@@ -122,7 +122,6 @@ fn main() {
     let _rt = tokio::runtime::Builder::new_multi_thread()
         .enable_all()
         .worker_threads(4)
-        .max_blocking_threads(2)
         .thread_keep_alive(Duration::from_secs(60))
         .thread_name("florestad")
         .build()

crates/floresta-chain/src/pruned_utreexo/chain_state.rs

@@ -1256,23 +1256,29 @@ impl<PersistedState: ChainStore> UpdatableChainstate for ChainState<PersistedSta
         acc: Stump,
         assumed_hash: BlockHash,
     ) -> Result<bool, BlockchainError> {
-        let mut curr_header = self.get_disk_block_header(&assumed_hash)?;
+        let assumed_header = self.get_disk_block_header(&assumed_hash)?;
 
-        while let Ok(header) = self.get_disk_block_header(&curr_header.block_hash()) {
+        let mut header = assumed_header;
+        let mut hash = assumed_hash;
+        loop {
             if self.is_genesis(&header) {
                 break;
             }
 
             let height = header.try_height()?;
-            self.update_header(&DiskBlockHeader::FullyValid(*header, height))?;
-            curr_header = self.get_ancestor(&header)?;
-        }
+            self.update_header_and_index(
+                &DiskBlockHeader::FullyValid(*header, height),
+                hash,
+                height,
+            )?;
 
-        self.update_view(curr_header.try_height()?, &curr_header, acc.clone())?;
+            // Move to the previous block
+            header = self.get_ancestor(&header)?;
+            hash = header.block_hash();
+        }
 
-        let mut guard = write_lock!(self);
-        guard.best_block.validation_index = assumed_hash;
-        guard.acc = acc;
+        // Update the tip and accumulator data with our assumed tip
+        self.update_view(assumed_header.try_height()?, &assumed_header, acc)?;
 
         Ok(true)
     }

crates/floresta-chain/src/pruned_utreexo/consensus.rs

@@ -569,10 +569,25 @@ impl Consensus {
         height: u32,
         unspent_indexes: HashSet<u32>,
         salt: &SipHashKeys,
-    ) -> Result<(SwiftSyncAgg, Amount), BlockchainError> {
+    ) -> Result<(SwiftSyncAgg, Amount, Vec<BitcoinNodeHash>), BlockchainError> {
         let txids = self.check_block(block, height)?;
 
-        Consensus::verify_block_transactions_swiftsync(height, block, txids, unspent_indexes, salt)
+        let utreexo_adds = udata::proof_util::get_block_adds_with_hints(
+            block,
+            &txids,
+            height,
+            block.block_hash(),
+            &unspent_indexes,
+        );
+        let (agg, amount) = Consensus::verify_block_transactions_swiftsync(
+            height,
+            block,
+            txids,
+            unspent_indexes,
+            salt,
+        )?;
+
+        Ok((agg, amount, utreexo_adds))
     }
 
     /// Returns the TxOut being spent by the given input.
@@ -729,7 +744,7 @@ impl Consensus {
         let adds = udata::proof_util::get_block_adds(block, height, block_hash);
 
         // Update the accumulator
-        let acc = acc.modify(&adds, &del_hashes, &proof)?.0;
+        let acc = acc.modify(&adds, &del_hashes, &proof)?;
         Ok(acc)
     }
 
@@ -1566,7 +1581,7 @@ mod tests {
                 // We add the only TxOut in this block to the aggregator (spent later).
                 9 => {
                     let unspent_indexes = HashSet::new();
-                    let (agg_blk_9, amount) = consensus
+                    let (agg_blk_9, amount, _) = consensus
                         .process_block_swiftsync(block, 9, unspent_indexes, &salt)
                         .unwrap();
 
@@ -1580,7 +1595,7 @@ mod tests {
                 // This block spends the TxOut that was added to the aggregator in block 9.
                 170 => {
                     let unspent_indexes = HashSet::from_iter(vec![0, 1, 2]);
-                    let (agg_blk_170, amount) = consensus
+                    let (agg_blk_170, amount, _) = consensus
                         .process_block_swiftsync(block, 170, unspent_indexes, &salt)
                         .unwrap();
 
@@ -1593,7 +1608,7 @@ mod tests {
                 }
                 i => {
                     let unspent_indexes = default_unspent_idx.clone();
-                    let (agg_i, amount) = consensus
+                    let (agg_i, amount, _) = consensus
                         .process_block_swiftsync(block, i as u32, unspent_indexes, &salt)
                         .unwrap();
 
@@ -1626,10 +1641,10 @@ mod tests {
         let block_9 = &mainnet_blocks[9];
         let block_170 = &mainnet_blocks[170];
 
-        let (agg_9, _) = consensus
+        let (agg_9, _, _) = consensus
             .process_block_swiftsync(block_9, 9, HashSet::new(), &salt)
             .unwrap();
-        let (agg_170, _) = consensus
+        let (agg_170, _, _) = consensus
             .process_block_swiftsync(block_170, 170, HashSet::from_iter(vec![0, 1, 2]), &salt)
             .unwrap();
 

crates/floresta-chain/src/pruned_utreexo/udata.rs

@@ -212,6 +212,7 @@ pub mod proof_util {
     use crate::BlockchainError;
     use crate::CompactLeafData;
     use crate::ScriptPubKeyKind;
+    use crate::extensions::Bip30UnspendableExt;
     use crate::prelude::*;
     use crate::pruned_utreexo::consensus::Consensus;
     use crate::pruned_utreexo::consensus::UTREEXO_TAG_V1;
@@ -348,6 +349,14 @@ pub mod proof_util {
         sha256::Hash::from_byte_array(leaf_hash.into())
     }
 
+    fn get_input_prevouts(txdata: &[Transaction]) -> HashSet<OutPoint> {
+        txdata
+            .iter()
+            .flat_map(|tx| tx.input.iter())
+            .map(|i| i.previous_output)
+            .collect()
+    }
+
     /// From a block, gets the roots that will be included on the acc, certifying
     /// that any utxo will not be spent in the same block.
     pub fn get_block_adds(
@@ -357,12 +366,7 @@ pub mod proof_util {
     ) -> Vec<BitcoinNodeHash> {
         // Get inputs from the block, we'll need this HashSet to check if an output is spent
         // in the same block. If it is, we don't need to add it to the accumulator.
-        let mut spent = HashSet::new();
-        for tx in &block.txdata {
-            for input in &tx.input {
-                spent.insert((input.previous_output.txid, input.previous_output.vout));
-            }
-        }
+        let spent = get_input_prevouts(&block.txdata);
 
         // Get all leaf hashes that will be added to the accumulator
         let mut adds = Vec::new();
@@ -371,7 +375,7 @@ pub mod proof_util {
             let is_cb = tx.is_coinbase();
 
             for (vout, output) in tx.output.iter().enumerate() {
-                let utxo_id = (txid, vout as u32);
+                let utxo_id = OutPoint::new(txid, vout as u32);
 
                 if Consensus::is_unspendable(&output.script_pubkey) || spent.contains(&utxo_id) {
                     // Do not add unspendable nor already spent utxos
@@ -387,6 +391,64 @@ pub mod proof_util {
         adds
     }
 
+    pub fn get_block_adds_with_hints(
+        block: &Block,
+        txids: &[Txid],
+        height: u32,
+        block_hash: BlockHash,
+        unspent_indexes: &HashSet<u32>,
+    ) -> Vec<BitcoinNodeHash> {
+        let transactions = &block.txdata;
+        assert_eq!(transactions.len(), txids.len());
+
+        let mut output_index = 0;
+
+        // Get inputs from the block, we'll need this HashSet to check if an output is spent
+        // in the same block. If it is, we don't need to add it to the accumulator.
+        let spent = get_input_prevouts(transactions);
+
+        // Get all leaf hashes that will be added to the accumulator
+        let mut adds = Vec::new();
+        for (tx, txid) in transactions.iter().zip(txids) {
+            let is_cb = tx.is_coinbase();
+
+            for (vout, output) in tx.output.iter().enumerate() {
+                // Special case: unspendable outputs do not count for the block `output_index`
+                if Consensus::is_unspendable(&output.script_pubkey) {
+                    continue;
+                }
+
+                let utxo_id = OutPoint::new(*txid, vout as u32);
+                if spent.contains(&utxo_id) {
+                    output_index += 1;
+                    // Do not add UTXOs spent in the same block
+                    continue;
+                }
+
+                let is_bip30_unspendable = is_cb && block.is_bip30_unspendable(height);
+
+                if is_bip30_unspendable || unspent_indexes.contains(&output_index) {
+                    // Add unspent outputs to the accumulator, according to the hints
+                    let leaf_hash =
+                        get_leaf_hashes(*txid, is_cb, vout as u32, output, height, block_hash);
+                    adds.push(BitcoinNodeHash::Some(leaf_hash.to_byte_array()));
+                } else {
+                    // Hinted as spent: add empty leaf hash
+                    adds.push(BitcoinNodeHash::Empty);
+                }
+
+                // BIP-30 unspendable outputs do not count for the block `output_index`
+                if is_bip30_unspendable {
+                    continue;
+                }
+
+                output_index += 1;
+            }
+        }
+
+        adds
+    }
+
     /// A hash map that provides the UTXO data given the outpoint. We will get this data
     /// from either our own cache or the Utreexo proofs, and use it to validate blocks
     /// and transactions.

crates/floresta-wire/Cargo.toml

@@ -20,6 +20,7 @@ categories = ["cryptography::cryptocurrencies", "network-programming"]
 bip324 = { version = "=0.10.0", features = [ "tokio" ] }
 bitcoin = { workspace = true }
 dns-lookup = { workspace = true }
+hintsfile = "0.1.1"
 rand = { workspace = true }
 rustls = { version = "=0.23.40", default-features = false, features = ["ring", "std", "tls12"] }
 rustreexo = { workspace = true }

crates/floresta-wire/src/p2p_wire/mod.rs

@@ -96,6 +96,7 @@ pub mod node_context;
 pub mod node_interface;
 pub mod peer;
 pub mod socks;
+mod stump_updater;
 #[cfg(test)]
 #[doc(hidden)]
 pub mod tests;

crates/floresta-wire/src/p2p_wire/node/blocks.rs

@@ -1,5 +1,6 @@
 // SPDX-License-Identifier: MIT OR Apache-2.0
 
+use std::sync::Arc;
 use std::time::Instant;
 
 use bitcoin::Block;
@@ -43,12 +44,15 @@ pub(crate) struct InflightBlock {
     pub peer: PeerId,
 
     /// The block itself.
-    pub block: Block,
+    pub block: Arc<Block>,
 
     /// Auxiliary data needed for validating this block. Currently, it includes utreexo
     /// leaf data (previous UTXOs spent in the block), the corresponding accumulator
     /// inclusion proof, and the peer id that provided them.
     pub aux_data: Option<UtreexoData>,
+
+    /// If this block is currently being processed in a worker, this is the start time.
+    pub processing_since: Option<Instant>,
 }
 
 impl InflightBlock {
@@ -57,7 +61,7 @@ impl InflightBlock {
     /// If the block doesn't spend any output (i.e., coinbase transaction only) this method adds
     /// empty auxiliary data, which marks this inflight block as ready to process. Blocks with
     /// transactions require [`UtreexoData`] (see [`InflightBlock::add_utreexo_data`]).
-    fn new(block: Block, peer: PeerId) -> Self {
+    fn new(block: Arc<Block>, peer: PeerId) -> Self {
         let aux_data = match block.txdata.len() {
             1 => Some((Vec::new(), Proof::default(), peer)),
             _ => None, // we need auxiliary data for the txs
@@ -67,6 +71,7 @@ impl InflightBlock {
             peer,
             block,
             aux_data,
+            processing_since: None,
         }
     }
 
@@ -82,6 +87,29 @@ where
     Chain: ChainBackend + 'static,
     WireError: From<Chain::Error>,
 {
+    /// Returns `true` only if we can request `BLOCKS_PER_GETDATA` without exceeding the maximum
+    /// unprocessed blocks allowed.
+    pub(crate) fn can_request_more_blocks(&self) -> bool {
+        let max_inflight_blocks = T::BLOCKS_PER_GETDATA * T::MAX_CONCURRENT_GETDATA;
+
+        // If we do a GETDATA request, this will be the new unprocessed count
+        let next_unprocessed = self.unprocessed_blocks() + T::BLOCKS_PER_GETDATA;
+
+        next_unprocessed <= max_inflight_blocks
+    }
+
+    /// Returns the number of blocks awaiting processing (in memory or requested).
+    pub(crate) fn unprocessed_blocks(&self) -> usize {
+        let blocks_in_mem = self.blocks.len();
+        let requested_blocks = self
+            .inflight
+            .keys()
+            .filter(|inflight| matches!(inflight, InflightRequests::Blocks(_)))
+            .count();
+
+        blocks_in_mem + requested_blocks
+    }
+
     pub(crate) fn request_blocks(&mut self, blocks: Vec<BlockHash>) -> Result<(), WireError> {
         let should_request = |block: &BlockHash| {
             let is_inflight = self
@@ -98,8 +126,8 @@ where
             return Ok(());
         }
 
-        let peer =
-            self.send_to_fast_peer(NodeRequest::GetBlock(blocks.clone()), ServiceFlags::NETWORK)?;
+        let block_req = NodeRequest::GetBlock(blocks.clone(), self.witnessless);
+        let peer = self.send_to_fast_peer(block_req, ServiceFlags::NETWORK)?;
 
         for block in blocks.iter() {
             self.inflight
@@ -126,7 +154,7 @@ where
         debug!("Received block {block_hash} from peer {peer}, with {txdata_len} txs");
 
         self.blocks
-            .insert(block_hash, InflightBlock::new(block, peer));
+            .insert(block_hash, InflightBlock::new(Arc::new(block), peer));
 
         // We only need auxiliary utreexo data if there are non-coinbase transactions
         if txdata_len != 1 {
@@ -311,7 +339,7 @@ where
     }
 
     /// Returns the inner [`BlockValidationErrors`] of this chain error, if any.
-    fn block_validation_err(e: BlockchainError) -> Option<BlockValidationErrors> {
+    pub(crate) fn block_validation_err(e: BlockchainError) -> Option<BlockValidationErrors> {
         match e {
             BlockchainError::TransactionError(tx_err) => Some(tx_err.error),
             BlockchainError::BlockValidation(block_err) => Some(block_err),
@@ -329,7 +357,7 @@ where
     fn handle_validation_errors(
         &mut self,
         e: BlockValidationErrors,
-        block: Block,
+        block: Arc<Block>,
         block_peer: PeerId,
         utreexo_peer: PeerId,
     ) -> Option<PeerId> {