feat: Compute authentication structure from leafs

In particular, it is not required to construct the full Merkle tree at
any point. This is useful when it is impossible to hold the full tree
in memory.

Author: jan-ferdinand

.cargo-husky/hooks/pre-commit

@@ -4,8 +4,4 @@
 # It's intended to be copied to `.git/hooks/` by `cargo-husky`.
 
 set -e
-
-echo '+cargo clippy --all-targets -- -D warnings'
-cargo clippy --all-targets -- -D warnings
-echo '+cargo fmt --all -- --check'
-cargo fmt --all -- --check
+exit 0

twenty-first/benches/merkle_tree_authenticate.rs

@@ -9,7 +9,8 @@ criterion_main!(merkle_tree_authenticate);
 criterion_group!(
     merkle_tree_authenticate,
     gen_auth_structure,
-    verify_auth_structure
+    verify_auth_structure,
+    recompute_auth_structure,
 );
 
 fn gen_auth_structure(c: &mut Criterion) {
@@ -38,6 +39,29 @@ fn verify_auth_structure(c: &mut Criterion) {
     });
 }
 
+fn recompute_auth_structure(c: &mut Criterion) {
+    let mut sampler = MerkleTreeSampler::default();
+    let leafs = sampler.leaf_digests();
+
+    let mut group = c.benchmark_group("recompute_auth_structure");
+    group.sample_size(10);
+    group.bench_function("sequential", |b| {
+        b.iter_batched(
+            || sampler.indices_to_open(),
+            |indices| MerkleTree::sequential_authentication_structure_from_leafs(&leafs, &indices),
+            BatchSize::SmallInput,
+        )
+    });
+    group.bench_function("parallel", |b| {
+        b.iter_batched(
+            || sampler.indices_to_open(),
+            |indices| MerkleTree::par_authentication_structure_from_leafs(&leafs, &indices),
+            BatchSize::SmallInput,
+        )
+    });
+    group.finish();
+}
+
 #[derive(Debug, Clone, PartialEq, Eq)]
 struct MerkleTreeSampler {
     rng: StdRng,

twenty-first/src/util_types/merkle_tree.rs

@@ -358,6 +358,71 @@ impl MerkleTree {
         Ok(set_difference.sorted_unstable().rev())
     }
 
+    /// Construct an [authentication structure][Self::authentication_structure]
+    /// without access to a full [`MerkleTree`], only the leafs. Particularly
+    /// useful in combination with [RAM-frugal root][frugal] computation.
+    ///
+    /// [frugal]: Self::sequential_frugal_root
+    pub fn sequential_authentication_structure_from_leafs(
+        leafs: &[Digest],
+        leaf_indices: &[MerkleTreeLeafIndex],
+    ) -> Result<Vec<Digest>> {
+        Self::authentication_structure_node_indices(leafs.len(), leaf_indices)?
+            .map(|node_index| Self::subtree_leafs(leafs, node_index))
+            .map(Self::sequential_frugal_root)
+            .collect()
+    }
+
+    /// Construct an [authentication structure][Self::authentication_structure]
+    /// without access to a full [`MerkleTree`], only the leafs. Particularly
+    /// useful in combination with [RAM-frugal root][frugal] computation.
+    ///
+    /// [frugal]: Self::par_frugal_root
+    pub fn par_authentication_structure_from_leafs(
+        leafs: &[Digest],
+        leaf_indices: &[MerkleTreeLeafIndex],
+    ) -> Result<Vec<Digest>> {
+        Self::authentication_structure_node_indices(leafs.len(), leaf_indices)?
+            .collect_vec()
+            .into_par_iter()
+            .map(|node_index| Self::subtree_leafs(leafs, node_index))
+            .map(Self::par_frugal_root)
+            .collect()
+    }
+
+    /// Given a list of leafs and a [`MerkleTreeNodeIndex`] within the tree
+    /// defined by those leafs, return the leafs that are part of the subtree
+    /// rooted at the given node index.
+    ///
+    /// For example, given 4 leafs and node index 3, returns the leafs with
+    /// node indices 6 and 7:
+    ///
+    /// ```markdown
+    ///         1
+    ///       ╱   ╲
+    ///      2      3
+    ///     ╱ ╲    ╱ ╲
+    ///    4   5  6   7
+    /// ```
+    ///
+    /// Because this is an internal helper function (and only for that reason),
+    /// it's the caller's responsibility to ensure that the arguments are
+    /// integral:
+    /// - the number of `leafs` must be a power of 2
+    /// - the `node_index` must be valid with respect to the tree induced by the
+    ///   `leafs`
+    fn subtree_leafs(leafs: &[Digest], node_index: MerkleTreeNodeIndex) -> &[Digest] {
+        let total_num_leafs = leafs.len();
+        let total_tree_height = total_num_leafs.ilog2();
+
+        let tree_height = total_tree_height - node_index.ilog2();
+        let left_leaf_node_index = node_index * (1 << tree_height);
+        let left_leaf_index = left_leaf_node_index - total_num_leafs;
+        let num_leafs = 1 << tree_height;
+
+        &leafs[left_leaf_index..left_leaf_index + num_leafs]
+    }
+
     /// Generate a de-duplicated authentication structure for the given
     /// [`MerkleTreeLeafIndex`]es.
     ///
@@ -1076,6 +1141,59 @@ pub mod merkle_tree_test {
         assert_eq!(auth_path_with_nodes([14, 6, 2]), auth_path_for_leaf(7));
     }
 
+    #[test]
+    fn authentication_paths_of_very_small_tree_are_identical_when_using_tree_or_only_leafs() {
+        let tree = MerkleTree::test_tree_of_height(3);
+        let leafs = tree.leafs().copied().collect_vec();
+
+        let tree_path = |i| tree.authentication_structure(&[i]).unwrap();
+        let seq_leaf_path =
+            |i| MerkleTree::sequential_authentication_structure_from_leafs(&leafs, &[i]).unwrap();
+        let par_leaf_path =
+            |i| MerkleTree::par_authentication_structure_from_leafs(&leafs, &[i]).unwrap();
+
+        for leaf_idx in 0..tree.num_leafs() {
+            dbg!(leaf_idx);
+            assert_eq!(tree_path(leaf_idx), seq_leaf_path(leaf_idx));
+            assert_eq!(tree_path(leaf_idx), par_leaf_path(leaf_idx));
+        }
+    }
+
+    #[proptest(cases = 100)]
+    fn auth_structure_is_independent_of_compute_method(test_tree: MerkleTreeToTest) {
+        let tree = test_tree.tree;
+        let selected_indices = test_tree.selected_indices;
+        let leafs = tree.leafs().copied().collect_vec();
+
+        let cached_auth_structure = tree.authentication_structure(&selected_indices)?;
+        let seq_auth_structure =
+            MerkleTree::sequential_authentication_structure_from_leafs(&leafs, &selected_indices)?;
+        let par_auth_structure =
+            MerkleTree::par_authentication_structure_from_leafs(&leafs, &selected_indices)?;
+
+        prop_assert_eq!(&cached_auth_structure, &seq_auth_structure);
+        prop_assert_eq!(&cached_auth_structure, &par_auth_structure);
+    }
+
+    #[test]
+    fn subtree_leafs_are_actually_sub_tree_leafs() {
+        let tree = MerkleTree::test_tree_of_height(5);
+        let leafs = tree.leafs().copied().collect_vec();
+        let subtree = |node_idx| MerkleTree::subtree_leafs(&leafs, node_idx);
+
+        // Check all node indices, one level at a time. Going level by level
+        // ensures that the expected subtrees (per level) have the same number
+        // of leafs.
+        for node_indices in [1..2, 2..4, 4..8, 8..16, 16..32, 32..64] {
+            // the expected number of leafs in any subtree at the current level
+            let num_leafs = tree.num_leafs() / node_indices.len();
+            for (i, node_index) in node_indices.enumerate() {
+                let expected_leafs = &leafs[i * num_leafs..(i + 1) * num_leafs];
+                assert_eq!(expected_leafs, subtree(node_index));
+            }
+        }
+    }
+
     #[proptest(cases = 10)]
     fn each_leaf_can_be_verified_individually(test_tree: MerkleTreeToTest) {
         let tree = test_tree.tree;