CompactProof: Deduplicate values (#233)

* trie-db: deduplicate detached values in compact proofs

encode_compact emits a detached value node once per referencing trie
node, so a value shared by N keys is sent N times (paritytech/polkadot-sdk#12565).
Add encode_compact_skip_duplicate_values to emit each distinct value
once; repeats are emitted unmodified and stay decodable by existing
hash-keyed decoders. For prefixed databases,
decode_compact_from_iter_with_known_values re-inserts deduplicated
values at every referencing position.

Author: skunert

compact-proof-oversized.scale.tar.xz

@@ -30,6 +30,7 @@ use trie_root::{Hasher, Value as TrieStreamValue};
 
 mod substrate;
 mod substrate_like;
+pub mod trie_db_0_31_decoder;
 pub mod node {
 	pub use trie_db::node::Node;
 }

compact-proof-worst-dedup-child.scale.tar.xz

@@ -0,0 +1,208 @@
+// Copyright 2019, 2021 Parity Technologies
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//! Verbatim compact-proof decoder of the released `trie-db` 0.31.0 (only imports adjusted).
+//!
+//! A frozen snapshot of deployed decoder behavior — **do not update**. It exists so tests and
+//! fuzzers can assert that deduplicated proofs (see `encode_compact_skip_duplicates`) stay
+//! decodable by decoders already in the field. Reconstructs into a hash-keyed database only; the
+//! 0.31.0 decoder inserts attached values under an incomplete prefix in a position-keyed database
+//! (fixed by #227), so prefixed databases are out of scope.
+
+use hash_db::HashDB;
+use std::{convert::TryInto, marker::PhantomData};
+use trie_db::{
+	nibble_ops::NIBBLE_LENGTH,
+	node::{Node, NodeHandle, Value},
+	CError, ChildReference, DBValue, NibbleVec, NodeCodec, TrieError, TrieHash, TrieLayout,
+};
+
+struct DecoderStackEntry<'a, C: NodeCodec> {
+	node: Node<'a>,
+	/// The next entry in the stack is a child of the preceding entry at this index. For branch
+	/// nodes, the index is in [0, NIBBLE_LENGTH] and for extension nodes, the index is in
+	/// [0, 1].
+	child_index: usize,
+	/// The reconstructed child references.
+	children: Vec<Option<ChildReference<C::HashOut>>>,
+	/// A value attached as a node. The node will need to use its hash as value.
+	attached_value: Option<&'a [u8]>,
+	_marker: PhantomData<C>,
+}
+
+impl<'a, C: NodeCodec> DecoderStackEntry<'a, C> {
+	fn advance_child_index(&mut self) -> trie_db::Result<bool, C::HashOut, C::Error> {
+		match self.node {
+			Node::Extension(_, child) if self.child_index == 0 => {
+				match child {
+					NodeHandle::Inline(data) if data.is_empty() => return Ok(false),
+					_ => {
+						let child_ref = child.try_into().map_err(|hash| {
+							Box::new(TrieError::InvalidHash(C::HashOut::default(), hash))
+						})?;
+						self.children[self.child_index] = Some(child_ref);
+					},
+				}
+				self.child_index += 1;
+			},
+			Node::Branch(children, _) | Node::NibbledBranch(_, children, _) => {
+				while self.child_index < NIBBLE_LENGTH {
+					match children[self.child_index] {
+						Some(NodeHandle::Inline(data)) if data.is_empty() => return Ok(false),
+						Some(child) => {
+							let child_ref = child.try_into().map_err(|hash| {
+								Box::new(TrieError::InvalidHash(C::HashOut::default(), hash))
+							})?;
+							self.children[self.child_index] = Some(child_ref);
+						},
+						None => {},
+					}
+					self.child_index += 1;
+				}
+			},
+			_ => {},
+		}
+		Ok(true)
+	}
+
+	fn push_to_prefix(&self, prefix: &mut NibbleVec) {
+		match self.node {
+			Node::Empty => {},
+			Node::Leaf(partial, _) | Node::Extension(partial, _) => {
+				prefix.append_partial(partial.right());
+			},
+			Node::Branch(_, _) => {
+				prefix.push(self.child_index as u8);
+			},
+			Node::NibbledBranch(partial, _, _) => {
+				prefix.append_partial(partial.right());
+				prefix.push(self.child_index as u8);
+			},
+		}
+	}
+
+	fn pop_from_prefix(&self, prefix: &mut NibbleVec) {
+		match self.node {
+			Node::Empty => {},
+			Node::Leaf(partial, _) | Node::Extension(partial, _) => {
+				prefix.drop_lasts(partial.len());
+			},
+			Node::Branch(_, _) => {
+				prefix.pop();
+			},
+			Node::NibbledBranch(partial, _, _) => {
+				prefix.pop();
+				prefix.drop_lasts(partial.len());
+			},
+		}
+	}
+
+	fn encode_node(self, attached_hash: Option<&[u8]>) -> Vec<u8> {
+		let attached_hash = attached_hash.map(|h| Value::Node(h));
+		match self.node {
+			Node::Empty => C::empty_node().to_vec(),
+			Node::Leaf(partial, value) =>
+				C::leaf_node(partial.right_iter(), partial.len(), attached_hash.unwrap_or(value)),
+			Node::Extension(partial, _) => C::extension_node(
+				partial.right_iter(),
+				partial.len(),
+				self.children[0].expect("required by method precondition; qed"),
+			),
+			Node::Branch(_, value) => C::branch_node(
+				self.children.into_iter(),
+				if attached_hash.is_some() { attached_hash } else { value },
+			),
+			Node::NibbledBranch(partial, _, value) => C::branch_node_nibbled(
+				partial.right_iter(),
+				partial.len(),
+				self.children.iter(),
+				if attached_hash.is_some() { attached_hash } else { value },
+			),
+		}
+	}
+}
+
+pub fn decode_compact_from_iter<'a, L, DB, I>(
+	db: &mut DB,
+	encoded: I,
+) -> trie_db::Result<(TrieHash<L>, usize), TrieHash<L>, CError<L>>
+where
+	L: TrieLayout,
+	DB: HashDB<L::Hash, DBValue>,
+	I: IntoIterator<Item = &'a [u8]>,
+{
+	let mut stack: Vec<DecoderStackEntry<L::Codec>> = Vec::new();
+
+	let mut prefix = NibbleVec::new();
+
+	let mut iter = encoded.into_iter().enumerate();
+	while let Some((i, encoded_node)) = iter.next() {
+		let mut attached_node = 0;
+		if let Some(header) = L::Codec::ESCAPE_HEADER {
+			if encoded_node.starts_with(&[header]) {
+				attached_node = 1;
+			}
+		}
+		let node = L::Codec::decode(&encoded_node[attached_node..])
+			.map_err(|err| Box::new(TrieError::DecoderError(<TrieHash<L>>::default(), err)))?;
+
+		let children_len = match node {
+			Node::Empty | Node::Leaf(..) => 0,
+			Node::Extension(..) => 1,
+			Node::Branch(..) | Node::NibbledBranch(..) => NIBBLE_LENGTH,
+		};
+		let mut last_entry = DecoderStackEntry {
+			node,
+			child_index: 0,
+			children: vec![None; children_len],
+			attached_value: None,
+			_marker: PhantomData::default(),
+		};
+
+		if attached_node > 0 {
+			// Read value
+			if let Some((_, fetched_value)) = iter.next() {
+				last_entry.attached_value = Some(fetched_value);
+			} else {
+				return Err(Box::new(TrieError::IncompleteDatabase(<TrieHash<L>>::default())))
+			}
+		}
+
+		loop {
+			if !last_entry.advance_child_index()? {
+				last_entry.push_to_prefix(&mut prefix);
+				stack.push(last_entry);
+				break
+			}
+
+			let hash = last_entry
+				.attached_value
+				.as_ref()
+				.map(|value| db.insert(prefix.as_prefix(), value));
+			let node_data = last_entry.encode_node(hash.as_ref().map(|h| h.as_ref()));
+			let node_hash = db.insert(prefix.as_prefix(), node_data.as_ref());
+
+			if let Some(entry) = stack.pop() {
+				last_entry = entry;
+				last_entry.pop_from_prefix(&mut prefix);
+				last_entry.children[last_entry.child_index] = Some(ChildReference::Hash(node_hash));
+				last_entry.child_index += 1;
+			} else {
+				return Ok((node_hash, i + 1))
+			}
+		}
+	}
+
+	Err(Box::new(TrieError::IncompleteDatabase(<TrieHash<L>>::default())))
+}

test-support/reference-trie/src/lib.rs

@@ -4,6 +4,19 @@ The format is based on [Keep a Changelog].
 
 [Keep a Changelog]: http://keepachangelog.com/en/1.0.0/
 
+## [Unreleased]
+- Fix the item count returned by `decode_compact`/`decode_compact_from_iter` when the last
+  decoded node carries an attached (detached-at-encoding) value: the value item was not counted,
+  so continuing to decode concatenated encodings at the returned offset re-read the value as a
+  node.
+- Add `encode_compact_skip_duplicates`, emitting each distinct item in a compact proof — trie
+  node or detached value node — only once instead of once per referencing position. Deduplicated
+  encodings reconstruct a readable hash-keyed database with any decoder, including already
+  released ones; decoding them into position-keyed (prefixed) databases, or relying on
+  reconstructed reference counts, is unsupported — a proof is a record of state accesses, not a
+  database to mutate
+  ([polkadot-sdk#12565](https://github.com/paritytech/polkadot-sdk/issues/12565)).
+
 ## [0.30.0] - 2025-03-06
 - Improve `TrieCache` size by reducing size_of `NodeOwned` [#216](https://github.com/paritytech/trie/pull/216)
 

test-support/reference-trie/src/trie_db_0_31_decoder.rs

@@ -10,7 +10,7 @@ cargo-fuzz = true
 
 [dependencies]
 hash-db = { path = "../../hash-db", version = "0.16.0" }
-memory-db = { path = "../../memory-db", version = "0.33.0" }
+memory-db = { path = "../../memory-db", version = "0.34.0" }
 reference-trie = { path = "../../test-support/reference-trie", version = "0.29.1" }
 arbitrary = { version = "1.3.0", features = ["derive"] }
 array-bytes = "6.0.0"
@@ -61,6 +61,14 @@ path = "fuzz_targets/trie_proof_valid.rs"
 name = "trie_codec_proof"
 path = "fuzz_targets/trie_codec_proof.rs"
 
+[[bin]]
+name = "trie_codec_proof_dedup"
+path = "fuzz_targets/trie_codec_proof_dedup.rs"
+
+[[bin]]
+name = "trie_codec_proof_dedup_guided"
+path = "fuzz_targets/trie_codec_proof_dedup_guided.rs"
+
 [[bin]]
 name = "trie_proof_invalid"
 path = "fuzz_targets/trie_proof_invalid.rs"

trie-db/CHANGELOG.md

@@ -0,0 +1,18 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use trie_db_fuzz::{
+	fuzz_that_trie_codec_proofs, fuzz_that_trie_codec_proofs_with_shared_subtrees,
+	fuzz_that_trie_codec_proofs_with_shared_values,
+};
+
+fuzz_target!(|data: &[u8]| {
+	// Hashed-value layout, so value detachment and deduplication are exercised.
+	fuzz_that_trie_codec_proofs::<reference_trie::HashedValueNoExtThreshold<1>>(data);
+	fuzz_that_trie_codec_proofs_with_shared_values::<reference_trie::HashedValueNoExtThreshold<1>>(
+		data,
+	);
+	fuzz_that_trie_codec_proofs_with_shared_subtrees::<reference_trie::HashedValueNoExtThreshold<1>>(
+		data,
+	);
+});

trie-db/fuzz/Cargo.toml

@@ -0,0 +1,13 @@
+#![no_main]
+
+use libfuzzer_sys::fuzz_target;
+use trie_db_fuzz::{fuzz_dedup_scenario, DedupScenario};
+
+// Structure-aware, differential harness for deduplicated compact proofs. `DedupScenario` is built
+// via `arbitrary`, so libFuzzer mutates the trie/value structure directly instead of a raw byte
+// stream fed through `fuzz_to_data`.
+fuzz_target!(|scenario: DedupScenario| {
+	// Hashed-value layout: values are detachable value nodes, so value- and subtree-level
+	// deduplication are both exercised.
+	fuzz_dedup_scenario::<reference_trie::HashedValueNoExtThreshold<1>>(scenario);
+});