fix(mempool): cap SeenTxSet size to prevent OOM from SeenTx flooding (backport #2887) (#2927)

## Summary
- Cap `SeenTxSet` to 10M entries (~5 GB worst-case) to prevent unbounded
memory growth from malicious peers flooding `SeenTx` messages with
random tx keys on channel `0x31`.
- When the cap is reached and a new key arrives, one random existing
entry is evicted to make room. Updates to existing keys are unaffected.
- Added `TestSeenTxSetMaxSize` to verify the cap, eviction, and
existing-key update behavior.

Closes PROTOCO-1325

## Test plan
- [x] `TestSeenTxSet` passes (existing behavior preserved)
- [x] `TestSeenTxSetMaxSize` passes (cap enforced, eviction works,
existing-key updates work at capacity)
- [ ] Run full mempool/cat test suite: `go test ./mempool/cat/...`

🤖 Generated with [Claude Code](https://claude.com/claude-code)


---

<a
href="https://app.devin.ai/review/celestiaorg/celestia-core/pull/2887"
target="_blank">
  <picture>
<source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-open-in-devin-review-dark.svg?v=1">
<img
src="https://static.devin.ai/assets/gh-open-in-devin-review-light.svg?v=1"
alt="Open with Devin">
  </picture>
</a>

<hr>This is an automatic backport of pull request #2887 done by
[Mergify](https://mergify.com).
<!-- devin-review-badge-begin -->

---

<a
href="https://app.devin.ai/review/celestiaorg/celestia-core/pull/2927"
target="_blank">
  <picture>
<source media="(prefers-color-scheme: dark)"
srcset="https://static.devin.ai/assets/gh-open-in-devin-review-dark.svg?v=1">
<img
src="https://static.devin.ai/assets/gh-open-in-devin-review-light.svg?v=1"
alt="Open with Devin">
  </picture>
</a>
<!-- devin-review-badge-end -->

Co-authored-by: CHAMI Rachid <chamirachid1@gmail.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Rootul P <rootulp@gmail.com>
Co-authored-by: Mikhail Rakhmanov <rakhmanov.m@gmail.com>

Author: 5 people

.changelog/unreleased/bug-fixes/seentx-set-unbounded-growth.md

@@ -0,0 +1,4 @@
+- Cap `SeenTxSet` to 10,000,000 entries (~5 GB worst-case) to prevent
+  unbounded memory growth from malicious peers flooding SeenTx messages
+  with random tx keys.
+  ([\#CELESTIA-256](https://github.com/celestiaorg/celestia-core/issues/CELESTIA-256))

mempool/cat/cache.go

@@ -25,13 +25,26 @@ func NewSeenTxSet() *SeenTxSet {
 	}
 }
 
+// maxSeenTxSetSize limits the number of unique tx keys tracked in the SeenTxSet
+// to prevent unbounded memory growth from malicious peers flooding SeenTx messages.
+// Each entry costs ~500 bytes (including Go map overhead and GC pressure),
+// so 10M entries ≈ 5 GB worst-case.
+const maxSeenTxSetSize = 10_000_000
+
 func (s *SeenTxSet) Add(txKey types.TxKey, peer uint16) {
 	if peer == 0 {
 		return
 	}
 	s.mtx.Lock()
 	defer s.mtx.Unlock()
 	seenSet, exists := s.set[txKey]
+	if !exists && len(s.set) >= maxSeenTxSetSize {
+		// Evict one random entry to make room.
+		for k := range s.set {
+			delete(s.set, k)
+			break
+		}
+	}
 	if !exists {
 		s.set[txKey] = timestampedPeerSet{
 			peers: map[uint16]struct{}{peer: {}},

mempool/cat/cache_test.go

@@ -1,6 +1,7 @@
 package cat
 
 import (
+	"encoding/binary"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -35,3 +36,27 @@ func TestSeenTxSet(t *testing.T) {
 	require.Nil(t, seenSet.Get(tx2Key))
 	require.True(t, seenSet.Has(tx3Key, peer1))
 }
+
+func TestSeenTxSetMaxSize(t *testing.T) {
+	seenSet := NewSeenTxSet()
+
+	// Pre-fill the map to exactly maxSeenTxSetSize by inserting dummy entries directly.
+	for i := 0; i < maxSeenTxSetSize; i++ {
+		var key types.TxKey
+		binary.BigEndian.PutUint64(key[:8], uint64(i))
+		seenSet.set[key] = timestampedPeerSet{peers: map[uint16]struct{}{1: {}}}
+	}
+	require.Equal(t, maxSeenTxSetSize, seenSet.Len())
+
+	// New key should evict one entry and be added (size stays at cap).
+	var extraKey types.TxKey
+	binary.BigEndian.PutUint64(extraKey[:8], uint64(maxSeenTxSetSize+1))
+	seenSet.Add(extraKey, 1)
+	require.Equal(t, maxSeenTxSetSize, seenSet.Len())
+	require.True(t, seenSet.Has(extraKey, 1))
+
+	// Adding a new peer to an existing key should still work at capacity.
+	seenSet.Add(extraKey, 2)
+	require.True(t, seenSet.Has(extraKey, 2))
+	require.Equal(t, maxSeenTxSetSize, seenSet.Len())
+}