Merge branch 'master' into dode/compression-3x

Author: Dodecahedr0x

.github/packages/npm-package/mbTestValidator.ts

@@ -87,6 +87,15 @@ function runMbTestValidator(): void {
     "--account",
     "FRqXJqfCi3o6gF3Yqnkx1gKA3YnbRDJbBs6hKpme3NHJ",
     p("FRqXJqfCi3o6gF3Yqnkx1gKA3YnbRDJbBs6hKpme3NHJ.json"),
+    "--account",
+    "Sc9MJUngNbQXSXGP3F67KvKwVnhaYn6kcioxXNVowYT",
+    p("Sc9MJUngNbQXSXGP3F67KvKwVnhaYn6kcioxXNVowYT.json"),
+    "--account",
+    "9yvg9551MmE8mhWd88jAPLE3noTXHoopYG1BDhmtkCeR",
+    p("9yvg9551MmE8mhWd88jAPLE3noTXHoopYG1BDhmtkCeR.json"),
+    "--account",
+    "7L9eCRv52UpGVePGj9P1zop8kzmh4SpYzYn6YhoAKHBg",
+    p("7L9eCRv52UpGVePGj9P1zop8kzmh4SpYzYn6YhoAKHBg.json"),
   ];
 
   const expectedFiles = [
@@ -108,6 +117,9 @@ function runMbTestValidator(): void {
     "CXMc1eCiEp9YXjanBNB6HUvbWCmxeVmhcR3bPXw8exJA.json",
     "GKE6d7iv8kCBrsxr78W3xVdjGLLLJnxsGiuzrsZCGEvb.json",
     "FRqXJqfCi3o6gF3Yqnkx1gKA3YnbRDJbBs6hKpme3NHJ.json",
+    "Sc9MJUngNbQXSXGP3F67KvKwVnhaYn6kcioxXNVowYT.json",
+    "9yvg9551MmE8mhWd88jAPLE3noTXHoopYG1BDhmtkCeR.json",
+    "7L9eCRv52UpGVePGj9P1zop8kzmh4SpYzYn6YhoAKHBg.json",
   ];
   const missingFiles = expectedFiles
     .map((f) => p(f))

.github/packages/npm-package/scripts/fetch-local-dumps.sh

@@ -17,13 +17,16 @@ accounts=(
   mAGicPQYBMvcYveUZA5F5UNNwyHvfYh5xkLS2Fr1mev
   EpJnX7ueXk7fKojBymqmVuCuwyhDQsYcLVL1XMsBbvDX
   7JrkjmZPprHwtuvtuGTXp9hwfGYFAQLnLeFM52kqAgXg
-  Cuj97ggrhhidhbu39TijNVqE74xvKJ69gDervRUXAxGh
-  5hBR571xnXppuCPveTrctfTU7tJLSN94nq7kv7FRK5Tc
+  Cuj97ggrhhidhbu39TijNVqE74xvKJ69gDervRUXAxGh  # vrf oracle queue
+  5hBR571xnXppuCPveTrctfTU7tJLSN94nq7kv7FRK5Tc  # vrf oracle ephemeral queue
   F72HqCR8nwYsVyeVd38pgKkjXmXFzVAM8rjZZsXWbdE
   paywJiVATrVDLYLmowJqzG6MsaCt77L8WyTnBb2754t   # vrf oracle identity
   CXMc1eCiEp9YXjanBNB6HUvbWCmxeVmhcR3bPXw8exJA  # vrf oracle data
   GKE6d7iv8kCBrsxr78W3xVdjGLLLJnxsGiuzrsZCGEvb  # queue for paywJiVATrVDLYLmowJqzG6MsaCt77L8WyTnBb2754t
   FRqXJqfCi3o6gF3Yqnkx1gKA3YnbRDJbBs6hKpme3NHJ  # deleted queue for paywJiVATrVDLYLmowJqzG6MsaCt77L8WyTnBb2754t
+  Sc9MJUngNbQXSXGP3F67KvKwVnhaYn6kcioxXNVowYT  # ephemeral (delegated) test queue, index 1 for paywJ
+  9yvg9551MmE8mhWd88jAPLE3noTXHoopYG1BDhmtkCeR  # delegation record for Sc9MJ ephemeral test queue
+  7L9eCRv52UpGVePGj9P1zop8kzmh4SpYzYn6YhoAKHBg  # delegation metadata for Sc9MJ ephemeral test queue
 )
 
 for acc in "${accounts[@]}"; do

Cargo.lock

@@ -199,6 +199,7 @@ solana-program-error = { version = "3.0" }
 solana-program-option = { version = "3.0" }
 solana-program-pack = { version = "3.0" }
 solana-program-runtime = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "b91ab7dad6642c63b9ea177fb318840b4c64c6cb" }
+solana-packet = { version = "3.0" }
 solana-pubkey = { version = "3.0" }
 solana-pubsub-client = { version = "3.1" }
 solana-rent = { version = "3.0" }

Cargo.toml

@@ -12,7 +12,6 @@ doctest = false
 
 [dependencies]
 async-trait = { workspace = true }
-base64 = { workspace = true }
 bincode = { workspace = true }
 borsh = { workspace = true }
 compressed-delegation-api = { workspace = true }
@@ -41,6 +40,7 @@ solana-hash = { workspace = true }
 solana-instruction = { workspace = true }
 solana-keypair = { workspace = true }
 solana-message = { workspace = true }
+solana-packet = { workspace = true }
 solana-program = { workspace = true }
 solana-pubkey = { workspace = true }
 solana-rpc-client = { workspace = true }

magicblock-committor-service/Cargo.toml

@@ -767,7 +767,7 @@ fn test_close_buffer_limit() {
     use crate::{
         test_utils,
         transactions::{
-            serialize_and_encode_base64, MAX_ENCODED_TRANSACTION_SIZE,
+            serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE,
         },
     };
 
@@ -797,9 +797,9 @@ fn test_close_buffer_limit() {
         .collect();
 
     let tx = Transaction::new_with_payer(&ixs, Some(&authority.pubkey()));
-    let tx_size = serialize_and_encode_base64(&tx).len();
+    let tx_size = serialized_transaction_size(&tx);
     info!(transaction_size = tx_size, "Cleanup task transaction size");
-    assert!(tx_size <= MAX_ENCODED_TRANSACTION_SIZE);
+    assert!(tx_size <= MAX_TRANSACTION_WIRE_SIZE);
 
     // One more unique task should overflow
     let overflow_task = CleanupTask {
@@ -810,7 +810,5 @@ fn test_close_buffer_limit() {
     ixs.push(overflow_task.instruction(&authority.pubkey()));
 
     let tx = Transaction::new_with_payer(&ixs, Some(&authority.pubkey()));
-    assert!(
-        serialize_and_encode_base64(&tx).len() > MAX_ENCODED_TRANSACTION_SIZE
-    );
+    assert!(serialized_transaction_size(&tx) > MAX_TRANSACTION_WIRE_SIZE);
 }

magicblock-committor-service/src/tasks/mod.rs

@@ -12,7 +12,7 @@ use crate::{
         commit_task::CommitDelivery, utils::TransactionUtils, BaseActionTask,
         BaseTask, BaseTaskImpl,
     },
-    transactions::{serialize_and_encode_base64, MAX_ENCODED_TRANSACTION_SIZE},
+    transactions::{serialized_transaction_size, MAX_TRANSACTION_WIRE_SIZE},
 };
 
 #[derive(Default)]
@@ -222,7 +222,7 @@ impl TaskStrategist {
     ) -> TaskStrategistResult<TransactionStrategy> {
         // Attempt optimizing tasks themselves(using buffers)
         if Self::try_optimize_tx_size_if_needed(&mut tasks)?
-            <= MAX_ENCODED_TRANSACTION_SIZE
+            <= MAX_TRANSACTION_WIRE_SIZE
         {
             // Persist tasks strategy
             if let Some(persistor) = persistor {
@@ -292,8 +292,7 @@ impl TaskStrategist {
             return false;
         };
 
-        let encoded_alt_tx = serialize_and_encode_base64(&alt_tx);
-        encoded_alt_tx.len() <= MAX_ENCODED_TRANSACTION_SIZE
+        serialized_transaction_size(&alt_tx) <= MAX_TRANSACTION_WIRE_SIZE
     }
 
     pub fn collect_lookup_table_keys(
@@ -393,10 +392,10 @@ impl TaskStrategist {
         }
     }
 
-    /// Optimizes tasks so as to bring the transaction size within the limit [`MAX_ENCODED_TRANSACTION_SIZE`]
+    /// Optimizes tasks so as to bring the transaction size within the limit [`MAX_TRANSACTION_WIRE_SIZE`]
     /// Returns Ok(size of tx after optimizations) else Err(SignerError).
     /// Note that the returned size, though possibly optimized one, may still not be under
-    /// the limit MAX_ENCODED_TRANSACTION_SIZE. The caller needs to check and make decision accordingly.
+    /// the limit MAX_TRANSACTION_WIRE_SIZE. The caller needs to check and make decision accordingly.
     fn try_optimize_tx_size_if_needed(
         tasks: &mut [BaseTaskImpl],
     ) -> Result<usize, SignerError> {
@@ -408,7 +407,7 @@ impl TaskStrategist {
                 u64::default(), // placeholder
                 &[],
             ) {
-                Ok(tx) => Ok(serialize_and_encode_base64(&tx).len()),
+                Ok(tx) => Ok(serialized_transaction_size(&tx)),
                 Err(TaskStrategistError::FailedToFitError) => Ok(usize::MAX),
                 Err(TaskStrategistError::SignerError(err)) => Err(err),
             }
@@ -417,7 +416,7 @@ impl TaskStrategist {
         // Get initial transaction size
         let mut current_tx_length = calculate_tx_length(tasks)?;
 
-        if current_tx_length <= MAX_ENCODED_TRANSACTION_SIZE {
+        if current_tx_length <= MAX_TRANSACTION_WIRE_SIZE {
             return Ok(current_tx_length);
         }
 
@@ -439,7 +438,7 @@ impl TaskStrategist {
 
         // We keep popping heaviest el-ts & try to optimize while heap is non-empty
         while let Some((_, index)) = map.pop() {
-            if current_tx_length <= MAX_ENCODED_TRANSACTION_SIZE {
+            if current_tx_length <= MAX_TRANSACTION_WIRE_SIZE {
                 break;
             }
 

magicblock-committor-service/src/tasks/task_strategist.rs

@@ -1,9 +1,9 @@
-use base64::{prelude::BASE64_STANDARD, Engine};
+use solana_packet::PACKET_DATA_SIZE;
 use solana_rpc_client::rpc_client::SerializableTransaction;
 use static_assertions::const_assert;
 
-/// From agave rpc/src/rpc.rs [MAX_BASE64_SIZE]
-pub(crate) const MAX_ENCODED_TRANSACTION_SIZE: usize = 1644;
+/// Maximum serialized transaction size that can be sent over the wire.
+pub(crate) const MAX_TRANSACTION_WIRE_SIZE: usize = PACKET_DATA_SIZE;
 
 /// How many process and commit buffer instructions fit into a single transaction
 #[allow(unused)] // serves as documentation as well
@@ -34,7 +34,7 @@ pub const MAX_PROCESS_AND_CLOSE_PER_TX: u8 = 2;
 /// close buffer instructions fit into a single transaction when
 /// using lookup tables but not including the buffer account
 #[allow(unused)] // serves as documentation as well
-pub const MAX_PROCESS_AND_CLOSE_PER_TX_USING_LOOKUP: u8 = 5;
+pub const MAX_PROCESS_AND_CLOSE_PER_TX_USING_LOOKUP: u8 = 4;
 
 /// How many finalize instructions fit into a single transaction
 #[allow(unused)] // serves as documentation as well
@@ -54,7 +54,7 @@ pub const MAX_UNDELEGATE_PER_TX: u8 = 3;
 /// when using address lookup tables
 /// NOTE: that we assume the rent reimbursement account to be the delegated account
 #[allow(unused)] // serves as documentation as well
-pub const MAX_UNDELEGATE_PER_TX_USING_LOOKUP: u8 = 16;
+pub const MAX_UNDELEGATE_PER_TX_USING_LOOKUP: u8 = 15;
 
 // Allows us to run undelegate instructions without rechunking them since we know
 // that we didn't process more than we also can undelegate
@@ -66,12 +66,11 @@ const_assert!(
     MAX_PROCESS_PER_TX_USING_LOOKUP <= MAX_UNDELEGATE_PER_TX_USING_LOOKUP
 );
 
-pub fn serialize_and_encode_base64(
+pub fn serialized_transaction_size(
     transaction: &impl SerializableTransaction,
-) -> String {
-    // SAFETY: runs statically
-    let serialized = bincode::serialize(transaction).unwrap();
-    BASE64_STANDARD.encode(serialized)
+) -> usize {
+    // SAFETY: runs on transactions we already serialize before sending.
+    usize::try_from(bincode::serialized_size(transaction).unwrap()).unwrap()
 }
 
 #[cfg(test)]
@@ -131,7 +130,7 @@ mod test {
         NoLookupTable,
         UseLookupTable,
     }
-    fn encoded_tx_size(
+    fn transaction_wire_size(
         auth: &Keypair,
         ixs: &[Instruction],
         opts: &TransactionOpts,
@@ -165,8 +164,7 @@ mod test {
             )
         })?;
 
-        let encoded = serialize_and_encode_base64(&versioned_tx);
-        Ok(encoded.len())
+        Ok(serialized_transaction_size(&versioned_tx))
     }
 
     // -----------------
@@ -309,7 +307,7 @@ mod test {
                         .collect::<Vec<_>>();
 
                     let tx_size =
-                        encoded_tx_size(auth, &ixs, &tx_opts).unwrap();
+                        transaction_wire_size(auth, &ixs, &tx_opts).unwrap();
                     tx_sizes.push((size, tx_size));
                 }
                 tx_lines.push(tx_sizes);
@@ -349,51 +347,8 @@ mod test {
         run(auth, 10);
         run(auth, 15);
         run(auth, 20);
-        /*
-         0 ixs:
-          0:  184|  10:  184|  20:  184|  50:  184| 100:  184| 200:  184| 500:  184|1024:  184
-          0:  184|  10:  184|  20:  184|  50:  184| 100:  184| 200:  184| 500:  184|1024:  184
-         1 ixs:
-          0:  620|  10:  636|  20:  648|  50:  688| 100:  756| 200:  888| 500: 1288|1024: 1988
-          0:  336|  10:  348|  20:  364|  50:  404| 100:  472| 200:  604| 500: 1004|1024: 1704
-         2 ixs:
-          0:  932|  10:  960|  20:  984|  50: 1064| 100: 1200| 200: 1468| 500: 2268|1024: 3664
-          0:  400|  10:  424|  20:  452|  50:  532| 100:  668| 200:  936| 500: 1736|1024: 3132
-         5 ixs:
-          0: 1864|  10: 1932|  20: 1996|  50: 2196| 100: 2536| 200: 3204| 500: 5204|1024: 8696
-          0:  588|  10:  652|  20:  720|  50:  920| 100: 1260| 200: 1928| 500: 3928|1024: 7420
-         8 ixs:
-          0: 2796|  10: 2904|  20: 3008|  50: 3328| 100: 3872| 200: 4940| 500: 8140|1024:13728
-          0:  776|  10:  880|  20:  988|  50: 1308| 100: 1852| 200: 2920| 500: 6120|1024:11708
-        10 ixs:
-          0: 3416|  10: 3552|  20: 3684|  50: 4084| 100: 4764| 200: 6096| 500:10096|1024:17084
-          0:  900|  10: 1032|  20: 1168|  50: 1568| 100: 2248| 200: 3580| 500: 7580|1024:14568
-        15 ixs:
-          0: 4972|  10: 5172|  20: 5372|  50: 5972| 100: 6992| 200: 8992| 500:14992|1024:25472
-          0: 1212|  10: 1412|  20: 1612|  50: 2212| 100: 3232| 200: 5232| 500:11232|1024:21712
-        20 ixs:
-          0: 6524|  10: 6792|  20: 7056|  50: 7856| 100: 9216| 200:11884| 500:19884|1024:33856
-          0: 1528|  10: 1792|  20: 2060|  50: 2860| 100: 4220| 200: 6888| 500:14888|1024:28860
-
-        Legend:
-
-        x ixs:
-            data size/ix: encoded size | ...
-            data size/ix: encoded size | ...  (using lookup tables)
-
-        Given that max transaction size is 1644 bytes, we can see that the max data size is:
-
-        -  1 ixs: slightly larger than 500 bytes
-        -  2 ixs: slightly larger than 200 bytes
-        -  5 ixs: slightly larger than 100 bytes
-        -  8 ixs: slightly larger than  50 bytes
-        - 10 ixs: slightly larger than  20 bytes
-        - 15 ixs: slightly larger than  10 bytes
-        - 20 ixs: no data supported (only lamport changes)
-
-        Also it is clear that using a lookup table makes a huge difference especially if we commit
-        lots of different accounts.
-        */
+        // This logs raw wire sizes. The sendability limit is
+        // MAX_TRANSACTION_WIRE_SIZE.
     }
 
     // -----------------
@@ -573,13 +528,13 @@ mod test {
                 &[&auth],
             )
             .unwrap();
-            let encoded = serialize_and_encode_base64(&versioned_tx);
+            let tx_size = serialized_transaction_size(&versioned_tx);
             info!(
                 chunks = chunks,
-                size_bytes = encoded.len(),
+                size_bytes = tx_size,
                 "Transaction size measured"
             );
-            if encoded.len() > MAX_ENCODED_TRANSACTION_SIZE {
+            if tx_size > MAX_TRANSACTION_WIRE_SIZE {
                 return chunks - 1;
             }
         }
@@ -662,13 +617,13 @@ mod test {
                 &[&auth],
             )
             .unwrap();
-            let encoded = serialize_and_encode_base64(&versioned_tx);
+            let tx_size = serialized_transaction_size(&versioned_tx);
             info!(
                 chunks = chunks,
-                size_bytes = encoded.len(),
+                size_bytes = tx_size,
                 "Transaction size measured with lookup table"
             );
-            if encoded.len() > MAX_ENCODED_TRANSACTION_SIZE {
+            if tx_size > MAX_TRANSACTION_WIRE_SIZE {
                 return chunks - 1;
             }
         }

magicblock-committor-service/src/transactions.rs

@@ -270,7 +270,8 @@ impl TransactionScheduler {
                 biased;
                 Ok(latest) = block_produced.recv() => {
                     if !self.coordinator.is_primary() && latest.slot >= self.slot {
-                        self.transition_to_new_slot(Some(latest)).await;
+                        let slot = self.transition_to_new_slot(Some(latest)).await;
+                        self.handle_superblock(slot).await;
                     }
                 }
                 _ = self.slot_ticker.tick() => {
@@ -342,8 +343,10 @@ impl TransactionScheduler {
             error!("failed to create accountsdb snapshot");
             return;
         };
-        let msg = Message::SuperBlock(SuperBlock { slot, checksum });
-        self.send_replication(msg).await;
+        if self.coordinator.is_primary() {
+            let msg = Message::SuperBlock(SuperBlock { slot, checksum });
+            self.send_replication(msg).await;
+        }
     }
 
     async fn pause_executors_for_snapshot(&mut self) -> OwnedSemaphorePermit {
@@ -380,7 +383,7 @@ impl TransactionScheduler {
             );
         }
 
-        let _permit = self
+        let permit = self
             .pause_permit
             .clone()
             .acquire_owned()
@@ -395,7 +398,17 @@ impl TransactionScheduler {
             .map_err(|error| error.to_string())?;
         self.accountsdb.set_slot(block.slot);
         self.update_sysvars(&block);
-        self.notify_executors_of_block(block).await
+        let slot = block.slot;
+        let result = self.notify_executors_of_block(block).await;
+        drop(permit);
+        // apply_replayed_block advances self.slot before the queued
+        // latest_block notification can be received. The block_produced path
+        // then skips transition_to_new_slot for this already-applied slot, so
+        // handle_superblock must run here for replayed superblock snapshots.
+        if result.is_ok() {
+            self.handle_superblock(slot).await;
+        }
+        result
     }
 
     async fn notify_executors_of_block(