Merge pull request #67 from bitfinity-network/upgrade_to_v1.3.4

Upgrade to v1.3.8 - New Engine

Author: ufoscout

.config/zepter.yaml

@@ -12,13 +12,13 @@ workflows:
         # Check that `A` activates the features of `B`.
         "propagate-feature",
         # These are the features to check:
-        "--features=std,optimism,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench",
-        # Do not try to add a new section into `[features]` of `A` only because `B` expose that feature. There are edge-cases where this is still needed, but we can add them manually.
+        "--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat",
+        # Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually.
         "--left-side-feature-missing=ignore",
         # Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on.
 
         "--left-side-outside-workspace=ignore",
-        # Auxilary flags:
+        # Auxiliary flags:
         "--offline",
         "--locked",
         "--show-path",

.github/CODEOWNERS

@@ -11,6 +11,7 @@ crates/engine               @rkrasiuk @mattsse @Rjected
 crates/e2e-test-utils/      @mattsse @Rjected
 crates/engine/              @rkrasiuk @mattsse @Rjected @fgimenez
 crates/errors/              @mattsse
+crates/era/                 @mattsse
 crates/ethereum/            @mattsse @Rjected
 crates/ethereum-forks/      @mattsse @Rjected
 crates/etl/                 @joshieDo @shekhirin
@@ -44,5 +45,6 @@ crates/tokio-util/          @fgimenez
 crates/tracing/             @onbjerg
 crates/transaction-pool/    @mattsse
 crates/trie/                @rkrasiuk @Rjected @shekhirin
+crates/ress                 @rkrasiuk
 etc/                        @Rjected @onbjerg @shekhirin
 .github/                    @onbjerg @gakonst @DaniPopes

.github/assets/check_rv32imac.sh

@@ -8,14 +8,28 @@ crates_to_check=(
     reth-network-peers
     reth-trie-common
     reth-chainspec
+    reth-consensus
+    reth-consensus-common
+    reth-prune-types
+    reth-static-file-types
+    reth-storage-errors
+    reth-execution-errors
+    reth-execution-types
+    reth-db-models
+    reth-evm
+    reth-storage-api
 
     ## ethereum
+    reth-evm-ethereum
     reth-ethereum-forks
     reth-ethereum-primitives
 
     ## optimism
     reth-optimism-chainspec
     reth-optimism-forks
+    reth-optimism-consensus
+    reth-optimism-primitives
+    reth-optimism-evm
 )
 
 # Array to hold the results

.github/assets/check_wasm.sh

@@ -10,9 +10,7 @@ crates=($(cargo metadata --format-version=1 --no-deps | jq -r '.packages[].name'
 exclude_crates=(
   # The following require investigation if they can be fixed
   reth-basic-payload-builder
-  reth-beacon-consensus
   reth-bench
-  reth-blockchain-tree
   reth-cli
   reth-cli-commands
   reth-cli-runner
@@ -45,7 +43,8 @@ exclude_crates=(
   reth-optimism-node
   reth-optimism-payload-builder
   reth-optimism-rpc
-  reth-optimism-primitives
+  reth-optimism-storage
+  reth-optimism-chain-registry
   reth-rpc
   reth-rpc-api
   reth-rpc-api-testing-util
@@ -56,18 +55,23 @@ exclude_crates=(
   reth-rpc-layer
   reth-stages
   reth-engine-local
+  reth-ress-protocol
+  reth-ress-provider
   # The following are not supposed to be working
   reth # all of the crates below
   reth-invalid-block-hooks # reth-provider
   reth-libmdbx # mdbx
   reth-mdbx-sys # mdbx
+  reth-payload-builder # reth-metrics
   reth-provider # tokio
   reth-prune # tokio
   reth-stages-api # reth-provider, reth-prune
   reth-static-file # tokio
   reth-transaction-pool # c-kzg
+  reth-payload-util # reth-transaction-pool
   reth-trie-parallel # tokio
   reth-testing-utils
+  reth-optimism-txpool # reth-transaction-pool
 )
 
 # Array to hold the results

.github/assets/hive/Dockerfile

@@ -1,7 +1,60 @@
-FROM ubuntu
+# syntax=docker.io/docker/dockerfile:1.7-labs
 
-COPY dist/reth /usr/local/bin
+# 
+# We'll use cargo-chef to speed up the build
+# 
+FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
+WORKDIR /app
 
+# Install system dependencies
+RUN apt-get update && apt-get -y upgrade && apt-get install -y libclang-dev pkg-config
+
+# 
+# We prepare the build plan
+# 
+FROM chef AS planner
+
+ARG CARGO_BIN
+
+COPY --exclude=.git --exclude=dist . .
+RUN cargo chef prepare --recipe-path recipe.json --bin ${CARGO_BIN}
+
+# 
+# And build the app
+# 
+FROM chef AS builder
+WORKDIR /app
+
+ARG CARGO_BIN
+ARG BUILD_PROFILE=hivetests
+ARG FEATURES=""
+ARG MANIFEST_PATH=""
+
+COPY --from=planner /app/recipe.json recipe.json
+
+RUN cargo chef cook \
+    --profile $BUILD_PROFILE \
+    --bin $CARGO_BIN \
+    ${FEATURES:+--features "$FEATURES"}  \
+    ${MANIFEST_PATH:+--manifest-path $MANIFEST_PATH}  \
+    --recipe-path recipe.json
+
+COPY --exclude=.git --exclude=dist . .
+RUN cargo build \
+    --profile $BUILD_PROFILE \
+    --bin $CARGO_BIN \
+    ${FEATURES:+--features "$FEATURES"}  \
+    ${MANIFEST_PATH:+--manifest-path $MANIFEST_PATH}  \
+    --locked
+
+# 
+# The runtime will then just use the build artifact without building anything
+# 
+FROM ubuntu AS runtime
+
+ARG CARGO_BIN
+
+COPY --from=builder /app/target/hivetests/$CARGO_BIN /usr/local/bin/reth
 COPY LICENSE-* ./
 
 EXPOSE 30303 30303/udp 9001 8545 8546

.github/assets/hive/build_simulators.sh

@@ -11,7 +11,7 @@ go build .
 
 # Run each hive command in the background for each simulator and wait
 echo "Building images"
-./hive -client reth --sim "pyspec" -sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/eest" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v4.1.0/fixtures_develop.tar.gz --sim.buildarg branch=v4.1.0 -sim.timelimit 1s || true &
 ./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
 ./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
 ./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &
@@ -20,17 +20,21 @@ echo "Building images"
 ./hive -client reth --sim "ethereum/sync" -sim.timelimit 1s || true &
 wait
 
-# Run docker save in parallel and wait
+# Run docker save in parallel, wait and exit on error
 echo "Saving images"
-docker save hive/hiveproxy:latest -o ../hive_assets/hiveproxy.tar &
-docker save hive/simulators/devp2p:latest -o ../hive_assets/devp2p.tar &
-docker save hive/simulators/ethereum/engine:latest -o ../hive_assets/engine.tar &
-docker save hive/simulators/ethereum/rpc-compat:latest -o ../hive_assets/rpc_compat.tar &
-docker save hive/simulators/ethereum/pyspec:latest -o ../hive_assets/pyspec.tar &
-docker save hive/simulators/smoke/genesis:latest -o ../hive_assets/smoke_genesis.tar &
-docker save hive/simulators/smoke/network:latest -o ../hive_assets/smoke_network.tar &
-docker save hive/simulators/ethereum/sync:latest -o ../hive_assets/ethereum_sync.tar &
-wait
+saving_pids=( )
+docker save hive/hiveproxy:latest -o ../hive_assets/hiveproxy.tar & saving_pids+=( $! )
+docker save hive/simulators/devp2p:latest -o ../hive_assets/devp2p.tar & saving_pids+=( $! )
+docker save hive/simulators/ethereum/engine:latest -o ../hive_assets/engine.tar & saving_pids+=( $! )
+docker save hive/simulators/ethereum/rpc-compat:latest -o ../hive_assets/rpc_compat.tar & saving_pids+=( $! )
+docker save hive/simulators/ethereum/eest/consume-engine:latest -o ../hive_assets/eest_engine.tar & saving_pids+=( $! )
+docker save hive/simulators/ethereum/eest/consume-rlp:latest -o ../hive_assets/eest_rlp.tar & saving_pids+=( $! )
+docker save hive/simulators/smoke/genesis:latest -o ../hive_assets/smoke_genesis.tar & saving_pids+=( $! )
+docker save hive/simulators/smoke/network:latest -o ../hive_assets/smoke_network.tar & saving_pids+=( $! )
+docker save hive/simulators/ethereum/sync:latest -o ../hive_assets/ethereum_sync.tar & saving_pids+=( $! )
+for pid in "${saving_pids[@]}"; do
+    wait "$pid" || exit
+done
 
 # Make sure we don't rebuild images on the CI jobs
 git apply ../.github/assets/hive/no_sim_build.diff

.github/assets/hive/expected_failures.yaml

@@ -1,20 +1,22 @@
-# https://github.com/paradigmxyz/reth/issues/7015
-# https://github.com/paradigmxyz/reth/issues/6332
+# tracked by https://github.com/paradigmxyz/reth/issues/13879
 rpc-compat:
   - debug_getRawBlock/get-invalid-number (reth)
   - debug_getRawHeader/get-invalid-number (reth)
   - debug_getRawReceipts/get-invalid-number (reth)
+  - debug_getRawReceipts/get-block-n (reth)
   - debug_getRawTransaction/get-invalid-hash (reth)
 
   - eth_call/call-callenv (reth)
-  - eth_feeHistory/fee-history (reth)
   - eth_getStorageAt/get-storage-invalid-key-too-large (reth)
   - eth_getStorageAt/get-storage-invalid-key (reth)
   - eth_getTransactionReceipt/get-access-list (reth)
   - eth_getTransactionReceipt/get-blob-tx (reth)
   - eth_getTransactionReceipt/get-dynamic-fee (reth)
+  - eth_getTransactionReceipt/get-legacy-contract (reth)
+  - eth_getTransactionReceipt/get-legacy-input (reth)
+  - eth_getTransactionReceipt/get-legacy-receipt (reth)
 
-# https://github.com/paradigmxyz/reth/issues/8732
+# no fix due to https://github.com/paradigmxyz/reth/issues/8732
 engine-withdrawals:
   - Withdrawals Fork On Genesis (Paris) (reth)
   - Withdrawals Fork on Block 1 (Paris) (reth)
@@ -37,17 +39,15 @@ engine-withdrawals:
 
 engine-api: []
 
-# https://github.com/paradigmxyz/reth/issues/8305
-# https://github.com/paradigmxyz/reth/issues/6217
-# https://github.com/paradigmxyz/reth/issues/8306
-# https://github.com/paradigmxyz/reth/issues/7144
+# no fix due to https://github.com/paradigmxyz/reth/issues/8732
 engine-cancun:
-  - Blob Transaction Ordering, Multiple Clients (Cancun) (reth)
   - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)
   - Invalid NewPayload, ExcessBlobGas, Syncing=True, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
-  - Invalid NewPayload, VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
-  - Invalid NewPayload, VersionedHashes Version, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
-  - Invalid NewPayload, Incomplete VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
-  - Invalid NewPayload, Extra VersionedHashes, Syncing=False, EmptyTxs=False, DynFeeTxs=False (Cancun) (reth)
 
 sync: []
+
+# no fix: it’s too expensive to check whether the storage is empty on each creation
+eest/consume-engine:
+  - tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_to_non_empty_storage[fork_Prague-blockchain_test_engine-zero_nonce]-reth
+eest/consume-rlp:
+  - tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_to_non_empty_storage[fork_Prague-blockchain_test-zero_nonce]-reth

.github/assets/hive/load_images.sh

@@ -11,6 +11,8 @@ IMAGES=(
     "/tmp/smoke_genesis.tar"
     "/tmp/smoke_network.tar"
     "/tmp/ethereum_sync.tar"
+    "/tmp/eest_engine.tar"
+    "/tmp/eest_rlp.tar"
     "/tmp/reth_image.tar"
 )
 
@@ -22,4 +24,4 @@ done
 
 wait
 
-docker image ls -a
+docker image ls -a

.github/assets/hive/run_simulator.sh

@@ -7,7 +7,7 @@ sim="${1}"
 limit="${2}"
 
 run_hive() {
-    hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism 4 --client reth 2>&1 | tee /tmp/log || true
+    hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism 8 --client reth 2>&1 | tee /tmp/log || true
 }
 
 check_log() {

.github/assets/kurtosis_op_network_params.yaml

@@ -1,7 +1,9 @@
 ethereum_package:
   participants:
     - el_type: reth
-      cl_type: lighthouse
+      el_extra_params:
+        - "--rpc.eth-proof-window=100"
+      cl_type: teku
 optimism_package:
   chains:
     - participants:
@@ -10,6 +12,3 @@ optimism_package:
       - el_type: op-reth
         el_image: "ghcr.io/paradigmxyz/op-reth:kurtosis-ci"
         cl_type: op-node
-      batcher_params:
-        extra_params:
-          - "--throttle-interval=0"