Reproducible ELF builds for SP1 program in Docker (#41)

Author: nuke-web3

Cargo.toml

@@ -17,7 +17,6 @@ zkvm-common = { path = "zkVM/common", default-features = false, version = "0.2.2
 anyhow = "1.0"
 clap = { version = "4.5", features = ["derive", "env"] }
 hex = "0.4"
-sha2 = "=0.10.8"
 sp1-zkvm = "5.1"
 sp1-sdk = "5.1"
 sp1-build = "5.1"
@@ -43,9 +42,10 @@ rustls = "0.23"
 tokio-rustls = "0.26"
 rustls-pemfile = "2.2"
 
+sha2 = "=0.10.8"
 [patch.crates-io]
 # TODO: update to 5.0?
 sha2 = { git = "https://github.com/sp1-patches/RustCrypto-hashes", tag = "patch-sha2-0.10.8-sp1-4.0.0" }
 
 [workspace.features]
-reproducible-elf = ["service/reproducible-elf"]
+reproducible-elf = ["service/reproducible-elf", "zkVM/sp1/reproducible-elf"]

Dockerfile

@@ -2,9 +2,9 @@
 FROM nvidia/cuda:12.9.1-devel-ubuntu24.04 AS base-dev
 
 RUN apt-get update && DEBIAN_FRONTEND=noninteractive \
-  apt-get install --no-install-recommends -y \
-  clang libclang-dev docker.io curl tar build-essential pkg-config git ca-certificates gnupg2 \
-  && rm -rf /var/lib/apt/lists/*
+    apt-get install --no-install-recommends -y \
+    clang libclang-dev docker.io curl tar build-essential pkg-config git ca-certificates gnupg2 \
+    && rm -rf /var/lib/apt/lists/*
 
 ENV GO_VERSION=1.24.5
 ENV GO_URL="https://go.dev/dl/go${GO_VERSION}.linux-amd64.tar.gz"
@@ -26,7 +26,7 @@ RUN cargo install cargo-chef
 
 # https://docs.succinct.xyz/docs/sp1/getting-started/install
 RUN curl -L https://sp1up.succinct.xyz | bash && \
-  /root/.sp1/bin/sp1up
+    /root/.sp1/bin/sp1up
 
 ####################################################################################################
 FROM base-dev AS planner
@@ -44,20 +44,20 @@ WORKDIR /app
 COPY --from=planner /app/recipe.json ./
 
 RUN --mount=type=cache,id=target_cache,target=/app/target \
-  cargo chef cook --release --recipe-path recipe.json
+    cargo chef cook --release --recipe-path recipe.json
 
 COPY . .
 
 # Build SP1 ELF to be proven (with optimizations)
-# TODO: need to use `cargo prove --docker` for repoducible builds?
+# WARNING: we do NOT assume `cargo prove build --docker` is required for identical ELF
 RUN --mount=type=cache,id=target_cache,target=/app/target \
-  RUSTFLAGS="-Copt-level=3 -Clto=fat -Ccodegen-units=1 -Cdebuginfo=1 -Cembed-bitcode=yes" /root/.sp1/bin/cargo-prove prove build -p chacha-program
-# Build the final binary
-# NOTE: default feature is to use --docker ELF
+    RUSTFLAGS="-Copt-level=3 -Clto=fat -Ccodegen-units=1 -Cdebuginfo=1 -Cembed-bitcode=yes"\
+    /root/.sp1/bin/cargo-prove prove build -p chacha-program
+# Build the final binary, embbeding ELF
 RUN --mount=type=cache,id=target_cache,target=/app/target \
-  cargo build --release --no-default-features && \
-  strip /app/target/release/pda-proxy && \
-  cp target/release/pda-proxy /app/pda-proxy # pop out of cache
+    cargo build --release && \
+    strip /app/target/release/pda-proxy && \
+    cp target/release/pda-proxy /app/pda-proxy # pop out of cache
 
 ####################################################################################################
 FROM nvidia/cuda:12.9.1-base-ubuntu24.04 AS runtime

justfile

@@ -3,13 +3,15 @@ default:
 
 alias r := run-debug
 alias rr := run-release
+alias rrr := run-release-reproducible
 alias bc := bench-cycles
 alias db := docker-build
 alias ds := docker-save
 alias dl := docker-load
 alias dr := docker-run
 alias b := build-debug
 alias br := build-release
+alias brr := build-release-reproducible
 alias f := fmt
 alias c := clean
 
@@ -30,7 +32,9 @@ initial-config-installs:
     @just sp1 initial-config-installs
 
 _pre-build:
-    # ALWAYS build with docker
+    @just sp1 build-elf
+
+_pre-build-reproducible:
     @just sp1 build-elf-reproducible
 
 _pre-run:
@@ -40,6 +44,10 @@ _pre-run:
 bench-cycles *FLAGS: _pre-build _pre-run
     cargo r -r -p sp1-util --bin cli -- --execute
 
+# Run in release mode, zkVM ELF stable with optimizations AND debug logs
+run-release-reproducible *FLAGS: _pre-build-reproducible _pre-run
+    RUST_LOG=pda_proxy=debug cargo r -r --features reproducible-elf -- {{ FLAGS }}
+
 # Run in release mode, with optimizations AND debug logs
 run-release *FLAGS: _pre-build _pre-run
     RUST_LOG=pda_proxy=debug cargo r -r -- {{ FLAGS }}
@@ -51,7 +59,11 @@ run-debug *FLAGS: _pre-build _pre-run
 
 # Build docker image & tag
 docker-build:
-    DOCKER_BUILDKIT=1 docker build --build-arg BUILDKIT_INLINE_CACHE=1 --tag "$DOCKER_CONTAINER_NAME" --progress=plain .
+    DOCKER_BUILDKIT=1 docker build \
+      --build-arg BUILDKIT_INLINE_CACHE=1 \
+      --tag "$DOCKER_CONTAINER_NAME" \
+      --progress=plain \
+      .
 
 # Save docker image to a tar.gz
 docker-save:
@@ -70,13 +82,15 @@ docker-run:
     docker run --rm -it \
       -v /var/run/docker.sock:/var/run/docker.sock \
       -v ./service/static:/app/static \
+      -v $HOME/.sp1/circuits:/root/.sp1/circuits \
       -v $PDA_DB_PATH:$PDA_DB_PATH \
       --env-file {{ env-settings }} \
-      --env TLS_CERTS_PATH=/app/static/sample.pem --env TLS_KEY_PATH=/app/static/sample.rsa \
+      --env TLS_CERTS_PATH=/app/static/sample.pem \
+      --env TLS_KEY_PATH=/app/static/sample.rsa \
       --env RUST_LOG=pda_proxy=debug \
       --network=host \
       -p $PDA_PORT:$PDA_PORT \
-      "$DOCKER_CONTAINER_NAME"
+      $DOCKER_CONTAINER_NAME
 
 # Build in debug mode, no optimizations
 build-debug: _pre-build
@@ -86,6 +100,10 @@ build-debug: _pre-build
 build-release: _pre-build
     cargo b -r
 
+# Build in release mode, enforce ELF build is reproducible includes optimizations
+build-release-reproducible: _pre-build-reproducible
+    cargo b -r --features reproducible-elf
+
 # Scrub build artifacts
 clean:
     cargo clean

service/Cargo.toml

@@ -30,6 +30,6 @@ rustls-pemfile.workspace = true
 hyper-rustls.workspace = true
 
 [features]
-default = ["reproducible-elf"]
+default = []
 
 reproducible-elf = []

service/src/internal/runner.rs

@@ -505,7 +505,7 @@ impl PdaRunner {
         job: &Job,
         job_key: &[u8],
     ) -> Result<util::SuccNetJobId, PdaRunnerError> {
-        debug!("Preparing local SP1 proving");
+        debug!("Preparing *REMOTE* SP1 proving");
         let zk_client_handle = self.get_zk_client_remote().await;
         let proof_setup = self
             .get_proof_setup_remote(program_id, zk_client_handle.clone())

zkVM/common/Cargo.toml

@@ -11,6 +11,5 @@ rand = { workspace = true, default-features = false, optional = true, features =
 sha2 = { workspace = true, optional = true }
 
 [features]
-default = ["std", "reproducible-elf"]
+default = ["std"]
 std = ["rand/os_rng", "sha2"]
-reproducible-elf = []

zkVM/common/src/lib.rs

@@ -1,15 +1,3 @@
-// Include the binary input file
-/// For reproducible builds, you need `cargo prove --docker`
-#[cfg(feature = "reproducible-elf")]
-pub const CHACHA_ELF: &[u8] = include_bytes!(
-    "../../../target/elf-compilation/docker/riscv32im-succinct-zkvm-elf/release/chacha-program"
-);
-
-#[cfg(not(feature = "reproducible-elf"))]
-pub const CHACHA_ELF: &[u8] = include_bytes!(
-    "../../../target/elf-compilation/riscv32im-succinct-zkvm-elf/release/chacha-program"
-);
-
 use chacha20::ChaCha20;
 use chacha20::cipher::{KeyIvInit, StreamCipher};
 

zkVM/sp1/Cargo.toml

@@ -33,6 +33,6 @@ codegen-units = 1
 debug = 1
 
 [features]
-default = ["reproducible-elf"]
+default = []
 
 reproducible-elf = []

zkVM/sp1/justfile

@@ -57,7 +57,7 @@ build-elf-reproducible:
     if [ ! -f "$ZK_PROGRAM_REPRODUCIBLE_ELF_PATH" ]; then \
         echo -e "Can't find ELF at \`$ZK_PROGRAM_REPRODUCIBLE_ELF_PATH\`.\nAttempting to build it..."; \
         RUSTFLAGS="-Copt-level=3 -Clto=fat -Ccodegen-units=1 -Cdebuginfo=1 -Cembed-bitcode=yes" \
-        echo -r "\n******\nNOTE: the first run will download an sp1 docker image ~5GB\ncheck network usage if you are not seeing any progress on this step...\n"; \
+        echo -e "\n******\nNOTE: the first run will download an sp1 docker image ~5GB\ncheck network usage if you are not seeing any progress on this step...\n******\n"; \
         RUSTFLAGS="-Copt-level=3 -Clto=fat -Ccodegen-units=1 -Cdebuginfo=1 -Cembed-bitcode=yes" cargo prove build --docker -p chacha-program; \
     else \
         echo "✅ REPRODUCIBLE - ELF Exists, skipping SP1 build"; \