chore: changes moby/stairway (#564)

Ended up being a big refactor of the NIST KAT + bench pipeline for baseline platform.

Author: titouantanguy

Cargo.toml

@@ -255,7 +255,7 @@ split-debuginfo = "unpacked"
 debug = "line-tables-only"
 
 [profile.bench]
-debug = true
+debug = "line-tables-only"
 
 # actual release profile
 [profile.release]

core/experiments/Cargo.toml

@@ -76,7 +76,7 @@ tfhe-zk-pok = { workspace = true, features = ["experimental"] }
 tonic-prost-build.workspace = true
 
 [lib]
-crate-type = ["lib", "cdylib"]
+crate-type = ["lib"]
 
 [[bin]]
 name = "gen-experiment"
@@ -223,6 +223,11 @@ path = "benches/non-threshold/tfhe-zk-pok/memory.rs"
 required-features = ["measure_memory"]
 harness = false
 
+[[bench]]
+name = "non-threshold_tfhe-zk-pok_size"
+path = "benches/non-threshold/tfhe-zk-pok/size.rs"
+harness = false
+
 
 
 [features]

core/experiments/Makefile.toml

@@ -60,56 +60,43 @@ args = ["build", "-f", "${THRESHOLD_FHE_DOCKER_FILE}", "--build-arg", "FEATURES=
 command = "docker"
 args = ["build", "-f", "${THRESHOLD_FHE_DOCKER_FILE}", "--build-arg", "FEATURES=measure_memory,extension_degree_8", "-t", "tfhe-core-degree-8-mem", "${THRESHOLD_FHE_DOCKER_ROOT}"]
 
+# Build all the images for TFHE parties with memory measurement feature
+[tasks.tfhe-docker-images-all-degrees-mem]
+run_task = { name = ["tfhe-docker-image-degree-3-mem", "tfhe-docker-image-degree-4-mem", "tfhe-docker-image-degree-5-mem", "tfhe-docker-image-degree-6-mem", "tfhe-docker-image-degree-7-mem", "tfhe-docker-image-degree-8-mem"]}
+
 #Build the image for BGV parties (with centralized DKG available)
 [tasks.bgv-docker-image]
 command = "docker"
 args = ["build", "-f", "${THRESHOLD_FHE_DOCKER_FILE}", "--build-arg", "FEATURES=extension_degree_4", "-t", "bgv-core", "${THRESHOLD_FHE_DOCKER_ROOT}"]
 
+[tasks.bgv-docker-image-mem]
+command = "docker"
+args = ["build", "-f", "${THRESHOLD_FHE_DOCKER_FILE}", "--build-arg", "FEATURES=measure_memory,extension_degree_4", "-t", "bgv-core-mem", "${THRESHOLD_FHE_DOCKER_ROOT}"]
+
 # Generate the certificate for TLS communication of the parties
 # Note: templating feature isn't required, but allows to compile the code only once since gen-experiment requires the feature
 [tasks.gen-test-certs]
 command = "cargo"
 args = ["run", "--bin", "threshold-gen-tls-certs","--release","--features", "templating",  "--", "--ca-prefix", "p", "--ca-count", "${NUM_PARTIES}", "-n", "1", "-o", "test_certs"]
 
-#Generate the toml config file for the choreographer for a cluster of parties accessible from localhost
-[tasks.gen-local-choreo]
-command = "cargo"
-args = ["run", "--bin", "gen-experiment", "--release", "--features", "templating", "--", "-n", "${NUM_PARTIES}", "-t", "${THRESHOLD}", "-f", "temp", "-o", "local-cluster", "choreographer"]
-
-#Generate the yml config file for the dockerized parties (tfhe)
-[tasks.tfhe-gen-local-cluster]
-command = "cargo"
-args = ["run", "--bin", "gen-experiment", "--release", "--features", "templating", "--", "-n", "${NUM_PARTIES}", "-o", "tfhe-local-cluster", "-f", "temp", "parties", "--protocol", "tfhe"]
-
-#Generate the yml config file for the dockerized parties (bgv)
-[tasks.bgv-gen-local-cluster]
-command = "cargo"
-args = ["run", "--bin", "gen-experiment", "--release", "--features", "templating", "--", "-n", "${NUM_PARTIES}", "-o", "bgv-local-cluster", "-f", "temp", "parties", "--protocol", "bgv"]
-
 #Generate docker-compose file for the TFHE parties as well as the toml config file for the choreogrpaher for a local deployment (tfhe)
 [tasks.tfhe-gen-experiment]
-command = "cargo"
-args = ["run", "--bin", "gen-experiment", "--release", "--features", "templating", "--", "-n", "${NUM_PARTIES}", "-t", "${THRESHOLD}", "-s", "${MOBY_STRATEGY}", "-f", "temp", "-o", "${EXPERIMENT_NAME}", "all", "--protocol", "tfhe" ]
+script = '''
+cargo run --bin gen-experiment --release --features templating -- -n ${NUM_PARTIES} -t ${THRESHOLD} -s ${MOBY_STRATEGY} -f temp -o ${EXPERIMENT_NAME} all --protocol tfhe ${MEASURE_MEMORY}
+'''
 
 #Generate docker-compose file for the BGV parties as well as the toml config file for the choreogrpaher for a local deployment (bgv)
 [tasks.bgv-gen-experiment]
-command = "cargo"
-args = ["run", "--bin", "gen-experiment", "--release", "--features", "templating", "--", "-n", "${NUM_PARTIES}", "-t", "${THRESHOLD}", "-s", "${MOBY_STRATEGY}", "-f", "temp", "-o", "${EXPERIMENT_NAME}" , "all", "--protocol", "bgv"]
-
-
-#Bring parties alive based on provided yml file
-[tasks.start-parties]
-command = "docker"
-args = ["compose", "--progress=quiet", "-f", "temp/${EXPERIMENT_NAME}.yml", "up", "--quiet-pull"]
-
-#Stop all parties
-[tasks.stop-parties]
 script = '''
-docker compose -f temp/${EXPERIMENT_NAME}.yml logs > temp/${EXPERIMENT_NAME}.log
-docker compose --progress=quiet -f temp/${EXPERIMENT_NAME}.yml down
+cargo run --bin gen-experiment --release --features templating -- -n ${NUM_PARTIES} -t ${THRESHOLD} -f temp -o ${EXPERIMENT_NAME} all --protocol bgv ${MEASURE_MEMORY}
 '''
 
 
+#Bring parties alive based on provided yml file, waiting for health check to pass
+[tasks.start-parties-healthy]
+command = "docker"
+args = ["compose", "--progress=quiet", "-f", "temp/${EXPERIMENT_NAME}.yml", "up", "--quiet-pull", "--wait"]
+
 ###One liner for benching everything
 [tasks.experiment-name]
 script = '''
@@ -138,82 +125,57 @@ mkdir -p temp/telemetry/archived
 mv temp/telemetry/export.json temp/telemetry/archived/trace_${EXPERIMENT_NAME}_$(date +%Y_%m_%d-%H:%M:%S).json
 '''
 
+# Setup the test, creating the necessary directories and files for telemetry and session stats, with appropriate permissions for the parties to write to them. Also adds a header to the session stats files to indicate the start of a new run with the experiment name and timestamp. This task is meant to be run before starting the parties in an experiment.
 [tasks.setup]
 script = '''
 mkdir -p temp/telemetry
 echo "" > temp/telemetry/export.json
 chmod o+w temp/telemetry/export.json
+mkdir -p temp/session_stats
+RUN_DATE="$(date -u +%Y%m%dT%H%M%SZ)-$$"
+for i in $(seq 1 ${NUM_PARTIES}); do
+    touch temp/session_stats/session_stats_$i.txt
+    printf "\nNEW_RUN: ${EXPERIMENT_NAME} ${RUN_DATE} ${MEASURE_MEMORY}" >> temp/session_stats/session_stats_$i.txt
+    chmod o+w temp/session_stats/session_stats_$i.txt
+done
 '''
 
-#Bring parties alive based on provided yml file, waiting for health check to pass
-[tasks.start-parties-healthy]
-command = "docker"
-args = ["compose", "--progress=quiet", "-f", "temp/${EXPERIMENT_NAME}.yml", "up", "--quiet-pull", "--wait"]
 
-#Run the full test suite for tfhe with real dkg
-[tasks.tfhe-test-small]
-command = "sh"
-args = ["test_scripts/tfhe_test_small_session.sh", "temp/${EXPERIMENT_NAME}.toml"]
+# Runs a cluster of 4 parties with the tfhe protocol, with a threshold of 1, and the "secure" strategy (no malicious behavior)
+[tasks.tfhe-bench-run-4p]
+env = {"NUM_PARTIES" = "4", THRESHOLD = "1", MOBY_STRATEGY="secure", EXPERIMENT_NAME="tfhe-bench-run-4p"}
+run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy"], fork = true}
 
-#Run the full test suite for tfhe with centralized dkg
-[tasks.tfhe-test-large]
-command = "sh"
-args = ["test_scripts/tfhe_test_large_session.sh", "temp/${EXPERIMENT_NAME}.toml"]
+# Runs a cluster of 5 parties with the tfhe protocol, with a threshold of 1, and the "secure" strategy (no malicious behavior)
+[tasks.tfhe-bench-run-5p]
+env = {"NUM_PARTIES" = "5", THRESHOLD = "1", MOBY_STRATEGY="secure", EXPERIMENT_NAME="tfhe-bench-run-5p"}
+run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy"], fork = true}
 
-#Run the full test suite for tfhe with real dkg
-[tasks.bgv-test-real-dkg]
-command = "sh"
-args = ["test_scripts/bgv_test_script_real_dkg.sh", "temp/${EXPERIMENT_NAME}.toml"]
+# Runs a cluster of 4 parties with the tfhe protocol, with a threshold of 1, and the "drop_all" strategy (malicious behavior where the malicious party does nothing)
+[tasks.tfhe-bench-run-4p-malicious-drop]
+env = {"NUM_PARTIES" = "4", THRESHOLD = "1", MOBY_STRATEGY="drop_all", EXPERIMENT_NAME="tfhe-bench-run-4p-malicious-drop"}
+run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy"], fork = true}
 
-#Run the full test suite for tfhe with centralized dkg
-[tasks.bgv-test-fake-dkg]
-command = "sh"
-args = ["test_scripts/bgv_test_script_fake_dkg.sh", "temp/${EXPERIMENT_NAME}.toml"]
+# Runs a cluster of 4 parties with the tfhe protocol, with a threshold of 1, and the "secure" strategy (no malicious behavior)
+[tasks.tfhe-bench-run-4p-mem]
+env = {"NUM_PARTIES" = "4", THRESHOLD = "1", MOBY_STRATEGY="secure", EXPERIMENT_NAME="tfhe-bench-run-4p-mem", MEASURE_MEMORY="--measure-memory"}
+run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy"], fork = true}
 
-[tasks.tfhe-bench-make]
-env = {MOBY_STRATEGY = "secure"}
+# Runs a cluster of 5 parties with the tfhe protocol, with a threshold of 1, and the "secure" strategy (no malicious behavior)
+[tasks.tfhe-bench-run-5p-mem]
+env = {"NUM_PARTIES" = "5", THRESHOLD = "1", MOBY_STRATEGY="secure", EXPERIMENT_NAME="tfhe-bench-run-5p-mem", MEASURE_MEMORY="--measure-memory"}
 run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy"], fork = true}
 
-[tasks.tfhe-bench-run]
-env = {"NUM_PARTIES" = "4", THRESHOLD = "1", MOBY_STRATEGY="secure", EXPERIMENT_NAME="tfhe-bench-run"}
+# Runs a cluster of 4 parties with the tfhe protocol, with a threshold of 1, and the "malicious_broadcast" strategy (MaliciousBroadcastSenderEcho strategy)
+[tasks.tfhe-bench-run-4p-malicious-bcast]
+env = {"NUM_PARTIES" = "4", THRESHOLD = "1", MOBY_STRATEGY="malicious_broadcast", EXPERIMENT_NAME="tfhe-bench-run-4p-malicious-bcast"}
 run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy"], fork = true}
 
+# Runs a cluster of 4 parties with the bgv protocol, with a threshold of 1, and the "secure" strategy (no malicious behavior)
 [tasks.bgv-bench-run]
 env = {"NUM_PARTIES" = "4", THRESHOLD = "1", EXPERIMENT_NAME="bgv-bench-run"}
 run_task = { name = ["setup", "gen-test-certs", "bgv-gen-experiment", "experiment-name", "start-parties-healthy"], fork = true}
 
-# For small session we run with N=4,t=1
-[tasks.tfhe-run-small-session]
-env = {"NUM_PARTIES" = "4", THRESHOLD = "1", MOBY_STRATEGY="secure", EXPERIMENT_NAME="tfhe-small-session"}
-run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy", "tfhe-test-small"], fork = true, cleanup_task = "shutdown" }
-
-[tasks.tfhe-run-small-session-drop]
-env = {"NUM_PARTIES" = "4", THRESHOLD = "1", MOBY_STRATEGY="drop_all", EXPERIMENT_NAME="tfhe-small-session-drop"}
-run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy", "tfhe-test-small"], fork = true , cleanup_task = "shutdown"}
-
-[tasks.tfhe-run-small-session-malicious-bcast]
-env = {"NUM_PARTIES" = "4", THRESHOLD = "1", MOBY_STRATEGY="malicious_broadcast", EXPERIMENT_NAME="tfhe-small-session-malicious-bcast"}
-run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy", "tfhe-test-small"], fork = true, cleanup_task = "shutdown" }
-
-# For large session we run with N=5,t=1
-[tasks.tfhe-run-large-session]
-env = {"NUM_PARTIES" = "5", THRESHOLD = "1", MOBY_STRATEGY="secure", EXPERIMENT_NAME="tfhe-large-session"}
-run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy", "tfhe-test-large"], fork = true, cleanup_task = "shutdown" }
-
-[tasks.tfhe-run-large-session-drop]
-env = {"NUM_PARTIES" = "5", THRESHOLD = "1", MOBY_STRATEGY="drop_all", EXPERIMENT_NAME="tfhe-large-session-drop"}
-run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy", "tfhe-test-large"], fork = true, cleanup_task = "shutdown" }
-
-
-[tasks.tfhe-run-large-session-malicious-bcast]
-env = {"NUM_PARTIES" = "5", THRESHOLD = "1", MOBY_STRATEGY="malicious_broadcast", EXPERIMENT_NAME="tfhe-large-session-malicious-bcast"}
-run_task = { name = ["setup", "gen-test-certs", "tfhe-gen-experiment", "experiment-name", "start-parties-healthy", "tfhe-test-large"], fork = true, cleanup_task = "shutdown" }
-
-##For BGV we set NUM_PARTIES to 4 as we only need small sessions
-[tasks.bgv-bench-fake-dkg]
-env = {"NUM_PARTIES" = "4", THRESHOLD = "1", EXPERIMENT_NAME="bgv"}
-run_task = { name = ["setup", "gen-test-certs", "bgv-gen-experiment", "experiment-name", "start-parties-healthy", "bgv-test-fake-dkg"], fork = true, cleanup_task = "shutdown" }
-
-[tasks.bgv-bench-real-dkg]
-env = {"NUM_PARTIES" = "4", THRESHOLD = "1", EXPERIMENT_NAME="bgv"}
-run_task = { name = ["setup", "gen-test-certs", "bgv-gen-experiment", "experiment-name", "start-parties-healthy", "bgv-test-real-dkg"], fork = true, cleanup_task = "shutdown" }
+[tasks.bgv-bench-run-mem]
+env = {"NUM_PARTIES" = "4", THRESHOLD = "1", EXPERIMENT_NAME="bgv-bench-run-mem", MEASURE_MEMORY="--measure-memory"}
+run_task = { name = ["setup", "gen-test-certs", "bgv-gen-experiment", "experiment-name", "start-parties-healthy"], fork = true}

core/experiments/NIST_scripts/.gitignore

@@ -1,2 +1,3 @@
 non_threshold_bench/
 kat/
+threshold/

core/experiments/NIST_scripts/build.sh

@@ -58,7 +58,7 @@ cargo install --force cargo-criterion
 
 # Clone the repository (replace with submission's repo URL)
 REPO_URL="https://github.com/zama-ai/kms.git"
-TARGET_DIR="$HOME/kms/core/experiments/
+TARGET_DIR="$HOME/kms/core/experiments/"
 
 if [ ! -d "$TARGET_DIR" ]; then
     cd $HOME

core/experiments/NIST_scripts/non-threshold-parser.py

@@ -104,6 +104,9 @@ def __init__(self):
         self.verify_batched_load_proof_memory:     float = -1
         self.verify_batched_load_verify_latency:   float = -1
         self.verify_batched_load_verify_memory:    float = -1
+        # Serialized proof sizes
+        self.proof_size_load_proof:                float = -1
+        self.proof_size_load_verify:               float = -1
 
     def all_missing(self):
         return all(v == -1 for v in [
@@ -114,6 +117,7 @@ def all_missing(self):
             self.verify_two_steps_load_verify_latency, self.verify_two_steps_load_verify_memory,
             self.verify_batched_load_proof_latency,    self.verify_batched_load_proof_memory,
             self.verify_batched_load_verify_latency,   self.verify_batched_load_verify_memory,
+            self.proof_size_load_proof,                self.proof_size_load_verify,
         ])
 
 
@@ -355,6 +359,37 @@ def parse_zk_memory(line):
         elif load == "load_verify":
             entry.verify_batched_load_verify_memory = memory
 
+# ---------------------------------------------------------------------------
+# ZK PoK size parsing
+# ---------------------------------------------------------------------------
+
+def parse_zk_size_line(line):
+    """Parse a line like 'proof size (B, serialized): non-threshold_zk-pok_PARAMS=1234'."""
+    match = re.search(r"proof size \(B, serialized\): (.+)=(\d+)", line)
+    if match is None:
+        return
+    name = match.group(1)
+    size_bytes = int(match.group(2))
+    size_kb = size_bytes * UNIT_CONV_TO_KB["B"]
+    for key in ZK_PARAMS_MAP:
+        if key in name:
+            entry = ZK_RESULT_MAP[key]
+            if name.endswith("_verify_load"):
+                entry.proof_size_load_verify = size_kb
+            elif name.endswith("_proof_load"):
+                entry.proof_size_load_proof = size_kb
+            else:
+                print("Unknown proof size line with name {}: {}".format(name, line))
+            return
+
+def parse_size_file():
+    if not os.path.isfile(SIZE_FILE):
+        return
+    with open(SIZE_FILE, "r") as f:
+        for line in f:
+            if "proof size" in line:
+                parse_zk_size_line(line)
+
 # ---------------------------------------------------------------------------
 # Shared memory-file entry point
 # ---------------------------------------------------------------------------
@@ -407,45 +442,55 @@ def output_zk_csv_files():
         file_name = os.path.join(OUTPUT_DIRECTORY, ZK_PARAMS_MAP[params])
         with open(file_name, "w") as f:
             w = csv.writer(f, delimiter=",")
-            w.writerow(["Operation", "avg_latency_ms", "max_memory_kBytes"])
+            w.writerow(["Operation", "avg_latency_ms", "max_memory_kBytes",
+                        "proof_size_kBytes"])
             w.writerow(["CRSGen",
                          result.crs_gen_latency,
-                         result.crs_gen_memory])
+                         result.crs_gen_memory,
+                         -1])
             w.writerow(["ProofGen_LoadProof",
                          result.proof_gen_load_proof_latency,
-                         result.proof_gen_load_proof_memory])
+                         result.proof_gen_load_proof_memory,
+                         result.proof_size_load_proof])
             w.writerow(["ProofGen_LoadVerify",
                          result.proof_gen_load_verify_latency,
-                         result.proof_gen_load_verify_memory])
+                         result.proof_gen_load_verify_memory,
+                         result.proof_size_load_verify])
             w.writerow(["VerifyTwoSteps_LoadProof",
                          result.verify_two_steps_load_proof_latency,
-                         result.verify_two_steps_load_proof_memory])
+                         result.verify_two_steps_load_proof_memory,
+                         result.proof_size_load_proof])
             w.writerow(["VerifyTwoSteps_LoadVerify",
                          result.verify_two_steps_load_verify_latency,
-                         result.verify_two_steps_load_verify_memory])
+                         result.verify_two_steps_load_verify_memory,
+                         result.proof_size_load_verify])
             w.writerow(["VerifyBatched_LoadProof",
                          result.verify_batched_load_proof_latency,
-                         result.verify_batched_load_proof_memory])
+                         result.verify_batched_load_proof_memory,
+                         result.proof_size_load_proof])
             w.writerow(["VerifyBatched_LoadVerify",
                          result.verify_batched_load_verify_latency,
-                         result.verify_batched_load_verify_memory])
+                         result.verify_batched_load_verify_memory,
+                         result.proof_size_load_verify])
 
 # ---------------------------------------------------------------------------
 # Entry point
 # ---------------------------------------------------------------------------
 
 def main():
-    global LATENCY_FILE, MEMORY_FILE, OUTPUT_DIRECTORY
+    global LATENCY_FILE, MEMORY_FILE, SIZE_FILE, OUTPUT_DIRECTORY
     if len(sys.argv) != 2:
         print("Usage: {} <folder>".format(sys.argv[0]))
         sys.exit(1)
     folder = sys.argv[1]
     LATENCY_FILE     = os.path.join(folder, "bench_results.json")
     MEMORY_FILE      = os.path.join(folder, "memory_bench_results.txt")
+    SIZE_FILE        = os.path.join(folder, "size_bench_results.txt")
     OUTPUT_DIRECTORY = os.path.join(folder, "output")
 
     parse_latency_file()
     parse_memory_file()
+    parse_size_file()
     output_result_csv_files()
     output_zk_csv_files()
 

core/experiments/NIST_scripts/non-threshold-tfhe-bench.sh

@@ -31,14 +31,22 @@ mkdir -p $TARGET_DIR
 : > "$MEMORY_OUTPUT_FILE"
 
 # Run the latency benchmarks
+echo "Running TFHE speed benchmarks → $OUTPUT_FILE"
 for bench in "${SPEED_BENCH_LIST[@]}"; do
     cargo-criterion --bench $bench --message-format json >> $OUTPUT_FILE
 done
 
 ## Run the memory benchmarks
+echo ""
+echo "Running TFHE memory benchmarks → $MEMORY_OUTPUT_FILE"
 for bench in "${MEMORY_BENCH_LIST[@]}"; do
     cargo bench --bench $bench --features=measure_memory >> $MEMORY_OUTPUT_FILE
 done
 
 
+echo ""
+echo "Parsing results into CSV → $TARGET_DIR/output/"
 python3 non-threshold-parser.py $TARGET_DIR
+
+echo ""
+echo "Done. Results saved to $TARGET_DIR"