perf(iconv): SIMD 2-byte UTF-8 decode for UTF-32 targets — 5x FASTER … #5323
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: FrankenLibC CI | |
| on: | |
| pull_request: | |
| push: | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| inputs: | |
| run_extended: | |
| description: "Run extended CI gates (slow)" | |
| required: false | |
| default: false | |
| type: boolean | |
| schedule: | |
| - cron: "0 6 * * *" | |
| concurrency: | |
| group: frankenlibc-ci-${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RUST_BACKTRACE: 1 | |
| jobs: | |
| core-gates: | |
| name: Core Gates | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: nightly | |
| components: rustfmt, clippy | |
| - name: Cache cargo and target | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Run core CI script | |
| run: | | |
| mkdir -p artifacts/ci | |
| scripts/ci.sh 2>&1 | tee artifacts/ci/core-gates.log | |
| - name: Run proptest gate (10k cases/property) | |
| run: | | |
| mkdir -p artifacts/ci | |
| PROPTEST_CASES=10000 \ | |
| cargo test -p frankenlibc-core prop_ -- --nocapture \ | |
| 2>&1 | tee artifacts/ci/proptest-10k.log | |
| - name: Upload core CI artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: core-ci-artifacts | |
| path: | | |
| artifacts/ci/** | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| release-builds: | |
| name: Release Builds | |
| needs: core-gates | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: nightly | |
| components: rustfmt, clippy | |
| - name: Cache cargo and target | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Build release workspace | |
| run: | | |
| mkdir -p artifacts/ci | |
| cargo build --workspace --all-targets --release 2>&1 | tee artifacts/ci/release-workspace.log | |
| - name: Build ABI release artifact | |
| run: | | |
| cargo build -p frankenlibc-abi --release 2>&1 | tee artifacts/ci/release-abi.log | |
| - name: Check replacement guard (interpose policy) | |
| run: | | |
| bash scripts/check_replacement_guard.sh interpose 2>&1 | tee artifacts/ci/replacement-guard-interpose.log | |
| - name: Check release replacement-level claim evidence | |
| run: | | |
| bash scripts/release/check_replacement_claim_evidence.sh 2>&1 | tee artifacts/ci/release-claim-evidence.log | |
| - name: Build standalone replacement artifact when feature exists | |
| run: | | |
| if grep -Eq '(^|[[:space:]])standalone([[:space:]]*=|[[:space:]])' crates/frankenlibc-abi/Cargo.toml; then | |
| cargo build -p frankenlibc-abi --release --features=standalone 2>&1 | tee artifacts/ci/release-standalone.log | |
| else | |
| echo "SKIP: frankenlibc-abi standalone feature not declared in Cargo.toml" | tee artifacts/ci/release-standalone.log | |
| fi | |
| - name: Upload release build artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: release-build-artifacts | |
| path: | | |
| artifacts/ci/release-*.log | |
| artifacts/ci/replacement-guard-interpose.log | |
| target/conformance/release_claim_evidence_gate.report.json | |
| target/conformance/release_claim_evidence_gate.log.jsonl | |
| target/release/libfrankenlibc_abi.so | |
| target/conformance/replacement_guard.report.json | |
| target/conformance/replacement_guard.log.jsonl | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| extended-gates: | |
| name: Extended Gates | |
| needs: core-gates | |
| if: > | |
| github.event_name == 'schedule' || | |
| github.ref == 'refs/heads/main' || | |
| (github.event_name == 'workflow_dispatch' && inputs.run_extended) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: nightly | |
| components: rustfmt, clippy | |
| - name: Cache cargo and target | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Run extended gate suite | |
| run: | | |
| mkdir -p artifacts/ci | |
| FRANKENLIBC_EXTENDED_GATES=1 scripts/ci.sh 2>&1 | tee artifacts/ci/extended-gates.log | |
| - name: Run LD_PRELOAD smoke suite | |
| run: | | |
| TIMEOUT_SECONDS=10 scripts/ld_preload_smoke.sh 2>&1 | tee artifacts/ci/ld-preload-smoke.log | |
| - name: Upload extended CI artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: extended-ci-artifacts | |
| path: | | |
| artifacts/ci/** | |
| target/conformance_golden/** | |
| target/runtime_math_golden/** | |
| target/ld_preload_smoke/** | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| fuzz-gates: | |
| name: Fuzz Gates | |
| needs: core-gates | |
| if: > | |
| github.event_name == 'schedule' || | |
| (github.event_name == 'workflow_dispatch' && inputs.run_extended) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 360 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust nightly | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: nightly | |
| components: rustfmt, clippy | |
| - name: Cache cargo and target | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Install cargo-fuzz | |
| run: | | |
| cargo install cargo-fuzz --locked | |
| - name: Run fuzz readiness gates | |
| run: | | |
| mkdir -p artifacts/ci | |
| scripts/check_fuzz_harness_architecture.sh 2>&1 | tee artifacts/ci/fuzz-harness-architecture.log | |
| scripts/check_fuzz_phase1_targets.sh 2>&1 | tee artifacts/ci/fuzz-phase1-targets.log | |
| scripts/check_fuzz_phase2_targets.sh 2>&1 | tee artifacts/ci/fuzz-phase2-targets.log | |
| scripts/check_fuzz_membrane_validation.sh 2>&1 | tee artifacts/ci/fuzz-membrane-validation.log | |
| - name: List fuzz targets | |
| run: | | |
| cargo fuzz list --fuzz-dir crates/frankenlibc-fuzz 2>&1 | tee artifacts/ci/fuzz-target-list.log | |
| - name: Run fuzz targets with nightly run budget | |
| run: | | |
| mkdir -p artifacts/ci/fuzz-runs | |
| FUZZ_RUNS_PER_TARGET="${FUZZ_RUNS_PER_TARGET:-1000000}" | |
| FUZZ_TIMEOUT_SECONDS="${FUZZ_TIMEOUT_SECONDS:-1800}" | |
| targets=( | |
| fuzz_string | |
| fuzz_malloc | |
| fuzz_membrane | |
| fuzz_printf | |
| fuzz_resolver | |
| fuzz_resolv | |
| fuzz_iconv | |
| fuzz_runtime_math | |
| ) | |
| for target in "${targets[@]}"; do | |
| log="artifacts/ci/fuzz-runs/${target}.log" | |
| echo "=== ${target}: -runs=${FUZZ_RUNS_PER_TARGET} ===" | tee "${log}" | |
| if command -v timeout >/dev/null 2>&1; then | |
| timeout "${FUZZ_TIMEOUT_SECONDS}" \ | |
| cargo fuzz run --fuzz-dir crates/frankenlibc-fuzz "${target}" -- \ | |
| -runs="${FUZZ_RUNS_PER_TARGET}" \ | |
| -print_final_stats=1 2>&1 | tee -a "${log}" | |
| else | |
| cargo fuzz run --fuzz-dir crates/frankenlibc-fuzz "${target}" -- \ | |
| -runs="${FUZZ_RUNS_PER_TARGET}" \ | |
| -print_final_stats=1 2>&1 | tee -a "${log}" | |
| fi | |
| done | |
| - name: Build fuzz execution summary | |
| if: always() | |
| run: | | |
| python3 - <<'PY' | |
| import glob | |
| import json | |
| import os | |
| import re | |
| from pathlib import Path | |
| run_id = os.environ.get("GITHUB_RUN_ID", "local") | |
| root = Path("artifacts/ci/fuzz-runs") | |
| rows = [] | |
| for path_str in sorted(glob.glob(str(root / "*.log"))): | |
| path = Path(path_str) | |
| text = path.read_text(encoding="utf-8", errors="replace") | |
| target = path.stem | |
| done = re.findall(r"Done\\s+([0-9]+)\\s+runs", text) | |
| executed = re.findall(r"number of executed units:\\s*([0-9]+)", text) | |
| timeout_hit = "Command exited with non-zero status 124" in text | |
| crash_markers = [ | |
| "==ERROR: libFuzzer: deadly signal", | |
| "ERROR: AddressSanitizer", | |
| "artifact_prefix=", | |
| "Test unit written to ", | |
| ] | |
| crash_detected = any(marker in text for marker in crash_markers) | |
| runs = 0 | |
| if done: | |
| runs = int(done[-1]) | |
| elif executed: | |
| runs = int(executed[-1]) | |
| rows.append( | |
| { | |
| "target": target, | |
| "runs_completed": runs, | |
| "timeout_hit": timeout_hit, | |
| "crash_detected": crash_detected, | |
| "log_path": str(path), | |
| } | |
| ) | |
| payload = { | |
| "schema_version": "fuzz.summary.v1", | |
| "run_id": run_id, | |
| "targets": rows, | |
| } | |
| out = Path("artifacts/ci/fuzz-summary.v1.json") | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text(json.dumps(payload, indent=2) + "\n", encoding="utf-8") | |
| print(f"wrote {out}") | |
| PY | |
| - name: Upload fuzz artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: fuzz-artifacts | |
| path: | | |
| artifacts/ci/fuzz-*.log | |
| artifacts/ci/fuzz-runs/** | |
| artifacts/ci/fuzz-summary.v1.json | |
| crates/frankenlibc-fuzz/artifacts/** | |
| crates/frankenlibc-fuzz/corpus/** | |
| tests/conformance/fuzz_*.json | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| multiarch-readiness: | |
| name: Multi-Arch Readiness | |
| needs: core-gates | |
| if: > | |
| github.event_name == 'schedule' || | |
| (github.event_name == 'workflow_dispatch' && inputs.run_extended) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 120 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Install Rust nightly with aarch64 target | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: nightly | |
| targets: aarch64-unknown-linux-gnu | |
| - name: Cache cargo and target | |
| uses: Swatinem/rust-cache@v2 | |
| - name: Multi-arch toolchain probe + build check | |
| run: | | |
| mkdir -p artifacts/ci | |
| if command -v aarch64-linux-gnu-gcc >/dev/null 2>&1; then | |
| export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc | |
| cargo check --workspace --all-targets --target aarch64-unknown-linux-gnu 2>&1 | tee artifacts/ci/multiarch-aarch64-check.log | |
| cargo build -p frankenlibc-abi --target aarch64-unknown-linux-gnu --release 2>&1 | tee artifacts/ci/multiarch-aarch64-build.log | |
| cargo test -p frankenlibc-core --test syscall_veneer_test --target aarch64-unknown-linux-gnu --no-run 2>&1 | tee artifacts/ci/multiarch-aarch64-syscall-veneer-build.log | |
| else | |
| echo "SKIP: aarch64-linux-gnu-gcc not available on runner (multi-arch checks are best-effort)." | tee artifacts/ci/multiarch-aarch64-check.log | |
| echo "SKIP: aarch64 build requires aarch64-linux-gnu-gcc linker." | tee artifacts/ci/multiarch-aarch64-build.log | |
| echo "SKIP: aarch64 syscall veneer test build requires aarch64-linux-gnu-gcc linker." | tee artifacts/ci/multiarch-aarch64-syscall-veneer-build.log | |
| fi | |
| if command -v qemu-aarch64 >/dev/null 2>&1; then | |
| if command -v aarch64-linux-gnu-gcc >/dev/null 2>&1; then | |
| cat > artifacts/ci/qemu_probe.c <<'C' | |
| #include <stdio.h> | |
| int main(void) { | |
| puts("qemu-aarch64-probe-ok"); | |
| return 0; | |
| } | |
| C | |
| aarch64-linux-gnu-gcc -O2 artifacts/ci/qemu_probe.c -o artifacts/ci/qemu_probe.aarch64 | |
| qemu-aarch64 -L /usr/aarch64-linux-gnu artifacts/ci/qemu_probe.aarch64 2>&1 | tee artifacts/ci/multiarch-qemu.log | |
| syscall_veneer_bin="$(find target/aarch64-unknown-linux-gnu/debug/deps -maxdepth 1 -type f -perm -111 -name 'syscall_veneer_test-*' ! -name '*.d' | head -n 1)" | |
| if [ -z "${syscall_veneer_bin}" ]; then | |
| echo "FAIL: aarch64 syscall_veneer_test binary not found after cargo test --no-run" | tee artifacts/ci/multiarch-aarch64-syscall-veneer-qemu.log | |
| exit 1 | |
| fi | |
| qemu-aarch64 -L /usr/aarch64-linux-gnu "${syscall_veneer_bin}" --nocapture 2>&1 | tee artifacts/ci/multiarch-aarch64-syscall-veneer-qemu.log | |
| else | |
| echo "SKIP: qemu probe needs aarch64-linux-gnu-gcc" | tee artifacts/ci/multiarch-qemu.log | |
| echo "SKIP: aarch64 syscall veneer qemu run needs aarch64-linux-gnu-gcc" | tee artifacts/ci/multiarch-aarch64-syscall-veneer-qemu.log | |
| fi | |
| else | |
| echo "SKIP: qemu-aarch64 not available on runner" | tee artifacts/ci/multiarch-qemu.log | |
| echo "SKIP: qemu-aarch64 not available on runner" | tee artifacts/ci/multiarch-aarch64-syscall-veneer-qemu.log | |
| fi | |
| - name: Upload multi-arch artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: multiarch-artifacts | |
| path: | | |
| artifacts/ci/multiarch-*.log | |
| if-no-files-found: warn | |
| retention-days: 14 | |
| glibc-version-matrix: | |
| name: glibc Version Matrix | |
| needs: core-gates | |
| if: > | |
| github.event_name == 'schedule' || | |
| github.ref == 'refs/heads/main' || | |
| (github.event_name == 'workflow_dispatch' && inputs.run_extended) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - glibc: "2.31" | |
| container: "ubuntu:20.04" | |
| distro: "Ubuntu 20.04" | |
| - glibc: "2.34" | |
| container: "ubuntu:22.04" | |
| distro: "Ubuntu 22.04 (baseline)" | |
| - glibc: "2.38" | |
| container: "ubuntu:24.04" | |
| distro: "Ubuntu 24.04" | |
| container: | |
| image: ${{ matrix.container }} | |
| steps: | |
| - name: Install dependencies | |
| run: | | |
| apt-get update | |
| apt-get install -y git curl build-essential jq | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Run glibc offset compatibility check | |
| id: glibc_check | |
| run: | | |
| mkdir -p artifacts/ci/glibc-matrix | |
| scripts/glibc_offset_check/check_glibc_offsets.sh 2>&1 | \ | |
| tee "artifacts/ci/glibc-matrix/glibc-${{ matrix.glibc }}.log" | |
| cp target/glibc_offset_check/glibc_offset_check.json \ | |
| "artifacts/ci/glibc-matrix/glibc-${{ matrix.glibc }}.json" || true | |
| - name: Verify glibc version matches expected | |
| run: | | |
| expected="${{ matrix.glibc }}" | |
| actual=$(cat target/glibc_offset_check/glibc_offset_check.json | jq -r '.glibc_version') | |
| major_minor=$(echo "${actual}" | cut -d. -f1,2) | |
| echo "Container glibc: ${actual} (major.minor: ${major_minor})" | |
| echo "Expected: ${expected}" | |
| if [[ "${major_minor}" != "${expected}" ]]; then | |
| echo "WARNING: Container glibc ${major_minor} differs from expected ${expected}" | |
| fi | |
| - name: Upload glibc matrix artifacts | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: glibc-matrix-${{ matrix.glibc }} | |
| path: | | |
| artifacts/ci/glibc-matrix/** | |
| if-no-files-found: warn | |
| retention-days: 14 |