Switch GitHub CI to C++ test runner (#1963)

.github/workflows/build.yml

@@ -45,12 +45,12 @@ jobs:
         os: [macos-15]
         cmake_flags: [""]
         lit_flags: [""]
-        test_runner_flags: [""]
+        test_runner_jit_flag: [""]
         include:
           - os: macos-15
             cmake_flags: "-DHERMESVM_ALLOW_JIT=2"
             lit_flags: "-Xjit=force"
-            test_runner_flags: "--vm-args='-Xjit=force'"
+            test_runner_jit_flag: "--jit=force"
     runs-on: ${{ matrix.os }}
     steps:
     - uses: maxim-lobanov/setup-xcode@v1
@@ -73,7 +73,8 @@ jobs:
         cmake -S hermes -B build -GNinja -DHERMES_ENABLE_INTL=ON -DCMAKE_BUILD_TYPE=Debug ${{ matrix.cmake_flags }}
         cmake --build ./build
         cmake --build ./build --target check-hermes
-        python3 hermes/utils/test_runner.py --test-intl test262/test -b build/bin ${{ matrix.test_runner_flags }}
+        cmake --build ./build --target test-runner
+        build/bin/test-runner --test-intl test262/test --skiplist hermes/utils/testsuite/skiplist.json ${{ matrix.test_runner_jit_flag }}
 
   test-linux-test262:
     strategy:
@@ -82,12 +83,12 @@ jobs:
         os: [4-core-ubuntu, ubuntu-24.04-arm]
         cmake_flags: [""]
         lit_flags: [""]
-        test_runner_flags: [""]
+        test_runner_jit_flag: [""]
         include:
           - os: ubuntu-24.04-arm
             cmake_flags: "-DHERMESVM_ALLOW_JIT=2"
             lit_flags: "-Xjit=force"
-            test_runner_flags: "--vm-args='-Xjit=force'"
+            test_runner_jit_flag: "--jit=force"
     runs-on: ${{ matrix.os }}
     steps:
     - uses: actions/checkout@v4.1.0
@@ -111,11 +112,12 @@ jobs:
 
         cmake -S hermes -GNinja -B build_intl -DHERMES_ENABLE_INTL=ON -DCMAKE_CXX_FLAGS=-O2 -DCMAKE_C_FLAGS=-O2 -DCMAKE_BUILD_TYPE=Debug ${{ matrix.cmake_flags }}
         cmake --build ./build_intl
+        cmake --build ./build_intl --target test-runner
         # Not running Hermes test with -DHERMES_ENABLE_INTL=ON  until more of
         # Intl is built out: toLocaleLowerCase and toLocaleUpperCase are the two
         # main ones.
         # cmake --build ./build_intl --target check-hermes
-        python3 hermes/utils/test_runner.py --test-intl test262/test -b build_intl/bin ${{ matrix.test_runner_flags }}
+        build_intl/bin/test-runner --test-intl test262/test --skiplist hermes/utils/testsuite/skiplist.json ${{ matrix.test_runner_jit_flag }}
 
   test-linux-armv7:
     runs-on: ubuntu-24.04-arm
@@ -148,11 +150,12 @@ jobs:
             -DCMAKE_CXX_FLAGS=-O2 -DCMAKE_C_FLAGS=-O2 -DCMAKE_BUILD_TYPE=Debug \
             -DCMAKE_SYSTEM_PROCESSOR=arm -DCMAKE_SYSTEM_NAME=Linux
           cmake --build ./build_intl
+          cmake --build ./build_intl --target test-runner
           # Not running Hermes test with -DHERMES_ENABLE_INTL=ON  until more of
           # Intl is built out: toLocaleLowerCase and toLocaleUpperCase are the two
           # main ones.
           # cmake --build ./build_intl --target check-hermes
-          python3 hermes/utils/test_runner.py --test-intl test262/test -b build_intl/bin
+          build_intl/bin/test-runner --test-intl test262/test --skiplist hermes/utils/testsuite/skiplist.json
 
   test-windows-test262:
     strategy:
@@ -201,5 +204,5 @@ jobs:
         cmake --build build --config ${{ matrix.build_type }} --target hermes -- \
           -m /p:UseMultiToolTask=true /p:EnforceProcessCountAcrossBuilds=true
 
-        # Run test262 suite
+        # Run test262 suite (using Python runner — C++ runner not yet ported to Windows)
         python3 hermes/utils/test_runner.py test262/test -b build/bin/${{ matrix.build_type }} --timeout 600

include/hermes/BCGen/HBC/HBC.h

@@ -68,6 +68,8 @@ struct CompileFlags {
   bool enableGenerator{true};
   /// Enable ES6 block scoping support
   bool enableES6BlockScoping{false};
+  /// Enable Temporal Dead Zone (TDZ) checking for let/const.
+  bool enableTDZ{false};
   /// Enable async generators support
   bool enableAsyncGenerators{false};
   /// Enable TypeScript support.

lib/BCGen/HBC/BCProviderFromSrc.cpp

@@ -104,6 +104,7 @@ BCProviderFromSrc::create(
 
   CodeGenerationSettings codeGenOpts{};
   codeGenOpts.test262 = compileFlags.test262;
+  codeGenOpts.enableTDZ = compileFlags.enableTDZ;
 
   OptimizationSettings optSettings;
   // If the optional value is not set, the parser will automatically detect

tools/CMakeLists.txt

@@ -23,6 +23,8 @@ add_subdirectory(shermes)
 add_subdirectory(synth)
 add_subdirectory(hcdp)
 
+add_subdirectory(test-runner)
+
 if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/facebook)
   add_subdirectory(facebook)
 endif()

tools/test-runner/CMakeLists.txt

@@ -0,0 +1,14 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+add_hermes_tool(test-runner
+  main.cpp
+  Executor.cpp
+  LINK_OBJLIBS hermesConsoleHost_obj
+  LINK_LIBS hermesvm_a)
+
+install(TARGETS test-runner
+  RUNTIME DESTINATION bin
+)

tools/test-runner/DESIGN.md

@@ -0,0 +1,147 @@
+# test-runner Design Document
+
+## 1. Motivation
+
+The C++ test-runner replaces the Python-based `test_runner.py` for running the
+test262 suite against Hermes. Key motivations:
+
+- **8x faster**: ~20s vs ~153s for the full test262 suite.
+- **Eliminates subprocess overhead**: The Python runner spawns 2 processes per
+  test (compile + execute) × ~50K tests = ~100K subprocesses. The C++ runner
+  does everything in-process.
+- **Better resource management**: In-process execution enables crash isolation
+  via signal handlers and direct control over memory and timeouts.
+
+## 2. Architecture Overview
+
+```
+┌─────────────────────────────────────────────────┐
+│                   test-runner                    │
+│                                                 │
+│  ┌───────────┐  ┌───────────┐  ┌─────────────┐ │
+│  │ Discovery │→│ Skiplist  │→│ Thread Pool │ │
+│  │           │  │ Filtering │  │  Execution  │ │
+│  └───────────┘  └───────────┘  └──────┬──────┘ │
+│                                       │        │
+│                              ┌────────▼───────┐ │
+│                              │   Per-Test:    │ │
+│                              │ Source → BCPro-│ │
+│                              │ viderFromSrc → │ │
+│                              │ runBytecode    │ │
+│                              └────────────────┘ │
+└─────────────────────────────────────────────────┘
+```
+
+- Single C++ binary linked against Hermes libraries.
+- In-process compilation and execution (no subprocess spawning).
+- Thread pool for parallel test execution.
+- Source → in-memory bytecode → execute directly (no `.hbc` serialization).
+
+## 3. Key Design Decisions
+
+### In-Process vs Subprocess
+
+Python runner (2 processes per test):
+```
+source → hermes -emit-binary → .hbc file → hermes -b .hbc
+```
+
+C++ runner (single process, in-memory):
+```
+source → BCProviderFromSrc (in-memory) → Runtime::runBytecode
+```
+
+### Compilation Path
+
+- Uses `hbc::BCProviderFromSrc::create()` directly.
+- Skips `CompilerDriver` overhead (no CLI parsing, no file I/O for bytecode).
+- Optional optimization passes via `-O` flag (default: off, matching Python runner).
+- Optional lazy compilation via `--lazy` flag.
+
+### Optimization Passes
+
+- `-O` flag enables `hbc::fullOptimizationPipeline` callback.
+- Default is no optimization (`-O0`), matching Python runner's default behavior.
+- The callback is passed to `BCProviderFromSrc::create()`, which sets
+  `opts.optimizationEnabled = !!runOptimizationPasses` internally.
+- Full test262 suite passes with both `-O` and `-O0`.
+
+### Lazy Compilation
+
+- `--lazy` flag sets `CompileFlags.lazy = true` for `BCProviderFromSrc::create()`.
+- Lazy mode is incompatible with persistent runtime modules. The runner sets
+  `RuntimeModuleFlags.persistent = !lazy` to avoid the fatal error
+  "Cannot enable persistent mode for lazy compilation."
+- `lazy_skip_list` tests are only skipped when `--lazy` is active, matching the
+  Python runner's conditional `if lazy: skip_categories.append(LAZY_SKIP_LIST)`.
+- When `--lazy` is off (default), `lazy_skip_list` entries are loaded but ignored
+  during filtering.
+
+### JIT Compilation
+
+- `--jit` flag accepts three modes: `off` (default), `on`, and `force`.
+- Maps to `RuntimeConfig::EnableJIT` and `RuntimeConfig::ForceJIT`.
+- JIT is a runtime-only setting — it does not affect compilation flags.
+- `--jit=force` matches the Python runner's `--vm-args='-Xjit=force'` behavior.
+
+### CompileFlags (matching Python's COMPILE_ARGS)
+
+```cpp
+compileFlags.test262 = true;
+compileFlags.enableES6BlockScoping = true;
+compileFlags.enableTDZ = true;
+compileFlags.enableAsyncGenerators = true;
+compileFlags.emitAsyncBreakCheck = true; // for timeout support
+```
+
+### RuntimeConfig (matching Python's run flags)
+
+```cpp
+ES6Proxy = true
+MicrotaskQueue = true
+EnableHermesInternalTestMethods = true
+Test262 = true
+```
+
+### Crash Isolation
+
+- `sigsetjmp`/`siglongjmp` crash guard catches `SIGABRT`/`SIGSEGV` per test.
+- Converts crashes to test failures instead of killing the process.
+- Trade-off: less safe than process isolation, but much faster.
+
+### Handle Sanitizer Support
+
+- Tests in `handlesan_skip_list` run with `GCSanitizeConfig::SanitizeRate = 0.0`.
+- Matches Python runner's `-gc-sanitize-handles=0` behavior.
+
+### Feature Detection
+
+- Uses compile-time `#ifdef HERMES_ENABLE_UNICODE_REGEXP_PROPERTY_ESCAPES`.
+- Mirrors Python's runtime `hermes --version` feature detection.
+
+## 4. Differences from Python Runner
+
+| Aspect              | Python runner              | C++ runner                 |
+|---------------------|----------------------------|----------------------------|
+| Lazy compilation    | `--lazy` flag              | `--lazy` flag              |
+| JIT compilation     | `--vm-args='-Xjit=force'`  | `--jit={off,on,force}`     |
+| staticBuiltins      | Explicitly disabled        | Default (off)              |
+| Bytecode path       | Serialized to `.hbc` file  | In-memory `BCProvider`     |
+| Crash recovery      | Process isolation          | Signal handler             |
+| stdout handling     | Normal (inherited)         | Suppressed during tests    |
+
+## 5. File Structure
+
+| File              | Purpose                                            |
+|-------------------|----------------------------------------------------|
+| `main.cpp`        | CLI, test discovery orchestration, result reporting |
+| `Executor.cpp/h`  | Compilation, execution, crash guard, timeout        |
+| `Skiplist.h`      | JSON skiplist loading, feature/path-based skip logic|
+| `TestDiscovery.h` | File enumeration, frontmatter parsing              |
+| `CMakeLists.txt`  | Build configuration                                |
+
+## 6. Testing
+
+- Full test262 suite: 38,418 passes, 0 failures.
+- Exact match with Python runner on pass/fail counts.
+- 8x wall-time speedup.