diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a78d0db4ffc..4f0afa6eaee 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,12 +45,12 @@ jobs: os: [macos-15] cmake_flags: [""] lit_flags: [""] - test_runner_flags: [""] + test_runner_jit_flag: [""] include: - os: macos-15 cmake_flags: "-DHERMESVM_ALLOW_JIT=2" lit_flags: "-Xjit=force" - test_runner_flags: "--vm-args='-Xjit=force'" + test_runner_jit_flag: "--jit=force" runs-on: ${{ matrix.os }} steps: - uses: maxim-lobanov/setup-xcode@v1 @@ -73,7 +73,8 @@ jobs: cmake -S hermes -B build -GNinja -DHERMES_ENABLE_INTL=ON -DCMAKE_BUILD_TYPE=Debug ${{ matrix.cmake_flags }} cmake --build ./build cmake --build ./build --target check-hermes - python3 hermes/utils/test_runner.py --test-intl test262/test -b build/bin ${{ matrix.test_runner_flags }} + cmake --build ./build --target test-runner + build/bin/test-runner --test-intl test262/test --skiplist hermes/utils/testsuite/skiplist.json ${{ matrix.test_runner_jit_flag }} test-linux-test262: strategy: @@ -82,12 +83,12 @@ jobs: os: [4-core-ubuntu, ubuntu-24.04-arm] cmake_flags: [""] lit_flags: [""] - test_runner_flags: [""] + test_runner_jit_flag: [""] include: - os: ubuntu-24.04-arm cmake_flags: "-DHERMESVM_ALLOW_JIT=2" lit_flags: "-Xjit=force" - test_runner_flags: "--vm-args='-Xjit=force'" + test_runner_jit_flag: "--jit=force" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4.1.0 @@ -111,11 +112,12 @@ jobs: cmake -S hermes -GNinja -B build_intl -DHERMES_ENABLE_INTL=ON -DCMAKE_CXX_FLAGS=-O2 -DCMAKE_C_FLAGS=-O2 -DCMAKE_BUILD_TYPE=Debug ${{ matrix.cmake_flags }} cmake --build ./build_intl + cmake --build ./build_intl --target test-runner # Not running Hermes test with -DHERMES_ENABLE_INTL=ON until more of # Intl is built out: toLocaleLowerCase and toLocaleUpperCase are the two # main ones. # cmake --build ./build_intl --target check-hermes - python3 hermes/utils/test_runner.py --test-intl test262/test -b build_intl/bin ${{ matrix.test_runner_flags }} + build_intl/bin/test-runner --test-intl test262/test --skiplist hermes/utils/testsuite/skiplist.json ${{ matrix.test_runner_jit_flag }} test-linux-armv7: runs-on: ubuntu-24.04-arm @@ -148,11 +150,12 @@ jobs: -DCMAKE_CXX_FLAGS=-O2 -DCMAKE_C_FLAGS=-O2 -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_SYSTEM_PROCESSOR=arm -DCMAKE_SYSTEM_NAME=Linux cmake --build ./build_intl + cmake --build ./build_intl --target test-runner # Not running Hermes test with -DHERMES_ENABLE_INTL=ON until more of # Intl is built out: toLocaleLowerCase and toLocaleUpperCase are the two # main ones. # cmake --build ./build_intl --target check-hermes - python3 hermes/utils/test_runner.py --test-intl test262/test -b build_intl/bin + build_intl/bin/test-runner --test-intl test262/test --skiplist hermes/utils/testsuite/skiplist.json test-windows-test262: strategy: @@ -201,5 +204,5 @@ jobs: cmake --build build --config ${{ matrix.build_type }} --target hermes -- \ -m /p:UseMultiToolTask=true /p:EnforceProcessCountAcrossBuilds=true - # Run test262 suite + # Run test262 suite (using Python runner — C++ runner not yet ported to Windows) python3 hermes/utils/test_runner.py test262/test -b build/bin/${{ matrix.build_type }} --timeout 600 diff --git a/include/hermes/BCGen/HBC/HBC.h b/include/hermes/BCGen/HBC/HBC.h index 986cec9c4c7..a01e5044270 100644 --- a/include/hermes/BCGen/HBC/HBC.h +++ b/include/hermes/BCGen/HBC/HBC.h @@ -68,6 +68,8 @@ struct CompileFlags { bool enableGenerator{true}; /// Enable ES6 block scoping support bool enableES6BlockScoping{false}; + /// Enable Temporal Dead Zone (TDZ) checking for let/const. + bool enableTDZ{false}; /// Enable async generators support bool enableAsyncGenerators{false}; /// Enable TypeScript support. diff --git a/lib/BCGen/HBC/BCProviderFromSrc.cpp b/lib/BCGen/HBC/BCProviderFromSrc.cpp index 4e820d54e6b..6badfa18a07 100644 --- a/lib/BCGen/HBC/BCProviderFromSrc.cpp +++ b/lib/BCGen/HBC/BCProviderFromSrc.cpp @@ -104,6 +104,7 @@ BCProviderFromSrc::create( CodeGenerationSettings codeGenOpts{}; codeGenOpts.test262 = compileFlags.test262; + codeGenOpts.enableTDZ = compileFlags.enableTDZ; OptimizationSettings optSettings; // If the optional value is not set, the parser will automatically detect diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 38f454effa3..2545f482d03 100644 --- a/tools/CMakeLists.txt +++ b/tools/CMakeLists.txt @@ -23,6 +23,8 @@ add_subdirectory(shermes) add_subdirectory(synth) add_subdirectory(hcdp) +add_subdirectory(test-runner) + if (EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/facebook) add_subdirectory(facebook) endif() diff --git a/tools/test-runner/CMakeLists.txt b/tools/test-runner/CMakeLists.txt new file mode 100644 index 00000000000..51ad306c4f5 --- /dev/null +++ b/tools/test-runner/CMakeLists.txt @@ -0,0 +1,14 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +add_hermes_tool(test-runner + main.cpp + Executor.cpp + LINK_OBJLIBS hermesConsoleHost_obj + LINK_LIBS hermesvm_a) + +install(TARGETS test-runner + RUNTIME DESTINATION bin +) diff --git a/tools/test-runner/DESIGN.md b/tools/test-runner/DESIGN.md new file mode 100644 index 00000000000..6edcff3bd01 --- /dev/null +++ b/tools/test-runner/DESIGN.md @@ -0,0 +1,147 @@ +# test-runner Design Document + +## 1. Motivation + +The C++ test-runner replaces the Python-based `test_runner.py` for running the +test262 suite against Hermes. Key motivations: + +- **8x faster**: ~20s vs ~153s for the full test262 suite. +- **Eliminates subprocess overhead**: The Python runner spawns 2 processes per + test (compile + execute) × ~50K tests = ~100K subprocesses. The C++ runner + does everything in-process. +- **Better resource management**: In-process execution enables crash isolation + via signal handlers and direct control over memory and timeouts. + +## 2. Architecture Overview + +``` +┌─────────────────────────────────────────────────┐ +│ test-runner │ +│ │ +│ ┌───────────┐ ┌───────────┐ ┌─────────────┐ │ +│ │ Discovery │→│ Skiplist │→│ Thread Pool │ │ +│ │ │ │ Filtering │ │ Execution │ │ +│ └───────────┘ └───────────┘ └──────┬──────┘ │ +│ │ │ +│ ┌────────▼───────┐ │ +│ │ Per-Test: │ │ +│ │ Source → BCPro-│ │ +│ │ viderFromSrc → │ │ +│ │ runBytecode │ │ +│ └────────────────┘ │ +└─────────────────────────────────────────────────┘ +``` + +- Single C++ binary linked against Hermes libraries. +- In-process compilation and execution (no subprocess spawning). +- Thread pool for parallel test execution. +- Source → in-memory bytecode → execute directly (no `.hbc` serialization). + +## 3. Key Design Decisions + +### In-Process vs Subprocess + +Python runner (2 processes per test): +``` +source → hermes -emit-binary → .hbc file → hermes -b .hbc +``` + +C++ runner (single process, in-memory): +``` +source → BCProviderFromSrc (in-memory) → Runtime::runBytecode +``` + +### Compilation Path + +- Uses `hbc::BCProviderFromSrc::create()` directly. +- Skips `CompilerDriver` overhead (no CLI parsing, no file I/O for bytecode). +- Optional optimization passes via `-O` flag (default: off, matching Python runner). +- Optional lazy compilation via `--lazy` flag. + +### Optimization Passes + +- `-O` flag enables `hbc::fullOptimizationPipeline` callback. +- Default is no optimization (`-O0`), matching Python runner's default behavior. +- The callback is passed to `BCProviderFromSrc::create()`, which sets + `opts.optimizationEnabled = !!runOptimizationPasses` internally. +- Full test262 suite passes with both `-O` and `-O0`. + +### Lazy Compilation + +- `--lazy` flag sets `CompileFlags.lazy = true` for `BCProviderFromSrc::create()`. +- Lazy mode is incompatible with persistent runtime modules. The runner sets + `RuntimeModuleFlags.persistent = !lazy` to avoid the fatal error + "Cannot enable persistent mode for lazy compilation." +- `lazy_skip_list` tests are only skipped when `--lazy` is active, matching the + Python runner's conditional `if lazy: skip_categories.append(LAZY_SKIP_LIST)`. +- When `--lazy` is off (default), `lazy_skip_list` entries are loaded but ignored + during filtering. + +### JIT Compilation + +- `--jit` flag accepts three modes: `off` (default), `on`, and `force`. +- Maps to `RuntimeConfig::EnableJIT` and `RuntimeConfig::ForceJIT`. +- JIT is a runtime-only setting — it does not affect compilation flags. +- `--jit=force` matches the Python runner's `--vm-args='-Xjit=force'` behavior. + +### CompileFlags (matching Python's COMPILE_ARGS) + +```cpp +compileFlags.test262 = true; +compileFlags.enableES6BlockScoping = true; +compileFlags.enableTDZ = true; +compileFlags.enableAsyncGenerators = true; +compileFlags.emitAsyncBreakCheck = true; // for timeout support +``` + +### RuntimeConfig (matching Python's run flags) + +```cpp +ES6Proxy = true +MicrotaskQueue = true +EnableHermesInternalTestMethods = true +Test262 = true +``` + +### Crash Isolation + +- `sigsetjmp`/`siglongjmp` crash guard catches `SIGABRT`/`SIGSEGV` per test. +- Converts crashes to test failures instead of killing the process. +- Trade-off: less safe than process isolation, but much faster. + +### Handle Sanitizer Support + +- Tests in `handlesan_skip_list` run with `GCSanitizeConfig::SanitizeRate = 0.0`. +- Matches Python runner's `-gc-sanitize-handles=0` behavior. + +### Feature Detection + +- Uses compile-time `#ifdef HERMES_ENABLE_UNICODE_REGEXP_PROPERTY_ESCAPES`. +- Mirrors Python's runtime `hermes --version` feature detection. + +## 4. Differences from Python Runner + +| Aspect | Python runner | C++ runner | +|---------------------|----------------------------|----------------------------| +| Lazy compilation | `--lazy` flag | `--lazy` flag | +| JIT compilation | `--vm-args='-Xjit=force'` | `--jit={off,on,force}` | +| staticBuiltins | Explicitly disabled | Default (off) | +| Bytecode path | Serialized to `.hbc` file | In-memory `BCProvider` | +| Crash recovery | Process isolation | Signal handler | +| stdout handling | Normal (inherited) | Suppressed during tests | + +## 5. File Structure + +| File | Purpose | +|-------------------|----------------------------------------------------| +| `main.cpp` | CLI, test discovery orchestration, result reporting | +| `Executor.cpp/h` | Compilation, execution, crash guard, timeout | +| `Skiplist.h` | JSON skiplist loading, feature/path-based skip logic| +| `TestDiscovery.h` | File enumeration, frontmatter parsing | +| `CMakeLists.txt` | Build configuration | + +## 6. Testing + +- Full test262 suite: 38,418 passes, 0 failures. +- Exact match with Python runner on pass/fail counts. +- 8x wall-time speedup. diff --git a/tools/test-runner/Executor.cpp b/tools/test-runner/Executor.cpp new file mode 100644 index 00000000000..a71b9404bd6 --- /dev/null +++ b/tools/test-runner/Executor.cpp @@ -0,0 +1,603 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "Executor.h" + +#include "hermes/BCGen/HBC/BCProviderFromSrc.h" +#include "hermes/BCGen/HBC/HBC.h" +#include "hermes/ConsoleHost/ConsoleHost.h" +#include "hermes/Public/RuntimeConfig.h" +#include "hermes/Support/MemoryBuffer.h" +#include "hermes/VM/Callable.h" +#include "hermes/VM/Domain.h" +#include "hermes/VM/Runtime.h" +#include "hermes/VM/TimeLimitMonitor.h" +#include "hermes/hermes.h" + +#include "jsi/jsi.h" + +#include "llvh/Support/MemoryBuffer.h" +#include "llvh/Support/raw_ostream.h" + +#include + +namespace hermes { +namespace testrunner { + +namespace { + +using Clock = std::chrono::steady_clock; + +/// Holds the runtime environment created for a single test execution. +struct TestRuntimeEnv { + std::unique_ptr jsiRuntime; + vm::Runtime *runtime; + std::shared_ptr timeLimitMonitor; +}; + +/// Capture the current exception message from the runtime and clear it. +std::string captureException(vm::Runtime &runtime) { + auto thrownVal = runtime.makeHandle(runtime.getThrownValue()); + std::string buf; + llvh::raw_string_ostream sos(buf); + runtime.printException(sos, thrownVal); + runtime.clearThrownValue(); + return sos.str(); +} + +/// Create a configured Hermes runtime for test262 execution. +/// When \p disableHandleSan is true, GC handle sanitization is disabled +/// (sanitize rate = 0), matching the Python runner's behavior for +/// handlesan_skip_list tests. +TestRuntimeEnv createTestRuntime( + unsigned timeoutSeconds, + bool disableHandleSan, + bool enableJIT, + bool forceJIT) { + auto gcConfigBuilder = vm::GCConfig::Builder(); + if (disableHandleSan) { + gcConfigBuilder.withSanitizeConfig( + vm::GCSanitizeConfig::Builder().withSanitizeRate(0.0).build()); + } + + auto runtimeConfig = vm::RuntimeConfig::Builder() + .withGCConfig(gcConfigBuilder.build()) + .withES6Proxy(true) + .withES6BlockScoping(true) + .withMicrotaskQueue(true) + .withEnableHermesInternal(true) + .withEnableHermesInternalTestMethods(true) + .withEnableAsyncGenerators(true) + .withTest262(true) + .withEnableEval(true) + .withAsyncBreakCheckInEval(true) + .withEnableJIT(enableJIT) + .withForceJIT(forceJIT) + .build(); + + auto hermesRuntime = facebook::hermes::makeHermesRuntime(runtimeConfig); + auto *runtime = static_cast( + facebook::jsi::castInterface( + hermesRuntime.get()) + ->getVMRuntimeUnsafe()); + + std::shared_ptr timeLimitMonitor; + if (timeoutSeconds > 0) { + timeLimitMonitor = vm::TimeLimitMonitor::getOrCreate(); + runtime->timeLimitMonitor = timeLimitMonitor; + timeLimitMonitor->watchRuntime( + *runtime, std::chrono::milliseconds(timeoutSeconds * 1000u)); + } + + return {std::move(hermesRuntime), runtime, std::move(timeLimitMonitor)}; +} + +/// Drain the setTimeout task queue, executing each callback and its +/// microtasks. Returns true and sets exceptionMsg if an exception was thrown. +bool drainTaskQueue( + vm::Runtime &runtime, + ConsoleHostContext &ctx, + vm::GCScope &scope, + std::string &exceptionMsg) { + vm::GCScopeMarkerRAII marker{scope}; + vm::MutableHandle task{runtime}; + while (auto optTask = ctx.dequeueTask()) { + task = std::move(*optTask); + auto callRes = vm::Callable::executeCall0( + task, runtime, vm::Runtime::getUndefinedValue(), false); + if (callRes == vm::ExecutionStatus::EXCEPTION) { + exceptionMsg = captureException(runtime); + return true; + } + microtask::performCheckpoint(runtime); + } + return false; +} + +/// Run compiled bytecode in a fresh runtime and evaluate the result +/// against negative expectations. +TestResult executeCompiledTest( + const std::string &testName, + std::unique_ptr bytecode, + const std::string &sourceURL, + const NegativeExpectation &negative, + unsigned timeoutSeconds, + bool disableHandleSan, + bool lazy, + bool enableJIT, + bool forceJIT, + Clock::time_point startTime) { + bool expectRuntimeError = + !negative.phase.empty() && negative.phase == "runtime"; + + auto makeResult = [&](ResultCode code, const std::string &msg) { + auto endTime = Clock::now(); + TestResult r; + r.testName = testName; + r.code = code; + r.message = msg; + r.duration = std::chrono::duration_cast( + endTime - startTime); + return r; + }; + + auto env = + createTestRuntime(timeoutSeconds, disableHandleSan, enableJIT, forceJIT); + + // Install console bindings (including $262, alert, setTimeout, etc.). + vm::GCScope scope(*env.runtime); + ConsoleHostContext ctx{*env.runtime}; + ctx.enableTestMethods_ = true; + installConsoleBindings(*env.runtime, ctx); + + // Override print/alert with no-ops to suppress stdout noise. + { + std::string suppressSrc = "print = function() {}; alert = function() {};"; + std::string suppressError; + auto suppressBC = compileSource( + suppressSrc, + "", + /*strict=*/false, + /*optimize=*/false, + /*lazy=*/false, + suppressError); + if (suppressBC) { + vm::RuntimeModuleFlags suppressFlags; + suppressFlags.persistent = false; + (void)env.runtime->runBytecode( + std::move(suppressBC), + suppressFlags, + "", + vm::Runtime::makeNullHandle()); + } + } + + // Run the test bytecode. + // Lazy compilation cannot use persistent mode. + vm::RuntimeModuleFlags rmFlags; + rmFlags.persistent = !lazy; + auto status = env.runtime->runBytecode( + std::move(bytecode), + rmFlags, + sourceURL, + vm::Runtime::makeNullHandle()); + + bool threwException = status == vm::ExecutionStatus::EXCEPTION; + std::string exceptionMsg; + if (threwException) { + exceptionMsg = captureException(*env.runtime); + } + + // Check for timeout. + if (threwException && + exceptionMsg.find("Javascript execution has timed out") != + std::string::npos) { + if (env.timeLimitMonitor) { + env.timeLimitMonitor->unwatchRuntime(*env.runtime); + } + return makeResult(ResultCode::ExecuteTimeout, "FAIL: Test timed out"); + } + + // Drain microtask queue and task queue while still under timeout protection. + if (!threwException) { + microtask::performCheckpoint(*env.runtime); + if (drainTaskQueue(*env.runtime, ctx, scope, exceptionMsg)) { + threwException = true; + } + } + + // Unwatch runtime from time limit monitor after all execution is complete. + if (env.timeLimitMonitor) { + env.timeLimitMonitor->unwatchRuntime(*env.runtime); + } + + // Check for timeout during microtask/task draining. + if (threwException && + exceptionMsg.find("Javascript execution has timed out") != + std::string::npos) { + return makeResult(ResultCode::ExecuteTimeout, "FAIL: Test timed out"); + } + + // Evaluate result based on expectations. + if (expectRuntimeError) { + if (!threwException) { + return makeResult( + ResultCode::ExecuteFailed, + "FAIL: Expected runtime " + negative.errorType + " but test passed"); + } + if (!negative.errorType.empty() && + exceptionMsg.find(negative.errorType) == std::string::npos) { + return makeResult( + ResultCode::ExecuteFailed, + "FAIL: Expected " + negative.errorType + " but got: " + exceptionMsg); + } + return makeResult(ResultCode::Passed, "PASS (expected runtime error)"); + } + + if (threwException) { + return makeResult(ResultCode::ExecuteFailed, "FAIL: " + exceptionMsg); + } + + return makeResult(ResultCode::Passed, "PASS"); +} + +/// Process a single test entry: read file, parse frontmatter, determine +/// variants, and execute each variant. +void processTestEntry( + const TestEntry &entry, + const HarnessCache &harness, + const Skiplist *skiplist, + const ExecConfig &config, + std::vector &results, + std::mutex &resultsMutex, + std::atomic &featureSkipped, + std::atomic &permanentFeatureSkipped) { + // Read test file. + auto fileBuf = llvh::MemoryBuffer::getFile(entry.path); + if (!fileBuf) { + TestResult r; + r.testName = entry.fullName; + r.code = ResultCode::Failed; + r.message = "FAIL: Cannot read file"; + std::lock_guard lock(resultsMutex); + results.push_back(std::move(r)); + return; + } + + llvh::StringRef content = (*fileBuf)->getBuffer(); + TestRecord record = parseFrontmatter(content); + + // Feature-based skipping. + if (skiplist) { + for (const auto &feat : record.features) { + SkipReason reason = skiplist->shouldSkipFeature(feat); + if (reason != SkipReason::NotSkipped) { + ++featureSkipped; + if (reason == SkipReason::PermanentUnsupportedFeature) + ++permanentFeatureSkipped; + return; + } + } + } + + // Skip module tests — Hermes doesn't support ES module execution. + if (record.isModule()) { + ++featureSkipped; + return; + } + + // Skip tests that include testIntl.js — Hermes doesn't implement all + // Intl constructors required by this harness (matching Python runner). + for (const auto &inc : record.includes) { + if (inc == "testIntl.js") { + ++featureSkipped; + return; + } + } + + // Check if handle sanitizer should be disabled for this test. + bool disableHandleSan = + skiplist && skiplist->shouldDisableHandleSan(entry.path); + + // Determine variants. + bool runStrict = !record.isNoStrict() && !record.isRaw(); + bool runNonStrict = !record.isOnlyStrict() && !record.isRaw(); + bool runRaw = record.isRaw(); + + std::vector includes = buildTestIncludes(entry, record); + + // Match the Python runner behavior: the compiler's strict mode flag + // is set based on whether the test CAN run in strict mode (runStrict), + // not on which variant is currently being run. + bool compileStrict = runStrict; + + // Run variants in order, short-circuiting on first failure (like the Python + // runner). Push exactly one result per test file. + auto runVariant = [&](const char *suffix, bool isStrict) -> TestResult { + std::string source = harness.buildSource(includes, record.src, isStrict); + std::string variantName = entry.fullName + " (" + suffix + ")"; + + return executeTestVariant( + variantName, + source, + entry.path, + compileStrict, + record.negative, + config.timeoutSeconds, + disableHandleSan, + config.optimize, + config.lazy, + config.enableJIT, + config.forceJIT); + }; + + TestResult lastResult; + if (runRaw) { + lastResult = runVariant("raw", false); + } else { + if (runNonStrict) { + lastResult = runVariant("default", false); + if (lastResult.code != ResultCode::Passed) { + std::lock_guard lock(resultsMutex); + results.push_back(std::move(lastResult)); + return; + } + } + if (runStrict) { + lastResult = runVariant("strict", true); + } + } + + { + std::lock_guard lock(resultsMutex); + results.push_back(std::move(lastResult)); + } +} + +/// Print percentage-based progress: "10.. 20.. 30.." etc. +void reportProgress( + std::atomic &completedCount, + size_t totalTests, + std::atomic &lastPrintedPct) { + size_t done = ++completedCount; + if (totalTests > 0) { + unsigned pct = (unsigned)(done * 100 / totalTests); + unsigned milestone = (pct / 10) * 10; + if (milestone > 0) { + unsigned expected = milestone - 10; + if (lastPrintedPct.compare_exchange_strong(expected, milestone)) { + if (milestone == 100) + llvh::outs() << " done."; + else + llvh::outs() << " " << milestone << ".."; + llvh::outs().flush(); + } + } + } +} + +} // namespace + +std::unique_ptr compileSource( + llvh::StringRef source, + llvh::StringRef sourceURL, + bool strict, + bool optimize, + bool lazy, + std::string &errorMsg) { + auto llvmBuf = llvh::MemoryBuffer::getMemBufferCopy(source, sourceURL); + auto buf = std::make_unique(std::move(llvmBuf)); + + hbc::CompileFlags flags; + flags.strict = strict; + flags.test262 = true; + flags.emitAsyncBreakCheck = true; + flags.enableGenerator = true; + flags.enableAsyncGenerators = true; + flags.enableES6BlockScoping = true; + flags.enableTDZ = true; + flags.lazy = lazy; + + auto [provider, error] = hbc::BCProviderFromSrc::create( + std::move(buf), + sourceURL, + /*sourceMap=*/"", + flags, + /*topLevelFunctionName=*/"global", + /*diagHandler=*/{}, + /*diagContext=*/nullptr, + optimize ? hbc::fullOptimizationPipeline + : std::function{}); + + if (!provider) { + errorMsg = error; + return nullptr; + } + + return std::move(provider); +} + +std::vector buildTestIncludes( + const TestEntry &entry, + const TestRecord &record) { + std::vector includes; + if (!record.isRaw()) { + if (entry.suiteKind == SuiteKind::Test262) { + includes.push_back("sta.js"); + includes.push_back("assert.js"); + } + for (const auto &inc : record.includes) { + includes.push_back(inc); + } + if (record.isAsync()) { + includes.push_back("doneprintHandle.js"); + } + } + return includes; +} + +TestResult executeTestVariant( + const std::string &testName, + const std::string &source, + const std::string &sourceURL, + bool isStrict, + const NegativeExpectation &negative, + unsigned timeoutSeconds, + bool disableHandleSan, + bool optimize, + bool lazy, + bool enableJIT, + bool forceJIT) { + auto startTime = Clock::now(); + + auto makeResult = [&](ResultCode code, const std::string &msg) { + auto endTime = Clock::now(); + TestResult r; + r.testName = testName; + r.code = code; + r.message = msg; + r.duration = std::chrono::duration_cast( + endTime - startTime); + return r; + }; + + bool hasNegative = !negative.phase.empty(); + bool expectCompileError = hasNegative && negative.phase == "parse"; + bool expectResolutionError = hasNegative && negative.phase == "resolution"; + + // Compile the source. + // Note: enableJIT/forceJIT are runtime-only settings and don't affect + // compilation — they are passed through to createTestRuntime instead. + std::string compileError; + auto bytecode = + compileSource(source, sourceURL, isStrict, optimize, lazy, compileError); + + if (!bytecode) { + if (expectCompileError || expectResolutionError) { + return makeResult(ResultCode::Passed, "PASS (expected compile error)"); + } + return makeResult( + ResultCode::CompileFailed, "FAIL: Compilation failed: " + compileError); + } + + if (expectCompileError || expectResolutionError) { + return makeResult( + ResultCode::ExecuteFailed, + "FAIL: Expected compile error but compilation succeeded"); + } + + // Check if compilation already exceeded the timeout budget. + if (timeoutSeconds > 0) { + auto elapsed = Clock::now() - startTime; + if (elapsed >= std::chrono::seconds(timeoutSeconds)) { + return makeResult( + ResultCode::CompileTimeout, "FAIL: Compilation timed out"); + } + } + + return executeCompiledTest( + testName, + std::move(bytecode), + sourceURL, + negative, + timeoutSeconds, + disableHandleSan, + lazy, + enableJIT, + forceJIT, + startTime); +} + +void WorkQueue::push(std::function task) { + std::lock_guard lock(mutex_); + tasks_.push_back(std::move(task)); + cv_.notify_one(); +} + +bool WorkQueue::pop(std::function &task) { + std::unique_lock lock(mutex_); + cv_.wait(lock, [this] { return !tasks_.empty() || done_; }); + if (tasks_.empty()) + return false; + task = std::move(tasks_.front()); + tasks_.pop_front(); + return true; +} + +void WorkQueue::finish() { + std::lock_guard lock(mutex_); + done_ = true; + cv_.notify_all(); +} + +void runAllTests( + const std::vector &tests, + const HarnessCache &harness, + const Skiplist *skiplist, + const ExecConfig &config, + std::vector &results, + std::atomic &featureSkipped, + std::atomic &permanentFeatureSkipped) { + std::mutex resultsMutex; + std::atomic completedCount{0}; + size_t totalTests = tests.size(); + std::atomic lastPrintedPct{0}; + + llvh::outs() << "Testing: 0 .."; + llvh::outs().flush(); + + WorkQueue queue; + std::vector workers; + + unsigned numWorkers = std::min(config.numThreads, (unsigned)tests.size()); + if (numWorkers == 0) + numWorkers = 1; + + for (unsigned i = 0; i < numWorkers; ++i) { + workers.emplace_back([&] { + std::function task; + while (queue.pop(task)) { + task(); + } + }); + } + + for (const auto &entry : tests) { + queue.push([&entry, + &harness, + skiplist, + &config, + &results, + &resultsMutex, + &completedCount, + totalTests, + &featureSkipped, + &permanentFeatureSkipped, + &lastPrintedPct] { + processTestEntry( + entry, + harness, + skiplist, + config, + results, + resultsMutex, + featureSkipped, + permanentFeatureSkipped); + reportProgress(completedCount, totalTests, lastPrintedPct); + }); + } + + queue.finish(); + for (auto &w : workers) { + w.join(); + } + + llvh::outs() << " \n"; +} + +} // namespace testrunner +} // namespace hermes diff --git a/tools/test-runner/Executor.h b/tools/test-runner/Executor.h new file mode 100644 index 00000000000..2a93f4c42ee --- /dev/null +++ b/tools/test-runner/Executor.h @@ -0,0 +1,163 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef HERMES_TOOLS_TESTRUNNER_EXECUTOR_H +#define HERMES_TOOLS_TESTRUNNER_EXECUTOR_H + +#include "Frontmatter.h" +#include "HarnessCache.h" +#include "Skiplist.h" +#include "TestDiscovery.h" + +#include "hermes/BCGen/HBC/BCProvider.h" + +#include "llvh/ADT/StringRef.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace hermes { +namespace testrunner { + +/// Result code for a single test variant. +enum class ResultCode { + Passed, + Failed, + Skipped, + PermanentlySkipped, + CompileFailed, + CompileTimeout, + ExecuteFailed, + ExecuteTimeout, +}; + +/// Display name for a result code. +inline const char *resultCodeName(ResultCode code) { + switch (code) { + case ResultCode::Passed: + return "PASS"; + case ResultCode::Failed: + return "FAIL"; + case ResultCode::Skipped: + return "SKIP"; + case ResultCode::PermanentlySkipped: + return "PERMANENTLY_SKIP"; + case ResultCode::CompileFailed: + return "COMPILE_FAIL"; + case ResultCode::CompileTimeout: + return "COMPILE_TIMEOUT"; + case ResultCode::ExecuteFailed: + return "EXECUTE_FAIL"; + case ResultCode::ExecuteTimeout: + return "EXECUTE_TIMEOUT"; + } + return "UNKNOWN"; +} + +/// Result of running a single test variant (strict or non-strict). +struct TestResult { + /// Full test name including variant suffix. + std::string testName; + ResultCode code; + std::string message; + /// Duration of this variant's execution. + std::chrono::microseconds duration{0}; +}; + +/// Configuration for test execution. +struct ExecConfig { + unsigned numThreads = 1; + unsigned timeoutSeconds = 30; + bool optimize = false; + bool lazy = false; + bool enableJIT = false; + bool forceJIT = false; +}; + +/// Compile JS source to bytecode in-memory. +/// +/// \p source the JavaScript source (must be null-terminated). +/// \p sourceURL filename for error messages. +/// \p strict whether to compile in strict mode. +/// \p optimize whether to run optimization passes. +/// \p lazy whether to enable lazy compilation. +/// \p[out] errorMsg set to the compile error message on failure. +/// +/// Returns the BCProvider on success, nullptr on failure. +std::unique_ptr compileSource( + llvh::StringRef source, + llvh::StringRef sourceURL, + bool strict, + bool optimize, + bool lazy, + std::string &errorMsg); + +/// Build the list of harness includes for a test entry. +/// Handles test262 default includes (sta.js, assert.js), test-specified +/// includes, and doneprintHandle.js for async tests. +std::vector buildTestIncludes( + const TestEntry &entry, + const TestRecord &record); + +/// Execute a single test variant (compile + run) in-process. +/// +/// Creates a fresh HermesRuntime, compiles the source to bytecode, +/// runs it, handles negative expectations, and drains microtasks. +/// When \p disableHandleSan is true, GC handle sanitization is disabled +/// for this test (sanitize rate set to 0), matching the Python runner's +/// behavior for handlesan_skip_list tests. +TestResult executeTestVariant( + const std::string &testName, + const std::string &source, + const std::string &sourceURL, + bool isStrict, + const NegativeExpectation &negative, + unsigned timeoutSeconds, + bool disableHandleSan = false, + bool optimize = false, + bool lazy = false, + bool enableJIT = false, + bool forceJIT = false); + +/// Thread-safe work queue for distributing tests to worker threads. +class WorkQueue { + std::mutex mutex_; + std::condition_variable cv_; + std::deque> tasks_; + bool done_ = false; + + public: + /// Add a task to the queue. + void push(std::function task); + + /// Get the next task. Returns false if the queue is done. + bool pop(std::function &task); + + /// Signal that no more tasks will be added. + void finish(); +}; + +/// Run all test entries using a thread pool. +void runAllTests( + const std::vector &tests, + const HarnessCache &harness, + const Skiplist *skiplist, + const ExecConfig &config, + std::vector &results, + std::atomic &featureSkipped, + std::atomic &permanentFeatureSkipped); + +} // namespace testrunner +} // namespace hermes + +#endif // HERMES_TOOLS_TESTRUNNER_EXECUTOR_H diff --git a/tools/test-runner/Frontmatter.h b/tools/test-runner/Frontmatter.h new file mode 100644 index 00000000000..c7fafc91583 --- /dev/null +++ b/tools/test-runner/Frontmatter.h @@ -0,0 +1,276 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef HERMES_TOOLS_TESTRUNNER_FRONTMATTER_H +#define HERMES_TOOLS_TESTRUNNER_FRONTMATTER_H + +#include "llvh/ADT/StringRef.h" + +#include +#include + +namespace hermes { +namespace testrunner { + +/// Expected failure specification from the `negative` frontmatter key. +struct NegativeExpectation { + /// Phase at which the error is expected: "parse", "resolution", or "runtime". + std::string phase; + /// The expected error constructor name (e.g. "SyntaxError"). + std::string errorType; +}; + +/// Parsed test262 frontmatter record. +struct TestRecord { + std::string description; + std::vector flags; + std::vector features; + std::vector includes; + NegativeExpectation negative; // empty phase means no negative expectation + std::string src; // test source with frontmatter stripped + + bool hasFlag(llvh::StringRef flag) const { + for (const auto &f : flags) + if (f == flag) + return true; + return false; + } + + bool isModule() const { + return hasFlag("module"); + } + bool isAsync() const { + return hasFlag("async"); + } + bool isRaw() const { + return hasFlag("raw"); + } + bool isNoStrict() const { + return hasFlag("noStrict"); + } + bool isOnlyStrict() const { + return hasFlag("onlyStrict"); + } + bool hasNegative() const { + return !negative.phase.empty(); + } +}; + +/// Parse a YAML value that is a list written as either: +/// - item1\n - item2\n (block style) +/// [item1, item2] (flow/inline style) +inline std::vector parseYamlList(llvh::StringRef value) { + std::vector result; + auto trimmed = value.trim(); + + // Flow style: [item1, item2] + if (trimmed.startswith("[") && trimmed.endswith("]")) { + auto inner = trimmed.drop_front(1).drop_back(1).trim(); + while (!inner.empty()) { + auto [item, rest] = inner.split(','); + auto t = item.trim(); + if (!t.empty()) + result.push_back(t.str()); + inner = rest.trim(); + if (rest.empty()) + break; + } + return result; + } + + // Block style: lines starting with " - " + llvh::SmallVector lines; + value.split(lines, '\n'); + for (auto line : lines) { + auto t = line.trim(); + if (t.startswith("- ")) { + result.push_back(t.drop_front(2).trim().str()); + } else if (t.startswith("-") && t.size() == 1) { + // bare "- " with nothing after + } + } + return result; +} + +/// Strip a leading license/copyright comment block from source text. +/// Returns the source with the license removed. +inline std::string stripLicenseHeader(llvh::StringRef src) { + if (!src.contains("Copyright") && !src.contains("Public Domain")) { + return src.str(); + } + if (!src.contains("All rights reserved") && !src.contains("Public Domain")) { + return src.str(); + } + + llvh::SmallVector srcLines; + src.split(srcLines, '\n'); + size_t pos = 0; + size_t licenseEnd = 0; + bool inLicense = false; + for (auto line : srcLines) { + auto t = line.trim(); + if (!inLicense && (t.startswith("// Copyright") || t.startswith("/*"))) { + if (t.contains("Copyright") || t.contains("Public Domain")) { + inLicense = true; + licenseEnd = pos + line.size() + 1; // +1 for newline + } + } else if (inLicense) { + if (t.startswith("//") || t.startswith("*") || t.endswith("*/")) { + licenseEnd = pos + line.size() + 1; + if (t.endswith("*/")) + inLicense = false; + } else { + break; + } + } + pos += line.size() + 1; + } + + if (licenseEnd > 0 && licenseEnd <= src.size()) + return src.substr(licenseEnd).str(); + return src.str(); +} + +/// Parse a test262 source file's YAML frontmatter. +/// +/// Extracts the YAML between /*--- and ---*/ markers. +/// Uses a simplified YAML parser sufficient for test262 frontmatter. +inline TestRecord parseFrontmatter(llvh::StringRef content) { + TestRecord record; + + // Find frontmatter markers. + auto startPos = content.find("/*---"); + if (startPos == llvh::StringRef::npos) { + record.src = content.str(); + return record; + } + + auto afterStart = startPos + 5; // "/*---" is 5 chars + auto endPos = content.find("---*/", afterStart); + if (endPos == llvh::StringRef::npos) { + record.src = content.str(); + return record; + } + + auto afterEnd = endPos + 5; // "---*/" is 5 chars + + // Extract YAML content. + auto yaml = content.slice(afterStart, endPos).trim(); + + // Strip frontmatter block from source (including trailing blank lines). + auto blockEnd = afterEnd; + auto remaining = content.substr(blockEnd); + while (!remaining.empty()) { + auto nl = remaining.find('\n'); + llvh::StringRef line; + if (nl == llvh::StringRef::npos) { + line = remaining; + } else { + line = remaining.substr(0, nl); + } + if (line.trim().empty()) { + blockEnd += line.size(); + if (nl != llvh::StringRef::npos) + blockEnd += 1; // newline + remaining = content.substr(blockEnd); + } else { + break; + } + } + + // Build source: everything before frontmatter + everything after. + std::string src = + content.substr(0, startPos).str() + content.substr(blockEnd).str(); + + // Parse simplified YAML first, so we know if this is a raw test. + // We handle keys at the top level, with values that are either: + // key: value + // key:\n - item1\n - item2 + // key:\n subkey: subvalue + llvh::SmallVector yamlLines; + yaml.split(yamlLines, '\n'); + + std::string currentKey; + std::string currentValue; + + auto finishKey = [&]() { + if (currentKey.empty()) + return; + llvh::StringRef key(currentKey); + llvh::StringRef val(currentValue); + + if (key == "description" || key == "info" || key == "commentary") { + record.description = val.trim().str(); + } else if (key == "flags") { + record.flags = parseYamlList(val); + } else if (key == "features") { + record.features = parseYamlList(val); + } else if (key == "includes") { + record.includes = parseYamlList(val); + } else if (key == "negative") { + // Parse sub-keys. + llvh::SmallVector negLines; + val.split(negLines, '\n'); + for (auto line : negLines) { + auto t = line.trim(); + if (t.startswith("phase:")) { + record.negative.phase = t.drop_front(6).trim().str(); + } else if (t.startswith("type:")) { + record.negative.errorType = t.drop_front(5).trim().str(); + } + } + } + // Ignore other keys (es5id, esid, es6id, etc.) + currentKey.clear(); + currentValue.clear(); + }; + + for (auto line : yamlLines) { + // Check if this is a top-level key (no leading whitespace, has colon). + if (!line.empty() && line[0] != ' ' && line[0] != '\t') { + finishKey(); + auto colonPos = line.find(':'); + if (colonPos != llvh::StringRef::npos) { + currentKey = line.substr(0, colonPos).trim().str(); + auto rest = line.substr(colonPos + 1).trim(); + // Handle multi-line values (> or |) + if (rest == ">" || rest == "|") { + currentValue = ""; + } else { + currentValue = rest.str(); + } + } + } else { + // Continuation line for current key. + if (!currentKey.empty()) { + if (!currentValue.empty()) + currentValue += "\n"; + currentValue += line.str(); + } + } + } + finishKey(); + + // For raw tests, preserve the source exactly as-is (minus frontmatter). + // For non-raw tests, strip the license header and leading blank lines. + if (!record.isRaw()) { + src = stripLicenseHeader(src); + + // Trim leading blank lines from result. + while (!src.empty() && src[0] == '\n') + src = src.substr(1); + } + + record.src = src; + + return record; +} + +} // namespace testrunner +} // namespace hermes + +#endif // HERMES_TOOLS_TESTRUNNER_FRONTMATTER_H diff --git a/tools/test-runner/HarnessCache.h b/tools/test-runner/HarnessCache.h new file mode 100644 index 00000000000..82aff19cf4b --- /dev/null +++ b/tools/test-runner/HarnessCache.h @@ -0,0 +1,129 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef HERMES_TOOLS_TESTRUNNER_HARNESSCACHE_H +#define HERMES_TOOLS_TESTRUNNER_HARNESSCACHE_H + +#include "llvh/ADT/StringRef.h" +#include "llvh/Support/FileSystem.h" +#include "llvh/Support/MemoryBuffer.h" +#include "llvh/Support/Path.h" +#include "llvh/Support/raw_ostream.h" + +#include +#include +#include + +namespace hermes { +namespace testrunner { + +/// Cached harness file content for test262. +/// +/// Loads all .js files from the test262/harness/ directory once at startup +/// so that test source assembly never touches the filesystem again. +class HarnessCache { + /// All .js files from test262/harness/, keyed by filename + /// (e.g. "sta.js", "assert.js", "sm/shell.js"). + std::unordered_map files_; + + public: + HarnessCache() = default; + + /// Load all .js harness files from the given test262 harness directory. + /// Returns true on success, false if the directory cannot be read. + bool load(llvh::StringRef harnessDir) { + std::error_code ec; + for (llvh::sys::fs::recursive_directory_iterator it(harnessDir, ec), end; + it != end; + it.increment(ec)) { + if (ec) { + llvh::errs() << "Error reading harness dir: " << ec.message() << "\n"; + return false; + } + + llvh::StringRef path = it->path(); + if (llvh::sys::path::extension(path) != ".js") + continue; + + auto fileBuf = llvh::MemoryBuffer::getFile(path); + if (!fileBuf) { + llvh::errs() << "Warning: cannot read harness file '" << path << "'\n"; + continue; + } + + // Key is the relative path from the harness directory. + llvh::StringRef rel = path; + if (rel.startswith(harnessDir)) { + rel = rel.drop_front(harnessDir.size()); + // Strip leading path separator. + if (!rel.empty() && (rel[0] == '/' || rel[0] == '\\')) + rel = rel.drop_front(1); + } + + files_[rel.str()] = (*fileBuf)->getBuffer().str(); + } + + return true; + } + + /// Look up a harness file by name. Returns nullptr if not found. + const std::string *get(llvh::StringRef name) const { + auto it = files_.find(name.str()); + if (it == files_.end()) + return nullptr; + return &it->second; + } + + /// Number of loaded harness files. + size_t size() const { + return files_.size(); + } + + /// Assemble a complete test source from harness includes and test code. + /// + /// \p includes - harness filenames to prepend (from frontmatter). + /// \p testSrc - the raw test source (frontmatter already stripped). + /// \p isStrict - whether to prepend 'use strict';. + /// + /// Returns the assembled source string. + std::string buildSource( + const std::vector &includes, + llvh::StringRef testSrc, + bool isStrict) const { + std::string result; + + // Estimate capacity. + size_t capacity = testSrc.size(); + if (isStrict) + capacity += 15; // "'use strict';\n" + for (const auto &inc : includes) { + if (auto *content = get(inc)) + capacity += content->size() + 1; + } + result.reserve(capacity); + + if (isStrict) + result += "'use strict';\n"; + + for (const auto &inc : includes) { + if (auto *content = get(inc)) { + result += *content; + result += '\n'; + } else { + llvh::errs() << "Warning: harness file '" << inc << "' not found\n"; + } + } + + result += testSrc.str(); + return result; + } +}; + +} // namespace testrunner +} // namespace hermes + +#endif // HERMES_TOOLS_TESTRUNNER_HARNESSCACHE_H diff --git a/tools/test-runner/Skiplist.h b/tools/test-runner/Skiplist.h new file mode 100644 index 00000000000..8fb7cc23fad --- /dev/null +++ b/tools/test-runner/Skiplist.h @@ -0,0 +1,322 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef HERMES_TOOLS_TESTRUNNER_SKIPLIST_H +#define HERMES_TOOLS_TESTRUNNER_SKIPLIST_H + +#include "hermes/Parser/JSONParser.h" +#include "hermes/Support/SourceErrorManager.h" + +#include "llvh/ADT/StringRef.h" +#include "llvh/Support/MemoryBuffer.h" +#include "llvh/Support/raw_ostream.h" + +#include +#include +#include + +namespace hermes { +namespace testrunner { + +/// Why a test was skipped. +enum class SkipReason { + NotSkipped, + ManualSkipList, + SkipList, + LazySkipList, + PermanentSkipList, + HandlesanSkipList, + UnsupportedFeature, + PermanentUnsupportedFeature, + IntlTests, + PlatformSkipList, +}; + +/// Display name for a skip reason. +inline const char *skipReasonName(SkipReason r) { + switch (r) { + case SkipReason::NotSkipped: + return "not_skipped"; + case SkipReason::ManualSkipList: + return "manual_skip_list"; + case SkipReason::SkipList: + return "skip_list"; + case SkipReason::LazySkipList: + return "lazy_skip_list"; + case SkipReason::PermanentSkipList: + return "permanent_skip_list"; + case SkipReason::HandlesanSkipList: + return "handlesan_skip_list"; + case SkipReason::UnsupportedFeature: + return "unsupported_features"; + case SkipReason::PermanentUnsupportedFeature: + return "permanent_unsupported_features"; + case SkipReason::IntlTests: + return "intl_tests"; + case SkipReason::PlatformSkipList: + return "platform_skip_list"; + } + return "unknown"; +} + +/// A category of skip paths with its associated reason. +struct SkipCategory { + SkipReason reason; + std::vector paths; +}; + +/// Pre-built lookup structure for fast skip checks. +/// +/// At load time, all skiplist JSON entries are flattened into per-category +/// path lists. The shouldSkip() method does substring matching (skiplist +/// path is a substring of the test path), matching the Python/Rust +/// implementations. +class Skiplist { + /// Path-based skip categories. + std::vector categories_; + /// Paths where handle sanitizer should be disabled (not skipped). + std::vector handlesanPaths_; + /// Features that cause a skip when present in test metadata. + std::unordered_set unsupportedFeatures_; + /// Features that cause a permanent skip. + std::unordered_set permanentUnsupportedFeatures_; + /// Features that the built Hermes binary supports, even if they appear in + /// unsupported_features. Matching the Python runner, which queries + /// `hermes --version` to discover supported features and excludes them + /// from feature-based skipping. + std::unordered_set supportedFeatures_; + + /// Extract all string values from a JSON array of skip entries into a + /// container. Each entry can be either a bare string or an object with a + /// "paths" array. The Inserter is called with each extracted std::string. + template + static void flattenStrings(parser::JSONArray *arr, Inserter inserter) { + if (!arr) + return; + for (size_t i = 0; i < arr->size(); ++i) { + auto *val = arr->at(i); + if (auto *str = llvh::dyn_cast(val)) { + inserter(str->str().str()); + } else if (auto *obj = llvh::dyn_cast(val)) { + auto *pathsVal = obj->get("paths"); + if (auto *pathsArr = + llvh::dyn_cast_or_null(pathsVal)) { + for (size_t j = 0; j < pathsArr->size(); ++j) { + if (auto *s = llvh::dyn_cast(pathsArr->at(j))) { + inserter(s->str().str()); + } + } + } + } + } + } + + /// Extract all path strings from a JSON array into a vector. + static void flattenEntries( + parser::JSONArray *arr, + std::vector &out) { + flattenStrings(arr, [&out](std::string s) { out.push_back(std::move(s)); }); + } + + /// Extract feature names from a JSON array into a set. + static void flattenFeatures( + parser::JSONArray *arr, + std::unordered_set &out) { + flattenStrings(arr, [&out](std::string s) { out.insert(std::move(s)); }); + } + + /// Get a JSON array from the root object by key, or nullptr. + static parser::JSONArray *getArray( + parser::JSONObject *root, + const char *key) { + auto *val = root->get(key); + return llvh::dyn_cast_or_null(val); + } + + /// Load named path-based skip categories from the root JSON object. + void loadPathCategories(parser::JSONObject *root) { + struct { + const char *key; + SkipReason reason; + } pathCategories[] = { + {"manual_skip_list", SkipReason::ManualSkipList}, + {"skip_list", SkipReason::SkipList}, + {"lazy_skip_list", SkipReason::LazySkipList}, + {"permanent_skip_list", SkipReason::PermanentSkipList}, + {"intl_tests", SkipReason::IntlTests}, + // handlesan_skip_list is loaded separately below — those tests + // run with handle sanitizer disabled, not skipped. + }; + + for (const auto &cat : pathCategories) { + SkipCategory sc; + sc.reason = cat.reason; + flattenEntries(getArray(root, cat.key), sc.paths); + if (!sc.paths.empty()) + categories_.push_back(std::move(sc)); + } + + // Load handlesan paths separately — these tests are run (not skipped) + // but with GC handle sanitization disabled. + flattenEntries(getArray(root, "handlesan_skip_list"), handlesanPaths_); + } + + /// Load platform-specific skip paths from the root JSON object. + void loadPlatformSkipList(parser::JSONObject *root) { +#if defined(__linux__) + const char *platform = "linux"; +#elif defined(__APPLE__) + const char *platform = "darwin"; +#elif defined(_WIN32) + const char *platform = "win32"; +#else + const char *platform = nullptr; +#endif + if (!platform) + return; + + auto *platformObj = llvh::dyn_cast_or_null( + root->get("platform_skip_list")); + if (!platformObj) + return; + + auto *platformArr = + llvh::dyn_cast_or_null(platformObj->get(platform)); + if (!platformArr) + return; + + SkipCategory sc; + sc.reason = SkipReason::PlatformSkipList; + flattenEntries(platformArr, sc.paths); + if (!sc.paths.empty()) + categories_.push_back(std::move(sc)); + } + + public: + Skiplist() = default; + + /// Load and parse a skiplist from a JSON file. + /// Returns true on success, false on parse error. + bool load(llvh::StringRef path) { + auto fileBuf = llvh::MemoryBuffer::getFile(path); + if (!fileBuf) { + llvh::errs() << "Error: cannot read skiplist file '" << path + << "': " << fileBuf.getError().message() << "\n"; + return false; + } + + parser::JSLexer::Allocator alloc; + parser::JSONFactory factory(alloc); + SourceErrorManager sm; + parser::JSONParser jsonParser(factory, *fileBuf.get(), sm); + auto parsed = jsonParser.parse(); + if (!parsed) { + llvh::errs() << "Error: failed to parse skiplist JSON\n"; + return false; + } + + auto *root = llvh::dyn_cast(parsed.getValue()); + if (!root) { + llvh::errs() << "Error: skiplist JSON root is not an object\n"; + return false; + } + + loadPathCategories(root); + loadPlatformSkipList(root); + + flattenFeatures( + getArray(root, "unsupported_features"), unsupportedFeatures_); + flattenFeatures( + getArray(root, "permanent_unsupported_features"), + permanentUnsupportedFeatures_); + + // Populate supported features based on compile-time configuration. + // This mirrors the Python runner's get_hermes_supported_test262_features(), + // which queries `hermes --version` and maps Hermes feature names to + // test262 feature names. Since the C++ runner runs in-process, we use + // compile-time preprocessor flags instead. +#ifdef HERMES_ENABLE_UNICODE_REGEXP_PROPERTY_ESCAPES + supportedFeatures_.insert("regexp-unicode-property-escapes"); +#endif + + return true; + } + + /// Check if a test path should be skipped based on path matching. + /// Uses substring matching: a skiplist entry matches if it is contained + /// within the test path (matching Python's `value in test_or_feature`). + SkipReason shouldSkipPath(llvh::StringRef testPath) const { + for (const auto &cat : categories_) { + for (const auto &skipPath : cat.paths) { + if (testPath.contains(skipPath)) { + return cat.reason; + } + } + } + return SkipReason::NotSkipped; + } + + /// Check if a test path should be skipped by any non-intl category. + /// This mirrors the Python runner's two-pass design: first check + /// skip_list/permanent_skip_list/manual_skip_list/platform_skip_list, + /// then check intl_tests separately. Used when --test-intl bypasses + /// IntlTests but platform-specific skips must still be honored. + SkipReason shouldSkipPathNonIntl(llvh::StringRef testPath) const { + for (const auto &cat : categories_) { + if (cat.reason == SkipReason::IntlTests) + continue; + for (const auto &skipPath : cat.paths) { + if (testPath.contains(skipPath)) { + return cat.reason; + } + } + } + return SkipReason::NotSkipped; + } + + /// Check if a test should be skipped because it uses an unsupported feature. + /// Features that the built binary actually supports (based on compile-time + /// config) are not skipped, matching the Python runner's behavior. + SkipReason shouldSkipFeature(llvh::StringRef feature) const { + std::string feat = feature.str(); + // If the binary supports this feature, don't skip regardless of skiplist. + if (supportedFeatures_.count(feat)) + return SkipReason::NotSkipped; + if (permanentUnsupportedFeatures_.count(feat)) + return SkipReason::PermanentUnsupportedFeature; + if (unsupportedFeatures_.count(feat)) + return SkipReason::UnsupportedFeature; + return SkipReason::NotSkipped; + } + + /// Check if a test should disable GC handle sanitization. + /// Tests in handlesan_skip_list should run but with sanitize rate = 0. + bool shouldDisableHandleSan(llvh::StringRef testPath) const { + for (const auto &p : handlesanPaths_) { + if (testPath.contains(p)) + return true; + } + return false; + } + + /// Get counts for reporting. + size_t totalSkipPaths() const { + size_t total = 0; + for (const auto &cat : categories_) + total += cat.paths.size(); + return total; + } + + size_t totalUnsupportedFeatures() const { + return unsupportedFeatures_.size() + permanentUnsupportedFeatures_.size(); + } +}; + +} // namespace testrunner +} // namespace hermes + +#endif // HERMES_TOOLS_TESTRUNNER_SKIPLIST_H diff --git a/tools/test-runner/TestDiscovery.h b/tools/test-runner/TestDiscovery.h new file mode 100644 index 00000000000..625efdc7d37 --- /dev/null +++ b/tools/test-runner/TestDiscovery.h @@ -0,0 +1,184 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#ifndef HERMES_TOOLS_TESTRUNNER_TESTDISCOVERY_H +#define HERMES_TOOLS_TESTRUNNER_TESTDISCOVERY_H + +#include "llvh/ADT/StringRef.h" +#include "llvh/Support/FileSystem.h" +#include "llvh/Support/Path.h" +#include "llvh/Support/raw_ostream.h" + +#include +#include +#include + +namespace hermes { +namespace testrunner { + +/// Supported test suite kinds. +enum class SuiteKind { + Test262, + Mjsunit, + CVEs, + Esprima, + Flow, +}; + +/// Short display name for a suite kind. +inline const char *suiteKindName(SuiteKind kind) { + switch (kind) { + case SuiteKind::Test262: + return "test262"; + case SuiteKind::Mjsunit: + return "mjsunit"; + case SuiteKind::CVEs: + return "CVEs"; + case SuiteKind::Esprima: + return "esprima"; + case SuiteKind::Flow: + return "flow"; + } + return "unknown"; +} + +/// A discovered test file with its suite classification. +struct TestEntry { + /// Absolute path to the .js test file. + std::string path; + /// Which test suite this file belongs to. + SuiteKind suiteKind; + /// Root directory of the suite (e.g., "/path/to/test262/"). + std::string suiteDir; + /// Human-readable test name: "{suite} :: {relative_path}". + std::string fullName; +}; + +/// Detect the suite kind from a file path by searching for known directory +/// markers. Returns true and sets kind/suiteDir if found. +inline bool +detectSuite(llvh::StringRef path, SuiteKind &kind, std::string &suiteDir) { + struct Marker { + const char *name; + SuiteKind kind; + }; + // Order matters: "flow/" must use trailing slash to avoid matching + // "flowtest/". + static const Marker markers[] = { + {"test262/", SuiteKind::Test262}, + {"mjsunit/", SuiteKind::Mjsunit}, + {"CVEs/", SuiteKind::CVEs}, + {"esprima/", SuiteKind::Esprima}, + {"flow/", SuiteKind::Flow}, + }; + + for (const auto &m : markers) { + // Use rfind to pick the innermost match for repeated folder names. + auto pos = path.rfind(m.name); + if (pos != llvh::StringRef::npos) { + kind = m.kind; + suiteDir = path.substr(0, pos + strlen(m.name)).str(); + return true; + } + } + return false; +} + +/// Check whether a file path is a valid test file. +/// Must end with .js and must not be a test262 fixture (*_FIXTURE.js). +inline bool isTestFile(llvh::StringRef path) { + if (llvh::sys::path::extension(path) != ".js") + return false; + + // test262 fixture files are helpers, not tests. + if (path.endswith("_FIXTURE.js") && path.contains("test262")) + return false; + + return true; +} + +/// Create a TestEntry from a file path and its detected suite info. +inline TestEntry makeTestEntry( + llvh::StringRef path, + SuiteKind kind, + const std::string &suiteDir) { + llvh::StringRef rel = path; + if (rel.startswith(suiteDir)) + rel = rel.drop_front(suiteDir.size()); + std::string fullName = std::string(suiteKindName(kind)) + " :: " + rel.str(); + return {path.str(), kind, suiteDir, fullName}; +} + +/// Recursively collect .js test files from a directory. +inline void collectTestFiles( + const llvh::Twine &dirPath, + std::vector &out) { + std::error_code ec; + for (llvh::sys::fs::recursive_directory_iterator it(dirPath, ec), end; + it != end; + it.increment(ec)) { + if (ec) { + llvh::errs() << "Error traversing " << dirPath << ": " << ec.message() + << "\n"; + break; + } + llvh::StringRef entry = it->path(); + if (!isTestFile(entry)) + continue; + + SuiteKind kind; + std::string suiteDir; + if (!detectSuite(entry, kind, suiteDir)) + continue; + + out.push_back(makeTestEntry(entry, kind, suiteDir)); + } +} + +/// Discover all test files under the given paths. +/// Each path can be a file or directory. Directories are walked recursively. +/// Returns test entries sorted by path for deterministic ordering. +inline std::vector discoverTests( + const std::vector &paths) { + std::vector entries; + + for (const auto &p : paths) { + llvh::sys::fs::file_status status; + if (llvh::sys::fs::status(p, status)) { + llvh::errs() << "Warning: cannot stat '" << p << "', skipping.\n"; + continue; + } + if (llvh::sys::fs::is_directory(status)) { + collectTestFiles(p, entries); + } else if (llvh::sys::fs::is_regular_file(status)) { + llvh::StringRef path(p); + if (!isTestFile(path)) + continue; + + SuiteKind kind; + std::string suiteDir; + if (!detectSuite(path, kind, suiteDir)) + continue; + + entries.push_back(makeTestEntry(path, kind, suiteDir)); + } else { + llvh::errs() << "Warning: '" << p + << "' is not a file or directory, skipping.\n"; + } + } + + // Sort for deterministic test ordering. + std::sort(entries.begin(), entries.end(), [](const auto &a, const auto &b) { + return a.path < b.path; + }); + return entries; +} + +} // namespace testrunner +} // namespace hermes + +#endif // HERMES_TOOLS_TESTRUNNER_TESTDISCOVERY_H diff --git a/tools/test-runner/main.cpp b/tools/test-runner/main.cpp new file mode 100644 index 00000000000..f83a894d54b --- /dev/null +++ b/tools/test-runner/main.cpp @@ -0,0 +1,490 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +#include "Executor.h" +#include "Frontmatter.h" +#include "HarnessCache.h" +#include "Skiplist.h" +#include "TestDiscovery.h" + +#include "llvh/Support/CommandLine.h" +#include "llvh/Support/FileSystem.h" +#include "llvh/Support/InitLLVM.h" +#include "llvh/Support/MemoryBuffer.h" +#include "llvh/Support/Path.h" +#include "llvh/Support/raw_ostream.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace hermes::testrunner; +namespace cl = llvh::cl; + +namespace { + +cl::list TestPaths( + cl::Positional, + cl::desc(""), + cl::OneOrMore); + +cl::opt NumThreads( + "j", + cl::desc("Number of parallel workers (default: hardware concurrency)"), + cl::init( + std::thread::hardware_concurrency() + ? std::thread::hardware_concurrency() + : 1)); + +cl::opt Timeout( + "timeout", + cl::desc("Per-test timeout in seconds (default: 30)"), + cl::init(30)); + +cl::opt ShowSlowestTests( + "show-slowest-tests", + cl::desc("Show N slowest tests (default: 0)"), + cl::init(0)); + +cl::opt DumpSource( + "dump-source", + cl::desc("Print preprocessed test source and exit"), + cl::init(false)); + +cl::opt + SkiplistPath("skiplist", cl::desc("Path to skiplist.json"), cl::init("")); + +cl::opt TestIntl( + "test-intl", + cl::desc("Include Intl (intl402) tests instead of skipping them"), + cl::init(false)); + +cl::opt Lazy( + "lazy", + cl::desc("Force lazy compilation (default: off)"), + cl::init(false)); + +cl::opt Optimize( + "O", + cl::desc("Enable optimization passes (default: off)"), + cl::init(false)); + +enum class JITMode { Off, On, Force }; +cl::opt JIT( + "jit", + cl::desc("JIT compilation mode (default: off)"), + cl::init(JITMode::Off), + cl::values( + clEnumValN(JITMode::Off, "off", "JIT is disabled"), + clEnumValN(JITMode::On, "on", "JIT is enabled"), + clEnumValN( + JITMode::Force, + "force", + "Force JIT compilation of every function"))); + +cl::opt TestSuiteDir( + "test-suite-dir", + cl::desc("Path to test262 suite root"), + cl::init("")); + +/// Try to auto-detect the test262 suite root by looking for a harness/ +/// directory. Walks up from each test path. +std::string findTest262Dir() { + if (!TestSuiteDir.empty()) + return TestSuiteDir; + + for (const auto &p : TestPaths) { + // Check if the path itself contains test262. + llvh::StringRef pathRef(p); + auto pos = pathRef.rfind("test262/"); + if (pos != llvh::StringRef::npos) { + std::string candidate = pathRef.substr(0, pos + 8).str(); // "test262/" + llvh::SmallString<256> harnessCheck(candidate); + llvh::sys::path::append(harnessCheck, "harness"); + if (llvh::sys::fs::is_directory(harnessCheck)) + return candidate; + } + + // Walk up looking for a test262 directory with harness/. + llvh::SmallString<256> dir(p); + for (int i = 0; i < 10; ++i) { + llvh::sys::path::append(dir, ".."); + llvh::SmallString<256> candidate(dir); + llvh::sys::path::append(candidate, "harness"); + if (llvh::sys::fs::is_directory(candidate)) { + // Verify this looks like a test262 root. + llvh::SmallString<256> staCheck(candidate); + llvh::sys::path::append(staCheck, "sta.js"); + if (llvh::sys::fs::exists(staCheck)) + return std::string(dir.str()); + } + } + } + return ""; +} + +/// Check if a test should be skipped due to unsupported features. +bool shouldSkipByFeature( + const TestRecord &record, + const Skiplist &skiplist, + bool hasSkiplist) { + if (!hasSkiplist) + return false; + for (const auto &feat : record.features) { + if (skiplist.shouldSkipFeature(feat) != SkipReason::NotSkipped) + return true; + } + return false; +} + +/// Result of filtering tests by the skiplist. +struct FilterResult { + std::vector tests; + size_t skippedCount = 0; + size_t permanentlySkippedCount = 0; +}; + +/// Filter test entries by the skiplist, separating tests to run from skipped. +FilterResult filterBySkiplist( + std::vector &allTests, + const Skiplist *skiplist, + bool testIntl, + bool lazy) { + FilterResult result; + for (auto &entry : allTests) { + if (skiplist) { + SkipReason reason = skiplist->shouldSkipPath(entry.path); + if (reason != SkipReason::NotSkipped) { + // When --test-intl is set, don't skip intl tests, but still + // honor other skip reasons (e.g. platform_skip_list). This + // mirrors the Python runner's two-pass design where it checks + // skip_list/platform_skip_list first, then intl_tests separately. + if (testIntl && reason == SkipReason::IntlTests) { + reason = skiplist->shouldSkipPathNonIntl(entry.path); + } + // Only skip lazy_skip_list tests when --lazy is enabled, + // matching the Python runner's conditional: + // if lazy: skip_categories.append(LAZY_SKIP_LIST) + if (!lazy && reason == SkipReason::LazySkipList) { + reason = SkipReason::NotSkipped; + } + if (reason != SkipReason::NotSkipped) { + ++result.skippedCount; + if (reason == SkipReason::PermanentSkipList) + ++result.permanentlySkippedCount; + continue; + } + } + } + result.tests.push_back(std::move(entry)); + } + return result; +} + +/// Tallied results from test execution. +/// Each test file produces exactly one result (short-circuit on first failure). +struct ResultTally { + size_t passed = 0; + size_t compileFail = 0; + size_t compileTimeout = 0; + size_t executeFail = 0; + size_t executeTimeout = 0; + std::vector failures; + + size_t totalFailed() const { + return compileFail + compileTimeout + executeFail + executeTimeout; + } +}; + +/// Tally results. Each entry is already one-per-file (the executor +/// short-circuits on first variant failure, matching the Python runner). +ResultTally tallyResults(const std::vector &results) { + ResultTally tally; + for (const auto &r : results) { + switch (r.code) { + case ResultCode::Passed: + ++tally.passed; + break; + case ResultCode::CompileFailed: + ++tally.compileFail; + tally.failures.push_back(r); + break; + case ResultCode::CompileTimeout: + ++tally.compileTimeout; + tally.failures.push_back(r); + break; + case ResultCode::Failed: + case ResultCode::ExecuteFailed: + ++tally.executeFail; + tally.failures.push_back(r); + break; + case ResultCode::ExecuteTimeout: + ++tally.executeTimeout; + tally.failures.push_back(r); + break; + case ResultCode::Skipped: + case ResultCode::PermanentlySkipped: + break; + } + } + return tally; +} + +/// Print the N slowest tests by execution time. +void printSlowestTests(const std::vector &results, unsigned count) { + if (count == 0 || results.empty()) + return; + + std::vector sorted; + sorted.reserve(results.size()); + for (const auto &r : results) + sorted.push_back(&r); + std::sort(sorted.begin(), sorted.end(), [](const auto *a, const auto *b) { + return a->duration > b->duration; + }); + unsigned n = std::min((unsigned)sorted.size(), count); + llvh::outs() << "\nSlowest " << n << " tests:\n"; + llvh::outs() << "-----------------------------------\n"; + for (unsigned i = 0; i < n; ++i) { + double secs = sorted[i]->duration.count() / 1000000.0; + llvh::outs() << llvh::format(" %.2fs ", secs) << sorted[i]->testName + << "\n"; + } + llvh::outs() << "-----------------------------------\n"; +} + +/// Print summary table and failure details. Returns the number of failed +/// test files. +size_t printResults( + const std::vector &results, + size_t skippedCount, + size_t permanentlySkippedCount, + size_t featureSkippedCount, + size_t permanentFeatureSkippedCount, + double wallSeconds) { + ResultTally tally = tallyResults(results); + + size_t totalPermanentlySkipped = + permanentlySkippedCount + permanentFeatureSkippedCount; + assert( + skippedCount + featureSkippedCount >= totalPermanentlySkipped && + "permanent skips must be a subset of total skips"); + size_t totalSkipped = + skippedCount + featureSkippedCount - totalPermanentlySkipped; + size_t totalFailed = tally.totalFailed(); + size_t totalTests = + tally.passed + totalFailed + totalSkipped + totalPermanentlySkipped; + size_t executed = tally.passed + totalFailed; + double passRate = + executed > 0 ? (double)tally.passed / (double)executed * 100.0 : 0.0; + + // Print result summary table. + llvh::outs() << llvh::format("Testing time: %.2fs\n", wallSeconds); + llvh::outs() << "-----------------------------------\n"; + if (totalFailed == 0) { + llvh::outs() << "| Results | PASS |\n"; + } else { + llvh::outs() << "| Results | FAIL |\n"; + } + llvh::outs() << "|----------------------+----------|\n"; + llvh::outs() << llvh::format("| Total | %8zu |\n", totalTests); + llvh::outs() << llvh::format( + "| Passes | %8zu |\n", tally.passed); + llvh::outs() << llvh::format( + "| Failures | %8zu |\n", totalFailed); + llvh::outs() << llvh::format( + "| Skipped | %8zu |\n", totalSkipped); + llvh::outs() << llvh::format( + "| Permanently Skipped | %8zu |\n", totalPermanentlySkipped); + llvh::outs() << llvh::format( + "| Pass Rate | %6.2f%% |\n", passRate); + llvh::outs() << "-----------------------------------\n"; + llvh::outs() << "| Failures | |\n"; + llvh::outs() << "|----------------------+----------|\n"; + llvh::outs() << llvh::format( + "| Compile fail | %8zu |\n", tally.compileFail); + llvh::outs() << llvh::format( + "| Compile timeout | %8zu |\n", tally.compileTimeout); + llvh::outs() << llvh::format( + "| Execute fail | %8zu |\n", tally.executeFail); + llvh::outs() << llvh::format( + "| Execute timeout | %8zu |\n", tally.executeTimeout); + llvh::outs() << "-----------------------------------\n"; + + // Print failure details. + if (!tally.failures.empty()) { + llvh::outs() << "\nFailed tests:\n"; + for (const auto &f : tally.failures) { + llvh::outs() << " " << resultCodeName(f.code) << ": " << f.testName + << "\n"; + llvh::outs() << " " << f.message << "\n"; + } + } + + printSlowestTests(results, ShowSlowestTests); + + return tally.totalFailed(); +} + +/// Print preprocessed test source for a single test (--dump-source mode). +void dumpTestSource( + const TestEntry &entry, + const HarnessCache &harness, + const Skiplist &skiplist, + bool hasSkiplist) { + auto fileBuf = llvh::MemoryBuffer::getFile(entry.path); + if (!fileBuf) { + llvh::errs() << "Error: cannot read '" << entry.path << "'\n"; + return; + } + + llvh::StringRef content = (*fileBuf)->getBuffer(); + TestRecord record = parseFrontmatter(content); + + if (shouldSkipByFeature(record, skiplist, hasSkiplist)) + return; + + // Skip module tests — Hermes doesn't support ES module execution. + if (record.isModule()) + return; + + bool runStrict = !record.isNoStrict() && !record.isRaw(); + bool runNonStrict = !record.isOnlyStrict() && !record.isRaw(); + bool runRaw = record.isRaw(); + + std::vector includes = buildTestIncludes(entry, record); + + auto printVariant = [&](const char *label, bool isStrict) { + std::string source = harness.buildSource(includes, record.src, isStrict); + llvh::outs() << "=== " << entry.fullName << " (" << label << ") ===\n"; + if (record.hasNegative()) { + llvh::outs() << "// negative: phase=" << record.negative.phase + << " type=" << record.negative.errorType << "\n"; + } + llvh::outs() << source << "\n"; + }; + + if (runRaw) { + printVariant("raw", false); + } else { + if (runNonStrict) { + printVariant("default", false); + } + if (runStrict) { + printVariant("strict", true); + } + } +} +} // namespace + +int main(int argc, char **argv) { + llvh::InitLLVM X(argc, argv); + cl::ParseCommandLineOptions( + argc, + argv, + "Hermes test262 runner\n\n" + " Runs test262 tests against the Hermes VM.\n" + " Accepts individual .js files or directories.\n"); + + // Discover test files. + std::vector allTests = discoverTests(TestPaths); + + if (allTests.empty()) { + llvh::errs() << "No test files found.\n"; + return 1; + } + + // Load skiplist if available. + Skiplist skiplist; + bool hasSkiplist = false; + if (!SkiplistPath.empty()) { + hasSkiplist = skiplist.load(SkiplistPath); + if (hasSkiplist) { + llvh::outs() << "Loaded skiplist: " << skiplist.totalSkipPaths() + << " skip paths, " << skiplist.totalUnsupportedFeatures() + << " unsupported features\n"; + } + } + + // Load harness files. + HarnessCache harness; + std::string test262Dir = findTest262Dir(); + if (!test262Dir.empty()) { + llvh::SmallString<256> harnessDir(test262Dir); + llvh::sys::path::append(harnessDir, "harness"); + if (harness.load(harnessDir.str())) { + llvh::outs() << "Loaded " << harness.size() << " harness files from " + << harnessDir << "\n"; + } + } + + // Filter tests by skiplist. + FilterResult filtered = filterBySkiplist( + allTests, hasSkiplist ? &skiplist : nullptr, TestIntl, Lazy); + + llvh::outs() << "-- Testing: " << filtered.tests.size() << " tests" + << ", max " << NumThreads << " concurrent tasks --\n"; + + // --dump-source mode: parse frontmatter, assemble source, print to stdout. + // Requires exactly one input path that is a .js file. + if (DumpSource) { + if (TestPaths.size() != 1 || + !llvh::sys::fs::is_regular_file(TestPaths[0])) { + llvh::errs() + << "Error: --dump-source requires exactly one .js file path.\n"; + return 1; + } + if (filtered.tests.empty()) { + llvh::outs() << "Test was skipped by skiplist.\n"; + return 0; + } + dumpTestSource(filtered.tests[0], harness, skiplist, hasSkiplist); + return 0; + } + + // Execute tests with thread pool. + ExecConfig execConfig; + execConfig.numThreads = NumThreads; + execConfig.timeoutSeconds = Timeout; + execConfig.optimize = Optimize; + execConfig.lazy = Lazy; + execConfig.enableJIT = JIT != JITMode::Off; + execConfig.forceJIT = JIT == JITMode::Force; + + std::vector results; + std::atomic featureSkippedCount{0}; + std::atomic permanentFeatureSkippedCount{0}; + + auto wallStart = std::chrono::steady_clock::now(); + + runAllTests( + filtered.tests, + harness, + hasSkiplist ? &skiplist : nullptr, + execConfig, + results, + featureSkippedCount, + permanentFeatureSkippedCount); + + auto wallEnd = std::chrono::steady_clock::now(); + double wallSeconds = + std::chrono::duration(wallEnd - wallStart).count(); + + size_t totalFailures = printResults( + results, + filtered.skippedCount, + filtered.permanentlySkippedCount, + featureSkippedCount.load(), + permanentFeatureSkippedCount.load(), + wallSeconds); + + return totalFailures > 0 ? 1 : 0; +}