Skip to content
Open
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion verifiers/v1/cli/eval/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -180,8 +180,13 @@ async def run_eval_server(config: EvalConfig) -> list[Trace]:
config.model,
)
logger.info("results: %s", out)
request_concurrency = config.max_concurrent
if request_concurrency and info.requires_group_scoring:
# max_concurrent is a rollout resource bound, not a request-throughput target.
# A group is indivisible, so one oversized group must still be allowed to run.
request_concurrency = max(1, request_concurrency // config.num_rollouts)
Comment thread
xeophon marked this conversation as resolved.
semaphore = (
asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
asyncio.Semaphore(request_concurrency) if request_concurrency else None
)

async def run_group_unit(idx: int) -> list[Trace]:
Expand Down
Loading