From f7cf82b1a1da0fb2e38f6c1082263d331c68a413 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Mon, 16 Nov 2026 21:45:24 +0800 Subject: [PATCH] Keep valid optimization candidates usable after known dependency failures Classify the optional TBE dependency failure at the Qwen candidate call boundary after a verified baseline. Require normal execution and bound evidence before a rejection can continue the shared lifecycle; preserve best, failed attempts and independent final. Agent-assisted-by: Codex (implementation) Signed-off-by: dengyunyang <584797741@qq.com> --- docs/optimization-benchmarks.md | 30 +++ .../tasks/qwen21-denoise-npu/TASK.md | 10 + .../tasks/qwen21-denoise-npu/benchmark.py | 71 +++++- .../tasks/qwen21-denoise-npu/optimization.py | 47 ++-- src/inferrsi/harness/optimization.py | 9 + tests/test_optimization_session.py | 50 ++++- tests/test_project_optimization.py | 17 +- tests/test_qwen21_task.py | 206 +++++++++++++++++- 8 files changed, 411 insertions(+), 29 deletions(-) diff --git a/docs/optimization-benchmarks.md b/docs/optimization-benchmarks.md index 5f3af4721..ec0504eb3 100644 --- a/docs/optimization-benchmarks.md +++ b/docs/optimization-benchmarks.md @@ -87,6 +87,36 @@ Repeat this operation until the declared round budget is consumed. Use evaluates a newly retained candidate snapshot and returns independent feedback. Profiling is available to the Agent as an optional diagnostic. +### A failed experiment can leave a usable best candidate + +A normally finished evaluation may explicitly reject a candidate. Harness keeps +that round and its evidence, assigns no score to it, and preserves the selected +best snapshot. The Agent can repair the candidate or, when the frozen round and +time rules permit, run `final` on the retained best without repairing the working +copy. Final still executes the full frozen protocol in a new process. + +For Qwen, a missing optional `tbe` module is a candidate dependency rejection only +when it is raised inside the candidate function and the same case's baseline +has already passed. This includes CANN's specific EC0010 TBE import failure at +`AclSetCompileopt`, which wraps the missing dependency in a `RuntimeError`. +Before rejecting the candidate, the worker reruns the same case's baseline +against the independent reference using pristine inputs and baseline parameters; +execution, synchronization and quality must all pass again. That check produces +no optimization score. A failed recheck remains an infrastructure failure and +retains both errors. Setup/import failures, failed baselines, other missing +modules and unknown device errors remain infrastructure failures. The task +assessor binds rejected results to the evaluated source snapshots and requires +the worker's normal rejection exit. A written rejection cannot override a +timeout, cancellation, signal or unknown execution state in the shared controller. + +This prevents a known candidate dependency failure from unnecessarily blocking +the campaign. It does not unlock existing `blocked` campaigns, bypass a failed +final or waive `exact` rounds. `recover` still only settles an existing attempt; +it is not an environment repair operation. Normal process exit does not certify +remote allocation release. Interruptions and unknown resource state remain +blocked and require external investigation. Historical frozen experiments and +their verdicts must not be rewritten with the new task protocol. + ## Independently retest and report ```bash diff --git a/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/TASK.md b/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/TASK.md index 00bfeeba2..335468e05 100644 --- a/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/TASK.md +++ b/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/TASK.md @@ -120,6 +120,16 @@ or baseline failure. Candidate import/execution consumes an attempt. A failure keeps its evidence and has no valid speed score. The caller must collect process termination and allocation-release evidence when cancelling or timing out a run. +A candidate-only missing optional TBE dependency (including the specific CANN +EC0010 import error) is rejected only after the same case's baseline has passed +and passes again after the error, with pristine inputs and its own parameters. +The recheck includes synchronization and reference quality; it produces no score. +The missing module and traceback remain in the result and log. A failed baseline +recheck remains an infrastructure failure. A normally finished rejection preserves the earlier +best implementation. Repair the candidate or complete independent final when +the frozen budget permits. Setup, baseline and unknown runtime failures still +block the session; a rejection result never overrides an interrupted execution. + ## Scope This task measures a scaled denoiser with synthetic weights and conditioning. diff --git a/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/benchmark.py b/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/benchmark.py index 848e5fa53..fbbd5cde1 100644 --- a/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/benchmark.py +++ b/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/benchmark.py @@ -24,6 +24,38 @@ class CandidateContractError(ValueError): """The candidate could not satisfy the Python entry contract before execution.""" +class CandidateDependencyError(RuntimeError): + """A known missing optional dependency; baseline recheck is still required.""" + + def __init__(self, module): + super().__init__(f"Candidate requires unavailable optional dependency: {module}") + self.module = module + + +def invoke_compute(function, inputs, params, config, *, candidate, baseline_verified): + # Catch only the submitted function call, never setup, synchronization or + # reference checks. TBE is optional for this task's trusted baseline; older + # native candidate kernels can require it. Other missing modules stay unknown. + try: + return function(*inputs, params, config=config) + except ModuleNotFoundError as error: + if (candidate and baseline_verified and isinstance(error.name, str) + and error.name.split(".", 1)[0] == "tbe"): + raise CandidateDependencyError(error.name) from error + raise + except RuntimeError as error: + # CANN wraps the Python import failure in an ACL RuntimeError. Require + # its specific compiler/import diagnostic, not an ACL code or "tbe" alone. + message = str(error) + if candidate and baseline_verified and all(part in message for part in ( + "AclSetCompileopt(aclCompileOpt::ACL_PRECISION_MODE", + "Environment_Error_Import_Python_Module_Failed(EC0010)", + "Failed to import Python module ModuleNotFoundError: No module named 'tbe'.", + )): + raise CandidateDependencyError("tbe") from error + raise + + def candidate_function(path): # Compile independently so only syntax errors in the submitted file are # attributed to the candidate, not failures importing its runtime dependencies. @@ -113,24 +145,41 @@ def synchronize(): record["phase"] = "candidate_import" record["round_consumed"] = True functions["candidate"] = candidate_function(source / "candidate.py") + verified_baselines = set() - def invoke(label, inputs, params, config, expected, phase, pair=None, sample=None): + def invoke(label, call_inputs, params, config, expected, phase, pair=None, sample=None): record["phase"] = label + "_" + phase original = [ (value, value.detach().cpu().clone(), value.stride(), value.device) - for value in tensors((inputs, params)) + for value in tensors((call_inputs, params)) ] synchronize() started = time.perf_counter_ns() - result = functions[label](*inputs, params, config=config) + try: + result = invoke_compute(functions[label], call_inputs, params, config, + candidate=label == "candidate", + baseline_verified=config["id"] in verified_baselines) + except CandidateDependencyError as error: + failed_phase = record["phase"] + record["dependency_failure"] = {"case": config["id"], "phase": failed_phase, + "module": error.module, "error": str(error.__cause__)} + # Retry only the trusted baseline on pristine request inputs and its + # own parameter storage. Any sync, execution or quality failure here + # keeps the baseline phase and blocks the campaign. Never retry the + # failed candidate or reuse its potentially modified tensors. + invoke("baseline", workload.map_tensors(inputs, lambda t: t.clone()), + separate_params["baseline"], config, expected, "recovery_check") + record["baseline_recheck"] = {"case": config["id"], "correctness": "PASS"} + record["phase"] = failed_phase + raise synchronize() elapsed = (time.perf_counter_ns() - started) / 1e6 - shape = (inputs[3].numel() - 1, *inputs[0].shape) + shape = (call_inputs[3].numel() - 1, *call_inputs[0].shape) if ( not isinstance(result, torch.Tensor) or tuple(result.shape) != shape - or result.dtype != inputs[0].dtype - or result.device != inputs[0].device + or result.dtype != call_inputs[0].dtype + or result.device != call_inputs[0].device ): raise OutputValidationError("Wrong trajectory shape, dtype or device") for value, previous, stride, device_before in original: @@ -173,6 +222,8 @@ def invoke(label, inputs, params, config, expected, phase, pair=None, sample=Non if not valid: raise OutputValidationError( f"Trajectory failed numerical checks, per-step NRMSE={nrmse}") + if label == "baseline": + verified_baselines.add(config["id"]) return elapsed preparation = 0.0 @@ -324,7 +375,10 @@ def main(): ): raise ValueError("Frozen source changed during evaluation") except Exception as error: - candidate_failure = (isinstance(error, (OutputValidationError, CandidateContractError)) + explicit_rejection = isinstance(error, (OutputValidationError, CandidateContractError)) + checked_dependency = (isinstance(error, CandidateDependencyError) + and record.get("baseline_recheck", {}).get("correctness") == "PASS") + candidate_failure = ((explicit_rejection or checked_dependency) and record["phase"].startswith("candidate_")) record.update( status="candidate_failure" if candidate_failure else "infrastructure_failure", @@ -332,6 +386,9 @@ def main(): error=str(error), speedup=None, ) + if isinstance(error, CandidateDependencyError): + record["failure_kind"] = "unsupported_candidate_dependency" + record["missing_module"] = error.module (output / "error.log").write_text(traceback.format_exc(), encoding="utf-8") record["elapsed_seconds"] = time.perf_counter() - started (output / "result.json").write_text(json.dumps(record, indent=2), encoding="utf-8") diff --git a/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/optimization.py b/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/optimization.py index ff8af1ef3..d3a8d3072 100644 --- a/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/optimization.py +++ b/src/inferrsi/benchmark/tasks/qwen21-denoise-npu/optimization.py @@ -34,22 +34,17 @@ def assess(directory: Path, settings, execution): "verdict": "INCONCLUSIVE", "reason": "missing_task_result"} result = json.loads(result_path.read_text(encoding="utf-8")) evidence = {"measurement/result.json": hashlib.sha256(result_path.read_bytes()).hexdigest()} - if result.get("status") == "candidate_failure": - return {"status": "candidate_failure", "score": None, "verdict": "FAIL", - "reason": result.get("error"), "evidence_files": evidence, - "quality_passed": False if result.get("correctness") == "FAIL" else None} - if (execution.get("status") != "completed" or execution.get("returncode") != 0 - or result.get("status") != "completed" or result.get("correctness") != "PASS"): + rejected = result.get("status") == "candidate_failure" + expected_exit = ("failed", 2) if rejected else ("completed", 0) + if (type(execution.get("returncode")) is not int + or (execution.get("status"), execution.get("returncode")) != expected_exit + or (not rejected and (result.get("status") != "completed" + or result.get("correctness") != "PASS"))): return {"status": "infrastructure_failure", "score": None, "verdict": "INCONCLUSIVE", "reason": "incomplete_execution_or_task", "evidence_files": evidence} - task = json.loads((Path(__file__).parent / "task.json").read_text(encoding="utf-8")) - expected_cases = {case["id"] for case in task["cases"]} - measured_cases = {case["id"] for case in task["cases"] if case["measured"]} - if (result.get("score_scope") != "complete_task" - or set(result.get("selected_cases", [])) != expected_cases - or result.get("protocol") != {**settings, "check_only": False}): - raise ValueError("Full-task scope or frozen sampling protocol changed") + # A rejection also needs source binding: it must not authorize continuation + # with a changed candidate or evaluator snapshot. identity = json.loads((output / "identity.json").read_text(encoding="utf-8")) required = {"candidate.py", "baseline.py", "reference.py", "workload.py", "task.json", "TASK.md", "benchmark.py", "LICENSE", "NOTICE.md"} @@ -71,6 +66,32 @@ def assess(directory: Path, settings, execution): evidence["measurement/snapshot/" + name] = expected evidence["measurement/identity.json"] = hashlib.sha256( (output / "identity.json").read_bytes()).hexdigest() + if rejected: + if result.get("failure_kind") == "unsupported_candidate_dependency": + case = result.get("dependency_failure", {}).get("case") + rechecks = [call for call in result.get("calls", []) + if call.get("label") == "baseline" + and call.get("phase") == "recovery_check" and call.get("case") == case] + if (not case or result.get("baseline_recheck") != {"case": case, "correctness": "PASS"} + or len(rechecks) != 1 or rechecks[0].get("correctness") != "PASS"): + return {"status": "infrastructure_failure", "score": None, + "verdict": "INCONCLUSIVE", "reason": "missing_baseline_recheck", + "evidence_files": evidence} + error_log = output / "error.log" + if error_log.is_file(): + evidence["measurement/error.log"] = hashlib.sha256(error_log.read_bytes()).hexdigest() + return {"status": "candidate_failure", "score": None, "verdict": "FAIL", + "reason": result.get("error"), "evidence_files": evidence, + "failure_kind": result.get("failure_kind"), + "missing_module": result.get("missing_module"), + "quality_passed": False if result.get("correctness") == "FAIL" else None} + task = json.loads((Path(__file__).parent / "task.json").read_text(encoding="utf-8")) + expected_cases = {case["id"] for case in task["cases"]} + measured_cases = {case["id"] for case in task["cases"] if case["measured"]} + if (result.get("score_scope") != "complete_task" + or set(result.get("selected_cases", [])) != expected_cases + or result.get("protocol") != {**settings, "check_only": False}): + raise ValueError("Full-task scope or frozen sampling protocol changed") cases = result.get("cases", []) if len(cases) != len(measured_cases) or {case["id"] for case in cases} != measured_cases: raise ValueError("Missing or duplicated performance cases") diff --git a/src/inferrsi/harness/optimization.py b/src/inferrsi/harness/optimization.py index bbd439227..8fd2a3fbc 100644 --- a/src/inferrsi/harness/optimization.py +++ b/src/inferrsi/harness/optimization.py @@ -228,6 +228,13 @@ def _settle(root, plan, state, pending, execution, assess): raise ValueError(state["integrity_error"]) if execution.get("coordinator_error"): raise ValueError(execution["coordinator_error"]) + # A task result cannot override an interrupted or unknown execution. + # A normal nonzero exit may carry a deliberate candidate rejection. + if (execution.get("status") not in ("completed", "failed") + or type(execution.get("returncode")) is not int + or execution["returncode"] < 0): + raise ValueError( + "Execution did not finish normally; candidate result cannot resume search") assessment = assess(root, directory, plan, execution) if not isinstance(assessment, dict) or assessment.get("status") not in ( "completed", "candidate_failure", "infrastructure_failure" @@ -235,6 +242,8 @@ def _settle(root, plan, state, pending, execution, assess): raise ValueError("Unknown task assessment status") score = assessment.get("score") if assessment["status"] == "completed": + if execution["status"] != "completed" or execution["returncode"] != 0: + raise ValueError("Completed assessment requires successful execution") if type(score) not in (int, float) or not math.isfinite(score) or score <= 0: raise ValueError("Completed assessment requires a finite positive score") if pending["kind"] == "round" and state["best"]["score"] is None: diff --git a/tests/test_optimization_session.py b/tests/test_optimization_session.py index 52bd3bbfc..608423637 100644 --- a/tests/test_optimization_session.py +++ b/tests/test_optimization_session.py @@ -132,30 +132,70 @@ def test_generic_job_precedes_child_start_and_collects_child_final(tmp_path, tas assert not (job / "plan.json").exists() # Host job and optimization stay separate. -def test_real_loop_rejects_bad_round_keeps_best_and_retests_independently(session, capsys): +@pytest.mark.parametrize("factors", [(2, -1, 1.5), (2, 1.5, -1)]) +def test_real_loop_rejects_bad_round_keeps_best_and_retests_independently(session, capsys, factors): observations = [] - for factor, input_tokens, output_tokens in ((2, 100, 20), (-1, 200, 10), (1.5, 20, 10)): + for factor, (input_tokens, output_tokens) in zip(factors, ((100, 20), (200, 10), (20, 10))): observations.append(submit(session, factor, capsys, { "input_tokens": input_tokens, "output_tokens": output_tokens, "cached_input_tokens": 10, })) - assert observations[1]["latest_observation"]["assessment"]["status"] == "candidate_failure" + rejected = factors.index(-1) + assert observations[rejected]["latest_observation"]["assessment"]["status"] == ( + "candidate_failure") assert observations[-1]["best"]["score"] == 2 assert observations[-1]["next_action"].startswith("independently_retest") + failed_record = (session / observations[rejected]["latest_observation"]["path"] + / "observation.json") + failed_bytes = failed_record.read_bytes() assert main(["benchmarks", "session", "final", "--session", str(session)]) == 0 final = json.loads(capsys.readouterr().out) assert final["state"] == "completed" - assert (session / "calls.log").read_text().splitlines() == ["2", "-1", "1.5", "2"] + assert (session / "calls.log").read_text().splitlines() == [*map(str, factors), "2"] + assert failed_record.read_bytes() == failed_bytes + state = json.loads((session / "state.json").read_text()) + assert state["final"]["candidate_sha256"] == observations[0]["best"]["sha256"] report = json.loads((session / "summary.json").read_text()) assert report["optimization_speedup"] == 2 assert report["completed_rounds"] == 3 assert report["usage"]["total_tokens"] == 360 assert [row["cumulative_usage"]["total_tokens"] for row in report["curve"]] == [120, 330, 360] - assert report["curve"][1]["assessment"]["score"] is None + assert report["curve"][rejected]["assessment"]["score"] is None assert all(row["best_score"] == 2 for row in report["curve"]) assert report["wall_elapsed_seconds"] > 0 +@pytest.mark.parametrize("status,code", [ + ("timeout", 0), ("cancelled", 0), ("error", None), ("failed", -9), +]) +def test_interrupted_execution_cannot_resume_using_a_written_candidate_rejection( + session, capsys, monkeypatch, status, code, +): + best = submit(session, 2, capsys)["best"] + original = optimization_session._session_execute + + def interrupted(*args): + record = original(*args) + record.update(status=status, returncode=code) + (args[1] / "execution/execution.json").write_text(json.dumps(record), encoding="utf-8") + return record + + monkeypatch.setattr(optimization_session, "_session_execute", interrupted) + candidate = session / "work/candidate.py" + candidate.write_text("FACTOR = -1\n", encoding="utf-8") + assert main(["benchmarks", "session", "round", "--session", str(session), + "--candidate", str(candidate), "--hypothesis", "Interrupted rejection"]) == 2 + view = json.loads(capsys.readouterr().out) + assert view["state"] == "blocked" and view["best"] == best + assert view["latest_observation"]["assessment"]["verdict"] == "INCONCLUSIVE" + assert not view["can_finalize"] + assert main(["benchmarks", "session", "recover", "--session", str(session)]) == 2 + assert json.loads(capsys.readouterr().out)["state"] == "blocked" + with pytest.raises(SystemExit): + main(["benchmarks", "session", "final", "--session", str(session)]) + assert (session / "calls.log").read_text().splitlines() == ["2", "-1"] + + def test_final_cannot_skip_optimization_rounds(session): with pytest.raises(SystemExit) as error: main(["benchmarks", "session", "final", "--session", str(session)]) diff --git a/tests/test_project_optimization.py b/tests/test_project_optimization.py index eb9f312fc..753b644dd 100644 --- a/tests/test_project_optimization.py +++ b/tests/test_project_optimization.py @@ -280,14 +280,25 @@ def test_bad_candidate_exception_is_classified_by_frozen_wrapper(project, capsys " result = {'status':'candidate_failure', 'reason':'compute raised RuntimeError'}\n" "Path(sys.argv[2]).write_text(json.dumps(result))\n", encoding="utf-8") start(project, capsys) + code, before = submit(project, capsys, 20) + assert code == 0 and before["best"]["kind"] == "candidate" (project[0] / "engine.py").write_text( "def compute(x):\n raise RuntimeError('bad candidate')\n", encoding="utf-8") code, view = submit(project, capsys, 20) - assert code == 0 and view["state"] == "active" and view["completed_rounds"] == 1 + assert code == 0 and view["state"] == "active" and view["completed_rounds"] == 2 + assert view["best"] == before["best"] assert view["latest_observation"]["assessment"]["verdict"] == "FAIL" assert len(view["latest_observation"]["execution"]["steps"]) == 2 # No wasted measurement. - (project[0] / "engine.py").write_text("def compute(x):\n return x*2\n", encoding="utf-8") - assert submit(project, capsys, 20)[1]["best"]["kind"] == "candidate" + failed_record = project[3] / view["latest_observation"]["path"] / "observation.json" + failed_bytes = failed_record.read_bytes() + assert main(["campaign", "final", "--directory", str(project[3]), + "--reason", "Keep the measured best after a rejected experiment"]) == 0 + capsys.readouterr() + _, state, summary = read_records(project[3]) + assert state["final"]["candidate_sha256"] == before["best"]["sha256"] + assert summary["final_assessment"]["verdict"] == "PASS" + assert failed_record.read_bytes() == failed_bytes + assert "raise RuntimeError" in (project[0] / "engine.py").read_text() def test_unclassified_process_crash_does_not_claim_candidate_failure(project, capsys): diff --git a/tests/test_qwen21_task.py b/tests/test_qwen21_task.py index 53d8a03a8..4ef85d00b 100644 --- a/tests/test_qwen21_task.py +++ b/tests/test_qwen21_task.py @@ -107,6 +107,8 @@ def test_unknown_runtime_errors_are_not_claimed_as_wrong_outputs( ): worker = load("benchmark") protocol = load("optimization") + candidate = tmp_path / "candidate.py" + shutil.copyfile(str(TASK.joinpath("baseline.py")), candidate) def fail(args, output, record): record["phase"] = phase @@ -117,7 +119,7 @@ def fail(args, output, record): monkeypatch.setattr(worker, "run", fail) output = tmp_path / "measurement" monkeypatch.setattr(sys, "argv", [ - "benchmark.py", "--candidate", str(TASK.joinpath("baseline.py")), + "benchmark.py", "--candidate", str(candidate), "--output-dir", str(output), "--device", "npu", "--pairs", "1", "--samples", "1", "--warmups", "0", ]) @@ -129,3 +131,205 @@ def fail(args, output, record): assessment = protocol.assess(tmp_path, {}, {"status": "failed", "returncode": code}) assert assessment["status"] == expected assert assessment["verdict"] == verdict + + +@pytest.mark.parametrize("candidate,baseline_verified,module,expected", [ + (True, True, "tbe", True), + (True, True, "tbe.common", True), + (False, True, "tbe", False), + (True, False, "tbe", False), + (True, True, "torch_npu", False), + (True, True, "tbe_other", False), + (True, True, None, False), +]) +def test_optional_dependency_classification_requires_candidate_and_verified_baseline( + candidate, baseline_verified, module, expected, +): + worker = load("benchmark") + original = ModuleNotFoundError("Synthetic missing dependency", name=module) + + def compute(*args, **kwargs): + raise original + + exception = worker.CandidateDependencyError if expected else ModuleNotFoundError + with pytest.raises(exception) as caught: + worker.invoke_compute(compute, (), {}, {}, candidate=candidate, + baseline_verified=baseline_verified) + if expected: + assert caught.value.__cause__ is original + assert caught.value.module == module + else: + assert caught.value is original + + +@pytest.fixture +def dependency_failure(tmp_path, monkeypatch): + """Exercise the actual worker receipt and task assessor without a device.""" + worker = load("benchmark") + candidate = tmp_path / "candidate.py" + candidate.write_text( + "def compute(*args, **kwargs):\n" + " raise ModuleNotFoundError('Synthetic missing TBE', name='tbe')\n", + encoding="utf-8", + ) + + def run(args, output, record): + record.update(phase="candidate_correctness", round_consumed=True) + function = worker.candidate_function(output / "snapshot/candidate.py") + # Controlled baseline precondition; this is a classification/lifecycle + # regression, not evidence that a real NPU baseline executed. + record["baseline_recheck"] = {"case": "smoke", "correctness": "PASS"} + record["dependency_failure"] = {"case": "smoke"} + record["calls"].append({"case": "smoke", "label": "baseline", + "phase": "recovery_check", "correctness": "PASS"}) + worker.invoke_compute(function, (), {}, {}, candidate=True, baseline_verified=True) + + monkeypatch.setattr(worker, "run", run) + monkeypatch.setattr(sys, "argv", [ + "benchmark.py", "--candidate", str(candidate), + "--output-dir", str(tmp_path / "measurement"), + "--pairs", "1", "--samples", "1", "--warmups", "0", + ]) + assert worker.main() == 2 + return tmp_path + + +def test_dependency_failure_retains_bound_sources_and_error(dependency_failure): + assessment = load("optimization").assess( + dependency_failure, {}, {"status": "failed", "returncode": 2}) + assert assessment["status"] == "candidate_failure" + assert assessment["score"] is None and assessment["verdict"] == "FAIL" + assert assessment["failure_kind"] == "unsupported_candidate_dependency" + assert assessment["missing_module"] == "tbe" + assert "measurement/snapshot/candidate.py" in assessment["evidence_files"] + assert "measurement/error.log" in assessment["evidence_files"] + + +def test_dependency_rejection_requires_retained_baseline_recheck(dependency_failure): + path = dependency_failure / "measurement/result.json" + result = json.loads(path.read_bytes()) + result.pop("baseline_recheck") + path.write_text(json.dumps(result), encoding="utf-8") + assessment = load("optimization").assess( + dependency_failure, {}, {"status": "failed", "returncode": 2}) + assert assessment["status"] == "infrastructure_failure" + assert assessment["reason"] == "missing_baseline_recheck" + + +CANN_IMPORT_ERROR = ( + "SetPrecisionMode: NPU function error: " + "at_npu::native::AclSetCompileopt(aclCompileOpt::ACL_PRECISION_MODE, precision_mode), " + "error code is 500001\n" + "Environment_Error_Import_Python_Module_Failed(EC0010): " + "Failed to import Python module ModuleNotFoundError: No module named 'tbe'." +) + + +@pytest.mark.parametrize("message,candidate,verified,recognized", [ + (CANN_IMPORT_ERROR, True, True, True), + (CANN_IMPORT_ERROR, False, True, False), + (CANN_IMPORT_ERROR, True, False, False), + (CANN_IMPORT_ERROR.replace("(EC0010)", "(EC9999)"), True, True, False), + (CANN_IMPORT_ERROR.replace("'tbe'", "'torch_npu'"), True, True, False), + (CANN_IMPORT_ERROR.replace("AclSetCompileopt", "DeviceSynchronize"), True, True, False), + ("ACL error 500001; tbe", True, True, False), +]) +def test_wrapped_cann_import_failure_requires_specific_diagnostic_and_baseline( + message, candidate, verified, recognized, +): + worker = load("benchmark") + original = RuntimeError(message) + + def compute(*args, **kwargs): + raise original + + with pytest.raises(RuntimeError) as caught: + worker.invoke_compute(compute, (), {}, {}, candidate=candidate, baseline_verified=verified) + assert isinstance(caught.value, worker.CandidateDependencyError) == recognized + if recognized: + assert caught.value.__cause__ is original + else: + assert caught.value is original + + +@pytest.mark.parametrize("break_baseline", [False, True]) +def test_real_math_baseline_recheck_controls_dependency_rejection(tmp_path, break_baseline): + pytest.importorskip("torch") + candidate = tmp_path / "candidate.py" + candidate.write_text( + "import torch.nn.functional as F\n" + "def broken(*args, **kwargs):\n raise RuntimeError('Synthetic unhealthy baseline')\n" + "def compute(latents, text, text_mask, sigmas, positions, params, *, config):\n" + " latents.zero_()\n params['image_in'].zero_()\n" + + (" F.linear = broken\n" if break_baseline else "") + + f" raise RuntimeError({CANN_IMPORT_ERROR!r})\n", encoding="utf-8", + ) + output = tmp_path / "measurement" + execution = subprocess.run([ + sys.executable, str(TASK.joinpath("benchmark.py")), "--candidate", str(candidate), + "--output-dir", str(output), "--device", "cpu", "--check-only", "--case", "smoke", + "--pairs", "1", "--samples", "1", "--warmups", "0", + ], capture_output=True, text=True, timeout=60) + assert execution.returncode == (1 if break_baseline else 2), execution.stderr + result = json.loads((output / "result.json").read_bytes()) + assert result["dependency_failure"]["module"] == "tbe" + assert result["speedup"] is None + assert result["status"] == ("infrastructure_failure" if break_baseline else "candidate_failure") + if break_baseline: + assert result["phase"] == "baseline_recovery_check" + assert "baseline_recheck" not in result + assert "Synthetic unhealthy baseline" in (output / "error.log").read_text() + else: + assert result["phase"] == "candidate_correctness" + assert result["baseline_recheck"] == {"case": "smoke", "correctness": "PASS"} + assert [call["phase"] for call in result["calls"]] == ["correctness", "recovery_check"] + assessment = load("optimization").assess( + tmp_path, {}, {"status": "failed", "returncode": execution.returncode}) + assert assessment["status"] == result["status"] + + +@pytest.mark.parametrize("status,code", [ + ("timeout", 2), ("cancelled", 2), ("error", 2), ("pending", None), + ("failed", None), ("failed", -9), ("failed", 1), ("completed", 0), + ("failed", True), +]) +def test_written_candidate_failure_does_not_override_execution_failure( + dependency_failure, status, code, +): + assessment = load("optimization").assess( + dependency_failure, {}, {"status": status, "returncode": code}) + assert assessment["status"] == "infrastructure_failure" + assert assessment["verdict"] == "INCONCLUSIVE" + + +@pytest.mark.parametrize("relative", [ + "candidate.py", "measurement/snapshot/candidate.py", "measurement/snapshot/benchmark.py", +]) +def test_candidate_rejection_cannot_resume_with_changed_sources(dependency_failure, relative): + path = dependency_failure / relative + path.write_bytes(path.read_bytes() + b"\n# changed\n") + with pytest.raises(ValueError, match="candidate|snapshot"): + load("optimization").assess( + dependency_failure, {}, {"status": "failed", "returncode": 2}) + + +@pytest.mark.parametrize("phase", ["setup", "candidate_import", "baseline_correctness"]) +def test_missing_tbe_outside_candidate_invocation_remains_infrastructure_failure( + tmp_path, monkeypatch, phase, +): + worker = load("benchmark") + + def fail(args, output, record): + record["phase"] = phase + raise ModuleNotFoundError("Synthetic missing TBE", name="tbe") + + monkeypatch.setattr(worker, "run", fail) + monkeypatch.setattr(sys, "argv", [ + "benchmark.py", "--candidate", str(TASK.joinpath("baseline.py")), + "--output-dir", str(tmp_path / "measurement"), + "--pairs", "1", "--samples", "1", "--warmups", "0", + ]) + assert worker.main() == 1 + result = json.loads((tmp_path / "measurement/result.json").read_text()) + assert result["status"] == "infrastructure_failure" + assert result["correctness"] == "INCONCLUSIVE"