diff --git a/src/evaluation/cli.py b/src/evaluation/cli.py index 9c4448fa..217d9e43 100644 --- a/src/evaluation/cli.py +++ b/src/evaluation/cli.py @@ -57,9 +57,9 @@ def _build_parser() -> argparse.ArgumentParser: p.add_argument( "--scorer-default", dest="scorer_default", - default="llm_judge", + default="static_json", help="Scorer name when scenario.scoring_method is unset. " - "Default: llm_judge.", + "Default: static_json.", ) p.add_argument( "--judge-model", diff --git a/src/evaluation/evaluator.py b/src/evaluation/evaluator.py index 924f784f..96eb9b17 100644 --- a/src/evaluation/evaluator.py +++ b/src/evaluation/evaluator.py @@ -33,7 +33,7 @@ class Evaluator: def __init__( self, - default_scorer: str = "llm_judge", + default_scorer: str = "static_json", judge_model: str | None = None, ) -> None: self.default_scorer = default_scorer diff --git a/src/evaluation/loader.py b/src/evaluation/loader.py index c799aaee..e05d2bcb 100644 --- a/src/evaluation/loader.py +++ b/src/evaluation/loader.py @@ -141,7 +141,7 @@ def _load_scenario_dir(path: Path) -> list[Scenario]: "type": "structured", "expected_answer": expected_answer, "evaluation_metadata": evaluation_metadata or None, - "scoring_method": scoring_method or "static_json", + "scoring_method": scoring_method, } ) ) diff --git a/src/evaluation/scorers/fmea.py b/src/evaluation/scorers/fmea.py index 3d6c418b..44d11c46 100644 --- a/src/evaluation/scorers/fmea.py +++ b/src/evaluation/scorers/fmea.py @@ -15,7 +15,7 @@ "utilities__get_failure_mode_catalog", } _DOMAIN_PREFIXES = ("fmsr__", "iot__", "tsfm__", "vibration__", "wo__") -_DEFAULT_F1_PASS_THRESHOLD = 0.70 +_DEFAULT_F1_PASS_THRESHOLD = 0.80 def _strict_json_object(answer: str) -> tuple[dict[str, Any] | None, list[str]]: @@ -207,7 +207,10 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR if isinstance(threshold_value, (int, float)) else _DEFAULT_F1_PASS_THRESHOLD ) - passed = semantic_f1 > f1_pass_threshold + # Compare at the reported precision: 2PR/(P+R) lands just under an exact + # boundary (6/7 and 6/8 give 0.7999999999999999), failing a 0.8 threshold. + score = round(semantic_f1, 6) + passed = score >= f1_pass_threshold strict_success = semantic_exact and json_only and exact_top_keys baseline_f1 = _metadata(scenario, "reference_answer").get("f1") details: dict[str, Any] = { @@ -217,7 +220,7 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR "weights": weights, "semantic_f1": semantic_f1, "f1_pass_threshold": f1_pass_threshold, - "f1_threshold_operator": ">", + "f1_threshold_operator": ">=", "rubric_weighted_score": rubric_weighted_score, "raw_weighted_score": rubric_weighted_score, "strict_success": strict_success, @@ -232,14 +235,14 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR } rationale = ( f"Semantic F1={semantic_f1:.3f} " - f"({'>' if passed else '<='} {f1_pass_threshold:.2f}); " + f"({'>=' if passed else '<'} {f1_pass_threshold:.2f}); " f"coverage={recall:.3f}, precision={precision:.3f}, " f"rubric_weighted_score={rubric_weighted_score:.3f}." ) return ScorerResult( scorer="fmea", passed=passed, - score=round(semantic_f1, 6), + score=score, rationale=rationale, details=details, ) diff --git a/src/evaluation/tests/test_fmea_scorer.py b/src/evaluation/tests/test_fmea_scorer.py index 98e89c52..26a1975c 100644 --- a/src/evaluation/tests/test_fmea_scorer.py +++ b/src/evaluation/tests/test_fmea_scorer.py @@ -103,7 +103,7 @@ def test_trajectory_reports_catalog_and_out_of_scope_tool_use(): assert result.details["catalog_process_compliant"] is False -def test_pass_uses_strict_greater_than_point_seven_f1(): +def test_pass_requires_at_least_point_eight_f1(): scenario = _scenario( {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(7)}} ) @@ -117,10 +117,27 @@ def test_pass_uses_strict_greater_than_point_seven_f1(): result = fmea_score(scenario, json.dumps(below), "") assert result.score == pytest.approx(4 / 7, abs=1e-6) assert result.passed is False - assert result.details["f1_pass_threshold"] == 0.7 + assert result.details["f1_pass_threshold"] == 0.8 -def test_f1_above_point_seven_passes_without_requiring_exact_match(): +def test_f1_exactly_at_point_eight_passes_despite_float_error(): + scenario = _scenario( + {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(7)}} + ) + at_threshold = { + "asset": "Electric Motor", + "failure_modes": { + **{str(i): "" for i in range(6)}, + **{f"extra-{i}": "" for i in range(2)}, + }, + } + result = fmea_score(scenario, json.dumps(at_threshold), "") + assert result.details["semantic_f1"] < 0.8 + assert result.score == 0.8 + assert result.passed is True + + +def test_f1_above_point_eight_passes_without_requiring_exact_match(): scenario = _scenario( {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(8)}} ) diff --git a/src/evaluation/tests/test_loader.py b/src/evaluation/tests/test_loader.py index eaf6a85f..a89eda8d 100644 --- a/src/evaluation/tests/test_loader.py +++ b/src/evaluation/tests/test_loader.py @@ -122,7 +122,7 @@ def test_load_scenarios_from_groundtruth_folders(tmp_path): assert len(scenarios) == 1 assert scenarios[0].id == "11" assert scenarios[0].expected_answer == "{'energy': 14, 'material': 48}" - assert scenarios[0].scoring_method == "static_json" + assert scenarios[0].scoring_method is None def test_load_scenarios_reads_groundtruth_eval_metadata(tmp_path):