fix(evaluation): let scenario_meta.json route scoring, and make the F…

src/evaluation/cli.py

@@ -57,9 +57,9 @@ def _build_parser() -> argparse.ArgumentParser:
     p.add_argument(
         "--scorer-default",
         dest="scorer_default",
-        default="llm_judge",
+        default="static_json",
         help="Scorer name when scenario.scoring_method is unset. "
-        "Default: llm_judge.",
+        "Default: static_json.",
     )
     p.add_argument(
         "--judge-model",

src/evaluation/evaluator.py

@@ -33,7 +33,7 @@ class Evaluator:
 
     def __init__(
         self,
-        default_scorer: str = "llm_judge",
+        default_scorer: str = "static_json",
         judge_model: str | None = None,
     ) -> None:
         self.default_scorer = default_scorer

src/evaluation/loader.py

@@ -141,7 +141,7 @@ def _load_scenario_dir(path: Path) -> list[Scenario]:
                     "type": "structured",
                     "expected_answer": expected_answer,
                     "evaluation_metadata": evaluation_metadata or None,
-                    "scoring_method": scoring_method or "static_json",
+                    "scoring_method": scoring_method,
                 }
             )
         )

src/evaluation/scorers/fmea.py

@@ -15,7 +15,7 @@
     "utilities__get_failure_mode_catalog",
 }
 _DOMAIN_PREFIXES = ("fmsr__", "iot__", "tsfm__", "vibration__", "wo__")
-_DEFAULT_F1_PASS_THRESHOLD = 0.70
+_DEFAULT_F1_PASS_THRESHOLD = 0.80
 
 
 def _strict_json_object(answer: str) -> tuple[dict[str, Any] | None, list[str]]:
@@ -207,7 +207,10 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR
         if isinstance(threshold_value, (int, float))
         else _DEFAULT_F1_PASS_THRESHOLD
     )
-    passed = semantic_f1 > f1_pass_threshold
+    # Compare at the reported precision: 2PR/(P+R) lands just under an exact
+    # boundary (6/7 and 6/8 give 0.7999999999999999), failing a 0.8 threshold.
+    score = round(semantic_f1, 6)
+    passed = score >= f1_pass_threshold
     strict_success = semantic_exact and json_only and exact_top_keys
     baseline_f1 = _metadata(scenario, "reference_answer").get("f1")
     details: dict[str, Any] = {
@@ -217,7 +220,7 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR
         "weights": weights,
         "semantic_f1": semantic_f1,
         "f1_pass_threshold": f1_pass_threshold,
-        "f1_threshold_operator": ">",
+        "f1_threshold_operator": ">=",
         "rubric_weighted_score": rubric_weighted_score,
         "raw_weighted_score": rubric_weighted_score,
         "strict_success": strict_success,
@@ -232,14 +235,14 @@ def fmea_score(scenario: Scenario, answer: str, trajectory_text: str) -> ScorerR
     }
     rationale = (
         f"Semantic F1={semantic_f1:.3f} "
-        f"({'>' if passed else '<='} {f1_pass_threshold:.2f}); "
+        f"({'>=' if passed else '<'} {f1_pass_threshold:.2f}); "
         f"coverage={recall:.3f}, precision={precision:.3f}, "
         f"rubric_weighted_score={rubric_weighted_score:.3f}."
     )
     return ScorerResult(
         scorer="fmea",
         passed=passed,
-        score=round(semantic_f1, 6),
+        score=score,
         rationale=rationale,
         details=details,
     )

src/evaluation/tests/test_fmea_scorer.py

@@ -103,7 +103,7 @@ def test_trajectory_reports_catalog_and_out_of_scope_tool_use():
     assert result.details["catalog_process_compliant"] is False
 
 
-def test_pass_uses_strict_greater_than_point_seven_f1():
+def test_pass_requires_at_least_point_eight_f1():
     scenario = _scenario(
         {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(7)}}
     )
@@ -117,10 +117,27 @@ def test_pass_uses_strict_greater_than_point_seven_f1():
     result = fmea_score(scenario, json.dumps(below), "")
     assert result.score == pytest.approx(4 / 7, abs=1e-6)
     assert result.passed is False
-    assert result.details["f1_pass_threshold"] == 0.7
+    assert result.details["f1_pass_threshold"] == 0.8
 
 
-def test_f1_above_point_seven_passes_without_requiring_exact_match():
+def test_f1_exactly_at_point_eight_passes_despite_float_error():
+    scenario = _scenario(
+        {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(7)}}
+    )
+    at_threshold = {
+        "asset": "Electric Motor",
+        "failure_modes": {
+            **{str(i): "" for i in range(6)},
+            **{f"extra-{i}": "" for i in range(2)},
+        },
+    }
+    result = fmea_score(scenario, json.dumps(at_threshold), "")
+    assert result.details["semantic_f1"] < 0.8
+    assert result.score == 0.8
+    assert result.passed is True
+
+
+def test_f1_above_point_eight_passes_without_requiring_exact_match():
     scenario = _scenario(
         {"asset": "Electric Motor", "failure_modes": {str(i): "" for i in range(8)}}
     )

src/evaluation/tests/test_loader.py

@@ -122,7 +122,7 @@ def test_load_scenarios_from_groundtruth_folders(tmp_path):
     assert len(scenarios) == 1
     assert scenarios[0].id == "11"
     assert scenarios[0].expected_answer == "{'energy': 14, 'material': 48}"
-    assert scenarios[0].scoring_method == "static_json"
+    assert scenarios[0].scoring_method is None
 
 
 def test_load_scenarios_reads_groundtruth_eval_metadata(tmp_path):