diff --git a/docs/static-json-evaluation.md b/docs/static-json-evaluation.md index 368ea7aa..4c9c5181 100644 --- a/docs/static-json-evaluation.md +++ b/docs/static-json-evaluation.md @@ -266,7 +266,7 @@ When the scenario's `groundtruth_eval.json` has `required_terms`, the score is `batteries`), appears in the value under the key. Irregular forms such as `criteria` / `criterion` still need listing. -The scenario passes when recall is at least 0.5 (`CAR_PASS_THRESHOLD`). +The scenario passes when recall is at least 0.75 (`CAR_PASS_THRESHOLD`). `precision` and `f1` are `null`: precision needs a count of what the model asserted, and a free-text value gives none, so it could only repeat recall. For the example above (two concepts): diff --git a/src/evaluation/scorers/static_json.py b/src/evaluation/scorers/static_json.py index 228df2af..3b7d8287 100644 --- a/src/evaluation/scorers/static_json.py +++ b/src/evaluation/scorers/static_json.py @@ -429,8 +429,10 @@ def similarity_score(gold_value: str, model_value: str) -> float: _MODE_KEYS = frozenset({"response", "clarification", "abstain"}) # A CAR answer scored against groundtruth_eval.json concepts passes at this -# recall. -CAR_PASS_THRESHOLD = 0.5 +# recall. At 0.75 a scenario with fewer than four required concepts cannot +# express the boundary: with two concepts coverage is only 0, 0.5 or 1.0, so +# the rule collapses onto full coverage there. +CAR_PASS_THRESHOLD = 0.75 _IMPORTANT_MODE_TERMS = ( "air conditioner", "cannot determine", diff --git a/src/evaluation/tests/test_static_json_scorer.py b/src/evaluation/tests/test_static_json_scorer.py index 5accea7d..6b0aee11 100644 --- a/src/evaluation/tests/test_static_json_scorer.py +++ b/src/evaluation/tests/test_static_json_scorer.py @@ -680,33 +680,73 @@ def test_car_without_concepts_keeps_extracted_term_scoring(): assert score.f1 == 0.5 -def test_static_json_scorer_passes_car_at_half_recall(): - scenario = Scenario.from_raw( +_CAR_GOLD_4 = { + "clarification": "Which asset, which site, which sensor and what time window?" +} +_CAR_METADATA_4 = { + "mode": "clarification", + "required_terms": [ + {"concept": "asset", "any_of": ["asset", "equipment"]}, + {"concept": "site", "any_of": ["site", "plant"]}, + {"concept": "sensor", "any_of": ["sensor", "tag"]}, + {"concept": "time window", "any_of": ["time window", "date range"]}, + ], +} + + +def _car_scenario(gold, metadata, scenario_id="1"): + return Scenario.from_raw( { - "id": "1", + "id": scenario_id, "text": "Synthetic CAR scenario.", - "expected_answer": json.dumps(_CAR_GOLD), - "evaluation_metadata": _CAR_METADATA, + "expected_answer": json.dumps(gold), + "evaluation_metadata": metadata, "scoring_method": "static_json", } ) + +def test_static_json_scorer_fails_car_at_half_recall(): + """Half coverage no longer passes: the threshold is 0.75.""" + scenario = _car_scenario(_CAR_GOLD, _CAR_METADATA) + scorer = StaticJsonScorer() half = scorer(scenario, '{"clarification": "Which asset?"}', "") + full = scorer(scenario, '{"clarification": "Which equipment, what date range?"}', "") none = scorer(scenario, '{"clarification": "Can you say more?"}', "") wrong_key = scorer(scenario, '{"abstain": "Which asset and time window?"}', "") - assert half.passed is True + assert half.passed is False assert half.score == 0.5 assert half.details["recall"] == 0.5 assert half.details["f1"] is None - assert half.rationale == "CAR recall 0.500 >= 0.5" + assert half.rationale == "CAR recall 0.500 < 0.75" + assert full.passed is True + assert full.score == 1.0 assert none.passed is False assert none.score == 0.0 assert wrong_key.passed is False assert wrong_key.score == 0.0 +def test_static_json_scorer_passes_car_at_three_quarter_recall(): + """Four concepts make the 0.75 boundary expressible: 3 of 4 passes, 2 of 4 does not.""" + scenario = _car_scenario(_CAR_GOLD_4, _CAR_METADATA_4, scenario_id="2") + + scorer = StaticJsonScorer() + three = scorer( + scenario, '{"clarification": "Which asset, which site, which sensor?"}', "" + ) + two = scorer(scenario, '{"clarification": "Which asset and which site?"}', "") + + assert three.passed is True + assert three.details["recall"] == 0.75 + assert three.rationale == "CAR recall 0.750 >= 0.75" + assert two.passed is False + assert two.details["recall"] == 0.5 + assert two.rationale == "CAR recall 0.500 < 0.75" + + def test_identifier_with_a_separator_is_not_read_as_a_count(): """An asset id is a label, not a number.