From 91e629269a80f1078626f163eb683618da766415 Mon Sep 17 00:00:00 2001 From: Niroj33 <86530493+professor3333@users.noreply.github.com> Date: Mon, 2 Nov 2026 08:44:22 +0545 Subject: [PATCH] Add pilot decision gate and annotation guidelines v0.2 `triggermap annotate gate` projects each role and group operator to the planned 800 documents from non-test pilot records (natural and enriched rates separately) and marks classes below 50 unsupported; document statuses are reported, not gated. Result in docs/pilot_gate.md: UNCERTAIN, ANY and UNRESOLVED are unsupported; EXCEPTION has no natural pilot examples. Median pace 5.6 min/doc (22 timed). Guidelines v0.2 resolve the pilot questions as rules R1-R8: input vs payload construction, attacker-set request parameters, environment events, hedging, ineffective protections (NOT_REQUIRED), exceptions remove exposure not impact, variant/escalation-specific conditions (UNCERTAIN), and references. Adds the pilot-2 migration procedure; v0.2 freezes when pilot-2 is imported. --- README.md | 7 +- annotations/README.md | 2 +- docs/annotation_guidelines.md | 99 ++++++++++++++++++-- docs/annotation_plan.md | 2 +- docs/pilot_gate.md | 28 ++++++ docs/pilot_protocol.md | 9 +- src/triggermap/annotate/gate.py | 161 ++++++++++++++++++++++++++++++++ src/triggermap/cli.py | 41 ++++++++ tests/test_gate.py | 57 +++++++++++ 9 files changed, 395 insertions(+), 11 deletions(-) create mode 100644 docs/pilot_gate.md create mode 100644 src/triggermap/annotate/gate.py create mode 100644 tests/test_gate.py diff --git a/README.md b/README.md index b50fd20..7d1b2b9 100644 --- a/README.md +++ b/README.md @@ -154,7 +154,7 @@ item can be highlighted in the original advisory. | Phase | Scope | Status | |---|---|:---:| | 1 · Feasibility | Pinned ingestion, corpus statistics, label schema, offset-preserving preprocessing, pilot sample, annotation tooling | ✅ built | -| 1 · Pilot | 100-document pilot imported (22 hand-labelled, 78 accepted model drafts); guideline revision, decision gate | 🟨 labels in · gate pending | +| 1 · Pilot | 100-document pilot imported (22 hand-labelled, 78 accepted model drafts); [decision gate](docs/pilot_gate.md) measured; guidelines v0.2 drafted, frozen at `pilot-2` | 🟨 revision in progress | | 2 · Baselines | Rules baseline, CRF + logistic-regression attributes and groups, evaluator with bootstrap CIs, evidence viewer | ✅ built · preliminary pilot dev scores | | 3 · Neural models | Frozen leakage-safe splits, annotation batches, DistilBERT and DeBERTa-v3 with span / role / group heads, experiment runner | 🟨 code ready · training awaits labels | | 4 · Reliability | Calibration and review policy, formatting robustness (rules: 100%), challenge-set and error-catalogue tooling | 🟨 tooling ready · thresholds await labels | @@ -251,13 +251,16 @@ JSON report. The same features are available as a JSON API: `POST /v1/extraction uv run triggermap corpus sample-pilot # reproduce the pilot sample (seed 20261020) uv run triggermap annotate tasks # Label Studio tasks for the sample uv run triggermap annotate import data/label_studio/export.json --annotator A1 +uv run triggermap annotate gate --export data/label_studio/export.json # docs/pilot_gate.md ``` The importer validates every exported annotation (offsets, labels, groups, document status), records its origin (manual, or a model draft accepted or edited), writes only clean records, and reports problems, label counts, origins and annotation pace. The procedure is in [`docs/pilot_protocol.md`](docs/pilot_protocol.md), and the labelling rules are in -[`docs/annotation_guidelines.md`](docs/annotation_guidelines.md). +[`docs/annotation_guidelines.md`](docs/annotation_guidelines.md). The gate projects each role +and group operator to 800 documents from the non-test pilot records and marks classes below 50 +as unsupported. ## How results will be measured diff --git a/annotations/README.md b/annotations/README.md index 2044093..8da5ab7 100644 --- a/annotations/README.md +++ b/annotations/README.md @@ -28,7 +28,7 @@ Each record's `origin` states how its labels were made, read from Label Studio's | Origin | Meaning | |---|---| -| `manual` | Labelled by hand, with no model draft. | +| `manual` | Submitted without a Label Studio pre-annotation (labelled by hand in the tool). | | `model_draft_accepted` | A model pre-annotation submitted with its labels unchanged (`origin_model` names it). | | `model_draft_edited` | A model pre-annotation that a person changed before submitting. | diff --git a/docs/annotation_guidelines.md b/docs/annotation_guidelines.md index 233444f..0815d1a 100644 --- a/docs/annotation_guidelines.md +++ b/docs/annotation_guidelines.md @@ -1,8 +1,10 @@ # Annotation guidelines -**Version 0.1 (pilot draft).** Label schema v1, preprocessing version 1. -These rules are tested by the 100-document pilot and will change. Every change is recorded in -the [changelog](#changelog), and annotations record the guideline version they followed. +**Version 0.2 (post-pilot draft).** Label schema v1 (record schema 1.1), preprocessing version 1. +Version 0.2 resolves the questions raised by the pilot (rules **R1–R8**, cite them in notes). It is +frozen when the revised pilot is imported as `pilot-2` (section 13), and batch annotation follows +it. Every change is recorded in the [changelog](#changelog), and annotations record the guideline +version they followed. ## 1. Goal @@ -42,6 +44,11 @@ matters for exposure?* If both answers are yes, it is a condition. | Workaround / mitigation | "disable the option immediately", "upgrade to 2.3.2" | Advice, even if it implies a condition. Annotate the condition only where the text states it as a condition. | | Generic best practice | "we recommend a reverse proxy" | Not stated as required for exposure. | | Severity, CVSS, credits, links | "CVSS 7.5", "thanks to …" | Not conditions. | +| Payload construction (R1) | "a ZIP whose entry names do not match", "chains containing duplicate self-signed certificates" | How the attacker builds the input. A developer cannot check their deployment for it. | +| Limits on impact (R6) | "access continues until the session expires" | Bounds how long or how far the impact goes, not whether the application is exposed. | +| Behaviour after the fix | "the patched version now marks it unsafe" | Version matching covers fixed releases. | +| Proof-of-concept specifics | "the PoC listens on port 8002" | Example values, unless the prose says the vulnerability requires them. | +| Unspecified hedges (R4) | "in some circumstances", "under certain configurations" | Names nothing a developer could check. | ## 4. Categories @@ -61,7 +68,22 @@ Tie-breakers, in order: 3. A property of the attacker → `ACCESS`. 4. Anything about the surrounding environment → `PLATFORM`. -Write a note if none fits well. The pilot will tell us whether the categories need changing. +Settled by the pilot: + +- **R1. Input.** *What kind of input the application processes* is `USAGE`: "processing + untrusted images", "parsing certificate chains supplied by clients". The span keeps the + adjectives inside that noun phrase (`untrusted`, `user-supplied`, `crafted`, `malicious`, + `invalid`) and stops before clauses that describe how the payload is built ("… containing + duplicate self-signed certificates" is left out). *The attacker's ability to deliver it* is + `ACCESS`: "attacker control of the chain input", "can upload files". +- **R2. Request parameters.** An option the attacker sets in their own request + (`trust_remote_code=true` sent in the request) is attacker capability, `ACCESS`. A server-side + setting that must *allow* that option is `CONFIGURATION`. +- **R3. Events in the environment.** Something that must happen in the environment (a server + restart, a scheduled job running, a package being reinstalled) is `PLATFORM`. Something the + victim must do stays `ACCESS`. + +Write a note if none fits well. ## 5. Roles @@ -84,6 +106,29 @@ Write a note if none fits well. The pilot will tell us whether the categories ne Ask: *is the negation part of what must be true (include it, `REQUIRED`), or is it the author saying something is unnecessary (leave it out, `NOT_REQUIRED`)?* +### Settled by the pilot + +- **R4. Hedging.** A hedge on *whether the vulnerability can be exploited* ("may allow", "can + lead to") does not change any role. A hedge on *whether a stated condition is needed* + ("mainly dangerous when X", "typically requires X", "in some configurations, such as X") makes + that condition `UNCERTAIN`, with a note. A hedge that names no checkable condition ("in some + circumstances") is not annotated. +- **R5. Ineffective protections.** "Even with X enabled", "X does not prevent this", + "regardless of X": the text says the *absence* of X is not needed. Annotate `X` (without the + negating words) in its own category with role `NOT_REQUIRED`. Never `EXCEPTION`: X does not + protect. Example: "restrictive url_fetcher configuration does not prevent exploitation" → + `restrictive url_fetcher configuration`, CONFIGURATION, NOT_REQUIRED. +- **R6. Exceptions remove exposure, not impact.** `EXCEPTION` is only for something that, if it + holds, means the application is not exposed at all. Something that only shortens or limits + the impact (session expiry, rate limits that slow an attack) is not a condition. +- **R7. Variant- and escalation-specific conditions.** When the advisory describes several + attack variants, or a main vulnerability plus an escalated impact, a condition that the text + ties to **only one variant or only the escalation** is `UNCERTAIN`. Write a note naming the + variant ("variant: URL fetch to internal hosts"), and do not group it with conditions of the + main vulnerability. If the whole advisory is about that variant, its conditions are + `REQUIRED` as usual. This keeps conditions visible without presenting them as prerequisites + for every exposure. + ## 6. Span boundaries A span is the **minimal contiguous phrase that fully states the condition**. @@ -105,8 +150,20 @@ Splitting: - Two conditions coordinated with their own predicates ("using X **and** enabling Y") → two spans, plus a group (section 7). - An "or"/"and" **inside one description** ("a caching proxy that does not strip cookies - **or** ignore responses with cookies") → one span. Rule of thumb: separate spans when a - developer would check them separately. + **or** ignore responses with cookies", "kernels and terminals disabled or delegated to remote + hosts") → one span. Rule of thumb: separate spans when a developer would check them separately. + +Qualifiers that the text leaves unspecified stay as written. "An authenticated user with +specific data permissions" is one `ACCESS` span. Never add the permission names, character +sets or sizes the text does not give. + +References (**R8**): +- When the only statement of a condition uses a reference ("the variable is not set", "that + page") and the referent is named **earlier in the same paragraph**, annotate the phrase as + written and name the referent in a note ("the variable = A2U_AUTH_TOKEN"). +- When the referent is outside the paragraph, annotate the phrase with role `UNCERTAIN`. +- When the condition is also stated with its name elsewhere, annotate both statements (see + Repetition below). Code: - Annotate inside code only when the prose presents the code as the vulnerable setup @@ -213,8 +270,38 @@ reported back for you to fix in Label Studio. Use the notes box whenever you hesitate: an unclear category, a borderline workaround, a span boundary you are unsure of. Notes are reviewed after the pilot to revise these guidelines. +## 12. Unsupported classes (pilot gate) + +The pilot gate (`docs/pilot_gate.md`, regenerated with `triggermap annotate gate`) projects each +role and group operator to the planned 800 documents. Classes projected below 50 examples are +**unsupported**: + +| Class | Pilot gate result | What it means | +|---|---|---| +| `UNCERTAIN` | unsupported | Keep annotating as before. Models do not claim it; it is a *Review required* state. | +| `ANY` | unsupported | Keep annotating. Predicted `ANY` groups are shown for review, and results are reported as low-support. | +| `UNRESOLVED` | unsupported | Keep annotating; always *Review required*. | +| `EXCEPTION` | supported in training only | No exceptions occurred in the natural pilot documents, so dev/test support will be very small. Training examples come from enriched batches. | + +Unsupported means *too few examples to learn or measure*, not *removed*. Never invent or +adapt examples to fill a class. + +## 13. Migrating the pilot to v0.2 + +`pilot-1` was labelled under v0.1 and stays as it is. Before batch-01: + +1. In Label Studio, re-check every pilot task that R1–R8 could affect: any note or draft review + flag about crafted input, ineffective protections, hedging, impact-specific or + variant-specific conditions, request parameters, lifecycle events or references. Edit, + resubmit, and cite the rule in the note ("R5"). +2. Export and import as a new version: + `triggermap annotate import data/label_studio/export.json --annotator A1 --annotation-version pilot-2 --out annotations/pilot/pilot-2.jsonl`. +3. Regenerate the gate (`triggermap annotate gate --gold annotations/pilot/pilot-2.jsonl --export data/label_studio/export.json`). + Version 0.2 is frozen at that point. + ## Changelog | Version | Date | Change | |---|---|---| | 0.1 | 2026-10-20 | Pilot draft. | +| 0.2 | 2026-11-02 | Post-pilot draft. R1 input vs payload construction; R2 attacker-set request parameters are ACCESS; R3 environment events are PLATFORM; R4 hedging; R5 ineffective protections are NOT_REQUIRED; R6 exceptions remove exposure, not impact; R7 variant/escalation-specific conditions are UNCERTAIN; R8 references. New "not conditions" rows, unspecified qualifiers, unsupported classes from the pilot gate (UNCERTAIN, ANY, UNRESOLVED), pilot migration to `pilot-2`. | diff --git a/docs/annotation_plan.md b/docs/annotation_plan.md index 8d750b2..2143c49 100644 --- a/docs/annotation_plan.md +++ b/docs/annotation_plan.md @@ -6,7 +6,7 @@ in `splits/pool-v1.json` (see [`splits.md`](splits.md)). Every batch is committe | Sample | Pool | n | Strategy | Usable for | When to annotate | |---|---|---:|---|---|---| -| `pilot/pilot_manifest.json` | mixed | 100 | 50 natural + 50 enriched | 39 train, 38 dev, 12 test (11 excluded) | now (guidelines v0.1) | +| `pilot/pilot_manifest.json` | mixed | 100 | 50 natural + 50 enriched | 39 train, 38 dev, 12 test (11 excluded) | done under v0.1 (`pilot-1`); revised to v0.2 as `pilot-2` | | `batches/batch-01.json` | train | 100 | natural | train | after guidelines v0.2 | | `batches/batch-02.json` | train | 100 | enriched (cue top quartile, ≤ 800 words) | train | after guidelines v0.2 | | `batches/batch-03.json` | dev | 112 | natural, ≤ 3 per leakage group | dev | after guidelines v0.2 | diff --git a/docs/pilot_gate.md b/docs/pilot_gate.md new file mode 100644 index 0000000..54a69cf --- /dev/null +++ b/docs/pilot_gate.md @@ -0,0 +1,28 @@ +# Pilot decision gate + +Generated by `triggermap annotate gate`; do not edit by hand. + +- Gold: `annotations/pilot/pilot-1.jsonl`, split `splits/pool-v1.json` +- Documents counted (test split excluded): 88 (enriched 50, random 38) +- Origin of counted records: manual 18, model_draft_accepted 70 +- Projection: 467 natural + 333 enriched-like documents; evaluation support assumes 300 natural dev + test documents +- Threshold: a class projected below 50 examples is **unsupported** + +| Group | Label | Random | Enriched | Manual only | Projected (800) | Projected in dev + test | Status | +|---|---|---:|---:|---:|---:|---:|---| +| Role | REQUIRED | 110 | 247 | 53 | 2997 | 868 | supported | +| Role | EXCEPTION | 0 | 15 | 3 | 100 | 0 | supported · no natural evaluation support | +| Role | NOT_REQUIRED | 3 | 8 | 2 | 90 | 24 | supported | +| Role | UNCERTAIN | 3 | 0 | 1 | 37 | 24 | **unsupported** | +| Group operator | ALL | 13 | 36 | 10 | 400 | 103 | supported | +| Group operator | ANY | 2 | 3 | 0 | 45 | 16 | **unsupported** | +| Group operator | UNRESOLVED | 1 | 2 | 0 | 26 | 8 | **unsupported** | +| Document status | HAS_CONDITIONS | 34 | 50 | 17 | 751 | 268 | reported, not gated | +| Document status | NO_EXPLICIT_CONDITION | 4 | 0 | 1 | 49 | 32 | reported, not gated | +| Document status | UNSUPPORTED_STRUCTURE | 0 | 0 | 0 | 0 | 0 | reported, not gated · no natural evaluation support | + +## Pace + +- Timed documents: 22 (Label Studio lead time; manual records only) +- Median minutes per document: 5.6 +- Projected hours for 800 documents at the median: 74 diff --git a/docs/pilot_protocol.md b/docs/pilot_protocol.md index e45942d..01c4ec9 100644 --- a/docs/pilot_protocol.md +++ b/docs/pilot_protocol.md @@ -70,7 +70,14 @@ agreement. ## Decision gate -These are proposed criteria for review together after the pilot: +The measured result is in [`pilot_gate.md`](pilot_gate.md), generated from the non-test pilot +records with: + +```bash +uv run triggermap annotate gate --export data/label_studio/export.json +``` + +The criteria: - **Pace.** The measured median minutes per document sets the realistic size of the gold set within the 80–130 hour budget. diff --git a/src/triggermap/annotate/gate.py b/src/triggermap/annotate/gate.py new file mode 100644 index 0000000..dead525 --- /dev/null +++ b/src/triggermap/annotate/gate.py @@ -0,0 +1,161 @@ +"""Pilot decision gate: projected support per role, group operator and document status. + +Counts come from pilot records **outside the test split** (the locked test documents are +never read). Per-document rates are taken separately for the natural (``random``) and +``enriched`` strata and projected onto the planned gold-set composition. A class projected +below the threshold is reported **unsupported**: it stays in the label inventory and is still +annotated, but no model claims it until targeted sampling finds more examples. +""" + +from __future__ import annotations + +import statistics +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Any + +from triggermap.schema.models import ( + AdvisoryAnnotation, + AnnotationOrigin, + DocumentStatus, + GroupOperator, + Role, +) + +# (group, labels, gated): document statuses are reported but not gated, because a model +# outputs NO_EXPLICIT_CONDITION whenever it finds nothing and never "claims" it as a class. +GATED: tuple[tuple[str, tuple[str, ...], bool], ...] = ( + ("Role", tuple(r.value for r in Role), True), + ("Group operator", tuple(o.value for o in GroupOperator), True), + ("Document status", tuple(s.value for s in DocumentStatus), False), +) + + +@dataclass(frozen=True) +class Plan: + """Planned gold-set composition (docs/annotation_plan.md).""" + + natural: int = 467 # pilot random 50, batches 01/03/04, 30% of batch-05 + enriched: int = 333 # pilot enriched 50, batch-02 100, 70% of batch-05 (uncertainty) + evaluation_natural: int = 300 # dev 150 + test 150, natural only + threshold: int = 50 + + +DEFAULT_PLAN = Plan() + + +@dataclass(frozen=True) +class Row: + group: str + label: str + counts: Mapping[str, int] # stratum -> count + manual: int + projected: float + projected_evaluation: float + supported: bool | None # None: reported, not gated + + +def _tally(records: Sequence[AdvisoryAnnotation]) -> Counter[str]: + counts: Counter[str] = Counter() + for record in records: + counts.update(c.role.value for c in record.conditions) + counts.update(g.operator.value for g in record.groups) + counts[record.document_status.value] += 1 + return counts + + +def gate( + records: Sequence[AdvisoryAnnotation], + strata: Mapping[str, str], + assignment: Mapping[str, str], + plan: Plan = DEFAULT_PLAN, +) -> tuple[list[Row], dict[str, int]]: + """Rows per gated label, plus document counts per stratum (test split excluded).""" + usable = [r for r in records if assignment.get(r.advisory_id) != "test"] + by_stratum: dict[str, list[AdvisoryAnnotation]] = {"random": [], "enriched": []} + for record in usable: + by_stratum.setdefault(strata[record.advisory_id], []).append(record) + tallies = {s: _tally(rs) for s, rs in by_stratum.items()} + manual = _tally([r for r in usable if r.origin == AnnotationOrigin.MANUAL]) + sizes = {s: len(rs) for s, rs in by_stratum.items()} + + def rate(stratum: str, label: str) -> float: + return tallies[stratum][label] / sizes[stratum] if sizes[stratum] else 0.0 + + rows = [] + for group, labels, gated in GATED: + for label in labels: + projected = rate("random", label) * plan.natural + rate("enriched", label) * ( + plan.enriched + ) + rows.append( + Row( + group, + label, + {s: tallies[s][label] for s in by_stratum}, + manual[label], + projected, + rate("random", label) * plan.evaluation_natural, + projected >= plan.threshold if gated else None, + ) + ) + return rows, sizes + + +def render_gate( + rows: Sequence[Row], + sizes: Mapping[str, int], + origins: Mapping[str, int], + lead_times: Sequence[float], + plan: Plan = DEFAULT_PLAN, + source: str = "", +) -> str: + lines = [ + "# Pilot decision gate", + "", + "Generated by `triggermap annotate gate`; do not edit by hand.", + "", + f"- Gold: {source}", + f"- Documents counted (test split excluded): {sum(sizes.values())} " + f"({', '.join(f'{s} {n}' for s, n in sorted(sizes.items()))})", + f"- Origin of counted records: " + f"{', '.join(f'{k} {v}' for k, v in sorted(origins.items())) or 'none'}", + f"- Projection: {plan.natural} natural + {plan.enriched} enriched-like documents; " + f"evaluation support assumes {plan.evaluation_natural} natural dev + test documents", + f"- Threshold: a class projected below {plan.threshold} examples is **unsupported**", + "", + "| Group | Label | Random | Enriched | Manual only | Projected (800) " + "| Projected in dev + test | Status |", + "|---|---|---:|---:|---:|---:|---:|---|", + ] + for row in rows: + if row.supported is None: + status = "reported, not gated" + else: + status = "supported" if row.supported else "**unsupported**" + if row.supported is not False and row.projected_evaluation < 1: + status += " · no natural evaluation support" + lines.append( + f"| {row.group} | {row.label} | {row.counts.get('random', 0)} " + f"| {row.counts.get('enriched', 0)} | {row.manual} | {row.projected:.0f} " + f"| {row.projected_evaluation:.0f} | {status} |" + ) + lines += ["", "## Pace", ""] + if lead_times: + minutes = [t / 60 for t in lead_times] + median = statistics.median(minutes) + lines += [ + f"- Timed documents: {len(minutes)} (Label Studio lead time; manual records only)", + f"- Median minutes per document: {median:.1f}", + f"- Projected hours for {plan.natural + plan.enriched} documents at the median: " + f"{median * (plan.natural + plan.enriched) / 60:.0f}", + ] + else: + lines.append("No lead times recorded.") + lines.append("") + return "\n".join(lines) + + +def strata_of(sample: Mapping[str, Any]) -> dict[str, str]: + return {str(item["advisory_id"]): str(item["stratum"]) for item in sample["items"]} diff --git a/src/triggermap/cli.py b/src/triggermap/cli.py index 01cdcb5..29e7628 100644 --- a/src/triggermap/cli.py +++ b/src/triggermap/cli.py @@ -170,6 +170,39 @@ def _annotate_import(args: argparse.Namespace) -> int: return 1 if problems else 0 +def _annotate_gate(args: argparse.Namespace) -> int: + from collections import Counter + + from triggermap.annotate.gate import gate, render_gate, strata_of + from triggermap.annotate.importer import load_export + from triggermap.corpus.splits import load_split + from triggermap.eval.runner import load_gold + from triggermap.schema import AdvisoryAnnotation + + assignment = load_split(args.split_manifest)["assignment"] + records = [ + r + for r in load_gold(args.gold).values() + if isinstance(r, AdvisoryAnnotation) and assignment.get(r.advisory_id) != "test" + ] + rows, sizes = gate(records, strata_of(json.loads(args.sample.read_text("utf-8"))), assignment) + origins = Counter(r.origin.value if r.origin else "unrecorded" for r in records) + lead_times: list[float] = [] + if args.export: + for task in load_export(args.export): + for ann in task.get("annotations", []): + lead = ann.get("lead_time") + manual = ann.get("parent_prediction") is None and not ann.get("was_cancelled") + if manual and isinstance(lead, int | float): + lead_times.append(float(lead)) + source = f"`{args.gold}`, split `{args.split_manifest}`" + args.out.parent.mkdir(parents=True, exist_ok=True) + args.out.write_text(render_gate(rows, sizes, origins, lead_times, source=source), "utf-8") + unsupported = [r.label for r in rows if r.supported is False] + print(f"wrote {args.out}; unsupported: {', '.join(unsupported) or 'none'}") + return 0 + + def _evaluate(args: argparse.Namespace) -> int: from triggermap.corpus.splits import load_split, select from triggermap.eval.report import render @@ -856,6 +889,14 @@ def build_parser() -> argparse.ArgumentParser: imp.add_argument("--report", type=Path) imp.set_defaults(handler=_annotate_import) + gate_cmd = annotate_cmds.add_parser("gate", help="pilot decision gate: projected support") + gate_cmd.add_argument("--gold", type=Path, default=Path("annotations/pilot/pilot-1.jsonl")) + gate_cmd.add_argument("--sample", type=Path, default=DEFAULT_PILOT) + gate_cmd.add_argument("--split-manifest", type=Path, default=DEFAULT_SPLIT) + gate_cmd.add_argument("--export", type=Path, help="Label Studio JSON export (for pace)") + gate_cmd.add_argument("--out", type=Path, default=Path("docs/pilot_gate.md")) + gate_cmd.set_defaults(handler=_annotate_gate) + evaluate = commands.add_parser("evaluate", help="score predictions against gold") evaluate.add_argument("gold", type=Path, help="gold AdvisoryAnnotation JSONL") evaluate.add_argument("pred", type=Path, help="predicted Extraction JSONL") diff --git a/tests/test_gate.py b/tests/test_gate.py new file mode 100644 index 0000000..6afb28b --- /dev/null +++ b/tests/test_gate.py @@ -0,0 +1,57 @@ +from triggermap.annotate.gate import Plan, gate, render_gate +from triggermap.schema import ( + AdvisoryAnnotation, + AnnotationOrigin, + Category, + Condition, + DocumentStatus, + Role, +) + +SHA = "0" * 64 +COMMIT = "1" * 40 + + +def record(aid: str, roles: list[Role], origin: AnnotationOrigin) -> AdvisoryAnnotation: + conditions = tuple( + Condition(id=f"c{i}", span=(i * 2, i * 2 + 1), category=Category.USAGE, role=role) + for i, role in enumerate(roles, 1) + ) + status = DocumentStatus.HAS_CONDITIONS if roles else DocumentStatus.NO_EXPLICIT_CONDITION + return AdvisoryAnnotation( + advisory_id=aid, + source_commit=COMMIT, + content_hash=SHA, + preprocessing_version="1", + annotation_version="pilot-1", + document_status=status, + conditions=conditions, + origin=origin, + ) + + +IDS = ["GHSA-2222-2222-2222", "GHSA-3333-3333-3333", "GHSA-4444-4444-4444"] + + +def test_test_split_is_never_counted_and_rates_are_projected() -> None: + records = [ + record(IDS[0], [Role.REQUIRED, Role.EXCEPTION], AnnotationOrigin.MANUAL), + record(IDS[1], [Role.REQUIRED], AnnotationOrigin.MODEL_DRAFT_ACCEPTED), + record(IDS[2], [Role.NOT_REQUIRED] * 9, AnnotationOrigin.MANUAL), # test split + ] + strata = {IDS[0]: "random", IDS[1]: "enriched", IDS[2]: "random"} + assignment = {IDS[0]: "dev", IDS[1]: "train", IDS[2]: "test"} + plan = Plan(natural=100, enriched=50, evaluation_natural=30, threshold=60) + rows, sizes = gate(records, strata, assignment, plan) + assert sizes == {"random": 1, "enriched": 1} + by_label = {r.label: r for r in rows} + assert by_label["NOT_REQUIRED"].projected == 0 # only in the test record + assert by_label["REQUIRED"].projected == 150 and by_label["REQUIRED"].supported + assert by_label["EXCEPTION"].projected == 100 and by_label["EXCEPTION"].manual == 1 + assert by_label["EXCEPTION"].projected_evaluation == 30 + assert by_label["ANY"].supported is False + assert by_label["UNSUPPORTED_STRUCTURE"].supported is None # statuses are not gated + text = render_gate(rows, sizes, {"manual": 1}, [300.0, 420.0], plan) + assert "| Role | ANY" not in text and "| Group operator | ANY | 0 | 0 | 0 | 0 | 0 |" in text + assert "reported, not gated" in text + assert "**unsupported**" in text and "Median minutes per document: 6.0" in text