feat(query): add temporal_relation and evidence_request plan steps

src/vidxp/application_models.py

@@ -1299,18 +1299,48 @@ def _unique_modalities(
         return values
 
 
+class OccurrenceMode(StrEnum):
+    first = "first"
+    best = "best"
+    all = "all"
+
+
 class SearchMomentsPlanStep(ApplicationModel):
     kind: Literal["search_moments"] = "search_moments"
     modality: Identifier
     query: SearchQuery
+    occurrence_mode: OccurrenceMode = OccurrenceMode.best
 
 
 class ActorOverviewPlanStep(ApplicationModel):
     kind: Literal["actor_overview"] = "actor_overview"
 
 
+class TemporalRelation(StrEnum):
+    before = "before"
+    after = "after"
+    during = "during"
+
+
+class TemporalRelationPlanStep(ApplicationModel):
+    kind: Literal["temporal_relation"] = "temporal_relation"
+    relation: TemporalRelation
+    reference_query: SearchQuery
+    target_modality: Identifier
+    target_query: SearchQuery
+
+
+class EvidenceRequestPlanStep(ApplicationModel):
+    kind: Literal["evidence_request"] = "evidence_request"
+    delivery_mode: EvidenceDeliveryMode
+    include_board: bool = False
+
+
 QueryPlanStep = Annotated[
-    SearchMomentsPlanStep | ActorOverviewPlanStep,
+    SearchMomentsPlanStep
+    | ActorOverviewPlanStep
+    | TemporalRelationPlanStep
+    | EvidenceRequestPlanStep,
     Field(discriminator="kind"),
 ]
 
@@ -1319,10 +1349,40 @@ class QueryPlan(ApplicationModel):
     steps: tuple[QueryPlanStep, ...] = Field(min_length=1, max_length=8)
 
 
+class PlannerSearchMomentsStep(ApplicationModel):
+    kind: Literal["search_moments"] = "search_moments"
+    modality: Identifier
+    query: SearchQuery
+
+
+class PlannerPlan(ApplicationModel):
+    """Planner output limited to the steps query execution runs today."""
+
+    steps: tuple[
+        Annotated[
+            PlannerSearchMomentsStep | ActorOverviewPlanStep,
+            Field(discriminator="kind"),
+        ],
+        ...,
+    ] = Field(min_length=1, max_length=8)
+
+    def to_query_plan(self) -> QueryPlan:
+        return QueryPlan(
+            steps=tuple(
+                SearchMomentsPlanStep(modality=step.modality, query=step.query)
+                if isinstance(step, PlannerSearchMomentsStep)
+                else step
+                for step in self.steps
+            )
+        )
+
+
 class QueryPlanningRequest(ApplicationModel):
     question: SearchQuery
     allowed_modalities: tuple[Identifier, ...]
     actor_overview_allowed: bool = False
+    temporal_relations_allowed: bool = False
+    evidence_requests_allowed: bool = False
 
 
 class QueryModelIdentity(ApplicationModel):

src/vidxp/infrastructure/ollama_query.py

@@ -10,6 +10,7 @@
 
 from vidxp.application_models import (
     DraftAnswer,
+    PlannerPlan,
     QueryModelIdentity,
     QueryPlan,
     QueryPlanningRequest,
@@ -59,7 +60,7 @@ def __init__(
         )
         self._planner = Agent(
             model,
-            output_type=NativeOutput(QueryPlan),
+            output_type=NativeOutput(PlannerPlan),
             instructions=_PLANNING_INSTRUCTIONS,
             retries=retries,
             model_settings={
@@ -87,7 +88,8 @@ def identity(self) -> QueryModelIdentity:
         return self._identity
 
     def plan(self, request: QueryPlanningRequest) -> QueryPlan:
-        return self._run(self._planner, request.model_dump_json())
+        planned = self._run(self._planner, request.model_dump_json())
+        return planned.to_query_plan()
 
     def synthesize(self, request: QuerySynthesisRequest) -> DraftAnswer:
         return self._run(self._synthesizer, request.model_dump_json())

src/vidxp/query_service.py

@@ -5,8 +5,10 @@
 
 from vidxp.application_models import (
     ActorEvidence,
+    ActorOverviewPlanStep,
     DraftAnswer,
     Evidence,
+    EvidenceRequestPlanStep,
     FusedSearchResult,
     GroundedClaim,
     IndexSnapshotReference,
@@ -18,7 +20,7 @@
     QuerySynthesisRequest,
     QueryVideoCommand,
     SearchMomentsPlanStep,
-    ActorOverviewPlanStep,
+    TemporalRelationPlanStep,
 )
 from vidxp.capabilities.actor.schemas import ActorClusterSummary
 from vidxp.ports import QueryModelPort, QueryProviderError
@@ -39,6 +41,8 @@ def _default_plan(
     *,
     search_modalities: tuple[str, ...],
     actor_overview: bool,
+    temporal_relations: bool = False,
+    evidence_requests: bool = False,
 ) -> QueryPlan:
     steps = [
         SearchMomentsPlanStep(
@@ -57,6 +61,8 @@ def _valid_plan(
     *,
     search_modalities: tuple[str, ...],
     actor_overview: bool,
+    temporal_relations: bool = False,
+    evidence_requests: bool = False,
 ) -> bool:
     searches = [
         step.modality
@@ -66,6 +72,28 @@ def _valid_plan(
     actor_steps = sum(
         isinstance(step, ActorOverviewPlanStep) for step in plan.steps
     )
+    temporal_steps = sum(
+        isinstance(step, TemporalRelationPlanStep) for step in plan.steps
+    )
+    evidence_steps = sum(
+        isinstance(step, EvidenceRequestPlanStep) for step in plan.steps
+    )
+    if any(
+        step.occurrence_mode.value != "best"
+        for step in plan.steps
+        if isinstance(step, SearchMomentsPlanStep)
+    ):
+        return False
+    if not temporal_relations and temporal_steps > 0:
+        return False
+    if not evidence_requests and evidence_steps > 0:
+        return False
+    if any(
+        step.target_modality not in search_modalities
+        for step in plan.steps
+        if isinstance(step, TemporalRelationPlanStep)
+    ):
+        return False
     return (
         len(searches) == len(set(searches))
         and set(searches) == set(search_modalities)
@@ -85,11 +113,15 @@ def plan(
         *,
         search_modalities: tuple[str, ...],
         actor_overview: bool,
+        temporal_relations: bool = False,
+        evidence_requests: bool = False,
     ) -> tuple[QueryPlan, str | None]:
         fallback = _default_plan(
             command,
             search_modalities=search_modalities,
             actor_overview=actor_overview,
+            temporal_relations=temporal_relations,
+            evidence_requests=evidence_requests,
         )
         if self.model is None:
             return fallback, "query_model_not_configured"
@@ -99,6 +131,8 @@ def plan(
                     question=command.question,
                     allowed_modalities=search_modalities,
                     actor_overview_allowed=actor_overview,
+                    temporal_relations_allowed=temporal_relations,
+                    evidence_requests_allowed=evidence_requests,
                 )
             )
         except QueryProviderError:
@@ -107,6 +141,8 @@ def plan(
             proposed,
             search_modalities=search_modalities,
             actor_overview=actor_overview,
+            temporal_relations=temporal_relations,
+            evidence_requests=evidence_requests,
         ):
             return fallback, "query_plan_rejected"
         return proposed, None

tests/test_ollama_query.py

@@ -7,6 +7,7 @@
 from vidxp.application_models import (
     DraftAnswer,
     MomentEvidence,
+    QueryPlan,
     QueryPlanningRequest,
     QuerySynthesisRequest,
     SearchHit,
@@ -132,6 +133,62 @@ def handler(request: httpx.Request) -> httpx.Response:
             self.assertEqual(request["model"], "contract-model")
             self.assertEqual(request["reasoning_effort"], "none")
 
+    def test_planner_schema_offers_only_executable_steps(self):
+        requests: list[dict[str, object]] = []
+
+        def handler(request: httpx.Request) -> httpx.Response:
+            requests.append(json.loads(request.content))
+            plan = {"steps": [{"kind": "actor_overview"}]}
+            return httpx.Response(
+                200,
+                request=request,
+                json={
+                    "id": "chatcmpl-1",
+                    "object": "chat.completion",
+                    "created": 1,
+                    "model": "contract-model",
+                    "choices": [
+                        {
+                            "index": 0,
+                            "message": {
+                                "role": "assistant",
+                                "content": json.dumps(plan),
+                            },
+                            "finish_reason": "stop",
+                        }
+                    ],
+                },
+            )
+
+        client = httpx.AsyncClient(transport=httpx.MockTransport(handler))
+        model = OllamaQueryModel(
+            base_url="http://ollama.test/v1",
+            model_name="contract-model",
+            timeout_seconds=2,
+            output_retries=0,
+            http_client=client,
+        )
+        try:
+            plan = model.plan(
+                QueryPlanningRequest(
+                    question="Who is in the video?",
+                    allowed_modalities=(),
+                    actor_overview_allowed=True,
+                )
+            )
+        finally:
+            asyncio.run(client.aclose())
+
+        self.assertIsInstance(plan, QueryPlan)
+        schema = json.dumps(requests[0]["response_format"]["json_schema"])
+        for hidden in (
+            "occurrence_mode",
+            "temporal_relation",
+            "evidence_request",
+        ):
+            self.assertNotIn(hidden, schema)
+        self.assertIn("search_moments", schema)
+
 
 if __name__ == "__main__":
     unittest.main()