-
Notifications
You must be signed in to change notification settings - Fork 50
Expand file tree
/
Copy pathgeneric.py
More file actions
2006 lines (1901 loc) · 109 KB
/
Copy pathgeneric.py
File metadata and controls
2006 lines (1901 loc) · 109 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""Generic coding-CLI driver: interactive sessions in tmux windows, observed via hooks.
Each pipeline step gets a fresh tmux window running the full interactive CLI
with the skill invocation as the initial prompt. Completion is detected
exclusively through hook-written event files (Stop/SessionEnd) plus the
presence of the skill-written result.json — the pane log's *contents* never
drive the wait loop (only tee'd for human debugging), though its *growth*
(mtime/size, never the bytes — see ``_log_activity_key``) is read as a liveness
signal to re-arm the dev-stall grace window. The one exception is post-mortem:
after the verdict and reconcile have settled, a single tail read of the log
classifies a transport-failure environment fault (#194, see
``_classify_env_fault``) — it labels the result, it never drives the wait loop.
Everything CLI-specific (binary, prompt rendering, bypass flags, usage
parser) comes from a declarative CLIProfile; each CLI's hook config registers
the shared relay script under its native event names but passes the canonical
event name as argv, so this adapter only ever sees canonical events. CLIs
without a SessionEnd hook (e.g. Codex) are covered by the window-death
fallback.
"""
from __future__ import annotations
import enum
import hashlib
import json
import shlex
import time
from collections.abc import Callable
from pathlib import Path
from typing import TYPE_CHECKING
from .. import devcontract, gates, runs
from ..bmadconfig import ProjectPaths
from ..journal import LOGS_DIR
from ..model import TokenUsage
from ..policy import Policy
from ..process_host import ProcessHostError, get_process_host
from ..signals import SignalWatcher
from ..tokens import read_usage as tally_usage
from ..verify import read_frontmatter, status_of
from .base import CodingCLIAdapter, SessionHandle, SessionResult, SessionSpec, SpecSnapshot
# Re-exported for importers that predate the env_fault module split (#194 landed
# these names on this module); the definitions now live in .env_fault. The
# redundant `X as X` form is the explicit-re-export spelling — it tells the linter
# these are deliberate pass-throughs, without an `__all__` that would read as a
# statement of this module's public API and understate it (callers also import
# GenericTmuxAdapter, the *_NUDGE_TEXT constants and HEARTBEAT_INTERVAL_S).
#
# READ-ONLY. An import copies the object binding, so these names are aliases, not
# a window onto env_fault's globals: reading them is exact, but REBINDING one here
# (`monkeypatch.setattr(generic, "ENV_FAULT_MATCH_TIMEOUT_S", ...)`) is invisible to
# the classifier, which resolves the constant from its own module at call time. That
# is not hypothetical — it silently defused the pathological-regex test the split
# inherited. Override at the definition site (`env_fault.<NAME>`) instead.
from .env_fault import _ANSI_RE as _ANSI_RE
from .env_fault import ENV_FAULT_EVIDENCE_MAX as ENV_FAULT_EVIDENCE_MAX
from .env_fault import ENV_FAULT_MATCH_TIMEOUT_S as ENV_FAULT_MATCH_TIMEOUT_S
from .env_fault import ENV_FAULT_STATUSES as ENV_FAULT_STATUSES
from .env_fault import ENV_FAULT_TAIL_BYTES as ENV_FAULT_TAIL_BYTES
from .env_fault import EnvFaultMixin
from .multiplexer import MultiplexerError, TerminalMultiplexer, get_multiplexer
from .profile import CLIProfile
if TYPE_CHECKING:
from ..process_host import ProcessHost
# Pane geometry for agent windows; mirrored in tui.data for log emulation.
PANE_COLUMNS = 220
PANE_LINES = 50
RESULT_GRACE_S = 15.0
RESULT_POLL_S = 0.5
KILL_POLL_S = 0.5
# Missing-marker fallback (#224): how many consecutive resultless-Stop
# observations of an IDENTICAL (path, mtime, status) fingerprint a marker-less
# terminal spec must survive before it is synthesized as this session's result.
# One observation is not enough: right after a review launch the spec still
# carries the dev pass's `done` frontmatter, and the review's first write can
# bump its mtime past the launch floor before the status flips to `in-review` —
# harvesting on that single sighting would score a review that never ran (#261).
# Two stable sightings bracket a full stall-grace + nudge with zero writes, which
# a session mid-edit cannot produce. A dead window skips the counter entirely:
# the kill settled liveness, so the frontmatter is as final as it will ever get.
FM_FALLBACK_MIN_OBS = 2
# Proof-of-work gate (#261): pane-log size, in bytes, above which a session counts
# as having produced SOMETHING — the floor a dead session must clear before a
# read-back artifact may upgrade its verdict to `completed`. Not zero: the three
# wedged sessions in #261 left logs of 0 and 2 bytes, so `size > 0` would have
# cleared one of them. The separation is wide in the observed data — that run's
# working dev session logged 1.4 MB against the wedged reviews' 0 and 2 — so the
# exact value is not load-bearing; it only has to sit above the noise a pane can
# accumulate without the CLI rendering anything. Note the floor measures the CLI's
# OWN output: the orchestrator's prompt is delivered by send-keys and a program
# that never echoes it leaves the log empty (measured), so this is not a proxy for
# "the session was launched" — only for "the CLI rendered something".
PROOF_OF_WORK_MIN_LOG_BYTES = 256
class _SnapVerdict(enum.Enum):
"""Launch-snapshot (#276 M1/M2) decision, shared by the mtime-scan fallback and
the stories read-back so the two completion paths can never drift.
NEUTRAL — no snapshot, a different file, or bytes changed since launch: fall
through to the path's normal accept logic.
PROVEN — a mid-session status transition (M2) was observed for this spec:
single-sighting harvest, and it OUTRANKS a byte-identical hash (a clean review
can round-trip back to the launch bytes yet provably ran).
REFUSE — bytes still byte-identical to the review-launch snapshot AND no
transition was observed (M1): the documented dead-window false positive
(a `done` spec re-opened for review, mtime-bumped but never re-driven).
"""
NEUTRAL = "neutral"
PROVEN = "proven"
REFUSE = "refuse"
# min spacing between heartbeat.json overwrites in wait_for_completion; the
# heartbeat's staleness is what makes a frozen orchestrator (#157) diagnosable.
HEARTBEAT_INTERVAL_S = 30.0
EVENT_KINDS = {"SessionStart", "Stop", "SessionEnd"}
NUDGE_TEXT = (
"You are running in bmad-loop automation mode. Finish the workflow now: "
"complete any remaining steps and write the result JSON file to "
"$BMAD_LOOP_RUN_DIR/tasks/$BMAD_LOOP_TASK_ID/result.json, then end your turn."
)
# Wake an idle dev session whose grace window elapsed with no output. bmad-loop
# has no background-completion re-invocation, so a turn ended to await a slow
# background process (a Unity PlayMode run, a long test) would otherwise wait
# forever; this nudge IS that re-invocation. Skill-agnostic: it must not assume a
# result.json (the bmad-build-auto skill writes none — see GenericDevAdapter).
STALL_NUDGE_TEXT = (
"You appear idle in bmad-loop automation mode, which cannot re-invoke you when "
"a background process finishes. If you are waiting on one (e.g. a Unity PlayMode "
"run or a long test), check its status now and continue the workflow; if it is "
"done, finalize the work and end your turn. If you are stuck, say so and stop. "
"Note: a prose reply cannot end this session — only your workflow's completion "
"artifact (the spec's terminal status / result file) does; if the work is "
"already complete, write it before ending your turn."
)
# Wrap-up demand for a session that crossed its token budget (#158, enforce
# mode): the guard arms a bounded grace window right after sending this, so the
# session must converge now — it will be terminated over_budget otherwise.
BUDGET_NUDGE_TEXT = (
"You have exceeded this session's token budget in bmad-loop automation mode. "
"Stop exploring and wrap up now: commit whatever is finished, write your "
"workflow's completion artifact (the spec's terminal status / result file), "
"and end your turn. Note: a prose reply cannot end this session — only the "
"completion artifact does; if you cannot finish, mark the work blocked in it "
"and end your turn."
)
# Targeted contract-repair nudge (#276 M4): a Stop found the spec at
# {spec_path} finalized to terminal frontmatter status {status} but WITHOUT the
# `## Auto Run Result` section bmad-loop's harvest scan keys on. Ask the skill to
# append that section itself so the omission is fixed at the source (a compliant
# append is then harvested by the normal scan; harness-side frontmatter synthesis
# stays the backstop). Sent at most once per session and never re-armed, so it is
# safe to be specific and directive. Guarded ("if this spec is not yours or the
# work is unfinished") so a session legitimately mid-workflow is not derailed.
CONTRACT_NUDGE_TEXT = (
"You are running in bmad-loop automation mode. The spec at {spec_path} now "
"carries a terminal frontmatter `status: {status}`, but it is missing the "
"`## Auto Run Result` section your contract requires — bmad-loop harvests "
"that section, not the frontmatter, so without it this finished story looks "
"unfinished. If this spec is yours and the work is done, append the section "
"to the spec now — the `## Auto Run Result` heading, a `Status: {status}` "
"line matching the frontmatter, and a brief summary — then end your turn. If "
"this spec is not yours, or the work is not actually finished, ignore this "
"and continue your workflow instead."
)
class _ResultFileMixin:
"""Result-file read-back and verdict finalization: acquire the
skill-written result dict and fold it into the session's final
``SessionResult``. Transport-agnostic — shared by the tmux adapters and
any adapter whose skill writes ``tasks/<task_id>/result.json``; needs
only ``self.tasks_dir`` and ``self.run_dir``."""
# Set by the concrete adapter's __init__; bare annotations (no runtime
# effect) tell the type checker the host attributes this mixin reads.
tasks_dir: Path
run_dir: Path
# Whether `_final` applies the #261 proof-of-work gate to its read-back. False
# here, and that is not a conservative default — it is the correct answer for
# this mixin's own read-back. `tasks/<task_id>/result.json` is task-unique and
# `start_session` unlinks it before launch, so its presence is already proof
# THIS session wrote it; a foreign writer cannot reach it. Gating it could only
# ever downgrade an authoritative completion. Overridden True by
# `_DevSynthesisMixin`, whose read-back scans a directory shared with every
# concurrent run — the one place a result can belong to somebody else.
_READBACK_NEEDS_PROOF_OF_WORK = False
def _hard_stop_requested(self) -> bool:
"""Has an operator lodged a *hard* stop request that this session must
honor (#319)? Either this run's own, or the owning run's.
Polled twice per wait-loop iteration by both real adapters — on either
side of the loop's own blocking wait — so a
``bmad-loop stop`` is honored mid-session on platforms where the
engine's SIGTERM path is unreachable. Read-only by contract: the
adapter never unlinks ``stop-request.json`` — the engine consumes it
when it raises, and must still see it to attribute the stop. A torn or
modeless read already leans ``"graceful"`` inside
``read_stop_request_mode``, so this can never abort a session
spuriously.
Both dirs are read because a nested auto-sweep is a first-class run *and*
somebody else's child: it mints its own id and appears in ``list``, so
``stop <child-id>`` must still reach it, while ``stop <parent-id>`` lodges
in a dir this adapter would otherwise never look at. The owner leg is
hard-only, like this whole predicate — a graceful request already
suppresses a child sweep from *starting*, and letting one already in flight
finish is exactly what graceful means."""
if runs.read_stop_request_mode(self.run_dir) == "hard":
return True
owner = runs.owner_run_dir()
# `!=` is a cheap dedupe for the common top-level case, not a correctness
# dependency: two spellings of one dir cost a redundant read, same answer.
return (
owner is not None
and owner != self.run_dir
and runs.read_stop_request_mode(owner) == "hard"
)
def _result_json(self, handle: SessionHandle, spec: SessionSpec, *, wait: bool) -> dict | None:
"""Acquire this session's result dict. Base behavior: read the
skill-written ``result.json`` (briefly awaiting it on the Stop event,
reading once otherwise). Subclasses whose skill writes no result.json
(GenericDevAdapter) override this to synthesize the dict from another
on-disk artifact."""
return self._await_result(handle.task_id) if wait else self._read_result(handle.task_id)
def _produced_work(self, handle: SessionHandle, stop_seen: bool) -> bool:
"""Whether this session shows ANY evidence it actually ran, for the #261
proof-of-work gate. Deliberately a very low bar — it separates "the CLI
wedged before it did anything" from "the CLI worked", not good work from bad.
Two independent signals, ORed, because each has a known blind spot: a `Stop`
event having arrived covers an adapter whose pane sink is misbound (#254/#217,
where a HEALTHY session still logs zero bytes), and pane-log growth covers a
profile whose hooks never fire. Requiring both to be absent is what makes the
gate safe to apply to a `completed` upgrade.
The hook signal is `Stop` specifically — a turn that ENDED — not "a hook
event arrived". Of the three canonical events, `SessionStart` fires before
the session does anything and `SessionEnd` fires when it stops being one;
both are emitted by a CLI that launched and wedged, so accepting either
would leave the gate satisfied in exactly the case it exists to catch. The
#254/#217 rationale is unaffected: a healthy session ends its turn.
Unknown never blocks: `_log_evidence` returns None when there is no signal at
all (no pane log — the opencode-http transport, and every unit-test fixture),
and that reads as evidence-present, preserving current behavior exactly."""
if stop_seen:
return True
evidence = self._log_evidence(handle)
return True if evidence is None else evidence
def _log_evidence(self, handle: SessionHandle) -> bool | None:
"""Tristate pane-log proof-of-work signal: True = the log grew past a
trivial floor, False = the log exists and did not, None = no such signal for
this transport. Base: None (inert). Overridden by `GenericAdapter`, which
tees a pane log."""
return None
def _session_vanished(self) -> bool:
"""Whether the whole multiplexer session is gone, asked only once a
crash verdict has already been reached (#489). Base: False — an adapter
with no session to lose (opencode-http) never vanishes. Overridden by
`GenericAdapter`.
Same failure convention as `_window_alive`: `MultiplexerError` is the
seam's declared "couldn't ask" and the override swallows it to False.
Anything else propagates, exactly as it does from the liveness probe —
this is a label on a verdict already made, so it degrades rather than
second-guessing the verdict, but it does not swallow unknown faults."""
return False
def _final(
self,
handle: SessionHandle,
spec: SessionSpec,
fallback: str,
session_id: str | None,
transcript: str | None,
*,
accept_result: bool = True,
budget_weighted: int | None = None,
stop_seen: bool = False,
) -> SessionResult:
"""Session is gone or done responding: completed if the result file
landed anyway, otherwise the fallback status. ``accept_result=False``
(a stall verdict reached under a live window) pins the fallback: an
artifact that appeared without a Stop or window death is not trusted.
``budget_weighted`` (a tripped session-budget guard's sample) rides
every exit so the engine can journal it whatever the verdict.
``stop_seen`` is the proof-of-work hook signal, threaded separately from
``session_id``/``transcript`` because those are also set by a mere launch."""
result_json = self._result_json(handle, spec, wait=False) if accept_result else None
if (
result_json is not None
and self._READBACK_NEEDS_PROOF_OF_WORK
and not self._produced_work(handle, stop_seen)
):
# Proof-of-work gate (#261): this session is gone and produced no
# observable output at all — no turn ever ended AND its pane log never
# grew. A read-back artifact is then not evidence THIS session finished;
# it is evidence that SOMETHING wrote a qualifying file in a directory we
# share. Keep the fallback verdict rather than upgrade a dead-on-arrival
# session to `completed`.
self._note_lifecycle(
handle.task_id,
"readback-refused-no-proof-of-work",
fallback=fallback,
spec=str(result_json.get("spec_file", "")),
status=str(result_json.get("status", "")),
)
result_json = None
status = "completed" if result_json is not None else fallback
# Diagnose the crash verdict only (#489) — see `_session_vanished`. A
# read-back upgrade to `completed` is deliberately not diagnosed: a
# session reaped AFTER flushing its result did produce something, and the
# verdict it earned is the honest one. `crashed` also covers the
# `SessionEnd` arm of `GenericAdapter.run()`, where the CLI announced
# its own exit rather than the window dying — the label stays truthful
# there because it reports what the mux answered, not how the window
# ended.
vanished = status == "crashed" and self._session_vanished()
if vanished:
# Evidence rides along like every neighbouring crumb: which session
# went missing (several runs share a host) and what verdict it lands.
# getattr because the mixin does not declare `session_name` (opencode-
# http has none) and only a mux-backed adapter can reach this branch
# (the base `_session_vanished` is a constant False). No default — an
# override on an adapter without a session name must fail loud here,
# not write evidence-free crumbs.
self._note_lifecycle(
handle.task_id,
"session-vanished",
session=getattr(self, "session_name"),
status=status,
)
return SessionResult(
status=status,
result_json=result_json,
session_id=session_id,
transcript_path=transcript,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
session_vanished=vanished,
)
def _result_path(self, task_id: str) -> Path:
return self.tasks_dir / task_id / "result.json"
def _append_diag_jsonl(self, task_id: str, filename: str, payload: dict) -> None:
"""Append ``payload`` as one JSON line to ``tasks/<task_id>/<filename>``.
Pure observability, best-effort: an unwritable run dir must never break
the completion loop. ``ensure_ascii=False`` is why the guard names more
than OSError: it leaves a lone surrogate — what a POSIX filename holding
a non-UTF-8 byte becomes, surrogate-escaped — in the dumped str, which
then hits the UTF-8 encode inside ``fh.write`` as a UnicodeEncodeError.
That is a ValueError, not an OSError; ``UnicodeError`` covers it and the
decode direction both (#380)."""
try:
path = self.tasks_dir / task_id / filename
path.parent.mkdir(parents=True, exist_ok=True)
line = json.dumps(payload, ensure_ascii=False)
with path.open("a", encoding="utf-8") as fh:
fh.write(line + "\n")
except (OSError, UnicodeError):
pass
def _note_resultless_stop(self, task_id: str, verdict: str, detail: str = "") -> None:
"""Append a diagnostic breadcrumb when a Stop's artifact read-back gives
up empty: one JSON line ({ts, verdict, detail}) in
``tasks/<task_id>/resultless-stops.jsonl`` — the #149 nudge livelock
was undiagnosable because nothing recorded *why* each Stop read as
result-less."""
self._append_diag_jsonl(
task_id,
"resultless-stops.jsonl",
{"ts": time.time_ns(), "verdict": verdict, "detail": detail},
)
def _note_lifecycle(self, task_id: str, event: str, **fields) -> None:
"""Append a session-lifecycle breadcrumb ({ts, event, ...}) to
``tasks/<task_id>/session-lifecycle.jsonl`` — issue #157's timeout fired
with zero record of *when* the adapter declared it or which clock had
elapsed, so a 2h19 journaling gap was unattributable."""
self._append_diag_jsonl(
task_id,
"session-lifecycle.jsonl",
{"ts": time.time_ns(), "event": event, **fields},
)
def _write_heartbeat(self, task_id: str, payload: dict) -> None:
"""Best-effort overwrite of ``tasks/<task_id>/heartbeat.json``: the wait
loop's proof-of-life. A heartbeat much staler than HEARTBEAT_INTERVAL_S
under a still-running session means the orchestrator itself was frozen
(host starvation, macOS sleep — #157), not the CLI."""
try:
(self.tasks_dir / task_id / "heartbeat.json").write_text(
json.dumps(payload, ensure_ascii=False), encoding="utf-8"
)
except OSError:
pass
def _read_result(self, task_id: str) -> dict | None:
path = self._result_path(task_id)
if not path.is_file():
return None
try:
data = json.loads(path.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError):
return None
return data if isinstance(data, dict) else None
def _await_result(self, task_id: str, grace_s: float = RESULT_GRACE_S) -> dict | None:
deadline = time.monotonic() + grace_s
while True:
result = self._read_result(task_id)
if result is not None:
return result
if time.monotonic() >= deadline:
self._note_resultless_stop(
task_id, "no-result-json", f"no readable {self._result_path(task_id)}"
)
return None
time.sleep(RESULT_POLL_S)
class GenericAdapter(_ResultFileMixin, EnvFaultMixin, CodingCLIAdapter):
injection = "tmux-initial-prompt"
observation = "hook-signal"
state = "local-jsonl"
def __init__(
self,
run_dir: Path,
policy: Policy,
profile: CLIProfile,
binary: str | None = None,
extra_args: tuple[str, ...] | None = None,
usage_grace_s: float | None = None,
stop_without_result_nudges: int | None = None,
mux: TerminalMultiplexer | None = None,
events_dir: Path | None = None,
):
self.run_dir = run_dir
self.policy = policy
self.profile = profile
# env-fault patterns compile lazily off self.profile — see EnvFaultMixin.
self.mux = mux or get_multiplexer()
# None = use the profile's default bypass flags; a tuple replaces them
self.extra_args = extra_args
# Effective timing knobs: an explicit [adapter]/[adapter.<stage>] override
# wins, else the CLI profile's shipped default, else the global fallback.
self._usage_grace_s = usage_grace_s if usage_grace_s is not None else profile.usage_grace_s
self._stop_nudges = (
stop_without_result_nudges
if stop_without_result_nudges is not None
else (
profile.stop_without_result_nudges
if profile.stop_without_result_nudges is not None
else policy.limits.stop_without_result_nudges
)
)
# Grace for a result-less Stop before declaring a stall. 0 (base default)
# keeps the fail-fast behavior; the dev adapter raises it so a session
# that ended its turn awaiting a background process isn't mis-stalled.
self._stall_grace_s = 0.0
# Wake-nudges to spend on grace expiry before stalling. 0 here is moot for
# the base adapter (grace 0 never opens the window); the dev adapter sets
# it from policy so an idle wait is re-invoked rather than killed outright.
self._stall_nudges = 0
self.name = f"{profile.name}-tmux"
self.binary = binary or profile.binary
self.session_name = f"bmad-loop-{run_dir.name}"
# The run's hook-event channel (#494): the out-of-tree directory the run
# bootstrap resolved, plus the legacy in-tree one kept under poll so a
# project whose installed relay predates the move still completes its
# sessions. `events_dir` is handed in rather than derived here because
# deriving it needs the PROJECT, and the only project this class can
# reach is `run_dir.parents[2]` — a shape real run dirs have and test run
# dirs do not, so a derivation would key the watcher off a directory that
# is not the project (see `_ensure_session`, which accepts exactly that
# weakness for a session tag but must not for the completion channel).
# Defaulting to the legacy dir keeps direct construction (tests, any
# caller outside `runsetup.make_adapters`) working unchanged; the
# bootstrap always passes one, pinned by a test.
self.watcher = SignalWatcher(events_dir or run_dir / "events", run_dir / "events")
self.tasks_dir = run_dir / "tasks"
self.logs_dir = run_dir / LOGS_DIR
self.tasks_dir.mkdir(parents=True, exist_ok=True)
self.logs_dir.mkdir(parents=True, exist_ok=True)
# --------------------------------------------------------- multiplexer
def _ensure_session(self, cwd: Path) -> None:
if not self.mux.has_session(self.session_name):
self.mux.new_session(self.session_name, cwd, PANE_COLUMNS, PANE_LINES)
# Tag the session with its project so a cleanup in another project
# never prunes this run (run_dir = <project>/.bmad-loop/runs/<id>).
project = self.run_dir.parents[2]
self.mux.set_session_option(
self.session_name, runs.PROJECT_OPTION, runs.project_tag(project)
)
def interactive_argv(self, spec: SessionSpec) -> list[str]:
extra = self.extra_args
if extra is None:
extra = self.profile.bypass_args
argv = [
self.binary,
*self.profile.launch_args,
self.profile.render_prompt(spec.prompt),
*extra,
]
if spec.model:
argv += [self.profile.model_flag, spec.model]
return argv
def interactive_env(self, spec: SessionSpec) -> dict[str, str]:
return {**self.profile.env, **spec.env}
def build_command(self, spec: SessionSpec) -> str:
return " ".join(shlex.quote(a) for a in self.interactive_argv(spec))
# --------------------------------------------------------------- adapter
def start_session(self, spec: SessionSpec) -> SessionHandle:
task_dir = self.tasks_dir / spec.task_id
task_dir.mkdir(parents=True, exist_ok=True)
(task_dir / "prompt.txt").write_text(spec.prompt + "\n", encoding="utf-8")
# A re-armed/resumed run reuses task_ids; drop any prior cycle's result
# so a session that writes nothing can't be read as a stale completion.
(task_dir / "result.json").unlink(missing_ok=True)
self._ensure_session(spec.cwd)
# Stamped before launch: hook events carry wall-clock ns, and
# wait_for_completion ignores anything older than this floor so a reused
# task_id's earlier Stop event cannot replay.
launched_ns = time.time_ns()
log_file = self.logs_dir / f"{spec.task_id}.log"
# A re-armed run reuses task_ids and both mux backends append; drop the prior
# cycle's tee so the #194 tail scan can't match a stale transport error (mirrors
# the result.json unlink above; journal.py already assumes "next session replaces it").
log_file.unlink(missing_ok=True)
# ...then create it EMPTY, before the window exists. `pipe_pane` below tolerates
# a window that already died and then attaches no tee, so without this a
# dead-on-arrival session leaves NO log at all — and an absent log is the
# `_log_evidence` "this transport has no pane signal" state, which the #261
# proof-of-work gate treats as inert. The gate would fail OPEN in exactly the
# case it exists to catch. A 0-byte file says something truer and stronger:
# this transport does tee a pane, and this session rendered nothing into it.
# (Both backends append, so pre-creating cannot truncate a live tee. Stall
# detection is unaffected: `_log_activity_key` reports (mtime, 0) instead of
# None, and every reader compares signatures rather than testing existence.)
log_file.touch()
window_id = self.mux.new_window(
self.session_name,
spec.task_id[-40:],
spec.cwd,
{**self.profile.env, **spec.env},
self.build_command(spec),
)
# pipe_pane tolerates the window having already died (a CLI that crashes on
# launch can take it down before the tee attaches); the dead window is then
# reported as a crash in wait_for_completion.
self.mux.pipe_pane(window_id, log_file)
return SessionHandle(task_id=spec.task_id, native_id=window_id, launched_ns=launched_ns)
def wait_for_completion(self, handle: SessionHandle, spec: SessionSpec) -> SessionResult:
deadline = time.monotonic() + spec.timeout_s
# Wall-clock co-bound (#157): a host suspend freezes time.monotonic(),
# silently extending the monotonic deadline by the nap's length. The
# wall clock keeps counting through a suspend, so it may EXPIRE the
# deadline — never extend it; all sub-waits below stay monotonic (a
# wall clock stepped backward must not stretch the session).
wall_deadline = time.time() + spec.timeout_s
session_id: str | None = None
transcript_path: str | None = None
nudges_left = self._stop_nudges
# Positive grace arms at launch for dev/review sessions, so a CLI that
# goes silent before its first Stop cannot burn the full wall timeout. A
# fresh Stop or later pane growth re-arms it; None = grace disabled.
stall_deadline = time.monotonic() + self._stall_grace_s if self._stall_grace_s > 0 else None
# pane-log activity signature captured when the grace window is armed; a
# session streaming output (a long productive turn, a streaming subagent)
# advances it and re-arms the window, so only genuine silence stalls.
last_activity = (
self._log_activity_key(handle.task_id) if stall_deadline is not None else None
)
# wake-nudges left to spend when the grace window elapses in silence: the
# session likely ended its turn awaiting a background process, so we prod
# it (bmad-loop has no background re-invocation) instead of stalling. A
# fresh Stop — proof it woke and acted — restores the budget; only an
# unresponsive session burns through it. Bounded overall by spec.timeout_s.
stall_nudges_left = self._stall_nudges
# monotonic total of stall nudges sent this session — never restored,
# unlike stall_nudges_left. When spec.stall_nudges_cap is set (the
# engine sets it for every session it drives), a session that keeps
# ending its turn without a result cannot ride the fresh-Stop refill
# forever: after cap total nudges it is declared stalled. cap=None
# (raw constructor default) skips the check.
stall_nudges_sent = 0
# latched on the first accepted `Stop`: the hook half of the #261 proof-of-work
# gate. Tracked apart from session_id/transcript_path — those are populated by
# SessionStart and SessionEnd too, which a CLI that launched and wedged emits
# without doing any work. Rides out on every exit (see SessionResult.stop_seen)
# so `_post_kill_reconcile` reads the same signal after run() kills the window.
stop_seen = False
# internal observability counter: counts ticks where the liveness probe
# raised a transport error (e.g. a 30s tmux hang). It deliberately does
# NOT escalate to "crashed" — a transient transport hiccup is not proof
# of death; spec.timeout_s already bounds a persistent failure to a
# timeout.
probe_failures = 0
# monotonic ts of the last heartbeat.json overwrite; None = not yet
# written, so the first tick always stamps one.
last_heartbeat: float | None = None
# Session-budget guard (#158): latched on the first cap crossing — the
# warn/nudge fires at most once per session. budget_deadline is the
# enforce-mode monotonic grace expiry (None = not armed); checked every
# tick, unlike the heartbeat-throttled sampling that arms it. The wall
# deadline is the #157 co-bound: a host suspend freezes
# time.monotonic(), silently stretching the "bounded" wrap-up window,
# so the wall clock may EXPIRE the grace — never extend it.
budget_tripped = False
budget_weighted: int | None = None
budget_deadline: float | None = None
budget_wall_deadline: float | None = None
while True:
remaining = deadline - time.monotonic()
wall_expired = time.time() >= wall_deadline
if remaining <= 0 or wall_expired:
if remaining <= 0 and wall_expired:
expired = "both"
elif remaining <= 0:
expired = "monotonic"
else:
# wall-only expiry with monotonic time to spare: the
# monotonic clock stood still — the suspend signature.
expired = "wall"
self._note_lifecycle(
handle.task_id,
"timeout-fired",
expired_clock=expired,
timeout_s=spec.timeout_s,
mono_remaining_s=round(remaining, 3),
)
return SessionResult(
status="timeout",
session_id=session_id,
transcript_path=transcript_path,
timeout_fired_at=time.time(),
timeout_expired_clock=expired,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
# Hard-stop poll (#319), per-iteration and deliberately NOT inside
# the heartbeat throttle below: the loop's own wait is capped at 5s
# (`watcher.wait_for(..., timeout_s=min(remaining, 5.0))`), so a stop
# normally lands well inside `stop_run`'s 10s grace window, while riding
# the 30s HEARTBEAT_INTERVAL_S would be worse than the status quo. Read
# that as the common case, not a bound: an iteration that goes on to
# wait RESULT_GRACE_S for an artifact, or to block on a tmux call under
# TMUX_TIMEOUT_S, exceeds the grace window on its own. See the second
# poll after the wait below for how the interval is split, and why it
# still cannot be made unconditionally short. Return the verdict — never raise `RunStopped` here: that would
# skip `run()`'s finally-kill + `_post_kill_reconcile`. The file is
# left on disk for the engine to consume and attribute the stop.
if self._hard_stop_requested():
self._note_lifecycle(handle.task_id, "stop-abort-fired")
return SessionResult(
status="aborted",
session_id=session_id,
transcript_path=transcript_path,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
now = time.monotonic()
if last_heartbeat is None or now - last_heartbeat >= HEARTBEAT_INTERVAL_S:
last_heartbeat = now
self._write_heartbeat(
handle.task_id,
{
"ts": time.time(),
"remaining_s": round(remaining, 3),
"stall_armed": stall_deadline is not None,
"stall_nudges_sent": stall_nudges_sent,
},
)
# Mid-session spec-status transition sampling (#276 M2) rides the
# same heartbeat cadence — a no-op unless this adapter drives the
# generic skill and the engine threaded a launch snapshot.
self._observe_tick(handle, spec)
# Budget sampling rides the heartbeat cadence — no extra knob.
# transcript_path is unknown until the first hook event carries
# it (SessionStart for claude); until then the guard is inert.
if (
not budget_tripped
and spec.token_budget is not None
and spec.token_budget_mode in ("warn", "enforce")
and transcript_path
):
weighted = self._sample_weighted_usage(transcript_path, spec)
if weighted is not None and weighted > spec.token_budget:
budget_tripped = True
budget_weighted = weighted
self._note_lifecycle(
handle.task_id,
"budget-tripped",
weighted=weighted,
budget=spec.token_budget,
mode=spec.token_budget_mode,
)
try:
gates.notify(
self.policy,
self.run_dir,
"bmad-loop session over token budget",
f"{handle.task_id}: weighted spend {weighted} crossed the "
f"{spec.token_budget} per-session cap "
f"(mode={spec.token_budget_mode})",
)
except OSError:
# observe-degrade: an unwritable ATTENTION file is
# observability, never a reason to break the loop
# (the _write_heartbeat doctrine).
pass
# nosec below: bandit B105 pattern-matches the "token"
# in token_budget_mode as a hardcoded-password compare;
# it is a mode enum, not a credential.
if spec.token_budget_mode == "enforce": # nosec B105
if spec.token_budget_grace_s <= 0:
# zero grace = terminate at trip, no nudge — but
# window death still wins (artifact honored via
# the crash path), exactly like grace expiry; a
# transport error is not proof of death.
try:
if not self._window_alive(handle):
return self._final(
handle,
spec,
"crashed",
session_id,
transcript_path,
budget_weighted=weighted,
stop_seen=stop_seen,
)
except MultiplexerError:
pass
self._note_lifecycle(
handle.task_id,
"over-budget-fired",
weighted=weighted,
budget=spec.token_budget,
grace_s=spec.token_budget_grace_s,
zero_grace=True,
)
return SessionResult(
status="over_budget",
session_id=session_id,
transcript_path=transcript_path,
budget_weighted=weighted,
stop_seen=stop_seen,
)
try:
self.send_text(handle, BUDGET_NUDGE_TEXT)
except MultiplexerError:
# a dead/hung window can't take the nudge; the
# grace still arms — the next tick's liveness
# probe scores a dead window crashed.
pass
budget_deadline = time.monotonic() + spec.token_budget_grace_s
budget_wall_deadline = time.time() + spec.token_budget_grace_s
if budget_deadline is not None and (
time.monotonic() >= budget_deadline
or (budget_wall_deadline is not None and time.time() >= budget_wall_deadline)
):
# Grace expired with no completion (wall co-bound included: a
# suspend-frozen monotonic clock must not stretch the window,
# #157). Window death is authoritative (its artifact is honored
# via the crash path); under a live window the session ends
# over_budget WITHOUT reading the result file — an artifact
# under a live window is never trusted (#48/#53). A transport
# error is not proof of death, so it falls through to the
# over_budget verdict.
try:
if not self._window_alive(handle):
return self._final(
handle,
spec,
"crashed",
session_id,
transcript_path,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
except MultiplexerError:
pass
self._note_lifecycle(
handle.task_id,
"over-budget-fired",
weighted=budget_weighted,
budget=spec.token_budget,
grace_s=spec.token_budget_grace_s,
zero_grace=False,
)
return SessionResult(
status="over_budget",
session_id=session_id,
transcript_path=transcript_path,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
event = self.watcher.wait_for(
handle.task_id,
EVENT_KINDS,
timeout_s=min(remaining, 5.0),
since_ns=handle.launched_ns,
)
# Second poll, and the reason there are two (#319). The arm at the top of
# the loop is separated from its next run by everything between: the 5s
# wait above, plus whichever dispatch leg the event selects — a
# `_window_alive` or `send_text` bounded only by TMUX_TIMEOUT_S (30s), or
# a `_result_json(wait=True)` that waits RESULT_GRACE_S (15s) for an
# artifact. The last of those alone outlasts `stop_run`'s 10s grace on a
# perfectly healthy box, with no transport fault anywhere. Polling here
# splits the iteration so at most one leg sits between two checks. It
# cannot make the interval unconditionally short — an in-flight
# subprocess is not interruptible from this thread — so a leg that does
# outlast the window still degrades to `stop_run`'s force-kill backstop:
# the pre-#319 outcome, never a worse one.
if self._hard_stop_requested():
self._note_lifecycle(handle.task_id, "stop-abort-fired")
return SessionResult(
status="aborted",
session_id=session_id,
transcript_path=transcript_path,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
if event is None:
try:
alive = self._window_alive(handle)
except MultiplexerError:
# transport hiccup (e.g. a 30s tmux hang), not proof of
# death: never roll back a possibly-working session. Skip the
# crash check this tick; hook events still complete it, and
# spec.timeout_s bounds a persistent transport failure to an
# honest "timeout".
probe_failures += 1
continue
probe_failures = 0
if not alive:
# died without a SessionEnd hook (killed, crashed hard)
return self._final(
handle,
spec,
"crashed",
session_id,
transcript_path,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
if stall_deadline is not None:
# No artifact shortcut here: the window is alive on this tick
# (a dead one returned "crashed" above), and a terminal
# artifact under a live window is advisory only — the agent
# may still be mid-turn (or the artifact stale from a prior
# drive), and run()'s finally-kill would terminate it before
# its remaining work flushes. Only a Stop event or window
# death completes the session.
# The grace window measures inactivity, not time-since-Stop:
# a session still streaming to the tee'd pane log (a long
# productive turn building a diff, a streaming subagent) is
# working, not stalled. Re-arm on any pane growth so only
# genuine silence for the full grace trips the stall below.
key = self._log_activity_key(handle.task_id)
if key is not None and key != last_activity:
last_activity = key
stall_deadline = time.monotonic() + self._stall_grace_s
continue
if stall_deadline is not None and time.monotonic() >= stall_deadline:
if stall_nudges_left > 0 and (
spec.stall_nudges_cap is None or stall_nudges_sent < spec.stall_nudges_cap
):
# The wake nudge IS the re-invocation bmad-loop otherwise
# lacks: prod the idle session and re-arm. Budget is
# restored only by a fresh Stop (a real turn-end), so the
# nudge's own echoed keystrokes can't be mistaken for the
# agent waking; an unresponsive session keeps draining it.
stall_nudges_left -= 1
stall_nudges_sent += 1
try:
self.send_text(handle, STALL_NUDGE_TEXT)
except MultiplexerError:
# A dead/hung window cannot take the nudge. The
# bounded attempt is still spent, and the next tick's
# ordinary liveness probe owns the verdict.
pass
stall_deadline = time.monotonic() + self._stall_grace_s
last_activity = self._log_activity_key(handle.task_id)
continue
# Re-probe liveness before finalizing: this return exits the
# loop, so a hard death (no SessionEnd) in the gap since the
# top-of-tick probe would otherwise never be caught. Window
# death is authoritative — a now-dead window flows through the
# crash path (which honors its artifact via accept_result=True)
# instead of a stall that discards a just-flushed result. A
# transport error is not proof of death (as at the top of the
# tick); fall through to the stall — spec.timeout_s bounds a
# persistent failure.
try:
if not self._window_alive(handle):
return self._final(
handle,
spec,
"crashed",
session_id,
transcript_path,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
except MultiplexerError:
pass
# Still alive: an artifact on disk cannot upgrade the stall to
# completed — it may be stale or mid-write; only a Stop or
# window death vouches for it.
return self._final(
handle,
spec,
"stalled",
session_id,
transcript_path,
accept_result=False,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
continue
if (
event.event == "Stop"
and self.profile.subagent_stop_without_transcript
and not event.transcript_path
):
# Copilot fires agentStop for each subagent turn with an empty
# transcriptPath and a tool-use session id; that is not the main
# session's turn-end. Ignore it (before accumulating the junk
# session id) so a subagent's premature Stop is not read as a
# result-less completion -> false stall, and the main session's
# real transcript is preserved for usage tallying.
continue
session_id = event.session_id or session_id
transcript_path = event.transcript_path or transcript_path
if event.event == "SessionStart":
continue
if event.event == "Stop":
# A turn ENDED — the one canonical event that proves the CLI did
# something, and so the hook half of the #261 proof-of-work gate.
# Latched after the subagent filter above, which rejects a stop that
# is not the main session's turn-end. Never cleared.
stop_seen = True
result_json = self._result_json(handle, spec, wait=True)
if result_json is not None:
return SessionResult(
status="completed",
result_json=result_json,
session_id=session_id,
transcript_path=transcript_path,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
if nudges_left > 0:
nudges_left -= 1
try:
self.send_text(handle, NUDGE_TEXT)
except MultiplexerError:
# The next deterministic liveness probe decides whether
# the un-nudgeable window is dead or merely unavailable.
pass
continue
if self._stall_grace_s <= 0:
return self._final(
handle,
spec,
"stalled",
session_id,
transcript_path,
budget_weighted=budget_weighted,
stop_seen=stop_seen,
)
# A result-less Stop, but the session may have ended its turn to