Repository navigation
Expand file tree
/
Copy pathindex.html
More file actions
3320 lines (3097 loc) · 375 KB
/
Copy pathindex.html
File metadata and controls
3320 lines (3097 loc) · 375 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!doctype html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>madengine — Codebase Wiki</title>
<style>
:root{
--bg:#0e1116;--bg2:#161b22;--bg3:#1f2630;--fg:#e6edf3;--mut:#8b949e;
--acc:#ff6a00;--acc2:#58a6ff;--ok:#3fb950;--warn:#d29922;--err:#f85149;
--layer-cli:#9d7cff;--layer-orch:#58a6ff;--layer-dep:#3fb950;
--layer-exec:#d29922;--layer-core:#f778ba;--layer-util:#79c0ff;
--layer-rep:#ff7b72;--border:#30363d;
--mono:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace;
}
*{box-sizing:border-box}
html,body{margin:0;background:var(--bg);color:var(--fg);font:14px/1.6 -apple-system,BlinkMacSystemFont,"Segoe UI",sans-serif}
a{color:var(--acc2);text-decoration:none}
a:hover{text-decoration:underline}
code,pre{font-family:var(--mono);font-size:.88em}
code{background:var(--bg3);padding:1px 5px;border-radius:4px;color:#ffd596}
pre{background:#010409;border:1px solid var(--border);border-radius:8px;padding:14px 16px;overflow:auto;color:#e6edf3;line-height:1.5}
pre code{background:none;padding:0;color:inherit}
/* layout */
.app{display:grid;grid-template-columns:290px 1fr;min-height:100vh}
nav.side{position:sticky;top:0;align-self:start;height:100vh;overflow:auto;
border-right:1px solid var(--border);background:var(--bg2);padding:18px 10px 40px;scrollbar-width:thin}
nav.side .brand{display:flex;align-items:center;gap:10px;margin:0 6px 14px;font-weight:700}
nav.side .brand .logo{width:30px;height:30px;border-radius:6px;background:linear-gradient(135deg,var(--acc),var(--acc2));display:grid;place-items:center;color:#111;font-weight:900;font-size:16px;flex:none}
nav.side .v{color:var(--mut);font-weight:400;font-size:.82em;margin-left:4px}
nav.side input{width:100%;background:#010409;color:var(--fg);border:1px solid var(--border);border-radius:6px;padding:7px 9px;margin:0 0 10px;font-size:.9em}
nav.side .grp{margin:16px 6px 4px;font-size:.68em;letter-spacing:.1em;text-transform:uppercase;color:var(--mut);font-weight:600}
nav.side a{display:flex;align-items:center;gap:8px;padding:4px 9px;border-radius:5px;color:#c9d1d9;font-size:.9em}
nav.side a:hover{background:#21262d;text-decoration:none}
nav.side a.active{background:#1f2937;color:#fff;border-left:2px solid var(--acc);padding-left:7px}
nav.side a .dot{width:7px;height:7px;border-radius:50%;flex:none}
main{padding:32px 44px 100px;max-width:1200px}
h1,h2,h3,h4{line-height:1.25;color:#fff}
h1{font-size:2.05em;margin:.2em 0 .1em}
h2{font-size:1.5em;margin:2.4em 0 .7em;border-bottom:1px solid var(--border);padding-bottom:6px}
h3{font-size:1.15em;margin:1.6em 0 .5em;color:#fafafa}
h4{font-size:.96em;margin:1.2em 0 .4em;color:#d0d7de;font-weight:600}
.sub{color:var(--mut)}
.pill{display:inline-block;padding:2px 8px;border-radius:999px;font-size:.72em;font-weight:600;background:#21262d;color:#c9d1d9;vertical-align:middle;white-space:nowrap}
.pill.cli{background:rgba(157,124,255,.15);color:var(--layer-cli)}
.pill.orch{background:rgba(88,166,255,.15);color:var(--layer-orch)}
.pill.dep{background:rgba(63,185,80,.15);color:var(--layer-dep)}
.pill.exec{background:rgba(210,153,34,.15);color:var(--layer-exec)}
.pill.core{background:rgba(247,120,186,.15);color:var(--layer-core)}
.pill.util{background:rgba(121,192,255,.15);color:var(--layer-util)}
.pill.rep{background:rgba(255,123,114,.15);color:var(--layer-rep)}
.pill.ok{background:rgba(63,185,80,.18);color:var(--ok)}
.pill.warn{background:rgba(210,153,34,.18);color:var(--warn)}
.pill.err{background:rgba(248,81,73,.18);color:var(--err)}
.pill.new{background:linear-gradient(90deg,#ff6a00,#ff3d83);color:#fff}
.grid{display:grid;gap:14px}
.cols-2{grid-template-columns:1fr 1fr}
.cols-3{grid-template-columns:repeat(3,1fr)}
@media(max-width:1000px){.cols-2,.cols-3{grid-template-columns:1fr}.app{grid-template-columns:1fr}nav.side{position:relative;height:auto}}
.card{background:var(--bg2);border:1px solid var(--border);border-radius:10px;padding:16px 18px}
.card h3,.card h4{margin-top:0}
.card p:last-child,.card ul:last-child,.card ol:last-child{margin-bottom:0}
.hero{background:radial-gradient(1100px 360px at 20% -10%,rgba(255,106,0,.18),transparent 60%),
radial-gradient(900px 320px at 90% 0%,rgba(88,166,255,.15),transparent 60%),
linear-gradient(180deg,#161b22,#0e1116);
border:1px solid var(--border);border-radius:14px;padding:30px 32px;margin-bottom:24px}
.hero h1{margin:0 0 8px;font-size:2.3em;background:linear-gradient(90deg,#fff,#ffaa55);-webkit-background-clip:text;background-clip:text;color:transparent}
.hero p{color:#c9d1d9;max-width:820px;margin:.6em 0}
.meta{display:flex;flex-wrap:wrap;gap:8px;margin-top:14px}
table{border-collapse:collapse;width:100%;margin:8px 0 16px;font-size:.9em}
th,td{border:1px solid var(--border);padding:7px 10px;text-align:left;vertical-align:top}
th{background:#161b22;color:#fff;font-weight:600}
tr:nth-child(even) td{background:#10151c}
td code{font-size:.85em}
details{background:var(--bg2);border:1px solid var(--border);border-radius:8px;padding:6px 14px;margin:8px 0}
details>summary{cursor:pointer;font-weight:600;padding:6px 0;user-select:none}
details[open]{padding-bottom:12px}
details[open]>summary{margin-bottom:8px}
.filepath{font-family:var(--mono);color:#79c0ff;font-size:.86em}
.tabs{display:flex;gap:2px;border-bottom:1px solid var(--border);margin:10px 0 0;flex-wrap:wrap}
.tabs button{background:transparent;border:0;color:var(--mut);padding:8px 14px;cursor:pointer;font-size:.9em;border-bottom:2px solid transparent;white-space:nowrap}
.tabs button.on{color:#fff;border-bottom-color:var(--acc)}
.tabpanel{display:none;padding-top:10px}
.tabpanel.on{display:block}
.legend{display:flex;flex-wrap:wrap;gap:14px;font-size:.85em;color:var(--mut);margin:6px 0 12px}
.legend span{display:inline-flex;align-items:center;gap:6px}
.legend .dot{width:10px;height:10px;border-radius:50%;flex:none}
.callout{border-left:3px solid var(--acc);background:rgba(255,106,0,.06);padding:10px 14px;border-radius:0 8px 8px 0;margin:12px 0}
.callout.info{border-color:var(--acc2);background:rgba(88,166,255,.06)}
.callout.ok{border-color:var(--ok);background:rgba(63,185,80,.06)}
.callout.warn{border-color:var(--warn);background:rgba(210,153,34,.06)}
.callout.err{border-color:var(--err);background:rgba(248,81,73,.06)}
.diag{background:#0a0d12;border:1px solid var(--border);border-radius:10px;padding:18px;overflow:auto}
.diag svg{display:block;min-width:720px;height:auto}
.footer{margin-top:70px;padding-top:18px;border-top:1px solid var(--border);color:var(--mut);font-size:.84em}
#modfilter,#ctxfilter,#envfilter{width:100%;padding:8px 10px;border-radius:6px;background:#010409;color:var(--fg);border:1px solid var(--border);margin-bottom:10px;font-size:.9em}
.tag-amd{color:#ff6a00}.tag-nv{color:#76b900}
@media print{nav.side{display:none}.app{grid-template-columns:1fr}main{padding:0}}
ul,ol{padding-left:22px}
li{margin:.2em 0}
/* accessibility & polish */
.skip{position:absolute;left:-999px;top:0;background:var(--acc);color:#111;padding:8px 12px;z-index:10;border-radius:0 0 6px 0}
.skip:focus{left:0}
:focus-visible{outline:2px solid var(--acc2);outline-offset:2px}
.tabs button:focus-visible{outline-offset:-2px}
html{scroll-behavior:smooth;scroll-padding-top:12px}
@media(prefers-reduced-motion:reduce){html{scroll-behavior:auto}}
nav.side a[aria-current="true"]{background:#1f2937;color:#fff;border-left:2px solid var(--acc);padding-left:7px}
.table-wrap{overflow-x:auto}
@media print{.tabpanel{display:block!important;page-break-inside:avoid}.tabs{display:none}details{display:block}}
</style>
</head>
<body>
<a class="skip" href="#main">Skip to content</a>
<div class="app">
<nav class="side" aria-label="Wiki sections">
<div class="brand"><div class="logo" aria-hidden="true">M</div>madengine<span class="v">v2.4.0+</span></div>
<input id="navfilter" type="search" placeholder="Filter sections…" aria-label="Filter sections" autocomplete="off">
<div class="grp">Start</div>
<a href="#overview"><span class="dot" style="background:var(--acc)" aria-hidden="true"></span>Overview</a>
<a href="#quickstart"><span class="dot" style="background:var(--acc2)" aria-hidden="true"></span>Quick start</a>
<a href="#install"><span class="dot" style="background:var(--mut)" aria-hidden="true"></span>Install & dev</a>
<div class="grp">Architecture</div>
<a href="#layers"><span class="dot" style="background:var(--layer-orch)" aria-hidden="true"></span>Layers</a>
<a href="#diagram"><span class="dot" style="background:var(--layer-cli)" aria-hidden="true"></span>Diagram</a>
<a href="#flows"><span class="dot" style="background:var(--layer-dep)" aria-hidden="true"></span>Data flows</a>
<div class="grp">CLI</div>
<a href="#cli-overview"><span class="dot" style="background:var(--layer-cli)" aria-hidden="true"></span>Commands</a>
<a href="#cli-discover"><span class="dot" style="background:var(--layer-cli)" aria-hidden="true"></span>discover</a>
<a href="#cli-build"><span class="dot" style="background:var(--layer-cli)" aria-hidden="true"></span>build</a>
<a href="#cli-run"><span class="dot" style="background:var(--layer-cli)" aria-hidden="true"></span>run</a>
<a href="#cli-report"><span class="dot" style="background:var(--layer-cli)" aria-hidden="true"></span>report / database</a>
<a href="#cli-ops"><span class="dot" style="background:var(--layer-cli)" aria-hidden="true"></span>cleanup / setup-page</a>
<a href="#config-frontend"><span class="dot" style="background:var(--layer-cli)" aria-hidden="true"></span>--config (Hydra)</a>
<a href="#exitcodes"><span class="dot" style="background:var(--err)" aria-hidden="true"></span>Exit codes</a>
<div class="grp">Configuration</div>
<a href="#context"><span class="dot" style="background:var(--layer-core)" aria-hidden="true"></span>additional_context</a>
<a href="#model-def"><span class="dot" style="background:var(--layer-util)" aria-hidden="true"></span>Model definition</a>
<a href="#manifest"><span class="dot" style="background:var(--layer-orch)" aria-hidden="true"></span>Build manifest</a>
<div class="grp">Execution</div>
<a href="#platforms"><span class="dot" style="background:var(--layer-exec)" aria-hidden="true"></span>Platforms</a>
<a href="#lifecycle"><span class="dot" style="background:var(--layer-exec)" aria-hidden="true"></span>Container lifecycle</a>
<a href="#build-context"><span class="dot" style="background:var(--layer-exec)" aria-hidden="true"></span>Docker build</a>
<div class="grp">Deployment</div>
<a href="#targets"><span class="dot" style="background:var(--layer-dep)" aria-hidden="true"></span>Target inference</a>
<a href="#slurm"><span class="dot" style="background:var(--layer-dep)" aria-hidden="true"></span>SLURM & Spur</a>
<a href="#k8s"><span class="dot" style="background:var(--layer-dep)" aria-hidden="true"></span>Kubernetes</a>
<a href="#k8s-slurm-multi"><span class="dot" style="background:var(--layer-dep)" aria-hidden="true"></span>slurm_multi on K8s</a>
<a href="#slurm-multi"><span class="dot" style="background:var(--layer-dep)" aria-hidden="true"></span>slurm_multi on SLURM</a>
<a href="#llm-d"><span class="dot" style="background:var(--layer-dep)" aria-hidden="true"></span>llm-d</a>
<div class="grp">Launchers</div>
<a href="#launchers"><span class="dot" style="background:var(--layer-util)" aria-hidden="true"></span>Launcher matrix</a>
<a href="#launcher-detail"><span class="dot" style="background:var(--layer-util)" aria-hidden="true"></span>Per-launcher config</a>
<a href="#recipes"><span class="dot" style="background:var(--ok)" aria-hidden="true"></span>Config recipes</a>
<div class="grp">Tools & GPU</div>
<a href="#profiling"><span class="dot" style="background:var(--warn)" aria-hidden="true"></span>Profiling tools</a>
<a href="#rocm"><span class="dot" style="background:var(--layer-core)" aria-hidden="true"></span>ROCm & GPU</a>
<a href="#envvars"><span class="dot" style="background:var(--layer-util)" aria-hidden="true"></span>Env variables</a>
<a href="#errors"><span class="dot" style="background:var(--err)" aria-hidden="true"></span>Errors & auth</a>
<div class="grp">Data</div>
<a href="#reporting"><span class="dot" style="background:var(--layer-rep)" aria-hidden="true"></span>Reporting</a>
<a href="#database"><span class="dot" style="background:var(--layer-rep)" aria-hidden="true"></span>Database</a>
<div class="grp">Reference</div>
<a href="#modules"><span class="dot" style="background:var(--layer-util)" aria-hidden="true"></span>Module reference</a>
<a href="#tests"><span class="dot" style="background:var(--ok)" aria-hidden="true"></span>Tests</a>
<a href="#contrib"><span class="dot" style="background:var(--mut)" aria-hidden="true"></span>Contributing</a>
<a href="#changes"><span class="dot" style="background:var(--acc)" aria-hidden="true"></span>Changelog</a>
</nav>
<main id="main">
<header class="hero">
<h1>madengine — Codebase Wiki</h1>
<p>AI/ML model automation & benchmarking platform for local Docker, bare-metal conda, SLURM (and Spur), Kubernetes, and llm-d.
A Typer-based CLI that discovers model cards, builds Docker images (or conda environments), runs them on a compute target,
and writes structured performance results.</p>
<p>Entry point: <span class="filepath">src/madengine/cli/app.py::cli_main</span>
→ console script <code>madengine</code> registered in <span class="filepath">pyproject.toml</span>.</p>
<div class="meta">
<span class="pill new">Reflects v2.4.0 + unreleased (develop)</span>
<span class="pill">Python ≥ 3.8</span>
<span class="pill cli">7 commands</span>
<span class="pill dep">Docker · bare-metal · SLURM/Spur · K8s · llm-d</span>
<span class="pill ok">Typer + Rich + Hydra</span>
<span class="pill warn">ROCm & CUDA</span>
<span class="pill">Jinja2 templates</span>
</div>
<p class="sub" style="margin-bottom:0">Version is derived from git tags at build time (<code>versioningit</code>); run <code>madengine --version</code> for the installed one.
Where this page and the code disagree, the code wins — each section names the file it was derived from.</p>
</header>
<section id="overview">
<h2>Overview</h2>
<div class="grid cols-2">
<div class="card">
<h3>What madengine does</h3>
<ol>
<li><strong>Discover</strong> — finds model cards from <code>models.json</code> or dynamic <code>get_models_json.py</code> scripts and resolves tags (<a href="#cli-discover">discover</a>).</li>
<li><strong>Build</strong> — builds a Docker image per card (or adopts a pre-built one, or prepares a conda prefix for bare-metal) and writes <code>build_manifest.json</code> (<a href="#cli-build">build</a>).</li>
<li><strong>Run</strong> — reads the manifest, infers the target from the context, runs the workload, and writes <code>perf.csv</code> (<a href="#cli-run">run</a>, <a href="#targets">targets</a>).</li>
<li><strong>Report & operate</strong> — HTML/email/TraceLens reports, MongoDB upload, orphaned-container cleanup, and a static model-picker page (<a href="#cli-report">report</a>, <a href="#cli-ops">cleanup / setup-page</a>).</li>
</ol>
<p>Build and run share one configuration spine — the <code>additional_context</code> dict — supplied as
<code>--additional-context</code>, <code>--additional-context-file</code>, or a Hydra-composed <code>--config</code>
(see <a href="#context">additional_context</a> and <a href="#config-frontend">--config</a>).</p>
</div>
<div class="card">
<h3>Execution targets at a glance</h3>
<table>
<thead><tr><th>Selected by</th><th>Target</th><th>Section</th></tr></thead>
<tbody>
<tr><td><em>(nothing below)</em></td><td>Local Docker</td><td><a href="#platforms">Platforms</a></td></tr>
<tr><td><code>"platform": "bare_metal"</code></td><td>Local conda/mamba prefix, no container</td><td><a href="#platforms">Platforms</a></td></tr>
<tr><td><code>"slurm"</code> key</td><td>SLURM sbatch (or Spur with <code>scheduler: spur</code>)</td><td><a href="#slurm">SLURM</a></td></tr>
<tr><td><code>"k8s"</code> / <code>"kubernetes"</code> key</td><td>Kubernetes Job</td><td><a href="#k8s">Kubernetes</a></td></tr>
<tr><td><code>"llm_d"</code> key</td><td>llm-d benchmark (K8s Job)</td><td><a href="#llm-d">llm-d</a></td></tr>
</tbody>
</table>
<p class="sub">Exact precedence and edge cases (including <code>slurm_multi</code>) are in <a href="#targets">Deployment target inference</a>.</p>
</div>
</div>
<h3>Where to start</h3>
<div class="grid cols-3">
<div class="card"><h4>Using madengine</h4><p><a href="#quickstart">Quick start</a> → <a href="#cli-run">run options</a> → <a href="#recipes">config recipes</a> → <a href="#exitcodes">exit codes</a> for CI.</p></div>
<div class="card"><h4>Writing a model card</h4><p><a href="#model-def">Model definition</a> → <a href="#manifest">manifest</a> → <a href="#profiling">profiling tools</a> → <a href="#reporting">reporting</a>.</p></div>
<div class="card"><h4>Changing madengine</h4><p><a href="#layers">Layers</a> → <a href="#flows">data flows</a> → <a href="#modules">module reference</a> → <a href="#tests">tests</a> → <a href="#contrib">contributing</a>.</p></div>
</div>
</section>
<section id="quickstart">
<h2>Quick start</h2>
<p class="sub">Run these from a MAD-style project directory (one that contains <code>models.json</code> and/or <code>scripts/</code>). Every flag and key below is documented in the CLI and configuration sections.</p>
<div class="tabs" data-tabs="qs">
<button class="on" data-tab="qs-local">Local Docker</button>
<button data-tab="qs-bare">Bare-metal</button>
<button data-tab="qs-slurm">SLURM</button>
<button data-tab="qs-k8s">Kubernetes</button>
<button data-tab="qs-llmd">llm-d</button>
<button data-tab="qs-file">Context file / --config</button>
</div>
<div class="tabpanel on" data-panel="qs-local">
<pre><code># 1. Install (from the madengine repo checkout)
pip install -e .
# 2. List the cards a tag selects
madengine discover --tags dummy
# 3. Build and run in one command
madengine run --tags dummy \
--additional-context '{"gpu_vendor":"AMD","guest_os":"UBUNTU"}'
# 4. Or build first (optionally push), then run from the manifest
madengine build --tags dummy --registry registry.example.com/ml
madengine run --manifest-file build_manifest.json \
--additional-context '{"docker_gpus":"0,1,2,3"}'</code></pre>
<p class="sub">No <code>slurm</code>, <code>k8s</code>, <code>kubernetes</code> or <code>llm_d</code> key and no <code>platform</code> → local Docker via <code>ContainerRunner</code>.
<code>gpu_vendor</code> / <code>guest_os</code> default to <code>AMD</code> / <code>UBUNTU</code> when omitted.</p>
</div>
<div class="tabpanel" data-panel="qs-bare">
<pre><code>madengine run --tags my_model --live-output \
--additional-context '{
"platform": "bare_metal",
"conda": {"conda_bin": "/path/to/micromamba"}
}'</code></pre>
<p class="sub">Runs inside a conda/mamba/micromamba prefix on this machine; cannot be combined with <code>slurm</code>, <code>k8s</code>, <code>kubernetes</code> or <code>llm_d</code>. See <a href="#platforms">Platforms</a>.</p>
</div>
<div class="tabpanel" data-panel="qs-slurm">
<pre><code>madengine build --tags llama3 --registry registry.example.com/ml
# Multi-node torchrun via sbatch
madengine run --manifest-file build_manifest.json \
--additional-context '{
"slurm": {"partition":"gpu","nodes":4,"gpus_per_node":8,"time":"24:00:00"},
"distributed": {"launcher":"torchrun","nnodes":4,"nproc_per_node":8}
}'
# A card that drives its own per-node containers (slurm_multi):
# skip the local build; "auto" reads DOCKER_IMAGE_NAME from the card's env_vars
madengine build --tags pyt_sglang_disagg --use-image auto</code></pre>
<p class="sub">A <code>slurm</code> key selects <code>SlurmDeployment</code>; add <code>"scheduler": "spur"</code> inside it for Spur. Launcher names are validated up front (unknown names exit 4). See <a href="#slurm">SLURM</a>, <a href="#slurm-multi">slurm_multi</a>, <a href="#launchers">launchers</a>.</p>
</div>
<div class="tabpanel" data-panel="qs-k8s">
<pre><code>madengine run --tags llama3 \
--additional-context '{"k8s":{"namespace":"ml-team","gpu_count":4}}'</code></pre>
<p class="sub">A <code>k8s</code> (or <code>kubernetes</code>) key selects <code>KubernetesDeployment</code>; preset defaults are merged under your keys. The <code>kubernetes</code> Python client is installed with madengine. See <a href="#k8s">Kubernetes</a> for the sub-key table, storage layouts and secrets.</p>
</div>
<div class="tabpanel" data-panel="qs-llmd">
<pre><code># Attach mode: benchmark an llm-d stack that already exists
madengine run --tags dummy_llm_d --additional-context '{
"k8s": {"namespace":"llm-d-bench","gpu_count":0},
"llm_d": {
"endpoint_url":"http://llm-d-inference-gateway.llm-d-bench.svc.cluster.local:80",
"model":{"name":"Qwen/Qwen3-32B"}
}
}'</code></pre>
<p class="sub">An <code>llm_d</code> key wins over <code>k8s</code>. Without <code>endpoint_url</code>, madengine runs in <em>managed</em> mode and installs and removes helm releases itself. See <a href="#llm-d">llm-d</a>.</p>
</div>
<div class="tabpanel" data-panel="qs-file">
<pre><code># Context file (JSON object)
madengine run --tags llama3 --additional-context-file my_run.json
# Hydra / YAML frontend (same keys as the context, plus config groups)
madengine run --config my_job.yaml
madengine run --config my_job.yaml --config distributed.nnodes=4
madengine run --config my_job.yaml --config +env=nccl_debug</code></pre>
<p class="sub"><code>--additional-context</code> is parsed as JSON first, then as a Python dict literal. <code>--additional-context-file</code> is merged under it (the inline string wins). <code>--config</code> cannot be combined with either. See <a href="#cli-overview">CLI overview</a> and <a href="#config-frontend">--config</a>.</p>
</div>
</section>
<section id="install">
<h2>Install & dev</h2>
<div class="grid cols-2">
<div class="card">
<h3>Setup</h3>
<pre><code>git clone https://github.com/ROCm/madengine.git
cd madengine
python3 -m venv venv && source venv/bin/activate
# Editable install: ALL runtime and dev dependencies come with it
pip install -e .
# Optional: host-side TraceLens analysis (see "madengine report tracelens")
pip install -e '.[tracelens]'
pre-commit install
madengine --version</code></pre>
<p class="sub">Run <code>madengine</code> from the root of a MAD-style model repository (the directory holding <code>models.json</code> and <code>scripts/</code>), or point <code>MODEL_DIR</code> at it (see <a href="#envvars">Environment variables</a>).</p>
</div>
<div class="card">
<h3>Requirements</h3>
<ul>
<li><strong>Python</strong> ≥ 3.8 (<code>requires-python</code>; Black targets py38–py311, mypy checks as 3.8).</li>
<li><strong>Docker</strong> with GPU access (ROCm: <code>/dev/kfd</code> + <code>/dev/dri</code>; NVIDIA: container toolkit) for the default container platform.</li>
<li><strong>conda / mamba / micromamba</strong> instead of Docker for <code>"platform": "bare_metal"</code> (see <a href="#platforms">Platforms</a>).</li>
<li><strong>Per target:</strong> SLURM (or Spur) CLI on the submit host; <code>kubectl</code> and a kubeconfig for Kubernetes; <code>helm</code> for managed llm-d stacks.</li>
</ul>
</div>
</div>
<h3>Dependencies and extras</h3>
<p>Everything is declared in <span class="filepath">pyproject.toml</span> under <code>[project] dependencies</code>, including the Kubernetes client, Hydra, MongoDB, and the dev toolchain. There is exactly one optional extra:</p>
<table>
<thead><tr><th>Install</th><th>Adds</th></tr></thead>
<tbody>
<tr><td><code>pip install -e .</code></td><td><code>pandas</code>, <code>typer</code>, <code>click</code>, <code>rich</code>, <code>tqdm</code>, <code>typing-extensions</code>, <code>jinja2</code>, <code>pyyaml</code>, <code>hydra-core</code>, <code>omegaconf</code>, <code>kubernetes</code>, <code>pymongo</code>, <code>paramiko</code>, <code>sqlalchemy</code>, <code>GitPython</code>, <code>jsondiff</code>, <code>toml</code>, plus pytest (<code>-cov</code>, <code>-xdist</code>, <code>-timeout</code>, <code>-mock</code>, <code>-asyncio</code>), black, flake8, mypy, isort, pre-commit</td></tr>
<tr><td><code>pip install -e '.[tracelens]'</code></td><td>TraceLens pinned to a git commit. Kept optional because it pins <code>protobuf>=6.31</code> and <code>xprof</code>, which can conflict with other packages in a shared environment. Needed only for <code>madengine report tracelens</code>; the in-container <code>tracelens</code> tool installs into its own venv instead.</td></tr>
</tbody>
</table>
<div class="callout warn">
<strong>No <code>[dev]</code>, <code>[kubernetes]</code> or <code>[all]</code> extras.</strong> They were removed when those dependencies moved into the base list; <code>pip install -e ".[dev]"</code> only produces a pip warning about an unknown extra. The wheel ships <code>scripts/**</code> and <code>configs/**</code> (Hydra groups for <code>--config</code>) via <code>[tool.hatch.build.targets.wheel] artifacts</code>.
</div>
<h3>Versioning</h3>
<p>The version is not stored in the source tree. <code>versioningit</code> derives it from git tags matching <code>v<version></code> (default tag <code>v2.0.0</code> when none exists). Untagged commits become <code><base>.post<distance>+g<sha></code>; a dirty tree adds a build date (<code><base>+d<YYYYMMDD></code> on a tag, <code>…+g<sha>.d<YYYYMMDD></code> after it). <code>madengine --version</code> and <code>madengine.__version__</code> read the installed package metadata, so an uninstalled checkout reports <code>unknown</code> / <code>dev</code>.</p>
<div class="grid cols-2" style="margin-top:14px">
<div class="card">
<h3>Test</h3>
<pre><code>pytest # testpaths: tests/unit, integration, e2e
pytest tests/unit -v # fast, mocked
pytest tests/unit/test_slurm_multi.py -v
pytest -k llm_d # name filter
pytest -m "not slow" # skip slow tests
pytest -m "amd or cpu" # hardware markers
pytest --cov=src/madengine --cov-report=html</code></pre>
<p class="sub"><code>addopts</code> = <code>-v --tb=short -ra --strict-markers -W default</code>, so an unregistered marker is a collection error. Markers are applied per test, not per directory: <code>-m unit</code> selects only tests tagged <code>unit</code>; use a path to run a whole directory. See <a href="#tests">Test layout</a>.</p>
</div>
<div class="card">
<h3>Quality</h3>
<pre><code>black src/ tests/
isort src/ tests/
flake8 src/ tests/
mypy src/madengine
pre-commit run --all-files</code></pre>
<p class="sub">The pre-commit hooks are black, isort, flake8, mypy (<code>src/madengine/</code> only), bandit, and file hygiene checks. See <a href="#contrib">Contributing</a>.</p>
</div>
</div>
</section>
<section id="layers">
<h2>Layered architecture</h2>
<p class="sub">Each layer calls only the layers below it. Layers are color-coded throughout this page.</p>
<div class="legend" aria-hidden="true">
<span><span class="dot" style="background:var(--layer-cli)"></span>CLI</span>
<span><span class="dot" style="background:var(--layer-orch)"></span>Orchestration</span>
<span><span class="dot" style="background:var(--layer-dep)"></span>Deployment</span>
<span><span class="dot" style="background:var(--layer-exec)"></span>Execution</span>
<span><span class="dot" style="background:var(--layer-core)"></span>Core</span>
<span><span class="dot" style="background:var(--layer-util)"></span>Utils</span>
<span><span class="dot" style="background:var(--layer-rep)"></span>Reporting</span>
</div>
<div class="table-wrap">
<table>
<thead><tr><th>Layer</th><th>Path</th><th>Responsibilities</th></tr></thead>
<tbody>
<tr>
<td><span class="pill cli">CLI</span></td>
<td><span class="filepath">src/madengine/cli/</span>, <span class="filepath">config/</span>, <span class="filepath">configs/</span>, <span class="filepath">setup_page/</span></td>
<td>Typer app with 7 commands, argument validation, Rich output, exit-code mapping, signal-handler install. The <code>--config</code> Hydra frontend (<code>config/</code> + YAML groups in <code>configs/</code>) translates into <code>additional_context</code>. <code>setup_page/</code> renders the model-picker HTML.</td>
</tr>
<tr>
<td><span class="pill orch">Orchestration</span></td>
<td><span class="filepath">src/madengine/orchestration/</span></td>
<td><code>BuildOrchestrator</code> (discover → build → manifest) and <code>RunOrchestrator</code> (manifest → target inference → dispatch), GPU-arch/vendor image filtering, and the Docker-vs-bare-metal platform decision.</td>
</tr>
<tr>
<td><span class="pill dep">Deployment</span></td>
<td><span class="filepath">src/madengine/deployment/</span></td>
<td><code>DeploymentFactory</code> + template-method <code>BaseDeployment</code>; SLURM, Spur, Kubernetes, and llm-d backends; presets, Jinja2 templates, layered config, K8s <code>slurm_multi</code> shims, monitoring and result collection.</td>
</tr>
<tr>
<td><span class="pill exec">Execution</span></td>
<td><span class="filepath">src/madengine/execution/</span></td>
<td>Local <code>ContainerRunner</code> (docker run, tools, log scanning, timeouts), <code>BareMetalRunner</code> + conda env management, <code>DockerBuilder</code>, run-result reporting.</td>
</tr>
<tr>
<td><span class="pill core">Core</span></td>
<td><span class="filepath">src/madengine/core/</span></td>
<td>Context merging & GPU detection, shell <code>Console</code>, <code>Docker</code> wrapper, error hierarchy, registry auth, timeouts, data providers, image digests, and the container <strong>lifecycle</strong> (heartbeat watchdog, signal-handler teardown).</td>
</tr>
<tr>
<td><span class="pill util">Utils</span></td>
<td><span class="filepath">src/madengine/utils/</span></td>
<td>Model discovery, GPU tool abstraction (AMD/NVIDIA), ROCm path resolution, GPU memory gate, config parsing, session tracking.</td>
</tr>
<tr>
<td><span class="pill rep">Reporting</span></td>
<td><span class="filepath">src/madengine/reporting/</span>, <span class="filepath">database/</span></td>
<td><code>perf.csv</code> / <code>perf_entry.csv</code> / <code>perf_super.*</code> writers, HTML/email/TraceLens reports, MongoDB upload.</td>
</tr>
</tbody>
</table>
</div>
<p class="sub">Packaged assets live beside the code: <span class="filepath">src/madengine/scripts/</span> (pre/post scripts, tools, SLURM/K8s helper scripts) is copied into a project's <code>scripts/common/</code> for a run and removed afterwards. See <a href="#modules">Module reference</a>.</p>
</section>
<section id="diagram">
<h2>Architecture diagram</h2>
<div class="diag">
<svg viewBox="0 0 1080 520" xmlns="http://www.w3.org/2000/svg" role="img" aria-labelledby="diag-t diag-d" font-family="ui-monospace,Menlo,Consolas,monospace" font-size="12">
<title id="diag-t">madengine architecture</title>
<desc id="diag-d">CLI on top of orchestration, which dispatches to five execution targets: local Docker, bare-metal conda, SLURM or Spur, Kubernetes, and llm-d. Core and utils support them; reporting and database sit at the bottom.</desc>
<defs>
<marker id="arr" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="6" markerHeight="6" orient="auto-start-reverse">
<path d="M0,0 L10,5 L0,10 z" fill="#8b949e"/>
</marker>
<style>.lbl{fill:#fff;font-weight:700}.sub{fill:#8b949e;font-size:10.5px}.box{stroke-width:1.5;fill-opacity:.15}.line{stroke:#8b949e;stroke-width:1.5;fill:none}</style>
</defs>
<!-- CLI -->
<rect x="20" y="20" width="1040" height="58" rx="8" class="box" stroke="#9d7cff" fill="#9d7cff"/>
<text x="40" y="44" class="lbl">CLI · Typer + Rich · --config (Hydra)</text>
<text x="40" y="62" class="sub">discover · build · run · report · database · cleanup · setup-page ExitCode: 0 OK · 1 FAILURE · 2 BUILD · 3 RUN · 4 INVALID_ARGS · 5 GPU_WEDGED</text>
<!-- Orchestration -->
<rect x="20" y="104" width="1040" height="76" rx="8" class="box" stroke="#58a6ff" fill="#58a6ff"/>
<text x="40" y="124" class="lbl">Orchestration</text>
<rect x="200" y="116" width="230" height="52" rx="6" stroke="#58a6ff" fill="#0a0d12"/>
<text x="218" y="138" fill="#fff">BuildOrchestrator</text>
<text x="218" y="154" class="sub">DiscoverModels → DockerBuilder / conda</text>
<text x="218" y="166" class="sub">→ build_manifest.json</text>
<rect x="450" y="116" width="230" height="52" rx="6" stroke="#58a6ff" fill="#0a0d12"/>
<text x="468" y="138" fill="#fff">RunOrchestrator</text>
<text x="468" y="154" class="sub">load manifest → merge context</text>
<text x="468" y="166" class="sub">→ infer target → dispatch</text>
<rect x="700" y="116" width="200" height="52" rx="6" stroke="#58a6ff" fill="#0a0d12"/>
<text x="718" y="138" fill="#fff">image_filtering · platform</text>
<text x="718" y="154" class="sub">GPU arch / vendor</text>
<text x="718" y="166" class="sub">docker vs bare_metal</text>
<!-- Targets -->
<rect x="20" y="208" width="1040" height="108" rx="8" class="box" stroke="#3fb950" fill="#3fb950"/>
<text x="40" y="228" class="lbl">Execution & deployment targets (inferred from context keys)</text>
<text x="40" y="246" class="sub">default → local · platform=bare_metal · "slurm" (+scheduler: spur) · "k8s" / "kubernetes" · "llm_d" (checked before k8s)</text>
<rect x="40" y="258" width="190" height="44" rx="6" stroke="#d29922" fill="#0a0d12"/>
<text x="56" y="278" fill="#d29922">ContainerRunner</text>
<text x="56" y="292" class="sub">local docker run</text>
<rect x="244" y="258" width="190" height="44" rx="6" stroke="#d29922" fill="#0a0d12"/>
<text x="260" y="278" fill="#d29922">BareMetalRunner</text>
<text x="260" y="292" class="sub">conda / mamba prefix</text>
<rect x="448" y="258" width="190" height="44" rx="6" stroke="#3fb950" fill="#0a0d12"/>
<text x="464" y="278" fill="#3fb950">Slurm / SpurDeployment</text>
<text x="464" y="292" class="sub">sbatch · job array</text>
<rect x="652" y="258" width="190" height="44" rx="6" stroke="#3fb950" fill="#0a0d12"/>
<text x="668" y="278" fill="#3fb950">KubernetesDeployment</text>
<text x="668" y="292" class="sub">Jobs · PVCs · slurm_multi shim</text>
<rect x="856" y="258" width="190" height="44" rx="6" stroke="#3fb950" fill="#0a0d12"/>
<text x="872" y="278" fill="#3fb950">LlmdDeployment</text>
<text x="872" y="292" class="sub">helm stack · benchmark Job</text>
<!-- Core / Utils -->
<rect x="20" y="344" width="500" height="58" rx="8" class="box" stroke="#f778ba" fill="#f778ba"/>
<text x="40" y="364" class="lbl">Core</text>
<text x="40" y="382" class="sub">Context · Console · Docker · errors · auth · lifecycle · data providers</text>
<rect x="540" y="344" width="520" height="58" rx="8" class="box" stroke="#79c0ff" fill="#79c0ff"/>
<text x="560" y="364" class="lbl">Utils</text>
<text x="560" y="382" class="sub">DiscoverModels · GPU tools · rocm_path_resolver · gpu_memory_gate</text>
<!-- Reporting / DB -->
<rect x="20" y="430" width="650" height="58" rx="8" class="box" stroke="#ff7b72" fill="#ff7b72"/>
<text x="40" y="450" class="lbl">Reporting</text>
<text x="40" y="468" class="sub">perf.csv · perf_entry.csv · perf_super · csv_to_html · csv_to_email · TraceLens</text>
<rect x="690" y="430" width="370" height="58" rx="8" class="box" stroke="#f778ba" fill="#f778ba"/>
<text x="710" y="450" class="lbl">Database</text>
<text x="710" y="468" class="sub">MongoDB upload · MongoDBConfig.from_env()</text>
<!-- Arrows -->
<path class="line" d="M540,78 V104" marker-end="url(#arr)"/>
<path class="line" d="M540,180 V208" marker-end="url(#arr)"/>
<path class="line" d="M270,316 V344" marker-end="url(#arr)"/>
<path class="line" d="M800,316 V344" marker-end="url(#arr)"/>
<path class="line" d="M540,402 V430" marker-end="url(#arr)"/>
<path class="line" d="M670,460 H690" marker-end="url(#arr)"/>
</svg>
</div>
</section>
<section id="flows">
<h2>Key data flows</h2>
<p class="sub">Derived from <span class="filepath">orchestration/build_orchestrator.py</span> and <span class="filepath">orchestration/run_orchestrator.py</span>. Both orchestrators receive a single merged <code>additional_context</code> dict from the CLI (see <a href="#context">additional_context</a>).</p>
<div class="grid cols-2">
<div class="card">
<h3>Build flow <span class="sub">(<code>BuildOrchestrator.execute</code>)</span></h3>
<ol>
<li>Constructor: merge context sources, fill build defaults (<code>gpu_vendor=AMD</code>, <code>guest_os=UBUNTU</code> when absent), run <code>ConfigLoader.load_config()</code> (infers target, rejects conflicting blocks such as <code>k8s</code> + <code>slurm</code> or <code>llm_d</code> + <code>slurm</code>, applies presets). The keys the user really set under <code>slurm</code> are remembered separately from preset defaults.</li>
<li><code>Context(build_only_mode=True)</code>: no GPU or NUMA detection. <code>MAD_SYSTEM_GPU_ARCHITECTURE</code> is auto-detected only when the build is part of <code>madengine run --tags</code> (<code>detect_local_gpu_arch=True</code>) and the user did not supply it.</li>
<li>Dispatch on mode, in this order:
<ul>
<li><code>platform: bare_metal</code> → create conda prefixes, no Docker (see <a href="#platforms">Execution platforms</a>).</li>
<li><code>--use-image</code> (or <code>auto</code>) → write a manifest for an existing image, no build.</li>
<li><code>--build-on-compute</code> → build and push from a SLURM compute node (needs <code>--registry</code>).</li>
<li>Otherwise the normal Docker build below.</li>
</ul>
<code>--use-image</code> together with <code>--build-on-compute</code> raises <code>ConfigurationError</code>.</li>
<li>Normal build: <code>DiscoverModels.run()</code> runs once and is reused. Cards whose <code>distributed.launcher</code> is a self-managed launcher (<code>slurm_multi</code>) need an image source: with no <code>--registry</code>, a single <code>env_vars.DOCKER_IMAGE_NAME</code> on the cards becomes an implicit <code>--use-image</code> (angle-bracket placeholders are rejected); otherwise <code>ConfigurationError</code>.</li>
<li>Warn when a selected Dockerfile declares <code>ARG MAD_SYSTEM_GPU_ARCHITECTURE</code> with no default and no value was resolved.</li>
<li><code>DockerBuilder.build_all_models()</code> builds every matching Dockerfile per card, optionally pushes, and never aborts on a single failure (see <a href="#build-context">Docker build</a>).</li>
<li>Manifest steps, always executed even after partial failure: <code>export_build_manifest()</code> → save <code>summary</code> → save <code>deployment_config</code> → merge the first built card's <code>distributed</code> / <code>slurm</code> fields into <code>deployment_config</code> (user-set values win).</li>
<li>Raise <code>BuildError</code> only when <em>all</em> builds failed. A partial build exits successfully with a warning.</li>
</ol>
</div>
<div class="card">
<h3>Run flow <span class="sub">(<code>RunOrchestrator.execute</code>)</span></h3>
<ol>
<li>Start a <code>SessionTracker</code>: records how many rows <code>perf.csv</code> already has (marker file <code>.madengine_session_start</code> beside it) so the CLI can show only this run's rows.</li>
<li>Manifest source, in priority order: <code>MAD_CONTAINER_IMAGE</code> in context (needs <code>--tags</code>; validates or pulls the image, writes a synthetic manifest with <code>local_image_mode</code>); else an existing <code>--manifest-file</code>; else build first (<code>--tags</code>). Neither tags nor a manifest → <code>ConfigurationError</code>.</li>
<li><code>_load_and_merge_manifest()</code>: runtime context overrides the stored <code>deployment_config</code> for <code>deploy slurm k8s kubernetes llm_d platform conda distributed vllm env_vars debug</code>; <code>tools</code>, <code>pre_scripts</code>, <code>post_scripts</code>, <code>encapsulate_script</code>, <code>require_pinned_image</code> are written into manifest <code>context</code>. A runtime <code>slurm</code> block replaces the stored one wholesale and resets the explicit-key provenance.</li>
<li>Stored <code>deployment_config</code> keys the runtime did not set are copied into the working context.</li>
<li>Target inference (see <a href="#targets">Deployment targets</a>): <code>platform: bare_metal</code> → <code>llm_d</code> → <code>k8s</code>/<code>kubernetes</code> → <code>slurm</code> (<code>spur</code> when <code>slurm.scheduler</code> or <code>deploy</code> is <code>spur</code>) → a self-managed launcher on the first built card implies <code>slurm</code> → otherwise <code>local</code>. A legacy <code>deployment_config.target</code> is the final fallback.</li>
<li>Dispatch:
<table>
<thead><tr><th>Target</th><th>Executor</th></tr></thead>
<tbody>
<tr><td><code>local</code> / <code>docker</code></td><td><code>ContainerRunner.run_models_from_manifest()</code></td></tr>
<tr><td><code>bare_metal</code></td><td><code>BareMetalRunner.run_models_from_manifest()</code></td></tr>
<tr><td><code>slurm</code>, <code>spur</code>, <code>k8s</code>, <code>llm-d</code></td><td><code>DeploymentFactory.create(DeploymentConfig).execute()</code></td></tr>
</tbody>
</table></li>
<li>Always, in a <code>finally</code>-style path: <code>_cleanup_model_dir_copies()</code> removes the madengine-provided files under <code>scripts/common/</code>.</li>
</ol>
</div>
</div>
<div class="grid cols-2">
<div class="card">
<h3>Local Docker path (<code>_execute_local</code>)</h3>
<ol>
<li>Runtime <code>Context</code> (full system + GPU detection), optional <code>data.json</code> loaded as <code>Data</code>, node info table.</li>
<li>Restore host-level fields from the manifest <code>context</code> without overriding runtime-detected values: <code>docker_mounts</code>, <code>docker_build_arg</code>, <code>docker_gpus</code>, <code>gpu_vendor</code>, <code>guest_os</code>, <code>docker_env_vars</code>; <code>tools</code> / scripts always come from the manifest unless the runtime context supplies its own.</li>
<li>Filter <code>built_images</code>: GPU vendor/architecture compatibility (<span class="filepath">orchestration/image_filtering.py</span>; an image without <code>gpu_vendor</code> is treated as compatible), then <code>skip_gpu_arch</code> from each card (unless <code>--disable-skip-gpu-arch</code>). Skipped cards are written to <code>perf.csv</code> with status <code>SKIPPED</code>. The whole step is best effort: any exception, including the <code>ExecutionError</code> raised when no image is compatible, is caught, printed as a warning, and the run proceeds with <em>all</em> images.</li>
<li><code>_copy_scripts()</code>: honours an external <code>MODEL_DIR</code> (merges its <code>docker/</code> and <code>scripts/</code> into the cwd) and copies <code>pre_scripts</code>, <code>post_scripts</code>, <code>tools</code>, <code>entrypoints</code>, <code>tools.json</code>, <code>test_echo.sh</code> from the madengine package into <code>scripts/common/</code>. The MAD project's own <code>scripts/</code> and <code>docker/</code> are never deleted.</li>
<li><code>--wait-for-gpu-memory</code> gate (local only), then <code>ContainerRunner</code> runs each manifest entry. Distributed targets print a warning and ignore <code>--keep-alive</code>, <code>--keep-model-dir</code>, <code>--skip-model-run</code> and <code>--wait-for-gpu-memory</code>.</li>
<li>For a full workflow the build and run logs are merged into one <code>*.live.log</code>.</li>
</ol>
</div>
<div class="card">
<h3>Distributed path (<code>_execute_distributed</code>)</h3>
<ol>
<li>For target <code>slurm</code> only, <code>skip_gpu_arch</code> is decided before submission (a filtered per-run copy of the manifest is submitted; the build manifest is left intact). Skipped cards appear as <code>SKIPPED</code> rows. An empty manifest returns a <code>FAILURE</code> row instead of exiting 0. Spur, Kubernetes and llm-d do not apply this pre-filter.</li>
<li>Manifest <code>context.docker_env_vars</code> are handed to the deployment layer (runtime keys win), because SLURM/K8s never enter the local restore step.</li>
<li>Session baseline is measured on the working <code>perf.csv</code> (what the deployment appends to), and this run's rows are exported to <code>--output</code> afterwards when it names another file.</li>
<li><code>DeploymentConfig</code> carries <code>timeout</code> (resolved against the default) and <code>cli_timeout</code> (verbatim, so the inner <code>madengine run</code> on the node still lets the model card outrank the default); <code>monitor</code> and <code>cleanup_on_failure</code> (both default <code>true</code>) are read from context.</li>
<li>If the scheduler fails before reporting per-model metrics, every manifest model not already reported is added to <code>failed_runs</code>, so the CLI exit code is not 0 for a failed job.</li>
</ol>
<p class="sub">Per-backend job flow (sbatch template, array jobs on Spur, Indexed Jobs on K8s, helm for llm-d): see <a href="#slurm">SLURM</a>, <a href="#k8s">Kubernetes</a>, <a href="#llm-d">llm-d</a>.</p>
</div>
</div>
<div class="card" style="margin-top:14px">
<h3>Runtime precedence cheat-sheet</h3>
<table>
<thead><tr><th>Setting</th><th>Order, highest first</th></tr></thead>
<tbody>
<tr><td>Run timeout</td><td>explicit <code>--timeout</code> > model card <code>timeout</code> > 7200 s (<span class="filepath">core/timeout.py</span>). Non-positive resolved value = no timeout.</td></tr>
<tr><td>GPU count</td><td>top-level runtime key (<code>gpus_per_node</code>, <code>gpu_count</code>, <code>n_gpus</code>, <code>num_gpus</code>, <code>ngpus</code>) > <code>slurm.gpus_per_node</code> / <code>k8s.gpu_count</code> > card <code>n_gpus</code> > 1 (<span class="filepath">utils/gpu_config.py</span>)</td></tr>
<tr><td>Distributed launcher (local Docker)</td><td><code>distributed.launcher</code> in context > card <code>distributed.launcher</code> > <code>MAD_LAUNCHER_TYPE</code> > <code>MAD_LAUNCHER</code></td></tr>
<tr><td>Docker env passed to the container</td><td>runtime-detected <code>docker_env_vars</code> are kept over manifest copies; <code>additional_context.docker_env_vars</code> then overwrite per key</td></tr>
<tr><td>Log error-scan settings</td><td>context > model card > built-in defaults</td></tr>
<tr><td>Deployment config</td><td>runtime <code>--additional-context</code> > manifest <code>deployment_config</code> > preset defaults</td></tr>
</tbody>
</table>
</div>
</section>
<section id="cli-overview">
<h2>CLI overview</h2>
<p>Entry point: <span class="filepath">src/madengine/cli/app.py::cli_main</span>, installed as the <code>madengine</code> console script. The Typer app registers <strong>seven top-level commands</strong>; <code>report</code> is a sub-app with its own sub-commands.</p>
<table>
<thead><tr><th>Command</th><th>Source</th><th>Purpose</th></tr></thead>
<tbody>
<tr><td><a href="#cli-discover"><code>discover</code></a></td><td><span class="filepath">cli/commands/discover.py</span></td><td>List / select model cards by tag; print JSON.</td></tr>
<tr><td><a href="#cli-build"><code>build</code></a></td><td><span class="filepath">cli/commands/build.py</span></td><td>Build (or adopt) Docker images, or prepare conda envs for bare-metal; write <code>build_manifest.json</code>.</td></tr>
<tr><td><a href="#cli-run"><code>run</code></a></td><td><span class="filepath">cli/commands/run.py</span></td><td>Run from a manifest, or build + run when no manifest exists; write <code>perf.csv</code>.</td></tr>
<tr><td><a href="#cli-report"><code>report</code></a> <span class="sub">(sub-app)</span></td><td><span class="filepath">cli/commands/report.py</span></td><td><code>to-html</code>, <code>to-email</code>, <code>tracelens</code>, <code>tracelens-compare</code>.</td></tr>
<tr><td><a href="#cli-report"><code>database</code></a></td><td><span class="filepath">cli/commands/database.py</span></td><td>Upload a CSV/JSON results file to MongoDB.</td></tr>
<tr><td><a href="#cli-ops"><code>cleanup</code></a></td><td><span class="filepath">cli/commands/cleanup.py</span></td><td>Remove benchmark containers orphaned by interrupted runs.</td></tr>
<tr><td><a href="#cli-ops"><code>setup-page</code></a></td><td><span class="filepath">cli/commands/setup_page.py</span></td><td>Generate a self-contained HTML "setup picker" from the repo's <code>models.json</code>.</td></tr>
</tbody>
</table>
<h3>Global behavior</h3>
<ul>
<li><code>madengine --version</code> prints the installed package version (<code>importlib.metadata</code>; <code>unknown</code> if not installed). With no sub-command the help is shown. Shell completion is disabled (<code>add_completion=False</code>).</li>
<li><code>cli_main()</code> first calls <code>core.lifecycle.install_signal_handlers()</code> so <code>SIGINT</code>/<code>SIGTERM</code>/<code>SIGHUP</code> tear down benchmark containers (see <a href="#lifecycle">container lifecycle</a>). A <code>KeyboardInterrupt</code> or an unhandled exception that escapes a command exits <code>1</code>.</li>
<li>A Rich traceback handler is installed with <code>show_locals=True</code>. An unhandled crash can therefore print local variable values; do not rely on the console log being free of credentials.</li>
</ul>
<h3>Context sources</h3>
<p><code>build</code> and <code>run</code> accept the same configuration spine through three <strong>mutually exclusive-in-part</strong> inputs:</p>
<table>
<thead><tr><th>Input</th><th>Format</th><th>Rules</th></tr></thead>
<tbody>
<tr><td><code>--additional-context</code>, <code>-c</code></td><td>String; parsed as JSON first, then as a Python dict literal (<code>ast.literal_eval</code>)</td><td>Must be an object at top level. Empty / <code>{}</code> counts as "not given".</td></tr>
<tr><td><code>--additional-context-file</code>, <code>-f</code></td><td>JSON file (object)</td><td>Loaded first; the <code>-c</code> string is <strong>deep-merged over it</strong> (CLI wins key by key, nested dicts merge rather than replace).</td></tr>
<tr><td><code>--config</code> <span class="sub">(repeatable)</span></td><td>YAML file and/or Hydra overrides</td><td>Cannot be combined with either input above (exit <code>4</code>). See <a href="#config-frontend">--config frontend</a>.</td></tr>
</tbody>
</table>
<h4>Validation applied to the merged context</h4>
<p>Implemented in <span class="filepath">cli/validators.py::finalize_additional_context_dict</span>; any failure exits <code>4</code> (<code>INVALID_ARGS</code>). <code>build</code> always validates; <code>run</code> validates only when a non-empty context was supplied.</p>
<ol>
<li>Missing <code>gpu_vendor</code> / <code>guest_os</code> are defaulted to <code>AMD</code> / <code>UBUNTU</code> (<code>core/additional_context_defaults.py</code>) and a banner says so.</li>
<li>Structure checks: <code>docker_build_arg</code>, <code>docker_env_vars</code>, <code>docker_mounts</code>, <code>env_vars</code>, <code>k8s</code>, <code>slurm</code>, <code>kubernetes</code>, <code>llm_d</code>, <code>distributed</code>, <code>vllm</code>, <code>deployment_config</code>, <code>deploy</code> must be objects; <code>tools</code> and <code>pre_scripts</code> arrays; <code>docker_gpus</code> and <code>MAD_CONTAINER_IMAGE</code> strings; <code>timeout</code> a number; <code>debug</code> a boolean; <code>log_error_patterns</code> a non-empty string array. Build-arg values are coerced to strings.</li>
<li><code>gpu_vendor</code> must be <code>AMD</code> or <code>NVIDIA</code>, <code>guest_os</code> must be <code>UBUNTU</code> or <code>CENTOS</code> (case-insensitive, normalized to upper case).</li>
<li><strong>Launcher:</strong> every <code>distributed.launcher</code> / <code>launcher.type</code> is validated and rewritten to its canonical spelling. An unknown name is rejected up front rather than running as a single process. A bare-string top-level <code>launcher</code> is rejected. The per-target rule (<code>slurm_multi</code> is not valid on llm-d) is enforced when the deployment is constructed. See <a href="#launchers">Launchers</a>.</li>
<li><strong>Platform:</strong> <code>platform</code> is a string; omitted means Docker, <code>"bare_metal"</code> selects conda. Unknown values, the retired top-level <code>bare_metal</code> object, and a <code>conda</code> block on a Docker run are rejected. See <a href="#platforms">Platforms</a>.</li>
</ol>
<h3>Common option letters</h3>
<table>
<thead><tr><th>Short</th><th>Long</th><th>Notes</th></tr></thead>
<tbody>
<tr><td><code>-t</code></td><td><code>--tags</code></td><td>Repeatable; each value may itself be comma-separated (<code>-t a -t b</code> = <code>-t a,b</code>).</td></tr>
<tr><td><code>-r</code></td><td><code>--registry</code></td><td>build, run.</td></tr>
<tr><td><code>-c</code> / <code>-f</code></td><td><code>--additional-context</code> / <code>--additional-context-file</code></td><td>build, run. On <code>database</code>, <code>-f</code> is <code>--file</code> and <code>-c</code> is <code>--collection</code>.</td></tr>
<tr><td><code>-m</code></td><td><code>--manifest-output</code> (build) / <code>--manifest-file</code> (run)</td><td>Same letter, different meaning.</td></tr>
<tr><td><code>-s</code>, <code>-l</code>, <code>-o</code>, <code>-v</code></td><td><code>--summary-output</code>, <code>--live-output</code>, <code>--output</code>, <code>--verbose</code></td><td><code>--live-output</code> and most other bool flags have no <code>--no-…</code> form.</td></tr>
</tbody>
</table>
<h3>How failures become exit codes</h3>
<p>Each command wraps its body and maps exception classes from <span class="filepath">core/errors.py</span> to an <a href="#exitcodes">exit code</a>. The mapping is not identical across commands:</p>
<table>
<thead><tr><th>Condition</th><th><code>build</code></th><th><code>run</code></th></tr></thead>
<tbody>
<tr><td>Mutually exclusive flags, bad context, <code>ConfigurationError</code></td><td><code>4</code></td><td><code>4</code></td></tr>
<tr><td><code>BuildError</code>, any failed build or failed registry push (including partial success)</td><td><code>2</code></td><td><code>2</code> (<code>BuildError</code> only)</td></tr>
<tr><td><code>ExecutionError</code>, any failed model run, GPU-memory gate refusal</td><td>n/a</td><td><code>3</code></td></tr>
<tr><td><code>DiscoveryError</code> (no models matched), missing file, permission error, Ctrl-C, other exceptions</td><td><code>1</code></td><td><code>1</code></td></tr>
</tbody>
</table>
</section>
<section id="cli-discover">
<h2>CLI — <code>discover</code></h2>
<p>Lists model cards and validates tag selection without building or running anything. Must be run from the MAD project root: discovery reads <code>./models.json</code> (required; <code>FileNotFoundError</code> otherwise) and walks <code>./scripts/</code>.</p>
<table>
<thead><tr><th>Option</th><th>Short</th><th>Default</th><th>Description</th></tr></thead>
<tbody>
<tr><td><code>--tags TEXT</code></td><td><code>-t</code></td><td>none</td><td>Tags / model names; repeatable and comma-separable. Omitted: print the list of all model names.</td></tr>
<tr><td><code>--full</code></td><td><code>-f</code></td><td>off</td><td>With no <code>--tags</code>: print the complete model card of every model (static and dynamic) as JSON. With <code>--tags</code>, the selected cards are printed either way.</td></tr>
<tr><td><code>--json</code></td><td><code>-j</code></td><td>off</td><td>Plain JSON on stdout only: no panels, no status lines; errors go to stderr. Without <code>--tags</code>/<code>--full</code> it prints a JSON array of model names. Use this when piping.</td></tr>
<tr><td><code>--verbose</code></td><td><code>-v</code></td><td>off</td><td>Debug logging; Rich traceback on failure.</td></tr>
</tbody>
</table>
<h3>Tag syntax</h3>
<table>
<thead><tr><th>Form</th><th>Example</th><th>Selects</th></tr></thead>
<tbody>
<tr><td>Tag, model name, or last path segment of a name</td><td><code>--tags llama3</code></td><td>Any card whose <code>name</code>, final <code>/</code>-separated name segment, or <code>tags</code> entry equals the value.</td></tr>
<tr><td>Several</td><td><code>--tags a,b</code> / <code>-t a -t b</code></td><td>Union. A tag that matches nothing aborts with <code>No models found corresponding to the given tag</code> (exit <code>1</code>).</td></tr>
<tr><td>Everything</td><td><code>--tags all</code></td><td>Every discovered model.</td></tr>
<tr><td>Scoped</td><td><code>--tags MAD-private/inference</code></td><td>Exactly one <code>/</code> and no <code>:</code>. Restricts to cards named <code>MAD-private/…</code> (from <code>scripts/MAD-private/</code>) and matches on tag, exact name, directory prefix, or any path component. <code>scope/all</code> selects the whole scope.</td></tr>
<tr><td>Per-run script args</td><td><code>--tags dummy3:batch=512:fp16</code></td><td>Text after the first <code>:</code> is split on <code>:</code>; each <code>=</code> becomes a space and the pieces are appended to the card's <code>args</code> as <code>--batch 512 --fp16</code>.</td></tr>
</tbody>
</table>
<div class="callout warn">
A single-slash name such as <code>MAD/foo</code> is always parsed as <em>scope/tag</em>, never as a plain name. To select by full name, use the scope form (<code>MAD/foo</code> matches the exact name) or the last-segment form (<code>foo</code>).
</div>
<h3>Discovery sources</h3>
<ol>
<li>Root <code>./models.json</code> — required.</li>
<li><code>scripts/**/models.json</code> — found by a <strong>recursive</strong> walk, so nested submodule layouts (<code>scripts/MAD/…</code>) work.</li>
<li><code>scripts/**/get_models_json.py</code> — dynamic; must define <code>list_models()</code> returning <code>CustomModel</code> objects (<code>utils/discover_models.py</code>).</li>
</ol>
<p>A directory containing both <code>models.json</code> and <code>get_models_json.py</code> is an error. If <code>MODEL_DIR</code> points elsewhere than the current directory, discovery first copies <code>docker/</code>, <code>scripts/</code>, <code>models.json</code>, <code>credential.json</code> and <code>data.json</code> from it into the working directory.</p>
</section>
<section id="cli-build">
<h2>CLI — <code>build</code></h2>
<p>Builds Docker images for the selected models (or creates conda environments when <code>platform</code> is <code>bare_metal</code>) and writes <a href="#manifest"><code>build_manifest.json</code></a>. Context is always validated; <code>gpu_vendor</code> / <code>guest_os</code> default to <code>AMD</code> / <code>UBUNTU</code> when absent.</p>
<table>
<thead><tr><th>Option</th><th>Short</th><th>Default</th><th>Description</th></tr></thead>
<tbody>
<tr><td><code>--tags</code></td><td><code>-t</code></td><td>none</td><td>Model selection (same syntax as <a href="#cli-discover">discover</a>). <strong>Effectively required:</strong> with no tags and no batch manifest, discovery selects nothing and the build ends with <code>No models discovered</code> (exit <code>1</code>). Use <code>--tags all</code> for everything.</td></tr>
<tr><td><code>--batch-manifest FILE</code></td><td></td><td>none</td><td>JSON list of <code>{"model_name", "build_new", "registry"?, "registry_image"?}</code>. Models with <code>build_new: true</code> are built; the rest get synthetic manifest entries so a later <code>run</code> sees the full set.</td></tr>
<tr><td><code>--target-archs</code></td><td><code>-a</code></td><td>none</td><td>GPU architectures to build for (<code>gfx90a,gfx942</code>); repeatable / comma-separated. Omitted: one image using <code>MAD_SYSTEM_GPU_ARCHITECTURE</code> from context or the detected GPU.</td></tr>
<tr><td><code>--registry</code></td><td><code>-r</code></td><td>none</td><td>Push built images here. A failed push is a build failure (exit <code>2</code>).</td></tr>
<tr><td><code>--use-image [NAME|auto]</code></td><td></td><td>none</td><td>Skip the Docker build and record a pre-built image in the manifest. <code>auto</code> reads <code>env_vars.DOCKER_IMAGE_NAME</code> from the model card. The image name is also written into each model's manifest <code>env_vars.DOCKER_IMAGE_NAME</code>.</td></tr>
<tr><td><code>--build-on-compute</code></td><td></td><td>off</td><td>Build on a SLURM compute node and push to the registry; the run phase pulls in parallel on all nodes.</td></tr>
<tr><td><code>--additional-context</code></td><td><code>-c</code></td><td><code>{}</code></td><td>See <a href="#cli-overview">context sources</a>.</td></tr>
<tr><td><code>--additional-context-file</code></td><td><code>-f</code></td><td>none</td><td>JSON file; <code>-c</code> is deep-merged over it.</td></tr>
<tr><td><code>--config</code></td><td></td><td>none</td><td>YAML / Hydra overrides; see <a href="#config-frontend">--config frontend</a>. Fills tags, registry, target archs, manifest path, live output, summary path and <code>docker.clean_cache</code> when the matching CLI flag was left at its default.</td></tr>
<tr><td><code>--clean-docker-cache</code></td><td></td><td>off</td><td>Rebuild without the Docker layer cache.</td></tr>
<tr><td><code>--manifest-output</code></td><td><code>-m</code></td><td><code>build_manifest.json</code></td><td>Manifest path.</td></tr>
<tr><td><code>--summary-output</code></td><td><code>-s</code></td><td>none</td><td>Write the build summary JSON here.</td></tr>
<tr><td><code>--live-output</code></td><td><code>-l</code></td><td>off</td><td>Stream build output.</td></tr>
<tr><td><code>--verbose</code></td><td><code>-v</code></td><td>off</td><td>Debug logging.</td></tr>
</tbody>
</table>
<div class="callout warn">
<strong>Rejected combinations (exit <code>4</code>):</strong>
<ul style="margin:.4em 0 0">
<li><code>--batch-manifest</code> with <code>--tags</code></li>
<li><code>--use-image</code> with <code>--registry</code></li>
<li><code>--use-image</code> with <code>--build-on-compute</code></li>
<li><code>--build-on-compute</code> without <code>--registry</code></li>
<li><code>--config</code> with <code>--additional-context</code> or <code>--additional-context-file</code></li>
</ul>
</div>
<h3>Self-managed launchers need an image source</h3>
<p>For a card whose launcher is <code>slurm_multi</code>, the build requires one of: <code>--registry</code>, <code>--use-image</code>, <code>--build-on-compute --registry</code>, or a single <code>env_vars.DOCKER_IMAGE_NAME</code> on the card(s). In the last case, with no registry given, it is used as an implicit <code>--use-image</code>. Otherwise the build raises <code>ConfigurationError</code> (exit <code>4</code>). An angle-bracket placeholder such as <code><your-image></code> in <code>DOCKER_IMAGE_NAME</code> is rejected.</p>
<h3>Exit status</h3>
<p><code>0</code> only if every build succeeded and every push succeeded. Any failure, including a partial one, exits <code>2</code>; images that did build remain in the manifest.</p>
</section>
<section id="cli-run">
<h2>CLI — <code>run</code></h2>
<p>Runs models and writes <code>perf.csv</code>. Mode is chosen by the manifest:</p>
<ul>
<li><code>--manifest-file</code> names an <strong>existing</strong> file → execution only (<code>--tags</code> is not used to select models).</li>
<li>Otherwise → full workflow: build, then run. A named-but-missing manifest only prints a warning. With neither a manifest nor <code>--tags</code> (and no <code>MAD_CONTAINER_IMAGE</code>) the orchestrator raises <code>ConfigurationError</code> (exit <code>4</code>).</li>
<li><code>MAD_CONTAINER_IMAGE</code> in context skips the build and synthesizes a manifest for the given image; <code>--tags</code> is then required.</li>
</ul>
<table>
<thead><tr><th>Option</th><th>Short</th><th>Default</th><th>Description</th></tr></thead>
<tbody>
<tr><td><code>--tags</code></td><td><code>-t</code></td><td>none</td><td>Models to build + run (full workflow). Same syntax as <a href="#cli-discover">discover</a>.</td></tr>
<tr><td><code>--manifest-file</code></td><td><code>-m</code></td><td><code>""</code> (unset)</td><td>Existing build manifest. <strong>No implicit default</strong>: if omitted, <code>build_manifest.json</code> is not picked up automatically.</td></tr>
<tr><td><code>--registry</code></td><td><code>-r</code></td><td>none</td><td>Registry URL (push in the build phase; image resolution at run).</td></tr>
<tr><td><code>--timeout INT</code></td><td></td><td><code>-1</code></td><td>Seconds per model. See <a href="#cli-run-timeout">timeout resolution</a>.</td></tr>
<tr><td><code>--additional-context</code> / <code>-c</code>, <code>--additional-context-file</code> / <code>-f</code>, <code>--config</code></td><td></td><td>see overview</td><td>Same context sources as <code>build</code>. With <code>--config</code>, <code>model.tags</code>, <code>model.timeout</code>, <code>model.manifest_file</code>, <code>model.skip_run</code>, <code>build.registry</code>, <code>output</code>, <code>summary_output</code>, <code>data_config</code>, <code>live_output</code>, <code>require_pinned_image</code>, <code>docker.keep_alive</code>, and <code>docker.clean_cache</code> are used when the matching flag is at its default.</td></tr>
<tr><td><code>--keep-alive</code></td><td></td><td>off</td><td>Leave the container running after the model finishes (also opts the container out of the heartbeat dead-man's switch).</td></tr>
<tr><td><code>--keep-model-dir</code></td><td></td><td>off</td><td>Keep the model directory copy. Docker-only; ignored for bare metal.</td></tr>
<tr><td><code>--clean-docker-cache</code></td><td></td><td>off</td><td>Rebuild without cache (full-workflow build phase only).</td></tr>
<tr><td><code>--skip-model-run</code></td><td></td><td>off</td><td>Start the container and run <code>pre_scripts</code>, but skip the model script. Pair with <code>--keep-alive</code> for a ready-to-exec container. Does not delete a stale <code>multiple_results</code> file.</td></tr>
<tr><td><code>--require-pinned-image</code></td><td></td><td>off</td><td>Pull registry images by the digest recorded in the manifest, not by tag; fails at once if an image has no digest. Same as context key <code>require_pinned_image</code>.</td></tr>
<tr><td><code>--manifest-output</code></td><td></td><td><code>build_manifest.json</code></td><td>Manifest path written by the build phase of a full workflow.</td></tr>
<tr><td><code>--summary-output</code></td><td><code>-s</code></td><td>none</td><td>Write the run (or combined workflow) summary JSON.</td></tr>
<tr><td><code>--live-output</code></td><td><code>-l</code></td><td>off</td><td>Stream container output.</td></tr>
<tr><td><code>--output</code></td><td><code>-o</code></td><td><code>perf.csv</code></td><td>Performance CSV path.</td></tr>
<tr><td><code>--ignore-deprecated</code></td><td></td><td>off</td><td>Accepted and stored on the args object, but nothing in <code>src/madengine</code> reads it; it currently has no effect.</td></tr>
<tr><td><code>--data-config</code></td><td></td><td><code>data.json</code></td><td>Data-provider config. Only loaded if the file exists.</td></tr>
<tr><td><code>--tools-config</code></td><td></td><td><code>./scripts/common/tools.json</code></td><td>Profiling-tools definition file.</td></tr>
<tr><td><code>--sys-env-details</code></td><td></td><td>on</td><td>Collect rocEnvTool system details. Declared as a single flag with default <code>True</code>, so there is <strong>no <code>--no-sys-env-details</code></strong>; turn it off with the <code>gen_sys_env_details</code> context key.</td></tr>
<tr><td><code>--force-mirror-local PATH</code></td><td></td><td>none</td><td>Takes a <strong>path</strong>, not a flag. Sets <code>mirrorlocal</code> to that path on every non-<code>local</code> provider in the data config (<code>core/dataprovider.py</code>). It does not change the deployment target.</td></tr>
<tr><td><code>--disable-skip-gpu-arch</code></td><td></td><td>off</td><td>Ignore each card's <code>skip_gpu_arch</code>.</td></tr>
<tr><td><code>--cleanup-perf</code></td><td></td><td>off</td><td>Intended to delete intermediate <code>perf_entry*</code> files after the run. See the warning below.</td></tr>
<tr><td><code>--wait-for-gpu-memory [SECONDS]</code></td><td></td><td>unset</td><td>Local runs only. Before any container starts, require every GPU to report at most <strong>512 MB</strong> in use. <code>0</code> checks once and fails if any GPU is over; a positive value polls until memory drops or the timeout elapses. A GPU that cannot be queried fails closed. Failure is an <code>ExecutionError</code> (exit <code>3</code>). SLURM and Kubernetes warn and ignore it.</td></tr>
<tr><td><code>--verbose</code></td><td><code>-v</code></td><td>off</td><td>Debug logging.</td></tr>
</tbody>
</table>
<div class="callout err">
<strong>Known defect — <code>--cleanup-perf</code>:</strong> <span class="filepath">cli/commands/run.py</span> imports <code>madengine.utils.perf_cleanup</code>, which does not exist in the source tree. The model runs and results are written, but the post-run cleanup raises <code>ImportError</code>, which the generic handler turns into exit <code>1</code> ("Run process failed"). Avoid the flag until the module is restored.
</div>
<h3 id="cli-run-timeout">Timeout resolution</h3>
<p>Single definition in <span class="filepath">core/timeout.py::resolve_run_timeout</span>, shared by local, SLURM and Kubernetes runs. Precedence, lowest to highest: <strong>default</strong> (<code>DEFAULT_RUN_TIMEOUT</code> = 7200 s) < <strong>model card <code>timeout</code></strong> < <strong>explicit <code>--timeout</code></strong>.</p>
<table>
<thead><tr><th><code>--timeout</code></th><th>Effect</th></tr></thead>
<tbody>
<tr><td><code>-1</code> (default)</td><td>"Not passed": falls through to the card's <code>timeout</code>, else 7200 s.</td></tr>
<tr><td><code>0</code></td><td>No timeout, overriding the card.</td></tr>
<tr><td>Positive integer</td><td>That many seconds, overriding the card (even <code>7200</code>, which is distinguishable from omitting the flag).</td></tr>
<tr><td>Below <code>-1</code></td><td>Rejected, exit <code>4</code>.</td></tr>
</tbody>
</table>
<p>A card may itself set <code>"timeout"</code> to <code>0</code> or any non-positive value to mean "unbounded"; <code>null</code> in an old manifest is ignored. Non-positive resolved values are mapped to "no timeout" for <code>subprocess</code> by <code>subprocess_timeout()</code> (since <code>subprocess</code> would treat <code>0</code> as "expire immediately").</p>
<h3>Exit status</h3>
<p><code>0</code> when every model succeeded; <code>3</code> if any model failed; <code>2</code> when every build of a full workflow failed (<code>BuildError</code>); <code>4</code> for invalid arguments/configuration; <code>1</code> otherwise. A full workflow whose build only <em>partly</em> failed still runs the images that built; if those runs all pass it exits <code>1</code> ("Workflow failed for unknown reasons"), not <code>2</code>. After the run, <code>run</code> prints the per-model results table and the <code>perf.csv</code> table (current-session rows highlighted via <code>utils/session_tracker.py</code>).</p>
</section>
<section id="cli-report">
<h2>CLI — <code>report</code> & <code>database</code></h2>
<h3><code>madengine report</code></h3>
<p>Sub-app (<code>report_app</code>, <code>no_args_is_help=True</code>). All sub-commands exit <code>1</code> on failure.</p>
<table>
<thead><tr><th>Sub-command</th><th>Options</th><th>Behavior</th></tr></thead>
<tbody>
<tr><td><code>to-html</code></td><td><code>--csv-file-path FILE</code> <span class="pill">required</span>, <code>-v</code></td><td>Converts one CSV to an HTML table written next to it (same name, <code>.html</code>). The file must exist and end in <code>.csv</code>.</td></tr>
<tr><td><code>to-email</code></td><td><code>--directory</code> / <code>--dir</code> (default <code>.</code>), <code>--output</code> / <code>-o</code> (default <code>run_results.html</code>), <code>-v</code></td><td>Consolidates every CSV in a directory into one email-ready HTML file with a section per CSV. Prints a warning, not an error, if the directory has no CSVs.</td></tr>
<tr><td><code>tracelens</code></td><td><code>--root</code>/<code>-r</code> (<code>.</code>), <code>--output-dir</code>/<code>-o</code> (<code>tracelens_output</code>), <code>--mode</code> (<code>auto</code>|<code>pytorch</code>|<code>rocprof</code>|<code>pftrace</code>|<code>collective</code>), <code>--python</code>, <code>--gpu-arch</code> (e.g. <code>MI300X</code>), <code>--world-size</code> (0 = number of traces), <code>--max-traces</code> (0 = no cap), <code>--discover-only</code>, <code>-v</code></td><td>Finds trace artifacts a run left behind (<code>rocprof_output/</code>, <code>torch_profiler_output/</code>, <code>slurm_results/</code>, <code>k8s_results/</code>) and runs the matching TraceLens report: operator + roofline for torch.profiler traces, kernel summaries for rocprofv3 JSON, activity/API/memcpy reports for pftrace, collective reports across ranks. Writes <code>tracelens_summary.csv</code> to the output dir. Exit <code>4</code> for an invalid <code>--mode</code>; <code>1</code> if TraceLens is missing, the root is not a directory, or any report failed. Requires <code>pip install 'madengine[tracelens]'</code> (or <code>--python</code> pointing at an interpreter that has it).</td></tr>
<tr><td><code>tracelens-compare</code></td><td><code>REPORTS...</code> (two or more <code>.xlsx</code> files or per-sheet CSV directories), <code>--output</code>/<code>-o</code> (<code>tracelens_comparison.xlsx</code>), <code>--names</code> (repeat per report), <code>--python</code>, <code>-v</code></td><td>Diff workbook: the first report is the baseline and every metric gains <code>_diff</code> and <code>_pct</code> columns.</td></tr>
</tbody>
</table>
<pre><code>madengine report to-html --csv-file-path perf.csv
madengine report to-email --dir ./results -o summary.html
madengine report tracelens --discover-only
madengine report tracelens --root slurm_results --mode collective --world-size 8
madengine report tracelens-compare base.xlsx cand.xlsx --names before --names after -o diff.xlsx</code></pre>
<h3><code>madengine database</code></h3>
<p>Uploads a CSV or JSON results file to MongoDB (<span class="filepath">database/mongodb.py</span>); the format is auto-detected.</p>
<table>
<thead><tr><th>Option</th><th>Short</th><th>Default</th><th>Description</th></tr></thead>
<tbody>
<tr><td><code>--file</code></td><td><code>-f</code></td><td><span class="pill">required</span></td><td>CSV or JSON file.</td></tr>
<tr><td><code>--database</code> / <code>--db</code></td><td></td><td><span class="pill">required</span></td><td>Database name.</td></tr>
<tr><td><code>--collection</code></td><td><code>-c</code></td><td><span class="pill">required</span></td><td>Collection name.</td></tr>
<tr><td><code>--unique-key</code></td><td><code>-k</code></td><td>auto-detected</td><td>Comma-separated deduplication fields.</td></tr>
<tr><td><code>--batch-size</code></td><td></td><td><code>1000</code></td><td>Bulk operation size.</td></tr>
<tr><td><code>--no-upsert</code></td><td></td><td>off</td><td>Insert only; do not update existing documents.</td></tr>
<tr><td><code>--no-index</code></td><td></td><td>off</td><td>Skip automatic index creation.</td></tr>
<tr><td><code>--dry-run</code></td><td></td><td>off</td><td>Validate without uploading.</td></tr>
<tr><td><code>--verbose</code></td><td><code>-v</code></td><td>off</td><td>Also prints up to 10 upload errors.</td></tr>
</tbody>
</table>
<p>Connection comes from the environment (<code>MongoDBConfig.from_env()</code>):</p>
<table>
<thead><tr><th>Variable</th><th>Default</th></tr></thead>
<tbody>
<tr><td><code>MONGO_HOST</code></td><td><code>localhost</code></td></tr>
<tr><td><code>MONGO_PORT</code></td><td><code>27017</code></td></tr>
<tr><td><code>MONGO_USER</code>, <code>MONGO_PASSWORD</code></td><td>empty (no auth unless both set)</td></tr>
<tr><td><code>MONGO_AUTH_SOURCE</code></td><td><code>admin</code></td></tr>
<tr><td><code>MONGO_TIMEOUT_MS</code></td><td><code>5000</code></td></tr>
</tbody>
</table>
<p>Exit <code>0</code> for a successful upload, or a partial one that inserted/updated at least one document; <code>1</code> otherwise (missing file, connection failure, nothing written). See also <a href="#database">Database</a> and <a href="#reporting">Reporting</a>.</p>
</section>
<section id="cli-ops">
<h2>CLI — <code>cleanup</code> & <code>setup-page</code></h2>
<h3><code>madengine cleanup</code></h3>
<p>Removes benchmark containers left behind by runs that died before they could clean up. Every container madengine starts carries the labels <code>madengine.session</code> and <code>madengine.owner_pid</code> (<span class="filepath">core/lifecycle.py</span>); cleanup lists containers (all states) with the session label and acts on those whose owner PID is no longer alive. It is safe to call from an <code>if: always()</code> CI step or from cron while other benchmarks are running, because containers with a live owner are skipped.</p>
<table>
<thead><tr><th>Option</th><th>Default</th><th>Description</th></tr></thead>
<tbody>
<tr><td><code>--all</code></td><td>off</td><td>Also remove containers whose owning madengine process is still running.</td></tr>
<tr><td><code>--dry-run</code></td><td>off</td><td>Print what would be removed; remove nothing (exit <code>0</code>).</td></tr>
</tbody>
</table>
<table>
<thead><tr><th>Exit</th><th>Condition</th></tr></thead>
<tbody>
<tr><td><code>0</code></td><td>Nothing found, nothing stale, dry run, or every stale container removed.</td></tr>
<tr><td><code>1</code></td><td><code>docker ps</code> timed out or failed (Docker unreachable), or a container could not be removed and was not wedged.</td></tr>
<tr><td><code>5</code> <code>GPU_WEDGED</code></td><td>A container could not be removed and there is evidence of a wedge (host PIDs in uninterruptible sleep, or the daemon still reports the container as live after repeated removal). The host needs a reboot; a <code>gpu_wedged.json</code> marker is written to the state dir (<code>~/.madengine</code>, override <code>MADENGINE_STATE_DIR</code>) so CI can pull the runner out of rotation.</td></tr>
</tbody>
</table>
<p>Removal escalates <code>docker stop</code> → <code>kill</code> → <code>rm -f</code> → direct <code>SIGKILL</code> of the container's host PIDs. See <a href="#lifecycle">Container lifecycle</a> for the heartbeat watchdog and signal handlers.</p>
<h3><code>madengine setup-page</code></h3>
<p>Generates a single, fully self-contained HTML page (no external assets, so it can be published to GitHub Pages) from the <strong>current directory's</strong> <code>models.json</code>. The page lists models and tags plus the madengine context schema (<span class="filepath">setup_page/schema.py::CONTEXT_SCHEMA</span>: general, deployment target, container, distributed, Kubernetes, SLURM, tools & scripts, logging, run flags) and shows the exact <code>madengine run</code> command to copy. Only values that differ from defaults and match the selected deployment target are emitted.</p>
<table>
<thead><tr><th>Option</th><th>Short</th><th>Default</th><th>Description</th></tr></thead>
<tbody>
<tr><td><code>--output</code></td><td><code>-o</code></td><td><code>index.html</code></td><td>Output path (parent directories are created).</td></tr>
<tr><td><code>--title</code></td><td></td><td><code>madengine Setup Picker</code></td><td>Page heading (HTML-escaped).</td></tr>
<tr><td><code>--repo-url</code></td><td></td><td>none</td><td>Model repository URL, used for the clone instructions (<code>.git</code> suffix and trailing slash are stripped).</td></tr>
<tr><td><code>--install-cmd</code></td><td></td><td><code>pip install git+https://github.com/ROCm/madengine.git</code></td><td>Install command shown in step 1.</td></tr>
<tr><td><code>--verbose</code></td><td><code>-v</code></td><td>off</td><td>Debug logging.</td></tr>
</tbody>
</table>
<p>Model collection uses <code>DiscoverModels</code> (so <code>scripts/**/models.json</code> and <code>get_models_json.py</code> models appear), falling back to the root <code>models.json</code> alone if full discovery fails. Missing <code>models.json</code> exits <code>1</code>. The schema is data-driven: adding a context key to the page is a one-entry change in <code>schema.py</code>, validated by <code>validate_schema()</code> at render time.</p>
</section>
<section id="config-frontend">
<h2><code>--config</code> — YAML / Hydra frontend</h2>
<p>An alternative to <code>--additional-context</code> for <code>build</code> and <code>run</code>. It composes a config with Hydra's Compose API, validates it, and <strong>translates it into the same <code>additional_context</code> dict</strong> the rest of the pipeline already consumes, so behavior is identical to the JSON form. Code: <span class="filepath">config/loader.py</span> (<code>HydraConfigLoader</code>), <span class="filepath">config/schema.py</span> (<code>ConfigValidator</code>), <span class="filepath">config/translator.py</span> (<code>ConfigTranslator</code>), package data in <span class="filepath">src/madengine/configs/</span> (shipped in the wheel).</p>
<h3>Argument forms</h3>
<p><code>--config</code> is repeatable; each value is one of:</p>
<table>
<thead><tr><th>Form</th><th>Example</th><th>Meaning</th></tr></thead>
<tbody>
<tr><td>YAML file</td><td><code>--config my_job.yaml</code></td><td>A <code>.yaml</code>/<code>.yml</code> argument without <code>=</code> and not starting with <code>+</code>. At most one file ("Only one YAML config file allowed").</td></tr>
<tr><td>Group selection</td><td><code>--config scheduler=slurm</code></td><td>Swap a default group. Only <code>platform</code>, <code>scheduler</code>, <code>hardware</code>, <code>launcher</code> are default groups.</td></tr>
<tr><td>Append group</td><td><code>--config +env=nccl_debug</code></td><td>Add a preset from an optional group: <code>+profile</code>, <code>+env</code>, <code>+tools</code>, <code>+data</code>, <code>+build</code>. One preset file per group per invocation.</td></tr>
<tr><td>Value override</td><td><code>--config distributed.nnodes=4</code></td><td>Dotted <code>key=value</code>; also <code>+env_vars.KEY=val</code> to add a new key.</td></tr>
</tbody>
</table>
<h3>Precedence (later wins)</h3>
<ol>
<li>Package defaults: <span class="filepath">configs/config.yaml</span> with default groups <code>platform=docker</code>, <code>scheduler=local</code>, <code>hardware=amd</code>, <code>launcher=none</code>.</li>
<li>The user YAML file, merged over the defaults.</li>
<li>Group selections and <code>+group</code> appends. Only values that <em>differ from the package defaults</em> are overlaid, so unrelated keys in your YAML are not reset. Corollary: selecting a group's default (<code>--config scheduler=local</code>) is a no-op.</li>
<li><code>key=value</code> overrides.</li>
</ol>
<p>Dicts merge key by key; <strong>lists are replaced</strong> (<code>+tools=…</code> replaces a YAML <code>tools</code> list, while <code>+env=…</code> adds to <code>env_vars</code>). Explicit CLI flags win over config-derived values wherever the flag was changed from its default.</p>
<h3>Groups shipped in <span class="filepath">configs/</span></h3>
<table>
<thead><tr><th>Group</th><th>Kind</th><th>Files</th></tr></thead>
<tbody>
<tr><td><code>platform</code></td><td>default</td><td><code>docker</code>, <code>bare_metal</code>, <code>podman</code>, <code>singularity</code>. Each writes <code>platform: {type: …}</code>; only <code>docker</code> passes validation (see below).</td></tr>
<tr><td><code>scheduler</code></td><td>default</td><td><code>local</code> (empty), <code>slurm</code>, <code>k8s</code></td></tr>
<tr><td><code>hardware</code></td><td>default</td><td><code>amd</code>, <code>nvidia</code>, <code>cpu</code></td></tr>
<tr><td><code>launcher</code></td><td>default</td><td><code>none</code> (<code>distributed.enabled: false</code>), <code>torchrun</code>, <code>deepspeed</code>, <code>megatron-lm</code>, <code>megatron</code> (alias of <code>megatron-lm</code>), <code>torchtitan</code>, <code>primus</code>, <code>vllm</code>, <code>sglang</code>, <code>sglang_disagg</code>, <code>slurm_multi</code>, <code>native</code></td></tr>
<tr><td><code>profile</code></td><td>append</td><td><code>mi300x_single</code>, <code>mi300x_8gpu</code>, <code>mi250x_4gpu</code>, <code>h100_8gpu</code>, <code>a100_8gpu</code> (GPU type/memory, per-node GPU count, <code>nproc_per_node</code>, AMD HSA/queue env vars)</td></tr>
<tr><td><code>env</code></td><td>append</td><td><code>infiniband</code>, <code>miopen_defaults</code>, <code>nccl_debug</code>, <code>nccl_tuned</code></td></tr>
<tr><td><code>tools</code></td><td>append</td><td><code>rocprofv3_lightweight</code>, <code>rocprofv3_comprehensive</code>, <code>rocm_trace_lite</code>, <code>power_profiler</code>, <code>vram_profiler</code></td></tr>
<tr><td><code>data</code></td><td>append</td><td><code>local</code>, <code>nas</code>, <code>s3</code>, <code>minio</code></td></tr>
<tr><td><code>build</code></td><td>append</td><td><code>default</code>, <code>ci</code> (<code>docker.clean_cache: true</code>), <code>multi_arch</code> (<code>gfx90a</code>, <code>gfx942</code>)</td></tr>
</tbody>
</table>
<p>Group files do not cover every deployment target: there is no group for <code>spur</code> or <code>llm_d</code>, and <code>llm_d</code> / <code>conda</code> are not accepted top-level YAML keys (see validation below). Use <code>--additional-context</code> for those.</p>
<h3>Job YAML shape</h3>
<p>A job file uses the same top-level keys as the JSON context (<code>slurm</code>, <code>k8s</code>, <code>distributed</code>, <code>env_vars</code>, <code>tools</code>, …), not a Hydra <code>defaults:</code> list. Reference templates and runnable demos live in <span class="filepath">examples/configs/templates/</span> and <span class="filepath">examples/configs/demo/{local,slurm,k8s}/</span>. Defaults provided by <code>config.yaml</code>:</p>
<table>
<thead><tr><th>Key</th><th>Default</th><th>Used for</th></tr></thead>
<tbody>
<tr><td><code>model.tags</code> / <code>model.timeout</code> / <code>model.manifest_file</code> / <code>model.skip_run</code> / <code>model.container_image</code></td><td><code>[]</code> / <code>null</code> / <code>null</code> / <code>false</code> / <code>null</code></td><td>CLI metadata. <code>container_image</code> becomes context <code>MAD_CONTAINER_IMAGE</code>.</td></tr>
<tr><td><code>build.registry</code> / <code>build.target_archs</code> / <code>build.manifest_output</code></td><td><code>null</code> / <code>[]</code> / <code>build_manifest.json</code></td><td>CLI metadata for <code>build</code>.</td></tr>
<tr><td><code>docker.build_args</code>, <code>env_vars</code>, <code>mounts</code>, <code>gpus</code>, <code>cpus</code>, <code>additional_run_options</code></td><td>empty / <code>null</code></td><td>Translated to <code>docker_build_arg</code>, <code>docker_env_vars</code>, <code>docker_mounts</code>, <code>docker_gpus</code>, <code>docker_cpus</code>, <code>additional_docker_run_options</code>. <code>docker.keep_alive</code> / <code>docker.clean_cache</code> become <code>docker_keep_alive</code> / <code>docker_clean_cache</code> only when true.</td></tr>
<tr><td><code>log_error.pattern_scan</code> / <code>benign_patterns</code> / <code>patterns</code></td><td><code>true</code> / <code>[]</code> / <code>[]</code></td><td><code>log_error_pattern_scan</code>, <code>log_error_benign_patterns</code>, <code>log_error_patterns</code> (empty lists omitted).</td></tr>
<tr><td><code>output</code>, <code>summary_output</code>, <code>data_config</code>, <code>live_output</code></td><td><code>perf.csv</code>, <code>null</code>, <code>data.json</code>, <code>false</code></td><td>CLI metadata.</td></tr>
<tr><td><code>env_vars</code>, <code>tools</code>, <code>pre_scripts</code>, <code>post_scripts</code>, <code>encapsulate_script</code>, <code>debug</code></td><td>empty / <code>false</code></td><td>Passed through unchanged (empty values dropped).</td></tr>
</tbody>
</table>
<h3>Validation and restrictions</h3>
<ul>
<li><strong>Unknown top-level keys are errors</strong> (<code>Unknown config key: '…'</code>). Allowed: <code>defaults</code>, <code>platform</code>, <code>scheduler</code>, <code>hardware</code>, <code>launcher</code>, <code>model</code>, <code>docker</code>, <code>build</code>, <code>env_vars</code>, <code>debug</code>, <code>live_output</code>, <code>log_error</code>, <code>tools</code>, <code>pre_scripts</code>, <code>post_scripts</code>, <code>encapsulate_script</code>, <code>data_config</code>, <code>output</code>, <code>summary_output</code>, <code>gpu_vendor</code>, <code>guest_os</code>, <code>runtime</code>, <code>slurm</code>, <code>k8s</code>, <code>kubernetes</code>, <code>distributed</code>, <code>vllm</code>, <code>sglang_disagg</code>, <code>shared_data</code>, <code>timeout</code>, <code>gpu_type</code>, <code>gpu_memory_gb</code>, <code>gpus_per_node</code>, <code>data</code>, <code>require_pinned_image</code>.</li>
<li><code>slurm</code> together with <code>k8s</code>/<code>kubernetes</code> is rejected.</li>
<li><code>distributed.enabled: true</code> requires <code>distributed.launcher</code>; <code>distributed.nnodes</code> must be a positive integer.</li>
<li><strong>Platform:</strong> the object form <code>platform: {type: …}</code> accepts only <code>docker</code>; <code>bare_metal</code>, <code>podman</code> and <code>singularity</code> are rejected as "not yet supported", so the <code>platform=bare_metal</code> group fails validation. A top-level <code>platform</code> value is CLI metadata and is not forwarded into <code>additional_context</code>, and <code>conda</code> is not an accepted key, so <strong>bare metal cannot be selected through <code>--config</code></strong>: pass <code>"platform": "bare_metal"</code> (and any <code>conda</code> block) with <code>--additional-context</code>.</li>
<li>Any validation problem is raised as <code>ConfigurationError</code> listing every error (exit <code>4</code>).</li>
</ul>
<h3>Launcher handling</h3>
<ul>
<li>The <code>launcher</code> top-level key as a <strong>string</strong> is the Hydra group selector and is dropped from the context. As an <strong>object</strong> (<code>type</code>, <code>master_port</code>, …) it is kept as <code>launcher</code> for Kubernetes.</li>
<li><code>distributed.launcher</code> is canonicalized by the same function the engine uses elsewhere (<code>sglang_disagg</code> → <code>sglang-disagg</code>, <code>slurm-multi</code> → <code>slurm_multi</code>). The group <code>launcher=megatron</code> yields <code>megatron-lm</code>; the bare string <code>megatron</code> is not a valid launcher name.</li>
<li><strong><code>distributed.enabled</code></strong> is passed through as written; the translator never sets it. Every launcher group writes <code>enabled: true</code>, and the default <code>launcher=none</code> writes <code>enabled: false</code>. Validation requires a launcher when it is <code>true</code>. Kubernetes profile selection treats either <code>enabled: true</code> or a launcher name as distributed.</li>
<li><code>launcher=native</code> (and <code>docker</code>) is a <code>perf.csv</code> reporting value, not a launcher. The translator forces <code>distributed.enabled: false</code> and <code>launcher: null</code> so a Kubernetes profile cannot inject <code>torchrun</code>. Do not copy that group into JSON.</li>
</ul>
<div class="callout warn">
<strong>Gotchas:</strong> <code>--config</code> needs <code>hydra-core</code> / <code>omegaconf</code> (declared dependencies). <code>runtime</code> from a hardware group is kept as metadata only and is not forwarded to <code>additional_context</code>. The groups disagree on the port key: <code>torchrun</code> writes both <code>port</code> and <code>master_port</code>, <code>sglang</code> / <code>sglang_disagg</code> write only <code>port</code>, the rest only <code>master_port</code>. SLURM reads <code>port</code> first and falls back to <code>master_port</code>; Kubernetes prefers <code>master_port</code> (including <code>launcher.master_port</code>) and falls back to <code>port</code>. A disagreeing pair is not rejected. The <code>sglang_disagg</code> group puts its split under a top-level <code>sglang_disagg</code> key, which the SLURM and Kubernetes launchers do not read; set <code>distributed.sglang_disagg.prefill_nodes</code> / <code>decode_nodes</code> to change the split. Likewise the <code>vllm</code> group's top-level <code>vllm</code> block is only stored in <code>deployment_config</code>.
</div>
<h3>Examples</h3>
<div class="tabs" data-tabs="config-frontend">
<button class="on" data-tab="config-frontend-yaml">YAML + overrides</button>
<button data-tab="config-frontend-groups">Groups only</button>
</div>
<div class="tabpanel on" data-panel="config-frontend-yaml">
<pre><code># Job file from the repo, scaled up and with NCCL debug env added
madengine run \
--config examples/configs/demo/slurm/multi-node-torchrun.yaml \
--config slurm.nodes=4 \
--config distributed.nnodes=4 \
--config +env=nccl_debug
# Local single GPU straight from a demo file
madengine run --config examples/configs/demo/local/single-gpu.yaml</code></pre>
</div>
<div class="tabpanel" data-panel="config-frontend-groups">
<pre><code># No YAML: compose from groups, then tweak values