From dc8b55f32f05937a37cc7afbaf63e5a0b35da5f4 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Mon, 16 Nov 2026 16:19:28 +0800 Subject: [PATCH 1/8] feat(knowledge): record what each case tried, and read cases without PyYAML Each casebook entry already recorded the directions tried, their outcome and when a retry would be justified, but only as prose or a Markdown table that ranking could not weigh. Add a `findings` list to all seven cases: direction, outcome, recorded reason, revisit condition, retrieval keywords and `scope`. Every text field is taken from the existing case body; no claim, cause or measurement is added and no existing field changes. `scope` states what would have to change for an observation to stop holding: `local` (that host or harness), `regime_bound` (the cohort the case names) or `general` (a property of the method). It is not derivable from the outcome -- a rejection can be bound to one resolution or hold everywhere. Of 35 findings, 24 are regime_bound, 6 general and 5 local. The source-contract cases are regime_bound: each reads one pinned upstream commit and says so in its own applicability, and a later revision can change the dispatch it describes. Reading a case parses no skill frontmatter, but importing the skill catalog required PyYAML, so a base install could not discover or open one. The parser is now imported where frontmatter is parsed. Its absence raises MissingSkillsExtra, a ValueError subclass in a module of its own so catching it never imports the parser, and `collect` lets it through rather than reporting "nothing found" for an install problem. The snapshot tool relied on the old import-time failure to stop before side effects, and now asks for the parser explicitly at that point. The reader exposes `cases()` and `case_resource_path()` so a caller ranking cases resolves them by the same rule `read` uses. Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=max] (author) Signed-off-by: Zhou ziheng --- src/inferrsi/knowledge/_knowledge_errors.py | 15 +++ .../cases/admission-drain-occupancy.json | 44 +++++++++ .../cases/drama3-rejected-directions.json | 97 +++++++++++++++++++ .../qwen-ascend-rejected-directions.json | 83 ++++++++++++++++ .../cases/qwen-b300-rejected-directions.json | 82 ++++++++++++++++ .../cases/sglang-gdn-source-contract.json | 58 +++++++++++ .../cases/vllm-gdn-source-contract.json | 57 +++++++++++ .../cases/sarathi-chunk-budget.json | 68 +++++++++++++ .../knowledge/knowledge_operations.py | 43 ++++++-- src/inferrsi/knowledge/skill_catalog.py | 45 +++++++-- tools/skill_eval_snapshot.py | 10 +- 11 files changed, 583 insertions(+), 19 deletions(-) create mode 100644 src/inferrsi/knowledge/_knowledge_errors.py diff --git a/src/inferrsi/knowledge/_knowledge_errors.py b/src/inferrsi/knowledge/_knowledge_errors.py new file mode 100644 index 000000000..540b36f30 --- /dev/null +++ b/src/inferrsi/knowledge/_knowledge_errors.py @@ -0,0 +1,15 @@ +"""Errors shared by knowledge readers and their optional dependencies.""" + + +class MissingSkillsExtra(ValueError): + """Reading a checkout skill requires the optional metadata parser. + + A ValueError subclass, so callers that already treat a bad skill as a + ValueError keep working, but distinct so per-item handlers can let it + through. A missing optional dependency is a property of the install, not + of the item being read: folding it into a per-item omission reports + "nothing found" where the truth is "install the skills extra". + + Kept in its own module so that raising or catching it never imports the + skill catalog, and with it the parser this error is about. + """ diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json index 11697819d..367265e27 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json @@ -18,6 +18,50 @@ } ], "applicability": ["Continuous admission, mixed prefill/decode and final drain", "Recorded Drama3 B300 C64 diagnostic only"], + "findings": [ + { + "direction": "Use the whole-run mean batch as sustained-admission occupancy", + "outcome": "caveat", + "scope": "general", + "reason": "A whole-run mean can be dominated by drain. In the recorded C64 diagnostic, pure decode averaged 60.718 of 64 active requests during client admission, final drain averaged 8.793, and the whole-trace pure-decode average was 28.146", + "revisit_when": "Separate the client-admission and drain intervals before reading occupancy; using the whole-run mean can misidentify scheduling headroom", + "keywords": [ + "whole run mean batch", + "sustained admission", + "drain", + "scheduling headroom", + "occupancy" + ] + }, + { + "direction": "Read a near-full batch in a selected interval as a service speedup", + "outcome": "caveat", + "scope": "general", + "reason": "A near-full batch in a selected interval does not establish a service speedup; client in-flight counts also include queueing and codec work, and CPU/GPU clock alignment was approximate", + "revisit_when": "Keep all requests and complete quality/repetition groups in the end-to-end metric rather than a selected-phase rate", + "keywords": [ + "selected phase rate", + "service speedup", + "in-flight count", + "queueing", + "codec work" + ] + }, + { + "direction": "Count PIECEWISE graph fragments instead of scheduler annotations", + "outcome": "caveat", + "scope": "regime_bound", + "reason": "The original investigation counted scheduler execution annotations rather than PIECEWISE graph fragments, and retained the full request cohort for acceptance. Scheduled prompt entries may be chunks, not unique requests", + "revisit_when": "Confirm which quantity the annotations represent; kernel sums, annotation unions and SM utilization are different quantities", + "keywords": [ + "scheduler steps", + "PIECEWISE graph", + "annotation union", + "SM utilization", + "chunk vs request" + ] + } + ], "limitations": ["Campaign INCONCLUSIVE", "No new measurement or trace reproduction", "Selected-phase occupancy is not service throughput", "Raw evidence remains under its existing owner"], "attribution": "InferRSI issue #10 diagnostic evidence; extraction retains the original observation and its limits" } diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json index 1fbe4f674..850c2e828 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json @@ -13,6 +13,103 @@ "issue": "https://github.com/InferMatrixLab/InferRSI/issues/10" }], "applicability": ["Drama3 standalone native-computation runtime on one B300", "Only each cited microbenchmark, fixed-state diagnostic or complete service trial"], + "findings": [ + { + "direction": "Fused tower projections / multi-step decode", + "outcome": "no_gain", + "scope": "regime_bound", + "reason": "Faster or correct in isolated checks but no full-service throughput gain; multi-step decode delayed admission", + "revisit_when": "Measure actual active batches, admission/drain and complete service latency, including sampling and codec. Kernel timing alone cannot select the runtime", + "keywords": [ + "fused tower projection", + "multi-step decode", + "admission delay", + "service throughput" + ] + }, + { + "direction": "Async service / overlap", + "outcome": "inconclusive", + "scope": "regime_bound", + "reason": "Early fixed-request controls passed while service WER failed in both arms; later bounded dynamic replay identified shared ATen cuBLAS workspace interference. Dedicated codec capture matched tested states and fresh private-capture service WER passed while shared capture failed", + "revisit_when": "Preserve the required private/dedicated capture setup and unchanged controls. This specific reproducer does not establish one cause for every historical WER failure, or qualify arbitrary overlap/throughput; all-concurrency acceptance remains unmet", + "keywords": [ + "async overlap", + "WER failure", + "cuBLAS workspace", + "shared capture", + "dedicated capture", + "bilateral" + ] + }, + { + "direction": "Production prefix reuse", + "outcome": "rejected", + "scope": "regime_bound", + "reason": "Extensive graph/state checks and both full quality groups passed, yet the complete C64 service continuation gate failed; default-off", + "revisit_when": "Retain final drain and all returned work. Investigate the actual schedule and GPU critical path before retrying; removed MoE rows and intake-only rates cannot replace the failed service score", + "keywords": [ + "prefix reuse", + "continuation gate", + "C64 service", + "default-off" + ] + }, + { + "direction": "Initial prefix-reuse harness comparison", + "outcome": "inconclusive", + "scope": "local", + "reason": "Compared different private padding histories; mismatch was confined to a private slot before later states were checked. Corrected identical-input controls reran it", + "revisit_when": "Reset complete physical input rows and run A/A before blaming production math. Do not reuse this superseded setup failure as evidence of model incompatibility", + "keywords": [ + "harness mismatch", + "padding history", + "A/A control", + "identical input" + ] + }, + { + "direction": "Transparent VMM weight compression", + "outcome": "no_gain", + "scope": "regime_bound", + "reason": "Native correctness/lifetime checks passed, but the fixed three-body screen missed its frozen timing gate; no runtime promotion", + "revisit_when": "First measure the current bottleneck and physical traffic/compression where possible. Mapped flags and logical capacity do not prove a bandwidth benefit", + "keywords": [ + "VMM", + "weight compression", + "memory compression", + "bandwidth", + "timing gate" + ] + }, + { + "direction": "Producer scale-tile cache", + "outcome": "no_gain", + "scope": "regime_bound", + "reason": "Complete-body reductions missed the source gate; stopped without model/service promotion", + "revisit_when": "New shapes or a different resource regime may justify another bounded screen; do not subtract unrelated historical screens to infer a cause", + "keywords": [ + "scale-tile cache", + "producer cache", + "bounded screen" + ] + }, + { + "direction": "Fixed eight-lane FP8 SIMT proposal", + "outcome": "rejected", + "scope": "regime_bound", + "reason": "Stopped at CPU/AOT cost analysis for that mapping, before a runtime benchmark. An analytical rejection, not an empirical GPU failure", + "revisit_when": "Revisit only with a changed mapping or supported cost model; this is not a universal ceiling", + "bounds": "That fixed eight-lane mapping and its cost model only", + "keywords": [ + "FP8", + "SIMT", + "eight-lane", + "AOT cost analysis", + "analysis-only" + ] + } + ], "tags": ["negative-evidence", "service-regression", "quality-rejection", "harness-mismatch", "prefix-reuse", "memory-compression", "analysis-only"], "limitations": ["Campaign remains INCONCLUSIVE and optimization is paused", "Individual completed gate failures do not complete the all-concurrency acceptance matrix", "Repository summary inspected; external raw traces and runtime artifacts not rerun or rehashed", "No root cause inferred from client accounting or cross-screen timing subtraction"], "attribution": "Issue #10 diagnostic lessons, including runtime 47c3d2ec4e3406d8a71fccc3cb4967783c2650f5; revisit checks are current suggestions" diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json index 81dba3ce7..587352abb 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json @@ -13,6 +13,89 @@ "upstream_head": "399cc01d1bc4542964d91aaa17fb43862a8cf3b3" }], "applicability": ["Author-reported single Ascend 910B1 Qwen-Image-2.1-Turbo cohort", "BF16, eager, TORCH_SDPA, CFG=1, 1280x704, native eight sigmas"], + "findings": [ + { + "direction": "Cache eager RoPE cos/sin or expanded SDPA masks", + "outcome": "no_gain", + "scope": "regime_bound", + "reason": "Pixel-identical, but discarded as slower in the source trials", + "revisit_when": "Profile current recomputation and retained-tensor costs; qualify a matched timing cohort before attributing a regression", + "keywords": [ + "RoPE", + "cos sin cache", + "SDPA mask", + "expanded mask", + "recomputation" + ] + }, + { + "direction": "Cache per-step masks and prompt tensors", + "outcome": "rejected", + "scope": "regime_bound", + "reason": "NPU OOM", + "revisit_when": "First inventory live tensor lifetimes and allocator headroom; a different residency policy or workload may change feasibility", + "keywords": [ + "per-step mask", + "prompt tensor", + "mask cache", + "OOM", + "allocator headroom" + ] + }, + { + "direction": "Fusion attention, including precise=2/0 and mask-free steps", + "outcome": "no_gain", + "scope": "regime_bound", + "reason": "No gain or a gain below the source keep criterion", + "revisit_when": "Check exact mask, dtype, shapes and device support; run operator quality then complete pipeline timing", + "keywords": [ + "fusion attention", + "precise", + "mask-free", + "operator quality" + ] + }, + { + "direction": "TASK_QUEUE_ENABLE=1/2 or enforce_eager=0", + "outcome": "no_gain", + "scope": "regime_bound", + "reason": "No gain or slower than the source eager configuration", + "revisit_when": "Inspect the actual execution path and scheduling overhead under the new stack; repeat with unchanged workload and warmups", + "keywords": [ + "TASK_QUEUE_ENABLE", + "enforce_eager", + "scheduling overhead", + "task queue" + ] + }, + { + "direction": "RoPE + VAE fastpath + fusion attention", + "outcome": "rejected", + "scope": "regime_bound", + "reason": "Initialization OOM for this triple stack", + "revisit_when": "Inventory initialization residency before attempting a new combination", + "keywords": [ + "RoPE", + "VAE fastpath", + "fusion attention", + "triple stack", + "initialization OOM" + ] + }, + { + "direction": "Separate kept RoPE + VAE stack on a later busy host", + "outcome": "inconclusive", + "scope": "local", + "reason": "Later requests failed resource reservation; a different stack/host cohort from the triple-stack initialization failure", + "revisit_when": "Qualify current headroom and outside contention before retrying; do not attribute this resource blockage to fusion attention", + "keywords": [ + "busy host", + "resource reservation", + "contention", + "VAE fastpath" + ] + } + ], "tags": ["negative-evidence", "resource-failure", "OOM", "no-demonstrated-gain", "RoPE", "fusion-attention", "task-queue"], "limitations": ["Imported author report; locally_verified=false", "Raw request timings, RGBA outputs and frozen validator are absent from this import", "InferRSI acceptance remains INCONCLUSIVE; the source 30 percent target was not met", "No cross-device or HTTP-serving qualification"], "attribution": "amy-why-3459 source report, retained by the existing Ascend guide; revisit checks are current suggestions" diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json index 1b3a729e4..9f4825006 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json @@ -13,6 +13,88 @@ "issue": "https://github.com/InferMatrixLab/InferRSI/issues/3" }], "applicability": ["Qwen-Image-2.1-Turbo, four B300, BF16, native eight steps", "Only the source ledger's individual trials and controls"], + "findings": [ + { + "direction": "H3: initial Q/K RMSNorm + complex RoPE fusion", + "outcome": "rejected", + "scope": "regime_bound", + "reason": "Operator tests passed; complete images failed; reverted. A quality rejection, not a kernel-speed result", + "revisit_when": "First match the actual reduction, complex FMA order and BF16 materialization; compare full native images after operator checks. H10/H13 show later, different implementations were retained, so the technique is not permanently excluded", + "keywords": [ + "RMSNorm", + "complex RoPE fusion", + "quality rejection", + "BF16 materialization", + "FMA order" + ] + }, + { + "direction": "H1: packed Q/K/V 5D exchange", + "outcome": "inconclusive", + "scope": "local", + "reason": "Quality passed but the exploratory trial was discarded", + "revisit_when": "Changed layouts/topology may justify another trial; measure staging/copies and the complete request against a matched control. Do not infer the cause from the one ledger timing", + "keywords": [ + "packed QKV", + "5D exchange", + "layout", + "staging copies" + ] + }, + { + "direction": "H6/H7: graph projection/norm/RoPE or post-attention work", + "outcome": "no_gain", + "scope": "regime_bound", + "reason": "Removed for no material E2E gain; H7 records replay/copy costs", + "revisit_when": "A new capture boundary or buffer lifetime may change the tradeoff; include capture/replay, copies and full request cost", + "keywords": [ + "CUDA Graph", + "graph projection", + "post-attention", + "replay cost", + "capture boundary" + ] + }, + { + "direction": "H14/H16: broadcast decode scale/gate or skip empty concatenation", + "outcome": "no_gain", + "scope": "regime_bound", + "reason": "No demonstrated benefit; removed/reverted", + "revisit_when": "Confirm these operations are on the current critical path before repeating; retain a matched E2E control", + "keywords": [ + "broadcast decode", + "scale gate", + "empty concatenation", + "critical path" + ] + }, + { + "direction": "H17 and the noisy H13 attempt", + "outcome": "inconclusive", + "scope": "local", + "reason": "Observed host load/concurrent diagnostics invalidate causal attribution", + "revisit_when": "Repeat on a qualified, uncontended host. Do not classify the numerical timing difference as a confirmed code regression", + "keywords": [ + "host load", + "contention", + "confounded run", + "causal attribution" + ] + }, + { + "direction": "H18: compiled launch API probe", + "outcome": "inconclusive", + "scope": "local", + "reason": "Host enqueue improved in a diagnostic; no E2E integration was performed", + "revisit_when": "Treat it as an unintegrated lead; verify actual execution, correctness and complete service overhead before accepting it", + "keywords": [ + "compiled launch", + "launch API", + "host enqueue", + "unintegrated lead" + ] + } + ], "tags": ["negative-evidence", "quality-rejection", "no-demonstrated-gain", "confounded-run", "RoPE", "CUDA-Graph"], "limitations": ["The final campaign PASS does not qualify each rejected direction", "Exploratory trial timings are not additive gains or causal comparisons", "No experiment was rerun for this extraction"], "attribution": "Historical issue #3 optimization ledger; revisit checks are current diagnostic suggestions" diff --git a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json index 670367025..552e012f6 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json @@ -49,6 +49,64 @@ "Pinned Qwen3.5/Qwen3-Next GDN caller/backend qualification", "Separate causal-conv phase/layout/state contracts and conditional one-shot classification-prefill transfer" ], + "findings": [ + { + "direction": "Use one backend label for both convolution and recurrence", + "outcome": "failure_signal", + "scope": "regime_bound", + "reason": "A prefill flag does not replace both operators; equating an explicit flag, package availability or a registered symbol with executed dispatch misreads the source, and sm89 compatibility cannot be claimed from an SM9/10 automatic default", + "revisit_when": "Read the backend aliases/policy and the ordinary decode path separately before naming a dispatch", + "keywords": [ + "backend label", + "prefill flag", + "executed dispatch", + "sm89", + "automatic default" + ] + }, + { + "direction": "Infer strides from shape for the Triton prefill path", + "outcome": "failure_signal", + "scope": "regime_bound", + "reason": "CUDA input can select stride-aware Triton prefill; assuming unconditional input aliasing or no mutation, letting packed requests read each other's history, or assuming the conv mask clears the recurrent matrix are all source misreadings", + "revisit_when": "Check the prefill caller, SRT wrapper and Triton wrapper for the actual stride and mutation contract", + "keywords": [ + "stride-aware", + "Triton prefill", + "input aliasing", + "packed request history", + "conv mask" + ] + }, + { + "direction": "Reuse batch index as persistent identity across reordered rows", + "outcome": "failure_signal", + "scope": "regime_bound", + "reason": "Equating temporal state with attention KV, losing gather/scatter write-back, or treating a successful output computation as proof that continuation, reset or recycling is correct all break the state contract", + "revisit_when": "Trace mapping/ownership and the state gather and write-back paths; both state writes must be kept", + "keywords": [ + "batch index", + "persistent identity", + "strided pool", + "gather scatter", + "write-back" + ] + }, + { + "direction": "Treat fresh classification prefill as decode or multi-item scoring", + "outcome": "failure_signal", + "scope": "regime_bound", + "reason": "Presenting source reasoning or packaged reads as a binding, native test, quality pass or service gain confuses the evidence level; a generic classifier is not MIS, and unused returned cache does not make internal state writes dispensable", + "revisit_when": "Read the MIS constraints before classifying a prefill path; keep source reasoning distinct from a native result", + "keywords": [ + "classification prefill", + "multi-item scoring", + "MIS", + "returned cache", + "internal state write" + ] + } + ], "limitations": [ "Native kernels, models and devices not_run", "Agent efficacy not_evaluated", diff --git a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json index 85910de4f..8c9e9513a 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json @@ -32,6 +32,63 @@ "Source-only qualification of the pinned Qwen3-Next CUDA-facing GDN caller", "Mixed prefill/decode, packed/speculative metadata and state-row contracts" ], + "findings": [ + { + "direction": "Treat every one-token row as decode in a mixed Qwen3-Next batch", + "outcome": "failure_signal", + "scope": "regime_bound", + "reason": "A first one-token chunk can require the prefill route to mask recycled state; selecting a decode-only convolution for the whole mixed batch, or passing unre-based offsets to the prefill recurrence, misreads the source", + "revisit_when": "Use prior-state/prefill metadata as well as query length; identify the varlen forward convolution when non-speculative prefill rows exist", + "keywords": [ + "one-token initialization", + "mixed phase", + "decode-only conv", + "rebased offsets", + "varlen forward" + ] + }, + { + "direction": "Infer the actual GDN branch from the requested backend", + "outcome": "failure_signal", + "scope": "regime_bound", + "reason": "Selection inferred from package presence, default flags or symbol names does not establish the branch; a separate-conv rule does not generalize to every model/backend, and a source predicate is not hardware compatibility or speedup", + "revisit_when": "Trace the actual branch in the pinned source rather than the requested backend label", + "keywords": [ + "requested backend", + "actual branch", + "FlashInfer", + "package presence", + "default flags" + ] + }, + { + "direction": "Equate batch index with cache row under reordering and speculation", + "outcome": "failure_signal", + "scope": "regime_bound", + "reason": "Copying standalone-package cursor semantics into this API, preserving stale first-chunk state, or treating convolution and recurrent state as attention KV all break the state-row contract", + "revisit_when": "Track reordered state rows and speculative progress explicitly; account for both state writes", + "keywords": [ + "reordered state rows", + "batch index", + "cache row", + "speculative progress", + "cursor semantics" + ] + }, + { + "direction": "Assert a model-to-HF binding from source reading alone", + "outcome": "failure_signal", + "scope": "regime_bound", + "reason": "Inventing a model-to-HF implementation mapping, prescribing an unverified installation, or asserting numeric correctness, hardware support or speedup all exceed what source classification establishes", + "revisit_when": "Keep the binding unproven until an identified caller is verified; native execution here is not_run", + "keywords": [ + "HF caller", + "binding", + "unverified installation", + "numeric correctness" + ] + } + ], "limitations": [ "Native kernel/model/device execution not_run", "Representative agent evaluation not_evaluated", diff --git a/src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json b/src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json index 4a918ae93..7757866c3 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json @@ -28,6 +28,74 @@ "Source-supported hypothesis selection when observed long prefills interfere with ongoing decodes", "Chunked-prefill token-budget tradeoffs under specified latency, workload, hardware and parallelism conditions" ], + "findings": [ + { + "direction": "Split a long prefill into chunks and combine them with ongoing decodes", + "outcome": "tradeoff", + "scope": "general", + "reason": "Source mechanism for long prefills interrupting token generation; chunking limits prefill work per iteration but does not make that work free. Figure 9 still shows added batch latency relative to decode-only execution", + "revisit_when": "Diagnose queueing, memory shortage or suspended requests first; only attribute the delay to prefill interference after that", + "keywords": [ + "chunked prefill", + "long prefill interrupts decode", + "TBT", + "time between tokens", + "prefill interference" + ] + }, + { + "direction": "Tighten the token budget with smaller chunks", + "outcome": "tradeoff", + "scope": "general", + "reason": "Smaller chunks reduce prefill work per iteration but can lower compute utilization and repeat fixed launch costs", + "revisit_when": "Profile the actual utilization and launch overhead for the target hardware before choosing a budget", + "keywords": [ + "token budget", + "chunk size", + "compute utilization", + "launch cost" + ] + }, + { + "direction": "Accept the KV re-read cost of later chunks", + "outcome": "caveat", + "scope": "general", + "reason": "Each later chunk attends to earlier chunks of the same prompt, rereading their KV cache. The source distinguishes these additional memory reads from unchanged attention computation", + "revisit_when": "Account for the extra memory reads separately from attention compute when estimating cost", + "keywords": [ + "KV cache reread", + "memory reads", + "attention computation", + "chunk attention" + ] + }, + { + "direction": "Pick a budget without accounting for tile boundaries or pipeline parallelism", + "outcome": "caveat", + "scope": "general", + "reason": "Matrix tile boundaries affect execution cost and neighboring token counts need not have neighboring runtimes; with pipeline parallelism, batch runtime differences can leave stages idle", + "revisit_when": "Measure runtime across neighboring token counts and check stage idling before fixing a budget", + "keywords": [ + "matrix tile boundary", + "pipeline parallelism", + "stage idle", + "batch runtime variation" + ] + }, + { + "direction": "Read Table 4 as an all-round improvement", + "outcome": "caveat", + "scope": "regime_bound", + "reason": "The combined design has lower P99 TBT than either component alone, but its P50 TTFT is higher than hybrid-batching-only. It trades first-token latency against tighter token gaps", + "revisit_when": "State which metric the deployment optimizes; the combined design does not improve every metric over each alternative", + "keywords": [ + "P99 TBT", + "P50 TTFT", + "hybrid batching", + "first token latency" + ] + } + ], "limitations": [ "Paper-author observations and proposed adaptation checks are not local measurements or current-engine support.", "The combined design does not beat hybrid-only P50 TTFT in either Table 4 trace.", diff --git a/src/inferrsi/knowledge/knowledge_operations.py b/src/inferrsi/knowledge/knowledge_operations.py index 275c93f6d..9c5e1e7b3 100644 --- a/src/inferrsi/knowledge/knowledge_operations.py +++ b/src/inferrsi/knowledge/knowledge_operations.py @@ -9,6 +9,7 @@ from importlib.resources import files from pathlib import Path, PurePosixPath +from ._knowledge_errors import MissingSkillsExtra from ._skill_selection import ( RELEVANCE_METHODS, TASK_RELEVANCE, @@ -48,6 +49,24 @@ def safe_path(value: str) -> str: return value +def case_resource_path(root: Path, record_path: Path, resource: object) -> Path: + """Resolve a resource a case record names, refusing anything outside `root`. + + `read` loads a case through this check, and `find` locates a case body for + ranking through it too, so a body that one would refuse is never treated as + readable by the other. + """ + if not isinstance(resource, str): + raise ValueError("Case resource must be a relative POSIX path") + safe_path(resource) + if resource == "case.json": + raise ValueError("Case resource collides with its case.json record") + path = record_path.parent / resource + if path.is_symlink() or not path.resolve().is_relative_to(root): + raise ValueError(f"Case resource escapes the skills root: {resource}") + return path + + def _json(value: object) -> bytes: return json.dumps(value, sort_keys=True, ensure_ascii=False, separators=(",", ":")).encode() @@ -314,6 +333,14 @@ def resolve(self, reference: str, owners: dict[str, list[str]] | None = None) -> ) return f"{found[0]}:{identity}" + def cases(self) -> dict[str, Path]: + """Case identities owned by this checkout's registered skills. + + The discovery `read` and `collect` resolve against, exposed so a caller + ranking cases cannot drift from the rule that decides whether they open. + """ + return self._cases() + def _cases(self) -> dict[str, Path]: result = {} if self.skill_root is not None: @@ -366,18 +393,11 @@ def _checkout(self, identity: str) -> Bundle: raise ValueError("Case required_resources must be a string list") if len(set(declared)) != len(declared): raise ValueError("Case required_resources must not repeat a path") - relative = safe_path(record["content"]) + relative = record["content"] required = [relative, "case.json", *(p for p in declared if p != relative)] content = {"case.json": path.read_bytes()} for resource in [relative, *required[2:]]: - safe_path(resource) - if resource == "case.json": - raise ValueError("Case resource collides with its case.json record") - resource_path = path.parent / resource - if resource_path.is_symlink() or not resource_path.resolve().is_relative_to( - self.skill_root - ): - raise ValueError(f"Case resource escapes the skills root: {resource}") + resource_path = case_resource_path(self.skill_root, path, resource) if not resource_path.is_file(): raise ValueError(f"Missing case resource: {resource}") content[resource] = resource_path.read_bytes() @@ -680,6 +700,11 @@ def select(reference: str, bundle: Bundle, view: dict) -> None: {"id": reference, "reason": "unavailable_revision", "detail": str(exc)} ) continue + except MissingSkillsExtra: + # The install cannot parse any checkout skill, not just this one. + # As an omission it would return `no_match`, which reads as + # "nothing found" rather than "install the skills extra". + raise except (ValueError, KeyError, TypeError, AttributeError, OSError, sqlite3.Error) as exc: omitted.append({"id": reference, "reason": "invalid", "detail": str(exc)}) continue diff --git a/src/inferrsi/knowledge/skill_catalog.py b/src/inferrsi/knowledge/skill_catalog.py index ad3ad25a2..08e307232 100644 --- a/src/inferrsi/knowledge/skill_catalog.py +++ b/src/inferrsi/knowledge/skill_catalog.py @@ -1,21 +1,47 @@ from __future__ import annotations import json +import os import re from pathlib import Path, PurePosixPath from typing import BinaryIO -try: - import yaml -except ImportError as exc: # The base install stays PyYAML-free by contract. - raise ValueError( - "Reading checkout skills requires the 'skills' extra (PyYAML); install it " - "from your InferRSI checkout, replacing the placeholder path: " - "python -m pip install '[skills]'" - ) from exc - +from ._knowledge_errors import MissingSkillsExtra from .catalog import BACKENDS +_SKILLS_EXTRA = ( + "Reading checkout skills requires the 'skills' extra (PyYAML); install it " + "from your InferRSI checkout, replacing the placeholder path: " + "python -m pip install '[skills]'" +) +# A bundled runtime has no checkout to install an extra from; what the caller +# can do there is put PyYAML into the interpreter running the launcher. +_SKILLS_EXTRA_BUNDLED = ( + "Reading checkout skill bodies requires PyYAML; install it into the Python " + "interpreter that runs this plugin, or use packaged knowledge, cases and " + "find, which do not need it." +) + + +def _install_hint() -> str: + return _SKILLS_EXTRA_BUNDLED if os.environ.get("INFERRSI_LAUNCHER") else _SKILLS_EXTRA + + +def _yaml(): + """Import PyYAML only for the calls that parse skill frontmatter. + + Resolving which skills a root registers reads skill-index.json or lists + directories; neither needs a YAML parser. Importing it at module load made + every caller depend on an optional package, so a base install could not + discover or open a case -- which parses no frontmatter -- merely because + discovery lives in this module. + """ + try: + import yaml + except ImportError as exc: # The base install stays PyYAML-free by contract. + raise MissingSkillsExtra(_install_hint()) from exc + return yaml + def _frontmatter(stream: BinaryIO) -> dict: if stream.readline().strip() != b"---": @@ -27,6 +53,7 @@ def _frontmatter(stream: BinaryIO) -> dict: lines.append(line) else: raise ValueError("Unclosed SKILL.md frontmatter") + yaml = _yaml() try: record = yaml.safe_load(b"".join(lines).decode("utf-8")) except yaml.YAMLError as exc: diff --git a/tools/skill_eval_snapshot.py b/tools/skill_eval_snapshot.py index bcf0593c9..98e2a293d 100644 --- a/tools/skill_eval_snapshot.py +++ b/tools/skill_eval_snapshot.py @@ -44,7 +44,12 @@ def selected_snapshot(repo: Path, revision: str, skill_root: str, skill: str, """ # Only materializing checkout skills requires the optional PyYAML parser; # reporting and controller imports must remain usable in the base install. - from inferrsi.knowledge.skill_catalog import _skill_name, _unique_fields, skill_metadata + from inferrsi.knowledge.skill_catalog import ( + _skill_name, + _unique_fields, + _yaml, + skill_metadata, + ) if not re.fullmatch(r"[0-9a-f]{40}", revision): raise ValueError("Selected snapshot requires a full immutable Git commit SHA") @@ -52,6 +57,9 @@ def selected_snapshot(repo: Path, revision: str, skill_root: str, skill: str, _skill_name(skill) if destination.exists() or destination.is_symlink(): raise ValueError("Selected snapshot destination must not exist") + # The parser is imported lazily now; require it before any Git or + # filesystem side effect, as the module-level import used to. + _yaml() version = subprocess.check_output(["git", "--version"], cwd=repo, text=True) match = re.match(r"git version (\d+)\.(\d+)", version) if match is None or tuple(map(int, match.groups())) < (2, 45): From 872b44f958bb88c7da150eea4188b1899c38a703 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Mon, 16 Nov 2026 16:22:35 +0800 Subject: [PATCH 2/8] chore(knowledge): regenerate skill cards for the cases that gained findings Mechanical follow-up. The generator pins each card's source hashes to a revision, so the three skills that own the edited cases -- the optimization casebook, the kernel-writing guide and the search policy -- held hashes for the case files as they were before findings were added, and `indexed_skill` rejected those projections as stale. Regenerated with tools/build_skill_index.py against the previous commit. Agent-assisted-by: Claude Code / claude-opus-5-5 [effort=max] (author) Signed-off-by: Zhou ziheng --- .../data/repo_packs/inferrsi/knowledge/_provenance.json | 2 +- .../common/skills/inference-optimization-casebook.json | 8 ++++---- .../knowledge/common/skills/kernel-writing-guide.json | 4 ++-- .../common/skills/optimization-search-policy.json | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json index 7ab45e60f..11d279ad7 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json @@ -1,7 +1,7 @@ { "schema_version": 1, "repository": "https://github.com/InferMatrixLab/InferRSI", - "source_revision": "23d27be8bde499f8dadd1436657017cf2f4077ab", + "source_revision": "dc8b55f32f05937a37cc7afbaf63e5a0b35da5f4", "registry": { "path": "src/inferrsi/knowledge/data/skill_library/skill-index.json", "sha256": "7904275c3383380db179287171e1430617aaa9ac4f439291eff8a71426987fbb" diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json index b2b9fdbb2..62535c9ac 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json @@ -25,13 +25,13 @@ "checkout_ref": "checkout:inference-optimization-casebook", "source_files": { "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/SKILL.md": "89a6a82c692f7f5192f4d7f4c3a5ff08a4df3b1c4bdaa0aafb7e04292d8aa5b4", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json": "3319e23bbe2bd40e2d45fb89dc87bdc92d4df2b5ddd8086fcf84813b43885b7c", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json": "77bcd2a5dcdf7892877114c993358234ad009dbde417b284c877cf8c1667389f", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.md": "e88bbac5d253a8e7e3a3d5e21047f23f2970c11ac23887d4303ab94fca52dd27", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json": "2cc0a526fe0509551607a03e7626dd3d7844e643c4afb194837f75cddead8b12", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json": "059f16c5a9dcd70532e1b7035bcf80f505086f665ca63407ba4bb6f4174c0db2", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.md": "6f143bc363a3290fc008c08e57cb075f1776d1c8cca8a6a59297d5b1fbbf61bf", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json": "b3d538f79c56724ccb2895cbf50cf2e6b0fec3979be62a684a8230cd932dff55", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json": "49cc9bb498a0e05515e902b2e549f6548259a828dca70f0856a2c7eced3a60d9", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.md": "be396983812262515899a3e2ba4f01b23ff92e1b836881882c7580372b8ae552", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json": "146ea1564a46ed03092c7dc5796b318fc05b04b1821105ee00eb7fa7fa1333e2", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json": "d463625c52c903e8b821772d8c05bddd0304f6cdb8b2454e73737f69f2bc0fb7", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.md": "cce2930f544a0f78f21797fba4d8ded298e4a81f76c34d7977b0e31d361bc738", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/metadata.json": "963488658f327d2379183d70357eabf832d70a6a70db2aa67f548493f4088d97" }, diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json index fc698838a..a5e7624fd 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json @@ -45,10 +45,10 @@ "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/SKILL.md": "360361f592ec3a24834198372cbf92aaedc1720404d182df94315eb0e01d58dd", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/causal-conv1d-dispatch-state.md": "2eb67dd522021bb2234616d7f660e6ba99acb43676f48cb23141bc69bf3e0029", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/hf-qwen35-gdn-binding.md": "b3a55af3f45f01c48e1cb18c8b029e5949a494c3e51dc444a5cccbad9acbba0a", - "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json": "6c2586ec80698dba848ed09024954d060307be71c8701719af19df3b735cf595", + "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json": "7eb01ffeb40cdd555a2b0012e863f13a09df4a13a00d0ba1fab555645b245a42", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.md": "8b093faf0ab775d5605ecdd9653fb3689c2b839cba2641c5d5cda17c721bb2ac", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-causal-conv-contract.md": "2f3dcee66e2dfe7a98bee21a36b892538998de5de3caf1bc63257137f9984236", - "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json": "73f1a2d8fa89ce05aead2091a05a285b13cbdc314d05452c35c6a2ad15ffeff7", + "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json": "68f1cb0dc99bc3eb11692fa030d5f785622d0ea1733a084a76d286c9540890bb", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.md": "7f9ebe8de7e55d3db2a119b6a56e15c37448fd7d4e022a47122a143206d31687", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/metadata.json": "766aa688d573c790789e2836f952315e91c4e06806fca3d04523a8a748bf9c32", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/references/akg-affine-tail-guards.md": "113deb2db10b171db3425bb3f1792bee763706b794c8cc2e222b4cc0d0114118", diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/optimization-search-policy.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/optimization-search-policy.json index 78a14091e..28d246f8e 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/optimization-search-policy.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/optimization-search-policy.json @@ -25,7 +25,7 @@ "checkout_ref": "checkout:optimization-search-policy", "source_files": { "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/SKILL.md": "493001e1266b91434d873d688c978486bbac4a4b86fb657492746877b652e4fb", - "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json": "f8448973cf1c9e506179c02c5e55bd804ea039b7276cf08071952f6c0be656ad", + "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json": "59f00bc62d0dfa19328a2e1c6f2a261af155c25f67fc185e94254a479bc66d17", "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.md": "2c2a8147001d24d1abc8672f15cf0ca759ae461b734413c8dcb7497634bb6821", "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/metadata.json": "43e6378cdc801a71fe510e6adc17079016220c94bd6debfa8366b3cc7117d493", "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/references/distserve-phase-placement/README.md": "56e16efce447dfd4990a024ff1d6f7cb428f8e3e1f7b32a7e4d05f57139b836a", From 0ed23a12ccf768352bbde127cdf84113530122a0 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Mon, 16 Nov 2026 16:39:45 +0800 Subject: [PATCH 3/8] feat(knowledge): unify discovery across cases, skills, and records Port the verified find and read flow to the current Knowledge package and Apps-facing CLI. Keep query fixtures, compact exact reads, and standalone plugin coverage together. Signed-off-by: Zhou ziheng Agent-assisted-by: Claude Code (author) Agent-assisted-by: Codex (author) --- README.md | 5 +- docs/knowledge-operations.md | 9 +- docs/operations.md | 80 ++ docs/quickstart.md | 21 +- plugins/inferrsi/skills/inferrsi/SKILL.md | 36 +- .../skills/inferrsi/scripts/inferrsi.py | 26 +- .../skills/inferrsi/scripts/inferrsi.py | 4 +- src/inferrsi/arguments.py | 30 +- src/inferrsi/cli.py | 101 +++ .../knowledge/data/query_aliases.json | 207 +++++ src/inferrsi/knowledge/knowledge_find.py | 726 ++++++++++++++++++ tests/fixtures/find/README.md | 50 ++ tests/fixtures/find/labelled-queries.json | 335 ++++++++ tests/test_codex_plugin.py | 68 +- tests/test_knowledge_cli.py | 17 + tests/test_knowledge_find.py | 646 ++++++++++++++++ 16 files changed, 2327 insertions(+), 34 deletions(-) create mode 100644 src/inferrsi/knowledge/data/query_aliases.json create mode 100644 src/inferrsi/knowledge/knowledge_find.py create mode 100644 tests/fixtures/find/README.md create mode 100644 tests/fixtures/find/labelled-queries.json create mode 100644 tests/test_knowledge_find.py diff --git a/README.md b/README.md index 5118084d2..61b9756ce 100644 --- a/README.md +++ b/README.md @@ -224,8 +224,9 @@ python -m venv .venv source .venv/bin/activate python -m pip install . -# Explore tuning knowledge, then export context for your coding agent. -inferrsi knowledge search 'Qwen-Image-2.1' --repo-type vllm_omni --backend ascend +# Find relevant tuning knowledge and follow the printed open command. +inferrsi find 'Qwen-Image-2.1 Ascend memory bottleneck' +# Export a context bundle for an agent that needs one. inferrsi context --repo-type vllm_omni --backend cuda --output context.md ``` diff --git a/docs/knowledge-operations.md b/docs/knowledge-operations.md index c82096f05..7003561a8 100644 --- a/docs/knowledge-operations.md +++ b/docs/knowledge-operations.md @@ -32,8 +32,9 @@ inferrsi collect 'Ascend OOM' --kind case --limit 3 --material --skill-root src/ inferrsi read calibrate-serving-simulation --skill-root src/inferrsi/knowledge/data/skill_library ``` -The `--skill-root` selections require the `skills` extra (`python -m pip install -'.[skills]'`); package-only collect/read work with the base install. +Reading checkout skill bodies requires the `skills` extra (`python -m pip install +'.[skills]'`). Package-only operations and checkout case reads work with the base +install. Every item has an owning `authority` (`package`, `checkout` or `store`) and a qualified `ref` of the form `:`. Public IDs are unchanged and may be used @@ -202,6 +203,10 @@ Selected resource reads require the exact returned revision. A changed checkout or installation rejects an old revision; it never substitutes new bytes. This reader does not itself retain history or claim standalone exports. +`read --material --omit-resources` returns the full entrypoint, provenance, +applicability and revision. Remove `--omit-resources` to list resource paths and +digests. Both forms use the same full-bundle revision. + - Local skills include their complete owner folder (except incidental `__pycache__`, `*.pyc` and `.DS_Store`) and declared cross-skill references, whether declared in the sidecar or in legacy frontmatter. Other links remain navigation in the required full checkout. diff --git a/docs/operations.md b/docs/operations.md index b5031ebe6..025f0b9cc 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -35,6 +35,7 @@ Effect classes used below: | Search packaged records | CLI `knowledge search` / `knowledge show`; catalog API | Read-only | Installed package catalog with the existing tier/repo/backend/scenario filters and IDs; these package-only filters exclude skills and cases from every other authority | | Prepare model-aware engine research | CLI `knowledge research`; Python `research_knowledge` | Read-only | Independent model/framework/task/modality/mechanism queries over the same package catalog; source-grounded known profiles, deduplicated matches and explicit target annotations; no engine/hardware qualification | | Discover/read packaged or custom skills | CLI `skills list` / `show` / `resource` / `reference` | Read-only | Metadata-first listing; `show` loads one named instruction body; `resource`/`reference` resolve paths for your file reader without loading or executing them | +| Rank relevant cases, skills and records | CLI `find` | Read-only | Search the registered library and package records in one list. Follow the emitted command to read a hit and its revision | | Collect normalized knowledge | CLI `collect`; Python `KnowledgeReader.collect` | Read-only | Explicit sources, authority-qualified identities, exact revisions, bounded output and explicit omissions | | Read exact material/resources | CLI `read` / `resource`; Python `KnowledgeReader.read` / `KnowledgeReader.resource` | Read-only | Returns the requested revision or an explicit unavailable/stale error; never substitutes newer bytes | | Export legacy cited material | CLI `context` | Read-only | Cited package export for manual use with a coding agent; not a conversation store and not a full Collect equivalent | @@ -166,6 +167,85 @@ upstream instructions. ## Normalized knowledge (Python) +### Find — `find` / `inferrsi.knowledge.knowledge_find.find` + +One ranked list for one question, across packaged skills, imported records and +checkout cases. Run a result's `open` command to read its complete entry and +receive a revision. Use that revision with the same ID and source selection +for a pinned `read` or `resource` call. + +Three properties distinguish it from the Collect selection path below. Cases are +ranked beside skills instead of being reachable only after their owner is +chosen, because a recorded rejected direction is the material retrieval can +supply that re-reading current source cannot. Nothing is filtered on the query: +requiring every token to appear returns nothing for a question phrased in +sentences, so ranking degrades and `--limit` truncates. Scoring reads corpus +files directly rather than Bundles, whose metadata lists the whole packaged +library as one dependency closure. The reader validates the returned material +without pinning a revision during search. + +Scoring is `idf x field weight x sqrt(occurrences)` summed over query terms, +with titles, summaries, tags and case findings weighted above bodies. Bodies +longer than the corpus median get less weight so a broad manual needs more +specific matches to outrank a short case. Separators +do not bind: `kv_cache`, `kv-cache` and `KV cache` all tokenise to the same two +terms, so a query in plain words reaches the punctuated spellings this corpus +also uses. Latin terms still match on boundaries, so `oom` does not match inside +`headroom`. CJK is cut into overlapping bigrams, because single characters make +idf treat grammatical particles as rare terms. Ties break towards the stronger +recorded state rather than alphabetical identity. This is deterministic lexical +matching, not a learned ranking, and a hit never asserts that an entry applies. + +Chinese query terms expand through `data/query_aliases.json` before ranking. +The packaged library is English by policy while its readers work in Chinese, and +the corpus holds too few Chinese characters for a Chinese query to score on its +subject. Expansion is one-way and query-side: 显存 reaches `device memory`, not +the reverse, and an entry is only listed when that English form actually occurs +in the corpus. A query whose terms were expanded reports them under `expanded`, +so a wrong expansion is visible in the result rather than only in the ranking. + +Three limits are inherent to lexical ranking and are recorded here rather than +worked around. A query made only of terms common to this corpus ("operator", +"test", "graph", "speedup") has little to rank on and may bury the intended +entry. A query that shares no words with its target does not reach it: asking +for "moving attention state to host memory" misses an entry that says `kv`, +`offload` and `tier` throughout, because nothing matches. Name the specific +symptom, operator or flag where you can, and when a plausible query returns +nothing useful, search the corpus directly rather than assuming it holds +nothing — `find` ranks words, not meanings. + +The third is that `find` always answers. It cannot tell a question the corpus +can help with from one the caller can already settle from the facts in front of +them, and it returns its best matches either way. On the labelled set, a +majority of queries were ones an engineer should decide without searching at +all, and `find` returned results for every one of them; their scores overlap +the answerable range too broadly for a cutoff to separate the two without +suppressing real answers. Deciding whether the question needs the corpus +remains the caller's judgement, and a returned list is not evidence that it +did. + +- **Inputs:** the query, `--limit`, `--kinds` (subset of `case,skill,record`), `--skill-root` to rank an explicit root, `--explain` for per-term contributions, `--json`. +- **Scope:** installed package records and the packaged (or selected) skill library, including its cases. Packaged-skill projections are excluded from the record side so one owner is not ranked under two identities. +- **Output.** Items identify their `id`, `authority` (`package` or `checkout`), + `kind`, recorded `state`, `score`, `title` and corpus-relative `path`. + Where recorded, items carry source, summary, applicability, limitations and + local verification status. Case findings are ordered by query overlap, + with `findings_total` and `findings_withheld` describing the available preview. + Each finding carries its outcome, reason, revisit condition and any assessed + scope and bounds. `bounds_source` identifies whether bounds apply to the + finding or the whole entry. The envelope reports `corpus_size`, `matched`, + `no_match`, any `expanded` terms and retrieval limitations. +- **Reading a hit.** Plain output prints a POSIX `open` command or a labelled + PowerShell command on Windows. JSON exposes the rendered command as `ref`, + full subprocess arguments as `open_argv` and InferRSI CLI arguments as + `read_args`. The arguments include the source-qualified ID and a skill root + where needed. They use `read --material --omit-resources` to return the + entrypoint, provenance and revision with a smaller response. Remove + `--omit-resources` to list the bundle's resources. Hits that fail the reader's + material check carry `unreadable` with the reason. +- **Side effects:** none. No revision is pinned; use `read`/`resource` when a capture or evidence record needs exact bytes. +- **Example:** `inferrsi find "NPU OOM after caching per-step masks" --limit 3` returns the casebook entry recording that attempt first. A vaguer phrasing ranks the procedural skills above it, which is the first limit below rather than a defect. + ### Collect — `collect` / `KnowledgeReader.collect` Optional `--relevance ` ranks generated packaged skills after the existing diff --git a/docs/quickstart.md b/docs/quickstart.md index 64bb180e9..c9547e2cd 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -47,8 +47,24 @@ Its common catalog ID and the separate MLX-LM procedure remain unchanged. ## Discover knowledge and skills -Read-only discovery and export — the [operation guide](operations.md) maps every -entry point, effect class and boundary: +Start from the problem. `find` ranks packaged skills, imported records and +casebook entries together and reports what each one's evidence covers, so a +direction already tried and dropped surfaces beside the procedure for trying it: + +```bash +inferrsi find 'mask caching ran out of device memory' +inferrsi find '算子测过了但整图质量没过' --limit 3 +``` + +Run the `open` command on a relevant hit to read the entry and its source, +applicability, limitations and revision. Add `--kinds case` to focus the search +on recorded observations and failed directions. + +Nothing is filtered out for an unmatched word, so a question phrased in +sentences still returns something; read the reported state, `scope` and bounds +before acting on a hit. The remaining commands are exact-lookup and export — +the [operation guide](operations.md) maps every entry point, effect class and +boundary: ```bash inferrsi knowledge search 'Qwen-Image-2.1' --repo-type vllm_omni --backend ascend @@ -113,6 +129,7 @@ plan/run: all doctor flags, plus --repetitions --min-improvement-pct --max-minutes --workload-id --model --model-revision --output-dir --dry-run status/report: --run-id --output-dir +find: QUERY --limit --kinds --skill-root --explain --json knowledge search: QUERY --tier --repo-type --backend --scenario --limit --json knowledge show: ID context: QUERY --tier --repo-type --backend --scenario --limit --output diff --git a/plugins/inferrsi/skills/inferrsi/SKILL.md b/plugins/inferrsi/skills/inferrsi/SKILL.md index 8a79ad7c1..cd18d471c 100644 --- a/plugins/inferrsi/skills/inferrsi/SKILL.md +++ b/plugins/inferrsi/skills/inferrsi/SKILL.md @@ -121,15 +121,33 @@ Use the CLI's help for exact flags. Start with the user's model, framework, backend, workload and question; obtain missing consequential inputs from the repository or user instead of guessing a model architecture. -- `knowledge research ` unions independent research dimensions. -- `knowledge search --json` searches packaged records with hard filters. -- `collect --relevance "" --limit 3 --json` discovers a bounded packaged - selection. Adjust the limit to the task. Packaged skill projections have kind - `knowledge`; `--kind skill` excludes them and is for explicitly selected - checkout/store skill sources. -- `read --revision --material` loads a selected exact revision. - Use the returned resource keys with `resource --help` when further material - is needed. +Use `find ""` to rank skills, imported records and casebook +entries. Add `--kinds case` when you specifically need a past observation. +Open a promising hit using its displayed `open` command or the JSON +`open_argv` argument list. If the result says `cannot open`, check the reason +and consult another hit when appropriate. + +`read` returns the material, its source, applicability, limitations and +revision. To cite or reopen exact evidence, add `--revision ` to the +same read arguments. Remove `--omit-resources` to list bundle resource keys. +Use those keys with `resource --help` for further material. + +A ranked hit is not a substitute for reading the repository. `find` supplies +what re-reading current source cannot: directions already tried, why they were +dropped, and the cohort each observation belongs to. Keep searching the tree +directly for anything about present-day code. + +`find` ranks words, not meanings. A query sharing no vocabulary with its target +will not reach it, so when a plausible query returns nothing useful, try the +terms the corpus itself would use, then grep the library directly. Do not report +that nothing is recorded on the strength of one query. Each result states how +far its evidence reaches: `scope` and `bounds` name the cohort an observation +belongs to, and their absence means not assessed, never "holds anywhere". + +`find` also always answers. It cannot tell that a question is already settled by +the measurements and source in front of you, so a returned list is not evidence +that searching was the right move. When the recorded facts decide the question, +decide it and say so rather than looking for support. Use qualified identities and revisions returned by the runtime. Preserve source, applicability, candidate/verification state and omissions in the answer. Package diff --git a/plugins/inferrsi/skills/inferrsi/scripts/inferrsi.py b/plugins/inferrsi/skills/inferrsi/scripts/inferrsi.py index f0aba0aef..9c93d305a 100644 --- a/plugins/inferrsi/skills/inferrsi/scripts/inferrsi.py +++ b/plugins/inferrsi/skills/inferrsi/scripts/inferrsi.py @@ -9,12 +9,13 @@ from pathlib import Path -def main() -> int: +def main(*, launcher_path: Path | None = None) -> int: if sys.version_info < (3, 10): print("InferRSI requires Python 3.10 or newer; select a compatible interpreter.", file=sys.stderr) return 2 - root = Path(__file__).resolve().parents[3] + script = Path(__file__).resolve() + root = script.parents[3] runtime = root / "runtime" if os.name == "nt" and not str(runtime).startswith("\\\\?\\"): runtime = Path("\\\\?\\UNC\\" + str(runtime)[2:] @@ -53,16 +54,17 @@ def main() -> int: "optional_skills_dependency": has_yaml, }, indent=2)) return 0 - needs_yaml = args[:1] == ["skills"] or any( - arg == "--skill-root" or arg.startswith("--skill-root=") for arg in args - ) - if needs_yaml and not has_yaml and not any(arg in ("--help", "-h") for arg in args): - print( - "This operation needs the optional skills dependency: install PyYAML in the " - "selected interpreter, or use packaged knowledge/collect/read without --skill-root.", - file=sys.stderr, - ) - return 2 + # No PyYAML precheck here. The runtime raises its own install hint, exit 2, + # from the one call that parses skill frontmatter; guessing from argument + # names refused operations that never parse any -- reading a case under + # --skill-root, which is exactly the reference `find` hands back. + + # Commands the CLI prints for a reader to run next must start the way this + # process was started. A plugin host has no `inferrsi` on PATH, so a + # printed `inferrsi read ...` fails where this launcher would succeed. The + # argv is passed as JSON so the CLI can quote it for the host's shell. + launcher = launcher_path.resolve() if launcher_path is not None else script + os.environ["INFERRSI_LAUNCHER"] = json.dumps([sys.executable, "-X", "utf8", str(launcher)]) from inferrsi.cli import main as cli_main diff --git a/plugins/workswarm/inferrsi/skills/inferrsi/scripts/inferrsi.py b/plugins/workswarm/inferrsi/skills/inferrsi/scripts/inferrsi.py index 7c7902fd2..95775914b 100644 --- a/plugins/workswarm/inferrsi/skills/inferrsi/scripts/inferrsi.py +++ b/plugins/workswarm/inferrsi/skills/inferrsi/scripts/inferrsi.py @@ -159,10 +159,10 @@ def main() -> int: entry = cache_runtime(manifest_bytes, launcher, content) # No wrapper subprocess: the host still owns the foreground CLI process. try: - runpy.run_path(str(entry), run_name="__main__") + runtime_main = runpy.run_path(str(entry))["main"] + return runtime_main(launcher_path=Path(__file__)) except SystemExit as exit_status: return exit_status.code if isinstance(exit_status.code, int) else 1 - return 0 except (OSError, ValueError, KeyError, zipfile.BadZipFile) as error: print(f"InferRSI plugin bundle is incomplete: {error}. " "See the package-integrity and runtime-cache repair steps in the bundled README.", diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 4d63775a5..5ddd53cd9 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -414,6 +414,26 @@ def parser() -> argparse.ArgumentParser: "--target-memory-gib", type=float, help="Caller-declared unified/device GiB" ) + find = commands.add_parser( + "find", + help="Rank packaged skills, imported records and checkout cases for one " + "question; returns corpus paths, not pinned revisions", + ) + find.add_argument("query", help="Describe the problem in your own words") + find.add_argument("--limit", type=int, default=5) + find.add_argument( + "--kinds", default="case,skill,record", + help="Comma-separated subset of case,skill,record; default is all three", + ) + find.add_argument( + "--skill-root", type=Path, + help="Skills root to rank; defaults to the installed library", + ) + find.add_argument( + "--explain", action="store_true", help="Report each query term's contribution" + ) + find.add_argument("--json", action="store_true") + collect = commands.add_parser( "collect", help="Collect explicitly selected sources as a revision-pinned envelope; " @@ -448,7 +468,7 @@ def parser() -> argparse.ArgumentParser: collect.add_argument( "--skill-root", type=Path, - help="Select an explicit skills root (requires the 'skills' extra)", + help="Select a skills root; reading checkout skills requires the 'skills' extra", ) collect.add_argument( "--store", @@ -464,10 +484,14 @@ def parser() -> argparse.ArgumentParser: read.add_argument("id") read.add_argument("--revision", help="Exact revision returned by collect or read") read.add_argument("--material", action="store_true") + read.add_argument( + "--omit-resources", action="store_true", + help="Omit the resource inventory from output; the revision still covers the full bundle", + ) read.add_argument( "--skill-root", type=Path, - help="Select an explicit skills root (requires the 'skills' extra)", + help="Select a skills root; reading checkout skills requires the 'skills' extra", ) read.add_argument( "--store", @@ -487,7 +511,7 @@ def parser() -> argparse.ArgumentParser: resource.add_argument( "--skill-root", type=Path, - help="Select an explicit skills root (requires the 'skills' extra)", + help="Select a skills root; reading checkout skills requires the 'skills' extra", ) resource.add_argument( "--store", diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 5c2880b12..7311df5c0 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -105,6 +105,87 @@ def _run_directory(output_dir: str, run_id: str) -> Path: return directory +_FIND_KIND = {"case": "case", "skill": "skill", "record": "record"} +_FIND_OUTCOME = { + "rejected": "rejected", + "no_gain": "no gain", + "inconclusive": "inconclusive", + "adopted": "adopted", + "tradeoff": "tradeoff", + "caveat": "caveat", + "failure_signal": "failure signal", +} +# How far the observation is claimed to reach. Absent means not assessed, which +# is printed as such rather than silently read as "holds anywhere". +_FIND_SCOPE = { + "local": "this run only", + "regime_bound": "this cohort only", + "general": "holds generally", +} + + +# Enough to carry the qualifications that decide whether a result is usable, +# without burying the finding it qualifies. +_MAX_SHOWN_LIMITS = 3 + + +def _clip(value: str, limit: int = 160) -> str: + text = _tsv(value).strip() + return text if len(text) <= limit else text[: limit - 1].rstrip() + "…" + + +def _print_find(result: dict) -> None: + """Lead with what was tried and how far the evidence goes, then the path.""" + if result["no_match"]: + print(f"no_match\t{result['corpus_size']} entries searched; " + f"rephrase, or search the corpus directly with rg") + return + for item in result["items"]: + labels = [_FIND_KIND.get(item["kind"], item["kind"]), _tsv(item["state"])] + if item.get("locally_verified") is False: + labels.append("not locally verified") + print(f"\n{item['id']}\t[{' · '.join(labels)}]\tscore {item['score']}") + if item.get("ref"): + label = "open (PowerShell)" if sys.platform == "win32" else "open" + print(f" {label}: {item['ref']}") + elif item.get("unreadable"): + print(f" cannot open: {_clip(item['unreadable'], 120)}") + if item.get("summary"): + print(f" {_clip(item['summary'])}") + for finding in item.get("findings", []): + outcome = _FIND_OUTCOME.get(finding["outcome"], finding["outcome"]) + label = outcome + if finding.get("scope"): + label += f", {_FIND_SCOPE.get(finding['scope'], finding['scope'])}" + print(f" ({label}) {_clip(finding['direction'], 90)}" + f" -> {_clip(finding['reason'], 110)}") + if finding.get("bounds"): + # Say whose bounds these are: an entry-level cohort describes + # the whole record, not a measurement of this direction alone. + whose = "entry" if finding.get("bounds_source") == "entry" else "observed" + print(f" bounds ({whose}): {_clip(finding['bounds'], 120)}") + if finding.get("revisit_when"): + print(f" revisit: {_clip(finding['revisit_when'], 120)}") + if item.get("findings_withheld"): + print(f" ... {item['findings_withheld']} more of " + f"{item['findings_total']} recorded directions in the file") + if item.get("applicability"): + print(f" applies to: {_clip(item['applicability'])}") + # How far the record itself says its evidence goes. The state label + # above says the claim is unverified; these say what is missing. + for caveat in item.get("limitations", [])[:_MAX_SHOWN_LIMITS]: + print(f" caveat: {_clip(caveat)}") + withheld = len(item.get("limitations", [])) - _MAX_SHOWN_LIMITS + if withheld > 0: + print(f" ... {withheld} more recorded limitation(s); see --json") + if item.get("path"): + print(f" -> {item['path']}") + if item.get("source"): + print(f" source: {item['source']}") + for token, gain in item.get("matched", {}).items(): + print(f" [{token}] {gain}") + + def _context(records: list[dict]) -> str: lines = [ "# InferRSI knowledge context", @@ -281,6 +362,23 @@ def main(argv: list[str] | None = None) -> int: f"{record['id']}\t{record['status']}\t{record['backend']}\t{record['title']}" ) return 0 + if args.command == "find": + from .knowledge.knowledge_find import find, load_corpus + + root = args.skill_root if args.skill_root is not None else Path( + files("inferrsi").joinpath("knowledge", "data", "skill_library") + ) + kinds = tuple(k.strip() for k in args.kinds.split(",") if k.strip()) + unknown = [k for k in kinds if k not in ("case", "skill", "record")] + if unknown or not kinds: + raise ValueError("--kinds takes a subset of case,skill,record") + result = find(args.query, load_corpus(root, kinds=kinds), + limit=args.limit, explain=args.explain) + if args.json: + _print(result) + else: + _print_find(result) + return 0 if args.command == "collect": result = _knowledge_reader(args).collect( args.query, @@ -347,6 +445,9 @@ def main(argv: list[str] | None = None) -> int: _, _, view = read_knowledge( _knowledge_reader(args), args.id, revision=args.revision, material=args.material, ) + if args.omit_resources: + view.pop("resources") + view.pop("required_resources", None) _print(view) return 0 if args.command == "resource": diff --git a/src/inferrsi/knowledge/data/query_aliases.json b/src/inferrsi/knowledge/data/query_aliases.json new file mode 100644 index 000000000..f7045c226 --- /dev/null +++ b/src/inferrsi/knowledge/data/query_aliases.json @@ -0,0 +1,207 @@ +{ + "schema_version": 1, + "_comment": [ + "Query-side term aliases for `inferrsi find`.", + "", + "The packaged library is written in English by repository policy, but the", + "people searching it work in Chinese. The corpus holds only a few hundred", + "distinct Chinese characters, almost all incidental, so a Chinese query", + "scores on whichever characters happen to appear rather than on its subject.", + "Every concept below already exists in the corpus under its English name;", + "what is missing is the route from one to the other.", + "", + "Direction is one-way: a query term expands to corpus terms, never the", + "reverse. Expanding a narrow term into a broad one (Ascend -> every NPU)", + "would widen results rather than find them, so entries stay specific.", + "", + "Values are spellings verified to occur in this corpus. A term whose English", + "form is absent does not belong here; it belongs in the corpus first.", + "Keep the list short: a broad alias costs precision on every query that", + "touches it, and an unused one is upkeep with no return. Add a term when a", + "real query misses because of it.", + "", + "JSON rather than YAML because the base install is PyYAML-free by contract.", + "", + "synonyms: English terms the corpus and its readers spell differently. Same one-way, query-side rule as aliases; listed separately so the Chinese map stays a translation table rather than a thesaurus." + ], + "aliases": { + "显存": [ + "device memory", + "hbm" + ], + "内存": [ + "host memory", + "memory" + ], + "昇腾": [ + "ascend" + ], + "分配器": [ + "allocator" + ], + "碎片": [ + "fragmentation" + ], + "驻留": [ + "residency" + ], + "占用": [ + "occupancy", + "utilization" + ], + "算子": [ + "operator", + "kernel" + ], + "启动": [ + "startup", + "initialization" + ], + "初始化": [ + "initialization", + "startup" + ], + "融合": [ + "fusion", + "fused" + ], + "图模式": [ + "graph mode", + "aclgraph", + "cuda graph" + ], + "捕获": [ + "capture" + ], + "重放": [ + "replay" + ], + "预热": [ + "warmup" + ], + "调度": [ + "scheduling", + "scheduler" + ], + "并发": [ + "concurrency", + "concurrent" + ], + "批量": [ + "batch" + ], + "吞吐": [ + "throughput" + ], + "时延": [ + "latency" + ], + "延迟": [ + "latency" + ], + "排空": [ + "drain" + ], + "准入": [ + "admission" + ], + "抢占": [ + "preemption" + ], + "量化": [ + "quantization" + ], + "精度": [ + "precision" + ], + "权重": [ + "weights", + "weight loading" + ], + "加载": [ + "loading" + ], + "缓存": [ + "cache", + "caching" + ], + "前缀缓存": [ + "prefix cache" + ], + "推理": [ + "inference" + ], + "服务": [ + "serving", + "service" + ], + "请求": [ + "request" + ], + "基线": [ + "baseline" + ], + "候选": [ + "candidate" + ], + "对照": [ + "control", + "controlled" + ], + "复现": [ + "reproduce", + "reproducibility" + ], + "回归": [ + "regression" + ], + "验收": [ + "acceptance", + "gate" + ], + "质量门": [ + "quality gate" + ], + "语音识别": [ + "asr", + "wer" + ], + "误码率": [ + "wer" + ], + "指标": [ + "metric", + "measurement" + ], + "证据": [ + "evidence" + ], + "冻结": [ + "frozen" + ], + "剖析": [ + "profile", + "profiling" + ], + "瓶颈": [ + "bottleneck" + ], + "上限": [ + "ceiling", + "roofline", + "headroom" + ] + }, + "synonyms": { + "startup": [ + "initialization" + ], + "initialization": [ + "startup" + ], + "vram": [ + "device memory", + "hbm" + ] + } +} diff --git a/src/inferrsi/knowledge/knowledge_find.py b/src/inferrsi/knowledge/knowledge_find.py new file mode 100644 index 000000000..49903d860 --- /dev/null +++ b/src/inferrsi/knowledge/knowledge_find.py @@ -0,0 +1,726 @@ +"""One ranked lookup across packaged skills, records and checkout cases. + +This is the discovery half of the knowledge interface. It answers "what is +already known about this problem" and returns corpus-relative paths, so the +caller can read, grep around, or stop. The audit half lives in ``read`` / +``resource``: exact identities, pinned revisions and content hashes. + +Three properties separate this from the existing ``collect`` selection path: + +* Cases participate. A rejected direction is the material a retrieval system + can supply that re-reading the current source cannot, so it is ranked beside + skills rather than excluded until its owner is already chosen. +* Nothing is filtered on the query. Requiring every token to appear is a hard + cut on short queries; ranking degrades instead, and ``limit`` truncates. +* Scoring reads the entries directly. Bundle closures may contain unrelated + files. The reader checks material only for results offered to the caller. + +Ranking is lexical, with query-side aliases. The caller assesses applicability. +""" + +from __future__ import annotations + +import json +import math +import os +import re +import shlex +import sys +from collections import Counter +from dataclasses import dataclass +from functools import lru_cache +from importlib.resources import files +from pathlib import Path +from statistics import median +from typing import Any + +# Author-written intent outranks bulk prose: a term in a title is evidence the +# document is about it, the same term in a long body often is not. +FIELD_WEIGHTS = { + "title": 5.0, + "summary": 4.0, + "tags": 4.0, + "findings": 4.0, + "applicability": 2.0, + "body": 1.0, +} + +# Ties break towards stronger evidence rather than alphabetical identity. +STATE_RANK = { + "validated": 3, + "author_reported": 2, + "historical_observation": 2, + "candidate": 1, + "imported": 1, + "unspecified": 0, +} + +FINDING_OUTCOMES = ( + "rejected", "no_gain", "inconclusive", "adopted", + "tradeoff", "caveat", "failure_signal", +) + +# How far an observation is being claimed to reach. The distinction a reader +# most needs is whether a result is bound to the cohort it was seen on, and +# that cannot be read off the outcome: a rejection can be local or general. +# Absent means not assessed, which is not the same as `general`. +FINDING_SCOPES = ("local", "regime_bound", "general") + +_CJK = re.compile(r"[㐀-䶿一-鿿豈-﫿]") +_CJK_RUN = re.compile(_CJK.pattern + "+") +_WORD = re.compile(r"[a-z0-9]+") +_MAX_SNIPPET = 400 +# A case can record a dozen directions; listing all of them buries the one the +# caller asked about. The rest stay available through the cited file. +_MAX_SHOWN_FINDINGS = 3 + + +def tokens(text: str) -> list[str]: + """Split on separators as well as whitespace; cut CJK into overlapping bigrams. + + Underscores, hyphens and dots separate rather than bind. Tags in this corpus + are written four ways for the same concept -- ``kv_cache``, ``kv-cache``, + ``KV cache``, ``CUDA-Graph`` -- and treating a separator as a word character + makes each spelling its own unsplittable token, so a query phrased in plain + words never reaches the punctuated ones. Splitting normalises all four to the + same terms. A compound still ranks above its parts because every part has to + match for the whole to score. + + There is no CJK segmenter here, so a run is cut every two characters. Two + characters carry a term -- 质量, 调度, 状态 -- while single characters mostly + carry grammar; this corpus holds so few CJK characters that idf treats every + one as rare, and a particle such as 和 would otherwise outscore the subject + of the query. Punctuation isolates such particles into runs of their own + ("任务、工具和输入"), so a one-character run is kept only when the whole + query is that character: a lone term the caller typed deliberately still + reaches the corpus, while a stranded conjunction does not score. + """ + folded = text.casefold() + found = _WORD.findall(folded) + runs = _CJK_RUN.findall(folded) + for run in runs: + if len(run) == 1: + if len(runs) == 1 and not found: + found.append(run) + continue + found.extend(run[index:index + 2] for index in range(len(run) - 1)) + return found + + +def _term_counts(text: str) -> Counter[str]: + folded = text.casefold() + counts = Counter(_WORD.findall(folded)) + for run in _CJK_RUN.findall(folded): + # Single characters are indexed for callers who deliberately query one. + counts.update(run) + counts.update(run[index:index + 2] for index in range(len(run) - 1)) + return counts + + +def _flatten(value: Any) -> str: + if value is None: + return "" + if isinstance(value, str): + return value + return json.dumps(value, ensure_ascii=False) + + +@dataclass(frozen=True) +class Finding: + direction: str + outcome: str + reason: str + revisit_when: str = "" + keywords: tuple[str, ...] = () + scope: str = "" + bounds: str = "" + + def search_text(self) -> str: + return " ".join(( + *self.keywords, self.direction, self.reason, self.bounds, self.revisit_when, + )) + + def view(self, *, inherited_bounds: str = "") -> dict: + """Carry the cohort alongside the result, not only on the parent entry. + + A reader who reaches one finding through search sees it on its own. If + the bounds stay at entry level, the most consequential qualifier -- the + hardware and configuration this was observed on -- is exactly what gets + dropped in transit. + """ + record = {"direction": self.direction, "outcome": self.outcome, "reason": self.reason} + if self.scope: + record["scope"] = self.scope + bounds = self.bounds or inherited_bounds + if bounds: + record["bounds"] = bounds + # An inherited bound describes the whole entry; saying so keeps a + # reader from reading it as measured for this direction alone. + record["bounds_source"] = "finding" if self.bounds else "entry" + if self.revisit_when: + record["revisit_when"] = self.revisit_when + return record + + +@dataclass +class Document: + """One retrievable unit with the fields scoring and display both read.""" + + identity: str + kind: str # case | skill | record + state: str + path: str | None # corpus-relative when an entry has a file + title: str + summary: str + fields: dict[str, str] + applicability: Any = None + limitations: tuple[str, ...] = () + findings: tuple[Finding, ...] = () + owner: str = "" + qualified: str = "" + root: Path | None = None + source_url: str = "" # where the observation was originally recorded + + def unverified(self) -> bool: + return any("locally_verified=false" in _flatten(item) for item in self.limitations) + + def cohort(self) -> str: + """The hardware and configuration this entry's observations belong to.""" + declared = self.applicability + if isinstance(declared, list): + return " · ".join(str(item) for item in declared) + if isinstance(declared, dict): + parts = [declared.get(key) for key in ("hardware", "scope")] + return " · ".join(str(part) for part in parts if part) + return str(declared) if declared else "" + + def relevant_findings(self, query_tokens: list[str]) -> tuple[list[Finding], int]: + """Rank this entry's findings by query overlap, then by recorded order. + + Scoring the whole entry says it is relevant; it does not say which of a + dozen recorded directions the caller asked about. Ranking per finding + keeps the answer first and reports how many were withheld. + + Weighting is computed over this entry's own findings, not the corpus. A + term shared by every direction in the entry cannot separate them however + rare it is elsewhere -- every direction in a Qwen case says Qwen -- while + a term in one direction identifies it even if the corpus sees it often. + Corpus-level idf gets this backwards at both ends, and collapses + entirely when the library holds a single entry, where it rates every + term identically and leaves the recorded order untouched. + + Findings that tie, including every finding when the query matches none, + keep their recorded order. That order is the author's, and for a case it + follows the sequence the directions were tried in. + """ + if not self.findings: + return [], 0 + terms = [_term_counts(finding.search_text()) for finding in self.findings] + appearances = Counter(token for counts in terms for token in counts) + total = len(terms) + ranked = [] + for position, counts in enumerate(terms): + overlap = 0.0 + for token in query_tokens: + hits = counts[token] + if not hits: + continue + carrying = appearances[token] + # A term in most of the entry's directions cannot pick one out, + # and in one-line findings its repetition would otherwise + # outweigh a single hit on the term actually asked about. + if total > 1 and carrying * 2 > total: + continue + # Rarer within the entry means more distinguishing. + overlap += math.log(total / carrying + 1.0) * math.sqrt(hits) + ranked.append((-overlap, position, self.findings[position])) + ranked.sort(key=lambda row: (row[0], row[1])) + shown = [finding for _, _, finding in ranked[:_MAX_SHOWN_FINDINGS]] + return shown, max(0, len(self.findings) - len(shown)) + + def view( + self, *, score: float, query_tokens: list[str] = (), unreadable: str | None = None + ) -> dict: + record = { + "id": self.identity, + "kind": self.kind, + "state": self.state, + "score": round(score, 2), + "title": self.title, + "path": self.path, + } + if self.qualified: + record["authority"] = self.qualified.partition(":")[0] + if unreadable is None: + read_args = _read_args(self.qualified, self.root) + argv = [*launcher(), *read_args] + record["read_args"] = list(read_args) + record["open_argv"] = argv + record["ref"] = shell_command(argv) + else: + record["unreadable"] = unreadable + if self.source_url: + record["source"] = self.source_url + if self.summary: + record["summary"] = self.summary.strip()[:_MAX_SNIPPET] + if self.findings: + shown, withheld = self.relevant_findings(list(query_tokens)) + record["findings"] = [ + finding.view(inherited_bounds=self.cohort()) for finding in shown + ] + record["findings_total"] = len(self.findings) + if withheld: + record["findings_withheld"] = withheld + scope = self.applicability + if isinstance(scope, dict): + scope = scope.get("scope") or _flatten(scope) + elif isinstance(scope, list): + scope = " · ".join(str(item) for item in scope) + if scope: + record["applicability"] = str(scope).strip()[:_MAX_SNIPPET] + if self.limitations: + record["limitations"] = list(self.limitations) + if self.unverified(): + record["locally_verified"] = False + return record + + +def _read_args(identity: str, root: Path | None = None) -> tuple[str, ...]: + """CLI arguments that return the entry's text, not just its metadata. + + Without `--material`, `read` reports the entrypoint's filename in `content` + rather than the file; a caller following the reference lands on a name. + Kept as an argument list so a program can run it without re-parsing, and + so quoting is decided once, for the shell the command will be shown in. + """ + parts = ["read", identity, "--material", "--omit-resources"] + if root is not None: + parts += ["--skill-root", str(root)] + return tuple(parts) + + +def launcher() -> list[str]: + """How to start this CLI from where the caller is. + + The plugin records its own launcher. Package invocations use the current + Python interpreter, including when the caller started with `python -m inferrsi`. + """ + declared = os.environ.get("INFERRSI_LAUNCHER") + if declared: + try: + argv = json.loads(declared) + except ValueError: + argv = None + if isinstance(argv, list) and argv and all(isinstance(a, str) for a in argv): + return argv + return [sys.executable, "-X", "utf8", "-m", "inferrsi"] + + +def shell_command(argv: list[str]) -> str: + """Render argv for a POSIX shell or, on Windows, PowerShell.""" + if os.name == "nt": + return "& " + " ".join("'" + arg.replace("'", "''") + "'" for arg in argv) + return shlex.join(argv) + + +def _source_url(record: dict) -> str: + """The recorded location of the original observation, if the entry cites one. + + A case carries the report it was extracted from. Handing that back lets a + reader check the finding against its source instead of taking the summary + on trust. + """ + for source in record.get("sources", []) or []: + if isinstance(source, dict): + for key in ("url", "evidence_url", "entry_url"): + value = source.get(key) + if isinstance(value, str) and value.startswith("https://"): + return value + return "" + + +def _findings(record: dict, identity: str) -> tuple[Finding, ...]: + declared = record.get("findings", []) + if not isinstance(declared, list): + raise ValueError(f"Case findings must be a list: {identity}") + result = [] + for item in declared: + if not isinstance(item, dict): + raise ValueError(f"Case finding must be a mapping: {identity}") + for key in ("direction", "outcome", "reason"): + if not isinstance(item.get(key), str) or not item[key].strip(): + raise ValueError(f"Case finding requires {key}: {identity}") + if item["outcome"] not in FINDING_OUTCOMES: + raise ValueError( + f"Unknown finding outcome {item['outcome']!r} in {identity}; " + f"expected one of {', '.join(FINDING_OUTCOMES)}" + ) + keywords = item.get("keywords", []) + if not isinstance(keywords, list) or any(not isinstance(k, str) for k in keywords): + raise ValueError(f"Case finding keywords must be a string list: {identity}") + scope = item.get("scope", "") + if scope and scope not in FINDING_SCOPES: + raise ValueError( + f"Unknown finding scope {scope!r} in {identity}; " + f"expected one of {', '.join(FINDING_SCOPES)}" + ) + bounds = item.get("bounds", "") + if not isinstance(bounds, str): + raise ValueError(f"Case finding bounds must be a string: {identity}") + revisit_when = item.get("revisit_when", "") + if not isinstance(revisit_when, str): + raise ValueError(f"Case finding revisit_when must be a string: {identity}") + result.append(Finding( + direction=item["direction"], outcome=item["outcome"], reason=item["reason"], + revisit_when=revisit_when, keywords=tuple(keywords), + scope=scope, bounds=bounds, + )) + return tuple(result) + + +def _case_documents(root: Path) -> list[Document]: + """Cases owned by the registered skills of this root. + + Discovery is delegated to the reader that `read` uses, rather than globbing + for `cases/*.json` at a guessed depth. Two layouts diverged under the glob: + a flat root, where each skill sits directly under it, put cases one level + too shallow to be found at all; and a directory nested at the right depth + but absent from skill-index.json was found here while `read` refused it, + so a reported reference could not be opened. The owning reader decides + which skills are registered, so both commands now agree by construction. + """ + from .knowledge_operations import KnowledgeReader, case_resource_path + + documents = [] + for identity, path in sorted(KnowledgeReader(skill_root=root).cases().items()): + record = json.loads(path.read_text(encoding="utf-8")) + try: + body_path = case_resource_path(root, path, record.get("content")) + except (ValueError, OSError): + body_path = None + findings = _findings(record, identity) + documents.append(Document( + identity=identity, + kind="case", + state=record.get("state", "unspecified"), + path=body_path.relative_to(root).as_posix() if body_path is not None else None, + title=identity.removeprefix("case."), + summary=_flatten(record.get("attribution")), + owner=path.parent.parent.name, + qualified=f"checkout:{identity}", + root=root, + source_url=_source_url(record), + fields={ + "title": identity.removeprefix("case.").replace("-", " "), + "summary": _flatten(record.get("attribution")), + "tags": " ".join(record.get("tags", [])), + "findings": " ".join(finding.search_text() for finding in findings), + "applicability": _flatten(record.get("applicability")), + "body": _body_text(body_path) if body_path is not None else "", + }, + applicability=record.get("applicability"), + limitations=tuple(record.get("limitations", [])), + findings=findings, + )) + return documents + + +def _body_text(path: Path) -> str: + try: + return path.read_text(encoding="utf-8") + except (OSError, UnicodeError): + # Metadata can still match; read reports why the material is unavailable. + return "" + + +def _skill_documents(root: Path) -> list[Document]: + """Skills registered by this root, resolved the way `skills` and `read` do. + + Reading skill-index.json directly assumed every root is a generated library. + A flat checkout registers its skills by having them directly under the root + and carries no index, so the default search raised on a library whose cases + `find` could already rank and `read` could already open. + """ + from .skill_catalog import _sidecar, skill_directories + + documents = [] + for name, directory in sorted(skill_directories(root).items()): + core = directory / "SKILL.md" + body = _body_text(core) + description = "" + frontmatter = body.split("\n---", 1)[0] if body.startswith("---\n") else "" + match = re.search(r"^description:[ \t]*(.+)$", frontmatter, re.MULTILINE) + if match: + description = match.group(1).strip().strip("'\"") + try: + metadata = _sidecar(directory) or {} + except (ValueError, OSError): + metadata = {} + applicability = metadata.get("applicability", {}) + scope = applicability.get("scope") if isinstance(applicability, dict) else applicability + documents.append(Document( + identity=name, + kind="skill", + state=metadata.get("status", "unspecified"), + path=core.relative_to(root).as_posix(), + title=name, + summary=description or _flatten(scope), + owner=metadata.get("backend", ""), + qualified=f"checkout:{name}", + root=root, + source_url=_source_url(metadata), + fields={ + "title": name.replace("-", " "), + "summary": description, + "tags": " ".join(metadata.get("tags", [])), + "findings": "", + "applicability": _flatten(applicability), + "body": body, + }, + applicability=applicability, + limitations=tuple(metadata.get("limitations", [])), + )) + return documents + + +def _record_documents() -> list[Document]: + """Imported engine records. Skill projections are skipped: the checkout + documents above already cover them, and counting both double-ranks one + owner under two identities.""" + from .catalog import knowledge_records + + documents = [] + for record in knowledge_records(): + if record.get("index_type") == "packaged-skill": + continue + identity = record["id"] + body = _flatten({ + key: value for key, value in record.items() + if key not in ("sources", "source_files", "skill_documents", "id") + }) + documents.append(Document( + identity=identity, + kind="record", + state=record.get("status", "unspecified"), + path=None, + title=record.get("title", identity), + summary=record.get("summary", ""), + owner=record.get("repo", ""), + qualified=f"package:{identity}", + source_url=_source_url(record), + fields={ + "title": f"{record.get('title', '')} {identity.replace('.', ' ')}", + "summary": record.get("summary", ""), + "tags": " ".join(record.get("tags", [])), + "findings": "", + "applicability": _flatten(record.get("applicability")), + "body": body, + }, + applicability=record.get("applicability"), + limitations=tuple(record.get("limitations", [])), + )) + return documents + + +class Corpus: + """Documents plus their inverse document frequencies, read once.""" + + def __init__(self, documents: list[Document]): + self.documents = documents + total = len(documents) + appearances = Counter() + self._counts = [ + {name: _term_counts(text) for name, text in document.fields.items()} + for document in documents + ] + lengths = [fields["body"].total() if "body" in fields else 0 for fields in self._counts] + nonempty_lengths = [length for length in lengths if length] + typical_length = median(nonempty_lengths) if nonempty_lengths else 1 + self._body_weights = [ + FIELD_WEIGHTS["body"] / math.sqrt(max(1, length / typical_length)) + for length in lengths + ] + for fields in self._counts: + appearances.update({token for counts in fields.values() for token in counts}) + self._idf = { + token: math.log((total + 1) / (count + 1)) + 1.0 + for token, count in appearances.items() + } + self._unseen = math.log(total + 1) + 1.0 + + def idf(self, token: str) -> float: + return self._idf.get(token, self._unseen) + + def score(self, index: int, query_tokens: list[str]) -> tuple[float, dict]: + """Sum of idf x field weight x sqrt(occurrences). + + The square root damps repetition. Bodies longer than the corpus median + get a lower weight so broad manuals compete with much shorter cases + through their specific matches. Every matched term contributes. + """ + total = 0.0 + detail: dict[str, float] = {} + body_weight = self._body_weights[index] + for token in query_tokens: + weight = self.idf(token) + for name, counts in self._counts[index].items(): + hits = counts[token] + if hits: + field_weight = body_weight if name == "body" else FIELD_WEIGHTS[name] + gain = weight * field_weight * math.sqrt(hits) + total += gain + detail[token] = round(detail.get(token, 0.0) + gain, 2) + return total, detail + + +def load_corpus( + skill_root: Path, *, kinds: tuple[str, ...] = ("case", "skill", "record") +) -> Corpus: + root = Path(skill_root).resolve() + if not root.is_dir(): + raise ValueError(f"Skill root is not a directory: {root}") + documents: list[Document] = [] + if "case" in kinds: + documents.extend(_case_documents(root)) + if "skill" in kinds: + documents.extend(_skill_documents(root)) + if "record" in kinds: + documents.extend(_record_documents()) + identities = [document.qualified for document in documents] + if len(set(identities)) != len(identities): + duplicate = next(i for i, c in Counter(identities).items() if c > 1) + raise ValueError(f"Duplicate identity in corpus: {duplicate}") + return Corpus(documents) + + +@lru_cache(maxsize=1) +def _aliases() -> tuple[tuple[str, tuple[str, ...]], ...]: + """Query-side term map, longest key first so 前缀缓存 wins over 缓存. + + Two blocks, same rule. `aliases` routes a Chinese term to the English the + corpus uses; `synonyms` covers English the corpus and its readers spell + differently -- a caller asking about "startup" should reach a finding that + says "initialization". Both expand one way, at query time only. + """ + path = files("inferrsi").joinpath("knowledge", "data", "query_aliases.json") + record = json.loads(path.read_text(encoding="utf-8")) + if record.get("schema_version") != 1: + raise ValueError("Query alias map requires schema_version 1") + entries = [] + for block in ("aliases", "synonyms"): + declared = record.get(block, {}) + if not isinstance(declared, dict): + raise ValueError(f"Query alias map requires a {block} mapping") + for term, expansions in declared.items(): + if not isinstance(expansions, list) or any( + not isinstance(value, str) or not value.strip() for value in expansions + ): + raise ValueError(f"Query alias expansions must be nonempty strings: {term}") + entries.append((term.casefold(), tuple(expansions))) + if len({term for term, _ in entries}) != len(entries): + raise ValueError("A term appears in both alias blocks") + return tuple(sorted(entries, key=lambda entry: -len(entry[0]))) + + +def expand(query: str) -> tuple[list[str], dict[str, tuple[str, ...]]]: + """Return query tokens plus the corpus terms its aliased terms stand for. + + Expansion is query-side and one-way. Longest match wins and consumes its + span, so 前缀缓存 does not also expand as 缓存 and score the narrower + concept twice. A matched term keeps its own tokens as well: dropping them + would lose a query that mixes a Chinese term with the English identifier + beside it, which is how these queries are actually written. + """ + folded = query.casefold() + found = tokens(query) + applied: dict[str, tuple[str, ...]] = {} + for term, expansions in _aliases(): + # A latin term must match as a word: "nvram" is not "vram" and + # "startups" is not "startup". CJK has no such boundaries, so a + # substring is the match there. + if _CJK.match(term): + pattern = re.escape(term) + else: + pattern = rf"(? str | None: + """Open the entry the way `read` would; return why not, or None if it opens. + + Ranking reads files directly, which is what lets it skip Bundle closures, + but it means a hit can be indexed and still refused by `read`: a source + record missing the version `read` requires, or a case whose body file was + never copied. Offering a command known to fail is worse than offering + none, so only the hits actually returned are checked -- a handful of + reads, not the corpus -- and a failing one reports why instead. + + A missing optional dependency is reported against the hit like any other + reason it cannot be opened here. Ranking itself needs no parser, so the + search must not fail for it; the caller learns which hits need PyYAML + to open, and can still open every case, which does not. + """ + from .knowledge_operations import KnowledgeReader + + if not document.qualified: + return None + reader = readers.get(document.root) + if reader is None: + reader = readers[document.root] = KnowledgeReader(skill_root=document.root) + try: + reader.read(document.qualified).view(material=True) + except (ValueError, KeyError, OSError, TypeError) as exc: + return str(exc) or type(exc).__name__ + return None + + +def find(query: str, corpus: Corpus, *, limit: int = 5, explain: bool = False) -> dict: + if type(limit) is not int or limit < 1: + raise ValueError("limit must be a positive integer") + query_tokens, applied = expand(query) + scored = [] + for index, document in enumerate(corpus.documents): + value, detail = corpus.score(index, query_tokens) + if value > 0: + scored.append((value, STATE_RANK.get(document.state, 0), index, detail)) + scored.sort(key=lambda row: ( + -row[0], -row[1], corpus.documents[row[2]].identity, corpus.documents[row[2]].qualified, + )) + items = [] + readers: dict = {} + for value, _, index, detail in scored[:limit]: + document = corpus.documents[index] + view = document.view( + score=value, query_tokens=query_tokens, unreadable=_why_unreadable(document, readers) + ) + if explain: + view["matched"] = detail + items.append(view) + result = { + "query": query, + "tokens": query_tokens, + "corpus_size": len(corpus.documents), + "matched": len(scored), + "items": items, + "no_match": not scored, + "limitations": [ + "Deterministic lexical ranking over titles, summaries, tags, findings " + "and bodies; not a learned ranking or a claim that material applies.", + "A match is textual. Read the cited state, applicability and " + "limitations before relying on an entry.", + "Paths are corpus-relative. Use read/resource for a pinned revision.", + ], + } + if applied: + # Report what a query term was taken to mean, so a wrong expansion is + # visible in the result rather than only in the ranking. + result["expanded"] = {term: list(values) for term, values in applied.items()} + return result diff --git a/tests/fixtures/find/README.md b/tests/fixtures/find/README.md new file mode 100644 index 000000000..06e56f8d6 --- /dev/null +++ b/tests/fixtures/find/README.md @@ -0,0 +1,50 @@ +# Externally labelled queries for `find` + +`labelled-queries.json` holds 36 queries derived from the nine frozen decision +points of the material-discovery experiment +([#380](https://github.com/InferMatrixLab/InferRSI/issues/380)), three to five +per decision point. + +## Why these labels are worth keeping + +The earlier query set for this tool was written by whoever wrote the corpus +metadata, so it matched by construction and overstated retrieval quality. These +were produced by a separate model working only from the task descriptions, with +instructions not to open the corpus. It could not phrase a query around a +document it had seen, and it was told that "no knowledge base entry would help +here" is a real answer rather than a failure. Nineteen of the 36 came back that +way, which is the single most informative part of the set. + +## Fields + +Each query carries: + +- `query` and `axis` — the wording, and which register it uses: `symptom`, + `terminology`, `question`, `terse` or `chinese`. At least one query per + decision point is `symptom` or `question`, and at least one is `chinese`. +- `expect` — `case`, `skill`, `record`, or `none` when the situation is already + decidable from the stated facts. +- `expect_reason` and `must_surface` — why that kind, and what the ideal top + result would tell the reader, in the labeller's own words rather than as a + path or an ID. +- `confidence` — the labeller's own `high` / `medium` / `low`. + +## What these numbers do and do not support + +Scoring counts whether an entry of the expected kind appears in the top three. +That is a coarse proxy: it does not check that the particular entry returned is +the most useful one, and it says nothing about whether acting on it leads +anywhere. The `none` queries are reported separately and never folded into a +hit rate — they test restraint, which this tool does not currently have. + +Known qualifications, retained rather than repaired after seeing results: + +- All nine decision points come from one experiment, built to study material + discovery. Day-to-day questions may be shaped differently. +- One query labelled `skill` is better served by a case: it asks how to measure + admission and drain separately, and a casebook entry records exactly that + observation. The label is kept as written. +- Labels are one model's judgement, unreviewed by a second. `confidence` is + self-reported. + +Reproduce with `tests/test_knowledge_find.py::test_external_label_set_retrieval`. diff --git a/tests/fixtures/find/labelled-queries.json b/tests/fixtures/find/labelled-queries.json new file mode 100644 index 000000000..1e8f3991d --- /dev/null +++ b/tests/fixtures/find/labelled-queries.json @@ -0,0 +1,335 @@ +[ + { + "case_id": "round1:qwen_b300_applicable", + "queries": [ + { + "query": "BF16 QK normalization complex RoPE fusion numerical order", + "axis": "terminology", + "expect": "record", + "expect_reason": "The proposed fusion makes the numerical effects of operation ordering and precision relevant to validation.", + "must_surface": "The record explains how changes to operation order or BF16 rounding in the fusion can produce numerical differences and under which conditions.", + "confidence": "medium" + }, + { + "query": "Has Qwen Image QK normalization and complex RoPE fusion been tried on four B300s, and what happened to speed and full-image quality?", + "axis": "question", + "expect": "case", + "expect_reason": "A prior attempt on the same accelerator cohort could inform the experiment without substituting for a new measurement.", + "must_surface": "The result describes the implementations tried, their operator, full-image quality, and latency outcomes, and their hardware, precision, and step-count conditions.", + "confidence": "high" + }, + { + "query": "QK norm RoPE B300", + "axis": "terse", + "expect": "case", + "expect_reason": "Past attempts at this fusion on B300s could reveal useful implementation and validation pitfalls.", + "must_surface": "A comparable B300 experiment reports the implementation, workload conditions, and observed quality and latency for the fusion.", + "confidence": "medium" + }, + { + "query": "图像生成里把 QK 归一化和旋转位置编码融合后,如何验证算子、整图质量和端到端时延?", + "axis": "chinese", + "expect": "skill", + "expect_reason": "The untested candidate needs a practical sequence for checking correctness and end-to-end performance.", + "must_surface": "Compare the fused and baseline implementations at operator and full-image levels before evaluating end-to-end latency.", + "confidence": "high" + } + ] + }, + { + "case_id": "round1:qwen_ascend_mismatch", + "queries": [ + { + "query": "Has RoPE cosine and sine caching been tried for Qwen Image on one Ascend 910B1 with BF16 eager execution, and what happened?", + "axis": "question", + "expect": "case", + "expect_reason": "A prior cache experiment with matching hardware and execution conditions could help plan the unmeasured candidate.", + "must_surface": "A comparable caching trial reports its implementation, latency and image-quality outcomes, and accelerator and workload conditions.", + "confidence": "high" + }, + { + "query": "Qwen Image single 910B1 BF16 eager eight-step RoPE cosine sine cache", + "axis": "terminology", + "expect": "case", + "expect_reason": "The component and cohort identify a potentially relevant prior attempt at the same optimization.", + "must_surface": "A comparable trial identifies whether cosine and sine caching was used and reports its speed and correctness outcomes.", + "confidence": "high" + }, + { + "query": "910B1 RoPE cache benchmark", + "axis": "terse", + "expect": "case", + "expect_reason": "A measured caching attempt on this accelerator could provide a useful starting point for a controlled local test.", + "must_surface": "A comparable 910B1 trial reports the RoPE cache’s measured effect and correctness together with its configuration.", + "confidence": "medium" + }, + { + "query": "四卡 B300 上的融合经验能直接推断单卡 910B1 缓存三角值的效果吗?", + "axis": "chinese", + "expect": "none", + "expect_reason": "The stated hardware and optimization both differ, so the B300 experience cannot establish the outcome of this candidate.", + "must_surface": "The earlier fusion result cannot establish the cache result on a different accelerator, so profile and compare the current candidate against its own baseline.", + "confidence": "high" + } + ] + }, + { + "case_id": "round1:qwen_gate_sufficient", + "queries": [ + { + "query": "operator test passes but image output fails the confirmed quality gate", + "axis": "symptom", + "expect": "none", + "expect_reason": "The verified full-image failure already decides whether this candidate meets the stated acceptance gate.", + "must_surface": "The candidate does not meet this round’s quality requirement and must be withdrawn or fixed and retested on full images.", + "confidence": "high" + }, + { + "query": "Can a candidate that failed the confirmed full-image quality gate be accepted because its operator test passed?", + "axis": "question", + "expect": "none", + "expect_reason": "A passing operator test does not change the given failure of the required full-image check.", + "must_surface": "The candidate cannot be accepted as passing and any repair needs a new full-image quality check.", + "confidence": "high" + }, + { + "query": "image quality gate failure", + "axis": "terse", + "expect": "none", + "expect_reason": "The current acceptance decision follows directly from the confirmed quality-gate result.", + "must_surface": "The confirmed image-quality failure rules out accepting the current candidate until a fix passes the full check.", + "confidence": "high" + }, + { + "query": "算子测过了但整图验收没过,这个版本还能入选吗,接下来怎么办?", + "axis": "chinese", + "expect": "none", + "expect_reason": "The problem already states that the required quality failure is valid and the decision concerns this round.", + "must_surface": "本轮不能把当前候选作为验收通过的版本,应撤回它或修复后重新验证整图质量。", + "confidence": "high" + } + ] + }, + { + "case_id": "round1:drama3_occupancy_holdout", + "queries": [ + { + "query": "continuously fed service has low full-run average decode batch below C64", + "axis": "symptom", + "expect": "case", + "expect_reason": "A comparable service run with phase-specific observations could help interpret an aggregate that includes drain.", + "must_surface": "A comparable run reports active decode batches during admission and drain, the phase boundaries, and the corresponding end-to-end outcome.", + "confidence": "medium" + }, + { + "query": "Does a low full-run average decode batch prove that the active batch stayed far below 64 while requests were still being admitted?", + "axis": "question", + "expect": "none", + "expect_reason": "The given average includes a drain phase, so it cannot establish the occupancy throughout admission by itself.", + "must_surface": "The aggregate alone does not prove sustained underfill, so inspect the current run by admission and drain while retaining whole-run evaluation.", + "confidence": "high" + }, + { + "query": "decode batch drain occupancy", + "axis": "terse", + "expect": "case", + "expect_reason": "A prior analysis of the same phases could illustrate how the workload affects the reported mean.", + "must_surface": "A comparable workload reports phase-specific occupancy, whole-run metrics, and enough workload detail to judge applicability.", + "confidence": "medium" + }, + { + "query": "怎么分开统计持续接收请求和尾部排空时的活跃批量,同时检查整个请求集的吞吐?", + "axis": "chinese", + "expect": "skill", + "expect_reason": "The current trace needs a phase-aware measurement procedure without dropping requests from end-to-end evaluation.", + "must_surface": "Define and measure admission and drain separately while reporting the full-run results for every request.", + "confidence": "high" + } + ] + }, + { + "case_id": "round2:N1", + "queries": [ + { + "query": "fused kernel passed unit checks but the finished image failed the verified acceptance test", + "axis": "symptom", + "expect": "none", + "expect_reason": "The confirmed end-to-end quality result already determines the acceptance decision.", + "must_surface": "The candidate cannot pass this round’s acceptance and needs to be withdrawn or repaired and retested on the complete image.", + "confidence": "high" + }, + { + "query": "Should I keep an operator-tested fusion as accepted when its confirmed full-image quality check failed?", + "axis": "question", + "expect": "none", + "expect_reason": "Operator-level success cannot override the stated mandatory image-quality failure.", + "must_surface": "Do not retain the candidate as accepted, and rerun the full-image quality gate after any fix.", + "confidence": "high" + }, + { + "query": "fusion quality acceptance failure", + "axis": "terse", + "expect": "none", + "expect_reason": "The verified failed gate supplies all facts needed for this acceptance decision.", + "must_surface": "The existing candidate has failed acceptance and must be withdrawn or corrected before full-image revalidation.", + "confidence": "high" + }, + { + "query": "融合版本的单算子测试合格,最终图片却没达到验收标准,该如何处理?", + "axis": "chinese", + "expect": "none", + "expect_reason": "The scenario already establishes that the final-image failure is valid for the current round.", + "must_surface": "当前版本不能算验收合格,应撤回,或者修复并重新完成整图质量验证。", + "confidence": "high" + } + ] + }, + { + "case_id": "round2:N2", + "queries": [ + { + "query": "candidate ran at 360 ms with diagnostics running versus an old 277 ms result on a quiet host", + "axis": "symptom", + "expect": "none", + "expect_reason": "The two timings are unpaired and were taken under different host conditions, so they cannot establish the cause.", + "must_surface": "The existing timings do not prove a code regression, and baseline and candidate should be remeasured in repeated matched runs.", + "confidence": "high" + }, + { + "query": "Do unpaired latency measurements under different host loads establish that a code change caused a regression?", + "axis": "question", + "expect": "none", + "expect_reason": "The stated comparison lacks a controlled baseline and has a host-load confounder.", + "must_surface": "A causal regression claim is unsupported until unchanged and modified code are compared under the same controlled conditions.", + "confidence": "high" + }, + { + "query": "unpaired latency host contention", + "axis": "terse", + "expect": "none", + "expect_reason": "The known difference in measurement conditions is enough to reject an immediate attribution to the code.", + "must_surface": "Treat the observed latency difference as inconclusive and collect matched repeated baseline and candidate measurements.", + "confidence": "high" + }, + { + "query": "主机忙闲不一致时,旧基线与新改动的耗时差能归因于代码吗?", + "axis": "chinese", + "expect": "none", + "expect_reason": "The old result and the busy-host candidate are not a controlled comparison of the code change.", + "must_surface": "现有耗时差不能证明代码导致回退,应在一致且受控的环境中重复对比未修改基线与候选。", + "confidence": "high" + } + ] + }, + { + "case_id": "round2:S1", + "queries": [ + { + "query": "fixed-request check passes but baseline and async candidate both fail speech-recognition quality in full service", + "axis": "symptom", + "expect": "case", + "expect_reason": "A comparable service experiment could suggest a focused diagnostic while showing which conditions limit its applicability.", + "must_surface": "A comparable full-service experiment reports its symptoms, shared conditions, diagnostic controls, and observed cause or remaining uncertainty.", + "confidence": "medium" + }, + { + "query": "If full-service speech recognition fails for both baseline and candidate, does that prove async overlap caused the failure?", + "axis": "question", + "expect": "none", + "expect_reason": "A failure present on both sides does not establish that the candidate caused the shared failure.", + "must_surface": "The shared failure alone does not identify the cause, so check shared conditions and use a controlled comparison before attributing a candidate effect.", + "confidence": "high" + }, + { + "query": "ASR baseline candidate failure", + "axis": "terse", + "expect": "skill", + "expect_reason": "The engineer needs a small controlled diagnostic that separates common failures from an effect of the change.", + "must_surface": "Hold inputs and environment constant, inspect the baseline quality failure, and compare the candidate under the same conditions.", + "confidence": "medium" + }, + { + "query": "异步服务的基线和候选语音识别都不达标,怎样用最少对照排查共同问题?", + "axis": "chinese", + "expect": "skill", + "expect_reason": "Both full-service results fail, so a procedure for isolating shared inputs, environment, and metric computation can guide the next experiment.", + "must_surface": "Reproduce the failure with shared inputs and environment, inspect the baseline recognition output and reference, and compare the candidate under the same conditions.", + "confidence": "medium" + } + ] + }, + { + "case_id": "round2:N3", + "queries": [ + { + "query": "quality checks passed but candidate service is 4307 frames/s versus 4371 baseline with longer drain", + "axis": "symptom", + "expect": "none", + "expect_reason": "The reported candidate throughput is lower in the only comparison, so the result does not demonstrate an improvement.", + "must_surface": "Do not claim a throughput improvement, and repeat the full-service comparison while recording output frame counts and the drain phase.", + "confidence": "high" + }, + { + "query": "Can I claim a full-service throughput improvement when the candidate measured 4307 frames/s and the baseline 4371 frames/s in one run?", + "axis": "question", + "expect": "none", + "expect_reason": "The sole measured candidate throughput is lower than the baseline even though the quality checks passed.", + "must_surface": "The comparison does not establish a throughput gain, and controlled repeated whole-service measurements are needed.", + "confidence": "high" + }, + { + "query": "throughput comparison drain mismatch", + "axis": "terse", + "expect": "none", + "expect_reason": "The given single comparison and its different frame counts and drain durations already rule out declaring an improvement.", + "must_surface": "Report the candidate as not shown to improve throughput and remeasure with comparable workloads while accounting for all output and drain time.", + "confidence": "high" + }, + { + "query": "两边语音质量检查都过了但候选帧吞吐更低,能宣布优化成功吗?", + "axis": "chinese", + "expect": "none", + "expect_reason": "Passing quality checks does not turn the reported lower candidate throughput into evidence of a gain.", + "must_surface": "不能宣称吞吐提升,应重复完整服务对照并记录生成帧数和排空时间。", + "confidence": "high" + } + ] + }, + { + "case_id": "round2:S2", + "queries": [ + { + "query": "Qwen Image 910B1 BF16 eager 1280x704 interleaved RoPE VAE fastpath fused attention", + "axis": "terminology", + "expect": "case", + "expect_reason": "Prior trials of these specific directions under matching conditions could inform which candidates to test first.", + "must_surface": "Relevant trials report the hardware, configuration, speed, and image-quality outcomes for each direction alone and for any combination tested.", + "confidence": "high" + }, + { + "query": "How should I isolate and then combine interleaved RoPE, VAE fastpath and fused attention when this build has not run the combination?", + "axis": "question", + "expect": "skill", + "expect_reason": "The unmeasured combination requires a procedure that can distinguish individual effects from interactions and check quality.", + "must_surface": "Run a matched baseline, test the candidates individually, and then test the combination for full-image quality and end-to-end latency.", + "confidence": "high" + }, + { + "query": "910B1 RoPE VAE attention", + "axis": "terse", + "expect": "case", + "expect_reason": "Comparable experiments on the target accelerator may reveal useful results or compatibility limits for these directions.", + "must_surface": "Comparable trials report which directions were tested, whether alone or together, and their measured outcomes and conditions.", + "confidence": "medium" + }, + { + "query": "昇腾单卡上这些 RoPE、VAE 和注意力改动有没有在相同配置组合试过,速度和整图质量怎样?", + "axis": "chinese", + "expect": "case", + "expect_reason": "The requested combination has no current-version measurement, so comparable prior attempts could provide bounded evidence.", + "must_surface": "A comparable combination reports its measured latency, full-image quality, and configuration separately from results for individual changes.", + "confidence": "high" + } + ] + } +] diff --git a/tests/test_codex_plugin.py b/tests/test_codex_plugin.py index 2613a87e2..599b83b2a 100644 --- a/tests/test_codex_plugin.py +++ b/tests/test_codex_plugin.py @@ -9,6 +9,7 @@ import shutil import subprocess import sys +import venv import zipfile from pathlib import Path from urllib.parse import unquote, urlsplit @@ -181,10 +182,18 @@ def test_missing_runtime_has_repair_guidance(tmp_path): @pytest.mark.parametrize("args", [ ("skills", "list", "--json"), - ("collect", "--skill-root", "."), - ("read", "example", "--skill-root=."), + ("read", "calibrate-serving-simulation", "--skill-root", "{skills}"), ]) def test_optional_dependency_guidance_does_not_require_a_checkout(distribution, tmp_path, args): + """Operations that parse skill frontmatter explain the missing dependency. + + The hint comes from the runtime, at the call that needs the parser, rather + than from the launcher guessing from argument names -- see the next test + for what that guess used to refuse. + """ + # Any registered root will do; the point is the parser, not this library. + skills = PROJECT / "src" / "inferrsi" / "knowledge" / "data" / "skill_library" + args = tuple(str(skills) if arg == "{skills}" else arg for arg in args) result = invoke(distribution, tmp_path, *args) assert result.returncode == 2 assert "PyYAML" in result.stderr @@ -193,6 +202,61 @@ def test_optional_dependency_guidance_does_not_require_a_checkout(distribution, assert help_result.returncode == 0 +def test_find_reference_opens_from_a_bare_plugin_host(distribution, tmp_path): + """Use the returned command unchanged in a bare plugin environment.""" + bare = tmp_path / "bare python" + venv.EnvBuilder().create(bare) + python = bare / ("Scripts/python.exe" if os.name == "nt" else "bin/python") + launcher = distribution / "skills/inferrsi/scripts/inferrsi.py" + cache = tmp_path / "runtime cache" + env = { + **os.environ, "PATH": os.pathsep.join((str(python.parent), os.defpath)), + "LOCALAPPDATA": str(cache), + } + env.pop("PYTHONHOME", None) + env.pop("PYTHONPATH", None) + assert shutil.which("inferrsi", path=env["PATH"]) is None + + def run(*command): + return subprocess.run( + command, cwd=tmp_path, capture_output=True, text=True, encoding="utf-8", + timeout=45, env=env, + ) + + prefix = (str(python), "-X", "utf8", str(launcher)) + info = run(*prefix, "--plugin-info") + assert info.returncode == 0, info.stderr + assert not json.loads(info.stdout)["optional_skills_dependency"] + search = ("find", "mask caching ran out of device memory", "--kinds", "case", "--limit", "1") + found = run(*prefix, *search, "--json") + assert found.returncode == 0, found.stderr + item, = json.loads(found.stdout)["items"] + assert item["id"] == "case.qwen-ascend-rejected-directions" + assert item["open_argv"] == [*prefix[:-1], str(launcher.resolve()), *item["read_args"]] + if cache.is_dir(): + # WorkSwarm's installed launcher recreates its private cache. + shutil.rmtree(_io_path(cache)) + opened = run(*item["open_argv"]) + assert opened.returncode == 0, opened.stderr + material = json.loads(opened.stdout) + assert "910B1" in material["content"] + assert material["sources"] and material["applicability"] and material["limitations"] + + plain = run(*prefix, *search) + assert plain.returncode == 0, plain.stderr + label = "open (PowerShell)" if os.name == "nt" else "open" + assert f" {label}: {item['ref']}" in plain.stdout.splitlines() + if os.name == "nt": + powershell = shutil.which("powershell.exe") + shell = (powershell, "-NoProfile", "-Command") if powershell else () + else: + shell = ("/bin/sh", "-c") + if shell: + pasted = run(*shell, item["ref"]) + assert pasted.returncode == 0, pasted.stderr + assert json.loads(pasted.stdout) == material + + def test_builder_refuses_to_replace_existing_files(tmp_path): output = tmp_path / "existing" output.mkdir() diff --git a/tests/test_knowledge_cli.py b/tests/test_knowledge_cli.py index b0e184145..915c85def 100644 --- a/tests/test_knowledge_cli.py +++ b/tests/test_knowledge_cli.py @@ -86,6 +86,23 @@ def test_read_returns_bundle_view_and_rejects_wrong_revision(): assert "unavailable" in stderr +def test_read_can_omit_inventory_without_changing_material_or_revision(): + args = ("read", "package:vllm_omni.common.diffusion.step_state_and_output_validation", + "--material") + code, stdout, stderr = run_cli(*args) + assert code == 0, stderr + full = json.loads(stdout) + assert "record.json" in full["resources"] + assert full["required_resources"] + + code, stdout, stderr = run_cli(*args, "--omit-resources", "--revision", full["revision"]) + assert code == 0, stderr + assert json.loads(stdout) == { + key: value for key, value in full.items() + if key not in ("resources", "required_resources") + } + + def test_resource_writes_exact_bytes_of_requested_revision(): bundle = KnowledgeReader(skill_root=SKILLS).read(SOURCE) entrypoint = bundle.metadata["entrypoint"] diff --git a/tests/test_knowledge_find.py b/tests/test_knowledge_find.py new file mode 100644 index 000000000..170241472 --- /dev/null +++ b/tests/test_knowledge_find.py @@ -0,0 +1,646 @@ +"""Behaviour of the single ranked lookup across skills, records and cases.""" + +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +from importlib.resources import files +from pathlib import Path + +import pytest + +from inferrsi.cli import main +from inferrsi.knowledge.knowledge_find import ( + FINDING_OUTCOMES, + FINDING_SCOPES, + Corpus, + Document, + Finding, + _aliases, + expand, + find, + load_corpus, + tokens, +) + +PROJECT = Path(__file__).resolve().parents[1] +ROOT = Path(files("inferrsi").joinpath("knowledge", "data", "skill_library")) + + +@pytest.fixture(scope="module") +def corpus(): + return load_corpus(ROOT) + + +def _ranked(result, identity): + for position, item in enumerate(result["items"], 1): + if item["id"] == identity: + return position + return None + + +def test_corpus_covers_all_three_kinds_without_double_counting(corpus): + kinds = {} + for document in corpus.documents: + kinds[document.kind] = kinds.get(document.kind, 0) + 1 + assert kinds["skill"] == 80 + assert kinds["case"] == 7 + # Packaged-skill projections are excluded so one owner is not ranked twice. + assert all( + document.kind != "record" or "skills." not in document.identity + for document in corpus.documents + ) + identities = [document.identity for document in corpus.documents] + assert len(set(identities)) == len(identities) + + +def test_cases_are_ranked_beside_skills(corpus): + result = find("word error rate regressed only under concurrent service", corpus, limit=5) + assert _ranked(result, "case.drama3-rejected-directions") == 1 + + +def test_findings_keywords_reach_a_problem_phrased_query(corpus): + """The case title names a campaign; the query names the symptom.""" + result = find("average batch size over the whole trace looks healthy", corpus, limit=5) + assert _ranked(result, "case.admission-drain-occupancy") == 1 + + +def test_word_boundary_keeps_oom_out_of_headroom(): + document = Document( + identity="headroom-only", kind="skill", state="candidate", path="x/SKILL.md", + title="headroom", summary="plenty of headroom remains", + fields={"title": "headroom", "summary": "plenty of headroom remains", + "tags": "", "findings": "", "applicability": "", "body": "headroom headroom"}, + ) + result = find("oom", Corpus([document]), limit=5) + assert result["no_match"] + + +def test_partial_query_still_ranks_instead_of_filtering(corpus): + """Requiring every token would return nothing for a long natural question.""" + result = find("why did the fused projection not help end to end service", corpus, limit=5) + assert not result["no_match"] + assert result["matched"] > 1 + + +def test_cjk_query_is_cut_into_bigrams(corpus): + """Single characters make idf treat particles as rare terms; pairs carry terms.""" + assert tokens("缓存 optimization") == ["optimization", "缓存"] + assert tokens("显存不足") == ["显存", "存不", "不足"] + assert tokens("算") == ["算"] + result = find("Qwen Ascend RoPE 缓存 优化", corpus, limit=5) + assert _ranked(result, "case.qwen-ascend-rejected-directions") is not None + + +def test_punctuation_does_not_strand_a_particle_as_a_term(): + """Commas isolate 和 into a run of its own, where it would score as a rare + term: this corpus has so little CJK that idf rates every character highly.""" + assert "和" not in tokens("让任务、工具和输入一致") + assert tokens("设计 A2 和 A3 的对照")[:3] == ["a2", "a3", "设计"] + # A query that is only that character is still the caller's actual term. + assert tokens("算") == ["算"] + + +def test_separators_do_not_bind_terms(): + """The corpus spells one concept four ways; a plain query must reach them all.""" + assert tokens("kv_cache") == tokens("kv-cache") == tokens("KV cache") == ["kv", "cache"] + assert tokens("CUDA-Graph") == ["cuda", "graph"] + assert tokens("TASK_QUEUE_ENABLE=1") == ["task", "queue", "enable", "1"] + + +def test_query_term_matches_a_punctuated_spelling(): + document = Document( + identity="tagged", kind="skill", state="candidate", path="x/SKILL.md", + title="x", summary="", + fields={"title": "x", "summary": "", "tags": "kv_cache sliding-window-attention", + "findings": "", "applicability": "", "body": ""}, + ) + result = find("kv cache", Corpus([document]), limit=1) + assert not result["no_match"] + assert find("sliding window attention", Corpus([document]), limit=1)["items"] + + +def test_ties_break_towards_stronger_evidence(): + shared = {"title": "same words", "summary": "", "tags": "", + "findings": "", "applicability": "", "body": ""} + weak = Document(identity="b-weak", kind="skill", state="candidate", + path="b/SKILL.md", title="b", summary="", fields=dict(shared)) + strong = Document(identity="a-strong", kind="case", state="validated", + path="a.md", title="a", summary="", fields=dict(shared)) + result = find("same words", Corpus([weak, strong]), limit=2) + assert [item["id"] for item in result["items"]] == ["a-strong", "b-weak"] + + +def test_findings_are_ordered_by_query_overlap_and_truncated(corpus): + result = find("cache per-step masks ran out of device memory", corpus, limit=5) + item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") + assert item["findings"][0]["direction"] == "Cache per-step masks and prompt tensors" + assert item["findings"][0]["outcome"] == "rejected" + assert len(item["findings"]) == 3 + assert item["findings_total"] == 6 + assert item["findings_withheld"] == 3 + + +def test_view_carries_evidence_qualifiers(corpus): + result = find("mask caching ran out of device memory", corpus, limit=10) + item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") + assert item["state"] == "author_reported" + assert item["locally_verified"] is False + assert "910B1" in item["applicability"] + assert item["path"].endswith("qwen-ascend-rejected-directions.md") + + +def test_packaged_cases_declare_known_outcomes(): + paths = set(ROOT.glob("*/*/cases/*.json")) | set(ROOT.glob("*/*/*/cases/*.json")) + total = 0 + for path in sorted(paths): + record = json.loads(path.read_text(encoding="utf-8")) + for finding in record.get("findings", []): + assert finding["outcome"] in FINDING_OUTCOMES + assert finding["direction"].strip() and finding["reason"].strip() + assert finding["keywords"] + total += 1 + assert total == 35 + + +def test_result_carries_a_reference_the_caller_can_open(corpus): + """A hit is only useful if it can be opened from where the caller is. + + A bare case identity does not resolve -- `read` defaults to package content + and cases live in the checkout -- and a corpus-relative path resolves + nowhere at all in a downstream project. + """ + result = find("mask caching ran out of device memory", corpus, limit=10) + item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") + assert item["read_args"][:2] == ["read", "checkout:case.qwen-ascend-rejected-directions"] + assert "--material" in item["read_args"] + assert item["open_argv"][-len(item["read_args"]):] == item["read_args"] + assert item["source"].startswith("https://") + + +def test_read_accepts_the_reference_find_reports(capsys): + """Run the reported command as given and check it returns the text. + + Metadata is not what a reader following a hit is after, and `read` reports + the entrypoint's filename in `content` unless asked for the material. + """ + result = find("mask caching ran out of device memory", load_corpus(ROOT), limit=10) + item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") + assert main(item["read_args"]) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["id"] == "case.qwen-ascend-rejected-directions" + body = payload["content"] + assert "rejected directions" in body.casefold() + assert "910B1" in body + + +def test_reference_survives_a_root_whose_path_has_spaces(tmp_path, capsys): + """An unquoted argument splits on the space and the command fails.""" + root = tmp_path / "a directory with spaces" / "skill_library" + shutil.copytree(ROOT, root) + corpus = load_corpus(root) + result = find("mask caching ran out of device memory", corpus, limit=10) + item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") + assert main(item["read_args"]) == 0 + assert "910B1" in json.loads(capsys.readouterr().out)["content"] + if os.name != "nt": + assert shlex.split(item["ref"]) == item["open_argv"] + + +def test_finding_order_is_not_decided_by_corpus_common_terms(corpus): + """"the" occurring three times in a one-line finding must not outrank the + term the caller asked about.""" + result = find( + "did the Qwen Ascend memory failure happen at startup or on later requests", + corpus, limit=5, + ) + item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") + shown = [f["direction"] for f in item["findings"]] + # Both memory failures the question distinguishes between must be present. + assert any("RoPE + VAE fastpath" in d for d in shown), shown + assert any("later busy host" in d for d in shown), shown + assert not any("TASK_QUEUE_ENABLE" in d for d in shown), shown + + +def _plant_case(root: Path, owner: str, *, registered: bool = True) -> None: + """Copy one packaged case under `owner`, optionally leaving it unregistered.""" + source = ROOT / "methods/inference-optimization-casebook/cases" + cases = root / owner / "cases" + cases.mkdir(parents=True) + for suffix in (".json", ".md"): + shutil.copy(source / f"qwen-ascend-rejected-directions{suffix}", cases) + if registered: + skill = root / owner / "SKILL.md" + body = (ROOT / "methods/inference-optimization-casebook/SKILL.md").read_text( + encoding="utf-8" + ) + skill.write_text( + re.sub(r"^name: .*$", f"name: {Path(owner).name}", body, count=1, flags=re.M), + encoding="utf-8", + ) + + +def test_default_search_needs_no_optional_dependency(): + """A base install has no PyYAML, and `find` must not require it. + + Resolving which skills a root registers reads an index or lists + directories. Neither parses a skill body, so the parser belongs behind the + call that needs it rather than at import time, where it made the default + search fail outright on a base install. + """ + source = (PROJECT / "src/inferrsi/knowledge/skill_catalog.py").read_text(encoding="utf-8") + assert "\nimport yaml" not in source, "module-level import reaches every caller" + assert "def _yaml()" in source + + probe = ( + "import builtins, sys\n" + "real = builtins.__import__\n" + "def guard(name, *a, **k):\n" + " if name == 'yaml':\n" + " raise ImportError('no yaml')\n" + " return real(name, *a, **k)\n" + "builtins.__import__ = guard\n" + "sys.modules.pop('yaml', None)\n" + "from importlib.resources import files\n" + "from pathlib import Path\n" + "from inferrsi.knowledge.knowledge_find import find, load_corpus\n" + "c = load_corpus(Path(files('inferrsi').joinpath('knowledge', 'data', 'skill_library')))\n" + "r = find('ascend oom allocator', c, limit=1)\n" + "print(len(c.documents), r['items'][0]['kind'])\n" + ) + result = subprocess.run( + [sys.executable, "-c", probe], capture_output=True, text=True, + env={**os.environ, "PYTHONPATH": str(PROJECT / "src")}, + ) + assert result.returncode == 0, result.stderr + assert int(result.stdout.split()[0]) == len(load_corpus(ROOT).documents) + + +def test_default_search_works_on_a_flat_root(tmp_path, capsys): + """A flat checkout carries no skill-index.json. + + Reading that file directly assumed every root is a generated library, so + the default search raised on a root whose cases `find` could already rank + with `--kinds case` and `read` could already open. + """ + _plant_case(tmp_path, "my-notes") + corpus = load_corpus(tmp_path) + assert {d.kind for d in corpus.documents} >= {"case", "skill"} + + result = find("mask caching ran out of device memory", corpus, limit=5) + item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") + assert main(item["read_args"]) == 0 + assert "910B1" in json.loads(capsys.readouterr().out)["content"] + + +def test_latin_aliases_match_whole_words_only(): + """`nvram` is not `vram`, and `startups` is not `startup`.""" + assert expand("nvram corruption")[1] == {} + assert expand("startups")[1] == {} + assert expand("vram usage")[1]["vram"] == ("device memory", "hbm") + # CJK has no word boundaries, so substring matching stays correct there. + assert expand("显存不足")[1]["显存"] == ("device memory", "hbm") + + +def test_text_output_states_what_the_record_withholds(capsys): + """The state label says the claim is unverified; limitations say what is + missing, and only the JSON mode used to carry them.""" + assert main(["find", "mask caching ran out of device memory", "--limit", "1"]) == 0 + output = capsys.readouterr().out + assert "caveat:" in output + + +def test_find_sees_a_case_under_a_flat_registered_root(tmp_path, capsys): + """A root whose skills sit directly under it is a supported layout for + `read`; globbing at a fixed depth missed its cases entirely.""" + _plant_case(tmp_path, "my-notes") + corpus = load_corpus(tmp_path, kinds=("case",)) + assert [d.identity for d in corpus.documents] == ["case.qwen-ascend-rejected-directions"] + + result = find("mask caching ran out of device memory", corpus, limit=1) + assert main(result["items"][0]["read_args"]) == 0 + assert "910B1" in json.loads(capsys.readouterr().out)["content"] + + +def test_a_hit_read_refuses_gets_a_reason_not_a_command(tmp_path): + """Ranking reads files directly, so a hit can be indexed and still refused + by `read`. Offering a command known to fail is worse than offering none.""" + _plant_case(tmp_path, "notes") + (tmp_path / "notes" / "cases" / "qwen-ascend-rejected-directions.md").unlink() + item = find("mask caching ran out of device memory", + load_corpus(tmp_path, kinds=("case",)), limit=1)["items"][0] + assert not {"ref", "read_args", "open_argv"} & item.keys() + assert "Missing case resource" in item["unreadable"] + + +@pytest.mark.parametrize("problem, reason", [ + ("encoding", "utf-8"), + ("provenance", "Bundle requires sources"), +]) +def test_invalid_material_has_no_open_command(tmp_path, problem, reason): + _plant_case(tmp_path, "notes") + cases = tmp_path / "notes" / "cases" + if problem == "encoding": + (cases / "qwen-ascend-rejected-directions.md").write_bytes(b"Note\x96legacy text") + else: + path = cases / "qwen-ascend-rejected-directions.json" + record = json.loads(path.read_text(encoding="utf-8")) + record["sources"] = [] + path.write_text(json.dumps(record), encoding="utf-8") + + result = find("mask caching ran out of device memory", load_corpus(tmp_path, kinds=("case",))) + item, = result["items"] + assert reason in item["unreadable"] + assert not {"ref", "read_args", "open_argv"} & item.keys() + + +def test_packaged_hits_with_an_incomplete_source_record_are_not_offered(corpus): + """`husky` cites an unversioned document, which `read` rejects.""" + result = find("husky evidence qualification negative results", corpus, limit=5) + husky = next(i for i in result["items"] if i["id"] == "husky") + assert "ref" not in husky + assert "version or snapshot" in husky["unreadable"] + # Everything that is offered must open. + for item in result["items"]: + if "read_args" in item: + assert main(item["read_args"]) == 0 + + +def test_find_skips_a_case_whose_owner_is_not_registered(tmp_path): + """`read` refuses these, so returning one would hand back a reference that + cannot be opened.""" + _plant_case(tmp_path, "methods/not-registered", registered=False) + (tmp_path / "skill-index.json").write_text( + json.dumps({"schema_version": 1, "skills": {}}), encoding="utf-8" + ) + assert load_corpus(tmp_path, kinds=("case",)).documents == [] + + +def test_finding_order_holds_when_the_library_has_one_entry(tmp_path): + """Weighting must come from the entry, not the corpus. + + A registered library can hold a single case. Corpus-level idf then rates + every term alike, so nothing separates the directions and the recorded + order survives untouched -- dropping exactly the two observations the + question distinguishes between. + """ + _plant_case(tmp_path, "casebook") + corpus = load_corpus(tmp_path, kinds=("case",)) + assert len(corpus.documents) == 1 + + result = find( + "did the Qwen Ascend memory failure happen at startup or on later requests", + corpus, limit=1, + ) + shown = [f["direction"] for f in result["items"][0]["findings"]] + assert any("RoPE + VAE fastpath" in d for d in shown), shown + assert any("later busy host" in d for d in shown), shown + + +def test_english_synonyms_bridge_a_spelling_the_corpus_does_not_use(corpus): + found, applied = expand("failure at startup") + assert applied["startup"] == ("initialization",) + assert "initialization" in found + + +def test_no_finding_claims_wider_reach_than_its_own_text(corpus): + """A finding whose text disclaims generality must not be scoped `general`. + + This caught a real mislabelling: the FP8 SIMT rejection says in so many + words that it is "not a universal ceiling", and was scoped `general` anyway. + """ + paths = set(ROOT.glob("*/*/cases/*.json")) | set(ROOT.glob("*/*/*/cases/*.json")) + disclaimers = ("not a universal", "not permanently", "for that mapping", + "this specific", "only the source") + offenders = [] + for path in sorted(paths): + record = json.loads(path.read_text(encoding="utf-8")) + for finding in record.get("findings", []): + if finding["scope"] != "general": + continue + text = (finding["reason"] + " " + finding.get("revisit_when", "")).casefold() + if any(phrase in text for phrase in disclaimers): + offenders.append(f"{record['id']}: {finding['direction']}") + assert not offenders, offenders + + +def test_packaged_findings_state_how_far_they_reach(): + """Scope is optional in the schema, but leaving it off here would hide the + difference between a result bound to one card and a source-contract fact.""" + paths = set(ROOT.glob("*/*/cases/*.json")) | set(ROOT.glob("*/*/*/cases/*.json")) + seen = {} + for path in sorted(paths): + record = json.loads(path.read_text(encoding="utf-8")) + for finding in record.get("findings", []): + assert "scope" in finding, f"{record['id']}: {finding['direction']}" + assert finding["scope"] in FINDING_SCOPES + seen[finding["scope"]] = seen.get(finding["scope"], 0) + 1 + # All three are in use: a uniform label would carry no information. + assert set(seen) == set(FINDING_SCOPES) + + +def _plant_custom_case(root: Path, record: dict) -> None: + """A registered owner holding one caller-supplied case record.""" + owner = root / "demo" + (owner / "cases").mkdir(parents=True) + owner.joinpath("SKILL.md").write_text( + "---\nname: demo\ndescription: demo owner\n---\n\nbody\n", encoding="utf-8" + ) + owner.joinpath("cases", "demo.md").write_text("body\n", encoding="utf-8") + owner.joinpath("cases", "demo.json").write_text( + json.dumps(record), encoding="utf-8" + ) + + +def test_find_opens_both_authorities_when_ids_overlap(tmp_path, capsys): + identity = "vllm_omni.ascend.diffusion.qwen_image_21_eager_t2i" + _plant_custom_case(tmp_path, { + "id": identity, "kind": "case", "content": "demo.md", + "state": "historical_observation", "applicability": [], + "limitations": [], "attribution": "Local Qwen eager observation", + "sources": [{"type": "internal", "issue": "local", "artifact": "fixture"}], + }) + assert main(["find", identity, "--skill-root", str(tmp_path), "--json"]) == 0 + found = json.loads(capsys.readouterr().out) + by_authority = {item["authority"]: item for item in found["items"] if item["id"] == identity} + assert set(by_authority) == {"package", "checkout"} + for authority, kind in (("package", "knowledge"), ("checkout", "case")): + args = by_authority[authority]["read_args"] + assert args[:2] == ["read", f"{authority}:{identity}"] + assert main(args) == 0 + opened = json.loads(capsys.readouterr().out) + assert opened["kind"] == kind + assert opened["revision"].startswith("sha256:") + if authority == "checkout": + assert opened["content"] == "body\n" + + +def test_unknown_outcome_is_rejected(tmp_path): + _plant_custom_case(tmp_path, { + "id": "case.demo", "kind": "case", "content": "demo.md", + "sources": [{"type": "unknown"}], + "findings": [{"direction": "d", "outcome": "invented", "reason": "r"}], + }) + with pytest.raises(ValueError, match="Unknown finding outcome"): + load_corpus(tmp_path, kinds=("case",)) + + +def test_limit_must_be_positive(corpus): + with pytest.raises(ValueError, match="limit must be a positive integer"): + find("ascend", corpus, limit=0) + + +def test_finding_view_omits_empty_revisit(): + assert Finding(direction="d", outcome="caveat", reason="r").view() == { + "direction": "d", "outcome": "caveat", "reason": "r" + } + + +def test_chinese_query_reaches_english_material_through_aliases(corpus): + """Every concept here exists in the corpus; only the route to it was missing.""" + found, applied = expand("语音识别的基线和候选都不达标") + assert applied["语音识别"] == ("asr", "wer") + assert applied["基线"] == ("baseline",) + assert {"asr", "wer", "baseline", "candidate"} <= set(found) + # The query's own tokens survive: these queries mix Chinese with identifiers. + assert "语音" in found + + +def test_longest_alias_wins(corpus): + _, applied = expand("前缀缓存命中率") + assert "前缀缓存" in applied + assert "缓存" not in applied + + +def test_expansion_is_reported_for_review(corpus): + result = find("显存不足", corpus, limit=1) + assert result["expanded"]["显存"] == ["device memory", "hbm"] + assert "expanded" not in find("device memory", corpus, limit=1) + + +def test_aliases_only_name_terms_present_in_the_corpus(corpus): + """An alias pointing at absent text is upkeep with no effect; catch it here.""" + blob = " ".join( + " ".join(document.fields.values()) for document in corpus.documents + ).casefold() + missing = { + term: [value for value in values if value not in blob] + for term, values in _aliases() + } + assert not {k: v for k, v in missing.items() if v} + + +def test_finding_inherits_the_entry_cohort_and_says_so(corpus): + """A finding reached through search must carry the cohort it was seen on.""" + result = find("cache per-step masks ran out of device memory", corpus, limit=5) + item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") + finding = item["findings"][0] + assert "910B1" in finding["bounds"] + assert finding["bounds_source"] == "entry" + + +def test_finding_bounds_override_the_entry_cohort(): + own = Finding(direction="d", outcome="rejected", reason="r", bounds="A100 80GB, TP=2") + view = own.view(inherited_bounds="whole-entry cohort") + assert view["bounds"] == "A100 80GB, TP=2" + assert view["bounds_source"] == "finding" + + +def test_absent_scope_is_not_reported_as_general(): + """Missing means not assessed; it must not read as a claim that it holds anywhere.""" + view = Finding(direction="d", outcome="rejected", reason="r").view() + assert "scope" not in view + assert "bounds" not in view + + +def test_unknown_scope_is_rejected(tmp_path): + _plant_custom_case(tmp_path, { + "id": "case.demo", "kind": "case", "content": "demo.md", + "sources": [{"type": "unknown"}], + "findings": [{"direction": "d", "outcome": "rejected", "reason": "r", + "scope": "everywhere"}], + }) + with pytest.raises(ValueError, match="Unknown finding scope"): + load_corpus(tmp_path, kinds=("case",)) + + +def test_external_label_set_retrieval(corpus): + """Measured on queries labelled without sight of the corpus. + + The threshold is a regression floor well below the current result, not a + target. Retrieval here depends on what the corpus happens to hold, so + adding or rewriting cases legitimately moves individual ranks; only a + collapse should fail the build. The `none` queries are excluded, since they + ask whether the tool should answer at all, which it cannot currently decide. + """ + labels = json.loads( + (PROJECT / "tests/fixtures/find/labelled-queries.json").read_text(encoding="utf-8") + ) + answerable, hits, by_kind = 0, 0, {} + for group in labels: + for entry in group["queries"]: + if entry["expect"] == "none": + continue + answerable += 1 + result = find(entry["query"], corpus, limit=3) + found = any(item["kind"] == entry["expect"] for item in result["items"]) + hits += found + kind = by_kind.setdefault(entry["expect"], [0, 0]) + kind[0] += found + kind[1] += 1 + + assert answerable == 17 + assert hits >= 12, f"{hits}/{answerable} expected kinds in top 3" + # Cases are the material this tool exists to reach and the older rankers + # could not return at all, so hold that line tighter than the overall floor. + assert by_kind["case"][0] >= 9, f"case {by_kind['case'][0]}/{by_kind['case'][1]}" + + +def test_every_none_query_still_gets_an_answer(corpus): + """Documents the limitation rather than asserting it is fixed. + + `find` answers every query, including the ones a labeller judged decidable + without searching. If that ever stops being true the restraint was added + deliberately, and this test should be rewritten to describe the new rule. + """ + labels = json.loads( + (PROJECT / "tests/fixtures/find/labelled-queries.json").read_text(encoding="utf-8") + ) + none_queries = [ + entry["query"] for group in labels + for entry in group["queries"] if entry["expect"] == "none" + ] + assert len(none_queries) == 19 + assert all(not find(query, corpus, limit=3)["no_match"] for query in none_queries) + + +def test_cli_reports_findings_and_path(capsys): + assert main(["find", "word error rate regressed under concurrent service", + "--limit", "1"]) == 0 + output = capsys.readouterr().out + assert "case.drama3-rejected-directions" in output + assert "historical_observation" in output + # Outcome and reach read together, so a cohort-bound result cannot be taken + # for a general one at a glance. + assert "(inconclusive, this cohort only)" in output or "(no gain, this cohort only)" in output + assert "bounds (entry): Drama3" in output + assert "drama3-rejected-directions.md" in output + assert "more of 7 recorded directions" in output + + +def test_cli_json_mode_states_its_limits(capsys): + assert main(["find", "ascend device health", "--limit", "1", "--json"]) == 0 + payload = json.loads(capsys.readouterr().out) + assert payload["items"][0]["id"] == "ascend-device-health" + assert payload["corpus_size"] == len(load_corpus(ROOT).documents) + assert any("not a learned ranking" in line for line in payload["limitations"]) + + +def test_cli_rejects_unknown_kind(capsys): + with pytest.raises(SystemExit) as caught: + main(["find", "ascend", "--kinds", "skill,invented"]) + assert caught.value.code == 2 + assert "--kinds takes a subset" in capsys.readouterr().err From 80ec573eb5b635a938cc6e215f1fd2dbed8c0d7f Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Mon, 16 Nov 2026 16:46:27 +0800 Subject: [PATCH 4/8] test(knowledge): account for packaged query aliases Agent-assisted-by: Codex (author) Signed-off-by: Zhou ziheng --- tests/check_installed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/check_installed.py b/tests/check_installed.py index 8672c273f..108c4ce73 100644 --- a/tests/check_installed.py +++ b/tests/check_installed.py @@ -540,7 +540,7 @@ def _inventory(root, prefix): **_inventory(files("inferrsi").joinpath("knowledge", "data"), "knowledge/data"), **_inventory(files("inferrsi").joinpath("harness", "data"), "harness/data"), } -assert len(inventory) == 927, f"installed resource count is {len(inventory)}, expected 927" +assert len(inventory) == 928, f"installed resource count is {len(inventory)}, expected 928" assert sum(name.startswith("harness/data/") for name in inventory) == len(REPO_TYPES) # Every pinned record path and every registered skill body resolves inside that # same enumeration, so the inventory and the records describe one resource set. From 1431037fe138679a0b225e1501dbbc43f97a2150 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Mon, 16 Nov 2026 17:34:52 +0800 Subject: [PATCH 5/8] Re-derive packaged findings from their source cases Use the existing case bodies to express the recorded decisions, limits, and follow-up checks. Keep the tests focused on which outcome is retrieved and the evidence it carries. Agent-assisted-by: Codex / gpt-6.1-sol [effort=max] (author) Signed-off-by: Zhou ziheng --- .../cases/admission-drain-occupancy.json | 67 ++++---- .../cases/drama3-rejected-directions.json | 144 +++++++++--------- .../qwen-ascend-rejected-directions.json | 107 ++++++------- .../cases/qwen-b300-rejected-directions.json | 119 ++++++++------- .../cases/sglang-gdn-source-contract.json | 100 ++++++------ .../cases/vllm-gdn-source-contract.json | 105 +++++++------ .../cases/sarathi-chunk-budget.json | 116 +++++++------- tests/test_knowledge_find.py | 19 +-- 8 files changed, 405 insertions(+), 372 deletions(-) diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json index 367265e27..7f793f216 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json @@ -18,50 +18,51 @@ } ], "applicability": ["Continuous admission, mixed prefill/decode and final drain", "Recorded Drama3 B300 C64 diagnostic only"], + "limitations": ["Campaign INCONCLUSIVE", "No new measurement or trace reproduction", "Selected-phase occupancy is not service throughput", "Raw evidence remains under its existing owner"], + "attribution": "InferRSI issue #10 diagnostic evidence; extraction retains the original observation and its limits", "findings": [ { - "direction": "Use the whole-run mean batch as sustained-admission occupancy", + "direction": "Split admitted traffic from the final drain when reading decode occupancy", "outcome": "caveat", - "scope": "general", - "reason": "A whole-run mean can be dominated by drain. In the recorded C64 diagnostic, pure decode averaged 60.718 of 64 active requests during client admission, final drain averaged 8.793, and the whole-trace pure-decode average was 28.146", - "revisit_when": "Separate the client-admission and drain intervals before reading occupancy; using the whole-run mean can misidentify scheduling headroom", + "reason": "Across the 256-request B300 diagnostic, pure decode averaged 60.718 of 64 active requests during admission, 8.793 in drain and 28.146 over the full trace. The trace-wide mean cannot identify how full decode was while requests were still entering.", + "revisit_when": "Mark the client admission interval and the final drain in the trace before using occupancy to select a scheduling hypothesis.", "keywords": [ - "whole run mean batch", - "sustained admission", - "drain", - "scheduling headroom", - "occupancy" - ] + "active decode batch", + "continuous admission", + "final drain", + "whole-run occupancy", + "C64" + ], + "scope": "local", + "bounds": "Drama3 B300, 256 requests and a C64 continuous-admission diagnostic; approximate CPU/GPU clock alignment." }, { - "direction": "Read a near-full batch in a selected interval as a service speedup", + "direction": "Count scheduler steps using owning CPU annotations", "outcome": "caveat", - "scope": "general", - "reason": "A near-full batch in a selected interval does not establish a service speedup; client in-flight counts also include queueing and codec work, and CPU/GPU clock alignment was approximate", - "revisit_when": "Keep all requests and complete quality/repetition groups in the end-to-end metric rather than a selected-phase rate", + "reason": "The owning CPU annotations count 1 prefill, 37 mixed, 762 decode and 2 empty steps, totaling 802. PIECEWISE fragments describe graph execution, and scheduled prompt entries may describe chunks.", + "revisit_when": "Verify annotation ownership, align the clocks and pair the counts with actual generated work when profiling another trace.", "keywords": [ - "selected phase rate", - "service speedup", - "in-flight count", - "queueing", - "codec work" - ] + "owning CPU annotation", + "PIECEWISE", + "mixed batch", + "prefill chunks", + "scheduler steps" + ], + "scope": "regime_bound" }, { - "direction": "Count PIECEWISE graph fragments instead of scheduler annotations", + "direction": "Carry the complete cohort into the service gate", "outcome": "caveat", - "scope": "regime_bound", - "reason": "The original investigation counted scheduler execution annotations rather than PIECEWISE graph fragments, and retained the full request cohort for acceptance. Scheduled prompt entries may be chunks, not unique requests", - "revisit_when": "Confirm which quantity the annotations represent; kernel sums, annotation unions and SM utilization are different quantities", + "reason": "Near-full decode occupancy in a selected interval is a diagnostic. The service decision still depends on all requests, complete quality and repetition groups, and the end-to-end metric.", + "revisit_when": "Use phase statistics to pick a hypothesis. Evaluate the resulting change with the unchanged complete cohort and matched frozen service controls.", "keywords": [ - "scheduler steps", - "PIECEWISE graph", - "annotation union", - "SM utilization", - "chunk vs request" - ] + "acceptance gate", + "service throughput", + "request cohort", + "quality groups", + "model-only microbenchmark" + ], + "scope": "general" } - ], - "limitations": ["Campaign INCONCLUSIVE", "No new measurement or trace reproduction", "Selected-phase occupancy is not service throughput", "Raw evidence remains under its existing owner"], - "attribution": "InferRSI issue #10 diagnostic evidence; extraction retains the original observation and its limits" + ] } diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json index 850c2e828..b1c1fa0c8 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json @@ -13,104 +13,108 @@ "issue": "https://github.com/InferMatrixLab/InferRSI/issues/10" }], "applicability": ["Drama3 standalone native-computation runtime on one B300", "Only each cited microbenchmark, fixed-state diagnostic or complete service trial"], + "tags": ["negative-evidence", "service-regression", "quality-rejection", "harness-mismatch", "prefix-reuse", "memory-compression", "analysis-only"], + "limitations": ["Campaign remains INCONCLUSIVE and optimization is paused", "Individual completed gate failures do not complete the all-concurrency acceptance matrix", "Repository summary inspected; external raw traces and runtime artifacts not rerun or rehashed", "No root cause inferred from client accounting or cross-screen timing subtraction"], + "attribution": "Issue #10 diagnostic lessons, including runtime 47c3d2ec4e3406d8a71fccc3cb4967783c2650f5; revisit checks are current suggestions", "findings": [ { - "direction": "Fused tower projections / multi-step decode", + "direction": "Qualify fused tower projections and longer decode groups at service scope", "outcome": "no_gain", - "scope": "regime_bound", - "reason": "Faster or correct in isolated checks but no full-service throughput gain; multi-step decode delayed admission", - "revisit_when": "Measure actual active batches, admission/drain and complete service latency, including sampling and codec. Kernel timing alone cannot select the runtime", + "reason": "The recorded isolated checks could be correct or faster while full-service throughput did not improve. Multi-step decode also deferred new admissions.", + "revisit_when": "Check active batch sizes through admission and drain, then time the complete path with sampling and codec under the same service workload.", "keywords": [ - "fused tower projection", + "tower projection fusion", "multi-step decode", - "admission delay", - "service throughput" - ] + "throughput", + "delayed admission", + "codec" + ], + "scope": "regime_bound" }, { - "direction": "Async service / overlap", - "outcome": "inconclusive", - "scope": "regime_bound", - "reason": "Early fixed-request controls passed while service WER failed in both arms; later bounded dynamic replay identified shared ATen cuBLAS workspace interference. Dedicated codec capture matched tested states and fresh private-capture service WER passed while shared capture failed", - "revisit_when": "Preserve the required private/dedicated capture setup and unchanged controls. This specific reproducer does not establish one cause for every historical WER failure, or qualify arbitrary overlap/throughput; all-concurrency acceptance remains unmet", + "direction": "Check ATen cuBLAS workspace sharing when codec capture changes WER", + "outcome": "caveat", + "reason": "The fixed-request controls initially passed despite service WER failures in both arms. In a later bounded replay, private codec capture passed the tested service control while shared capture failed; the runtime guarded dedicated capture.", + "revisit_when": "Preserve the private/dedicated capture setup and matched controls. Establish WER and throughput for the proposed overlap and concurrency separately.", "keywords": [ - "async overlap", - "WER failure", - "cuBLAS workspace", - "shared capture", - "dedicated capture", - "bilateral" - ] + "async service", + "codec graph capture", + "WER", + "shared ATen cuBLAS workspace", + "private capture" + ], + "scope": "regime_bound" }, { - "direction": "Production prefix reuse", + "direction": "Require the complete C64 continuation gate for production prefix reuse", "outcome": "rejected", - "scope": "regime_bound", - "reason": "Extensive graph/state checks and both full quality groups passed, yet the complete C64 service continuation gate failed; default-off", - "revisit_when": "Retain final drain and all returned work. Investigate the actual schedule and GPU critical path before retrying; removed MoE rows and intake-only rates cannot replace the failed service score", + "reason": "The graph/state checks and both full quality groups passed, while the complete C64 continuation gate failed. The recorded production default stayed off.", + "revisit_when": "Keep all returned work and final drain in the replay. Inspect the actual schedule and GPU critical path before proposing another production setting.", "keywords": [ "prefix reuse", - "continuation gate", - "C64 service", - "default-off" - ] + "C64 continuation gate", + "default-off", + "final drain", + "full quality groups" + ], + "scope": "regime_bound" }, { - "direction": "Initial prefix-reuse harness comparison", - "outcome": "inconclusive", - "scope": "local", - "reason": "Compared different private padding histories; mismatch was confined to a private slot before later states were checked. Corrected identical-input controls reran it", - "revisit_when": "Reset complete physical input rows and run A/A before blaming production math. Do not reuse this superseded setup failure as evidence of model incompatibility", + "direction": "Reset private padding rows before an A/A state comparison", + "outcome": "caveat", + "reason": "The initial prefix-reuse comparison paired distinct physical padding histories. The reported discrepancy was in a private slot, and an identical-input control superseded that setup.", + "revisit_when": "Recreate all physical input rows with identical histories and finish A/A before interpreting any later state mismatch.", "keywords": [ - "harness mismatch", - "padding history", - "A/A control", - "identical input" - ] + "prefix reuse control", + "private padding history", + "physical rows", + "identical-input", + "A/A" + ], + "scope": "local", + "bounds": "Initial Drama3 prefix-reuse harness comparison, later replaced by matched-input controls." }, { - "direction": "Transparent VMM weight compression", + "direction": "Require physical traffic and a timing win before enabling transparent VMM compression", "outcome": "no_gain", - "scope": "regime_bound", - "reason": "Native correctness/lifetime checks passed, but the fixed three-body screen missed its frozen timing gate; no runtime promotion", - "revisit_when": "First measure the current bottleneck and physical traffic/compression where possible. Mapped flags and logical capacity do not prove a bandwidth benefit", + "reason": "VMM compression passed native correctness and lifetime checks, yet its fixed three-body screen missed the frozen timing gate. The runtime did not promote it.", + "revisit_when": "Confirm the workload is bandwidth-sensitive and measure physical bytes or effective compression together with the complete timed body.", "keywords": [ - "VMM", + "transparent VMM", "weight compression", - "memory compression", - "bandwidth", - "timing gate" - ] + "physical traffic", + "native lifetime", + "three-body timing" + ], + "scope": "regime_bound" }, { - "direction": "Producer scale-tile cache", + "direction": "Measure complete kernels before retaining a producer scale-tile cache", "outcome": "no_gain", - "scope": "regime_bound", - "reason": "Complete-body reductions missed the source gate; stopped without model/service promotion", - "revisit_when": "New shapes or a different resource regime may justify another bounded screen; do not subtract unrelated historical screens to infer a cause", + "reason": "The producer scale-tile cache did not meet its complete-body reduction gate. The screen stopped before model or service promotion.", + "revisit_when": "Run a bounded screen for a materially different shape or resource constraint. Include a matched complete kernel as the control.", "keywords": [ - "scale-tile cache", - "producer cache", - "bounded screen" - ] + "producer scale tile cache", + "complete-body", + "source gate", + "kernel timing", + "resource regime" + ], + "scope": "regime_bound" }, { - "direction": "Fixed eight-lane FP8 SIMT proposal", + "direction": "Revisit FP8 SIMT only with an improved mapping or cost case", "outcome": "rejected", - "scope": "regime_bound", - "reason": "Stopped at CPU/AOT cost analysis for that mapping, before a runtime benchmark. An analytical rejection, not an empirical GPU failure", - "revisit_when": "Revisit only with a changed mapping or supported cost model; this is not a universal ceiling", - "bounds": "That fixed eight-lane mapping and its cost model only", + "reason": "The fixed eight-lane mapping was stopped by CPU/AOT analysis before runtime evaluation. No accelerator timing supports a hardware-wide conclusion.", + "revisit_when": "Supply a different lane mapping or a supported cost model before deciding whether a native run is warranted.", "keywords": [ - "FP8", - "SIMT", - "eight-lane", - "AOT cost analysis", - "analysis-only" - ] + "FP8 SIMT", + "eight-lane mapping", + "CPU/AOT", + "cost analysis", + "runtime benchmark" + ], + "scope": "regime_bound" } - ], - "tags": ["negative-evidence", "service-regression", "quality-rejection", "harness-mismatch", "prefix-reuse", "memory-compression", "analysis-only"], - "limitations": ["Campaign remains INCONCLUSIVE and optimization is paused", "Individual completed gate failures do not complete the all-concurrency acceptance matrix", "Repository summary inspected; external raw traces and runtime artifacts not rerun or rehashed", "No root cause inferred from client accounting or cross-screen timing subtraction"], - "attribution": "Issue #10 diagnostic lessons, including runtime 47c3d2ec4e3406d8a71fccc3cb4967783c2650f5; revisit checks are current suggestions" + ] } diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json index 587352abb..99352d96d 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json @@ -13,90 +13,93 @@ "upstream_head": "399cc01d1bc4542964d91aaa17fb43862a8cf3b3" }], "applicability": ["Author-reported single Ascend 910B1 Qwen-Image-2.1-Turbo cohort", "BF16, eager, TORCH_SDPA, CFG=1, 1280x704, native eight sigmas"], + "tags": ["negative-evidence", "resource-failure", "OOM", "no-demonstrated-gain", "RoPE", "fusion-attention", "task-queue"], + "limitations": ["Imported author report; locally_verified=false", "Raw request timings, RGBA outputs and frozen validator are absent from this import", "InferRSI acceptance remains INCONCLUSIVE; the source 30 percent target was not met", "No cross-device or HTTP-serving qualification"], + "attribution": "amy-why-3459 source report, retained by the existing Ascend guide; revisit checks are current suggestions", "findings": [ { - "direction": "Cache eager RoPE cos/sin or expanded SDPA masks", + "direction": "Check eager RoPE table and expanded SDPA mask retention against recomputation", "outcome": "no_gain", - "scope": "regime_bound", - "reason": "Pixel-identical, but discarded as slower in the source trials", - "revisit_when": "Profile current recomputation and retained-tensor costs; qualify a matched timing cohort before attributing a regression", + "reason": "The source trials produced identical pixels with retained cos/sin or expanded masks, but those alternatives were slower and were discarded.", + "revisit_when": "On the target shapes, measure the work avoided and the lifetime and cost of the retained tensors in a paired end-to-end run.", "keywords": [ - "RoPE", - "cos sin cache", + "eager RoPE", + "cos sin precompute", "SDPA mask", - "expanded mask", - "recomputation" - ] + "pixel identical", + "retention cost" + ], + "scope": "regime_bound" }, { - "direction": "Cache per-step masks and prompt tensors", + "direction": "Budget live NPU memory before retaining per-step masks and prompt tensors", "outcome": "rejected", - "scope": "regime_bound", - "reason": "NPU OOM", - "revisit_when": "First inventory live tensor lifetimes and allocator headroom; a different residency policy or workload may change feasibility", + "reason": "That cache configuration exhausted device memory on the reported run. Its feasibility depends on what stays resident alongside the cached tensors.", + "revisit_when": "Inspect tensor lifetimes, allocator headroom and peak residency. Retry a paired test when the request shape or residency policy materially changes.", "keywords": [ - "per-step mask", - "prompt tensor", - "mask cache", - "OOM", + "cache per-step masks", + "prompt tensors", + "NPU OOM", + "device memory", "allocator headroom" - ] + ], + "scope": "local" }, { - "direction": "Fusion attention, including precise=2/0 and mask-free steps", + "direction": "Qualify fusion attention with the actual mask, dtype and shape", "outcome": "no_gain", - "scope": "regime_bound", - "reason": "No gain or a gain below the source keep criterion", - "revisit_when": "Check exact mask, dtype, shapes and device support; run operator quality then complete pipeline timing", + "reason": "The recorded fusion-attention runs, including precise=2 and precise=0 and steps without masks, showed either no gain or a gain below the source experiment’s keep criterion.", + "revisit_when": "Verify dispatch and numerical behavior for the present mask and tensor layout. Follow operator checks with a matched pipeline timing.", "keywords": [ "fusion attention", - "precise", + "precise=2", + "precise=0", "mask-free", - "operator quality" - ] + "Ascend" + ], + "scope": "regime_bound" }, { - "direction": "TASK_QUEUE_ENABLE=1/2 or enforce_eager=0", + "direction": "Recheck queueing and graph mode under the target stack", "outcome": "no_gain", - "scope": "regime_bound", - "reason": "No gain or slower than the source eager configuration", - "revisit_when": "Inspect the actual execution path and scheduling overhead under the new stack; repeat with unchanged workload and warmups", + "reason": "TASK_QUEUE_ENABLE=1/2 and enforce_eager=0 failed to improve the recorded eager configuration; some variants were slower.", + "revisit_when": "Confirm the execution route and scheduling overhead after a runtime change, with a fixed workload and equal warmups across arms.", "keywords": [ "TASK_QUEUE_ENABLE", "enforce_eager", - "scheduling overhead", - "task queue" - ] + "host dispatch", + "task queue", + "eager execution" + ], + "scope": "regime_bound" }, { - "direction": "RoPE + VAE fastpath + fusion attention", + "direction": "Check initialization residency before combining RoPE, VAE and attention fast paths", "outcome": "rejected", - "scope": "regime_bound", - "reason": "Initialization OOM for this triple stack", - "revisit_when": "Inventory initialization residency before attempting a new combination", + "reason": "The run with RoPE, the VAE fastpath and fusion attention ran out of memory during initialization. This identifies a failure phase for that particular combination.", + "revisit_when": "Measure what each component reserves during initialization before assembling another multi-path configuration.", "keywords": [ - "RoPE", - "VAE fastpath", - "fusion attention", + "RoPE VAE fusion attention", "triple stack", - "initialization OOM" - ] + "initialization OOM", + "startup memory" + ], + "scope": "local", + "bounds": "Triple-stack initialization, separate from later two-path requests on a busy host." }, { - "direction": "Separate kept RoPE + VAE stack on a later busy host", + "direction": "Qualify host headroom for the later RoPE and VAE reservation failure", "outcome": "inconclusive", - "scope": "local", - "reason": "Later requests failed resource reservation; a different stack/host cohort from the triple-stack initialization failure", - "revisit_when": "Qualify current headroom and outside contention before retrying; do not attribute this resource blockage to fusion attention", + "reason": "A later run of the kept two-path stack on a busy host encountered resource-reservation failures. Fusion attention was not part of that stack.", + "revisit_when": "Check available device resources and contention on the target host before judging repeatability or timing.", "keywords": [ + "kept RoPE VAE fastpath", "busy host", "resource reservation", - "contention", - "VAE fastpath" - ] + "outside contention" + ], + "scope": "local", + "bounds": "Later resource-reservation failure for the separate RoPE plus VAE stack." } - ], - "tags": ["negative-evidence", "resource-failure", "OOM", "no-demonstrated-gain", "RoPE", "fusion-attention", "task-queue"], - "limitations": ["Imported author report; locally_verified=false", "Raw request timings, RGBA outputs and frozen validator are absent from this import", "InferRSI acceptance remains INCONCLUSIVE; the source 30 percent target was not met", "No cross-device or HTTP-serving qualification"], - "attribution": "amy-why-3459 source report, retained by the existing Ascend guide; revisit checks are current suggestions" + ] } diff --git a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json index 9f4825006..173b0afca 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json @@ -13,89 +13,96 @@ "issue": "https://github.com/InferMatrixLab/InferRSI/issues/3" }], "applicability": ["Qwen-Image-2.1-Turbo, four B300, BF16, native eight steps", "Only the source ledger's individual trials and controls"], + "tags": ["negative-evidence", "quality-rejection", "no-demonstrated-gain", "confounded-run", "RoPE", "CUDA-Graph"], + "limitations": ["The final campaign PASS does not qualify each rejected direction", "Exploratory trial timings are not additive gains or causal comparisons", "No experiment was rerun for this extraction"], + "attribution": "Historical issue #3 optimization ledger; revisit checks are current diagnostic suggestions", "findings": [ { - "direction": "H3: initial Q/K RMSNorm + complex RoPE fusion", + "direction": "Require full-image parity for fused Q/K RMSNorm and complex RoPE", "outcome": "rejected", - "scope": "regime_bound", - "reason": "Operator tests passed; complete images failed; reverted. A quality rejection, not a kernel-speed result", - "revisit_when": "First match the actual reduction, complex FMA order and BF16 materialization; compare full native images after operator checks. H10/H13 show later, different implementations were retained, so the technique is not permanently excluded", + "reason": "The H3 operator checks passed, yet its complete images failed and the change was reverted. The later kept H10/H13 implementations were different.", + "revisit_when": "Reproduce the reduction, complex FMA ordering and BF16 materialization, then check native full images after the operator comparisons.", "keywords": [ - "RMSNorm", - "complex RoPE fusion", - "quality rejection", + "H3", + "Q/K RMSNorm", + "complex RoPE", "BF16 materialization", - "FMA order" - ] + "image quality" + ], + "scope": "regime_bound" }, { - "direction": "H1: packed Q/K/V 5D exchange", - "outcome": "inconclusive", - "scope": "local", - "reason": "Quality passed but the exploratory trial was discarded", - "revisit_when": "Changed layouts/topology may justify another trial; measure staging/copies and the complete request against a matched control. Do not infer the cause from the one ledger timing", + "direction": "Pair a packed Q/K/V 5D layout with full request timing", + "outcome": "rejected", + "reason": "H1 passed quality but the exploratory exchange was discarded. The ledger’s standalone trial does not resolve the cost of layout staging or copies.", + "revisit_when": "For a changed topology or layout, keep the comparison matched and account for staging, transfers and request time separately.", "keywords": [ - "packed QKV", - "5D exchange", - "layout", - "staging copies" - ] + "H1", + "QKV", + "packed 5D exchange", + "layout staging", + "copies" + ], + "scope": "regime_bound" }, { - "direction": "H6/H7: graph projection/norm/RoPE or post-attention work", + "direction": "Measure complete graph replay before promoting projection or norm/RoPE changes", "outcome": "no_gain", - "scope": "regime_bound", - "reason": "Removed for no material E2E gain; H7 records replay/copy costs", - "revisit_when": "A new capture boundary or buffer lifetime may change the tradeoff; include capture/replay, copies and full request cost", + "reason": "H6 and H7 were removed after failing to produce a material end-to-end gain. H7 includes capture/replay and copy costs that matter to the final decision.", + "revisit_when": "Test a materially different capture boundary or buffer lifetime against a paired full request. Account for capture, replay and copies.", "keywords": [ - "CUDA Graph", - "graph projection", - "post-attention", - "replay cost", - "capture boundary" - ] + "H6", + "H7", + "graph replay", + "norm RoPE", + "capture boundary", + "post-attention" + ], + "scope": "regime_bound" }, { - "direction": "H14/H16: broadcast decode scale/gate or skip empty concatenation", + "direction": "Verify critical-path time for broadcast scales and empty concatenation removal", "outcome": "no_gain", - "scope": "regime_bound", - "reason": "No demonstrated benefit; removed/reverted", - "revisit_when": "Confirm these operations are on the current critical path before repeating; retain a matched E2E control", + "reason": "The H14/H16 decode scale/gate broadcast and empty concatenation changes were removed or reverted because the trials did not demonstrate a benefit.", + "revisit_when": "Profile the intended configuration to see whether these operations are on its critical path, and retain the matched service control.", "keywords": [ - "broadcast decode", + "H14", + "H16", + "decode broadcast", "scale gate", - "empty concatenation", + "empty concat", "critical path" - ] + ], + "scope": "regime_bound" }, { - "direction": "H17 and the noisy H13 attempt", + "direction": "Treat H17 and the noisy H13 run as host-confounded readings", "outcome": "inconclusive", - "scope": "local", - "reason": "Observed host load/concurrent diagnostics invalidate causal attribution", - "revisit_when": "Repeat on a qualified, uncontended host. Do not classify the numerical timing difference as a confirmed code regression", + "reason": "Concurrent diagnostics and host load were observed during these attempts, so the timing difference does not isolate the code change.", + "revisit_when": "Repeat the same request on a qualified host without overlapping diagnostics.", "keywords": [ + "H17", + "H13", "host load", - "contention", - "confounded run", - "causal attribution" - ] + "concurrent diagnostics", + "confounded timing" + ], + "scope": "local", + "bounds": "The cited exploratory H17 run and noisy H13 attempt." }, { - "direction": "H18: compiled launch API probe", + "direction": "Carry a compiled launch API probe through full-pipeline integration before judging gain", "outcome": "inconclusive", - "scope": "local", - "reason": "Host enqueue improved in a diagnostic; no E2E integration was performed", - "revisit_when": "Treat it as an unintegrated lead; verify actual execution, correctness and complete service overhead before accepting it", + "reason": "H18 improved host enqueue in a diagnostic. It never went through the complete pipeline, so the resulting service overhead was not measured.", + "revisit_when": "Confirm actual dispatch and full-image correctness in the integrated candidate, then measure request time.", "keywords": [ - "compiled launch", - "launch API", + "H18", + "compiled launch API", "host enqueue", - "unintegrated lead" - ] + "pipeline integration", + "dispatch" + ], + "scope": "regime_bound" } - ], - "tags": ["negative-evidence", "quality-rejection", "no-demonstrated-gain", "confounded-run", "RoPE", "CUDA-Graph"], - "limitations": ["The final campaign PASS does not qualify each rejected direction", "Exploratory trial timings are not additive gains or causal comparisons", "No experiment was rerun for this extraction"], - "attribution": "Historical issue #3 optimization ledger; revisit checks are current diagnostic suggestions" + ] } diff --git a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json index 552e012f6..80e9de5f2 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json @@ -49,72 +49,76 @@ "Pinned Qwen3.5/Qwen3-Next GDN caller/backend qualification", "Separate causal-conv phase/layout/state contracts and conditional one-shot classification-prefill transfer" ], + "limitations": [ + "Native kernels, models and devices not_run", + "Agent efficacy not_evaluated", + "No HF binding, sm89/backend qualification, numerical/model-quality or performance claim", + "Whole reset/cancellation/radix eviction, speculative and ReplaySSM lifecycles outside this readset" + ], + "attribution": "SGLang and Qwen contributors; Apache-2.0 root license. Conv files retain vLLM adaptation and Tri Dao notices. Caller contracts only; no external kernel implementation imported.", + "required_resources": [ + "sglang-gdn-source-contract.json" + ], "findings": [ { - "direction": "Use one backend label for both convolution and recurrence", + "direction": "Resolve prefill recurrence and decode convolution separately", "outcome": "failure_signal", - "scope": "regime_bound", - "reason": "A prefill flag does not replace both operators; equating an explicit flag, package availability or a registered symbol with executed dispatch misreads the source, and sm89 compatibility cannot be claimed from an SM9/10 automatic default", - "revisit_when": "Read the backend aliases/policy and the ordinary decode path separately before naming a dispatch", + "reason": "On the pinned path, the CUDA GDN prefill convolution alias can be rebound while ordinary decode keeps the Triton conv-update alias. A FlashInfer prefill setting or a CUDA update symbol does not show that decode was dispatched there.", + "revisit_when": "Trace runner-level overrides and effective aliases through the owning backend. Check the SM9/FP32 and SM10/BF16/CUDA13 prefill predicates before asking for an execution trace.", "keywords": [ - "backend label", - "prefill flag", - "executed dispatch", - "sm89", - "automatic default" - ] + "SGLang", + "GDN", + "FlashInfer prefill", + "Triton conv update", + "decode convolution", + "effective dispatch" + ], + "scope": "regime_bound" }, { - "direction": "Infer strides from shape for the Triton prefill path", + "direction": "Use the actual last-axis stride and sequence lengths to classify convolution prefill", "outcome": "failure_signal", - "scope": "regime_bound", - "reason": "CUDA input can select stride-aware Triton prefill; assuming unconditional input aliasing or no mutation, letting packed requests read each other's history, or assuming the conv mask clears the recurrent matrix are all source misreadings", - "revisit_when": "Check the prefill caller, SRT wrapper and Triton wrapper for the actual stride and mutation contract", + "reason": "Even a CUDA input can take Triton when last_stride != 1 and seq_lens_cpu is supplied. That path returns a distinct output while updating state; the CUDA branch returns the modified input buffer it uses.", + "revisit_when": "Inspect strides, packed token offsets and row IDs. Verify the prefix-history mask for convolution independently of recurrent state initialization.", "keywords": [ - "stride-aware", - "Triton prefill", - "input aliasing", - "packed request history", - "conv mask" - ] + "causal_conv1d", + "CUDA prefill", + "Triton stride", + "seq_lens_cpu", + "aliasing", + "initial-history mask" + ], + "scope": "regime_bound" }, { - "direction": "Reuse batch index as persistent identity across reordered rows", + "direction": "Preserve both state write-backs through a reorder and noncontiguous gather", "outcome": "failure_signal", - "scope": "regime_bound", - "reason": "Equating temporal state with attention KV, losing gather/scatter write-back, or treating a successful output computation as proof that continuation, reset or recycling is correct all break the state contract", - "revisit_when": "Trace mapping/ownership and the state gather and write-back paths; both state writes must be kept", + "reason": "Ordinary non-CPU extend maps requests to persistent Mamba rows. With noncontiguous pools it gathers indexed copies and scatters both convolution history and the GDN matrix back. A correct immediate output does not exercise their later use.", + "revisit_when": "Check fresh and continued requests after a reorder and compare both state tensors at their mapped rows. Account for separate dtypes and head sharding.", "keywords": [ - "batch index", - "persistent identity", - "strided pool", + "Mamba rows", + "strided state pool", "gather scatter", - "write-back" - ] + "conv history", + "temporal state", + "request reorder" + ], + "scope": "regime_bound" }, { - "direction": "Treat fresh classification prefill as decode or multi-item scoring", + "direction": "Prove HF classifier parity before transferring an SGLang prefill path", "outcome": "failure_signal", - "scope": "regime_bound", - "reason": "Presenting source reasoning or packaged reads as a binding, native test, quality pass or service gain confuses the evidence level; a generic classifier is not MIS, and unused returned cache does not make internal state writes dispensable", - "revisit_when": "Read the MIS constraints before classifying a prefill path; keep source reasoning distinct from a native result", + "reason": "A fresh whole-sequence classifier can use prefill reasoning only after identifying its worker and state contract. MIS additionally assumes delimiter-driven query/item branches, zero cached prefixes, Triton and a compatible layout.", + "revisit_when": "Pin the HF caller and check layout, gates, zero state, mutation behavior, output/state parity and classifier quality before timed comparison.", "keywords": [ - "classification prefill", - "multi-item scoring", + "HF classifier", "MIS", - "returned cache", - "internal state write" - ] + "multi-item scoring", + "one-shot prefill", + "zero state", + "caller binding" + ], + "scope": "regime_bound" } - ], - "limitations": [ - "Native kernels, models and devices not_run", - "Agent efficacy not_evaluated", - "No HF binding, sm89/backend qualification, numerical/model-quality or performance claim", - "Whole reset/cancellation/radix eviction, speculative and ReplaySSM lifecycles outside this readset" - ], - "attribution": "SGLang and Qwen contributors; Apache-2.0 root license. Conv files retain vLLM adaptation and Tri Dao notices. Caller contracts only; no external kernel implementation imported.", - "required_resources": [ - "sglang-gdn-source-contract.json" ] } diff --git a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json index 8c9e9513a..edcb5adf1 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json @@ -32,72 +32,77 @@ "Source-only qualification of the pinned Qwen3-Next CUDA-facing GDN caller", "Mixed prefill/decode, packed/speculative metadata and state-row contracts" ], + "limitations": [ + "Native kernel/model/device execution not_run", + "Representative agent evaluation not_evaluated", + "No concrete HF binding, external implementation equivalence, hardware compatibility or performance claim", + "Physical state allocation, release and recycling outside the readset" + ], + "attribution": "vLLM contributors; Apache-2.0 root and in-file notices retained in the source reference. The in-tree conv file additionally credits Tri Dao; vendored FLA carries its separate MIT notice.", + "required_resources": [ + "vllm-gdn-source-contract.json", + "vllm-gdn-causal-conv-contract.md" + ], "findings": [ { - "direction": "Treat every one-token row as decode in a mixed Qwen3-Next batch", + "direction": "Classify one-token GDN rows with prefill and prior-state metadata", "outcome": "failure_signal", - "scope": "regime_bound", - "reason": "A first one-token chunk can require the prefill route to mask recycled state; selecting a decode-only convolution for the whole mixed batch, or passing unre-based offsets to the prefill recurrence, misreads the source", - "revisit_when": "Use prior-state/prefill metadata as well as query length; identify the varlen forward convolution when non-speculative prefill rows exist", + "reason": "The first one-token chunk may require the prefill route to mask recycled state. Mixed non-speculative input with any prefill row uses varlen forward convolution; the recurrent split requires decode-first handling, rebased prefill lengths and an output stitch.", + "revisit_when": "Inspect first and resumed one-token rows and a mixed batch with longer prefill. Treat speculative-row classification as a separate test.", "keywords": [ - "one-token initialization", - "mixed phase", - "decode-only conv", - "rebased offsets", - "varlen forward" - ] + "vLLM", + "Qwen3-Next", + "one-token prefill", + "varlen", + "mixed batch", + "rebased seqlens" + ], + "scope": "regime_bound" }, { - "direction": "Infer the actual GDN branch from the requested backend", + "direction": "Verify actual GDN and convolution branches after requesting FlashInfer", "outcome": "failure_signal", - "scope": "regime_bound", - "reason": "Selection inferred from package presence, default flags or symbol names does not establish the branch; a separate-conv rule does not generalize to every model/backend, and a source predicate is not hardware compatibility or speedup", - "revisit_when": "Trace the actual branch in the pinned source rather than the requested backend label", + "reason": "Requested FlashInfer prefill does not resolve decode convolution. Inspect the packed decode guard and effective recurrence selection. Qwen3-Next’s interleaved GQA layout excludes the inspected fused CUDA MTP option, while XPU SYCL has its own whole-layer route.", + "revisit_when": "Record the effective operator on the intended backend and distinguish an automatic fallback from an explicitly rejected CUDA configuration.", "keywords": [ - "requested backend", - "actual branch", - "FlashInfer", - "package presence", - "default flags" - ] + "FlashInfer prefill", + "decode conv", + "packed decode", + "fused CUDA MTP", + "gqa_interleaved_layout", + "XPU SYCL" + ], + "scope": "regime_bound" }, { - "direction": "Equate batch index with cache row under reordering and speculation", + "direction": "Use mapped state rows and the accepted-token offset in speculative updates", "outcome": "failure_signal", - "scope": "regime_bound", - "reason": "Copying standalone-package cursor semantics into this API, preserving stale first-chunk state, or treating convolution and recurrent state as attention KV all break the state-row contract", - "revisit_when": "Track reordered state rows and speculative progress explicitly; account for both state writes", + "reason": "Batch position and cumulative packed offsets do not own persistent history. The inspected speculative read uses num_accepted_tokens minus one. Convolution history writes and the recurrent matrix must retain their own initialization, padding and row semantics.", + "revisit_when": "For padded and reordered requests, compare both writes using real cache-row mapping, accepted-token metadata and per-rank layout.", "keywords": [ - "reordered state rows", - "batch index", - "cache row", - "speculative progress", - "cursor semantics" - ] + "causal_conv1d_update", + "cache_seqlens", + "num_accepted_tokens", + "speculative", + "row mapping", + "conv state" + ], + "scope": "regime_bound" }, { - "direction": "Assert a model-to-HF binding from source reading alone", + "direction": "Resolve the concrete HF caller before attempting a vLLM conv binding", "outcome": "failure_signal", - "scope": "regime_bound", - "reason": "Inventing a model-to-HF implementation mapping, prescribing an unverified installation, or asserting numeric correctness, hardware support or speedup all exceed what source classification establishes", - "revisit_when": "Keep the binding unproven until an identified caller is verified; native execution here is not_run", + "reason": "A common symbol name does not establish equivalence between the vLLM in-tree call chain and standalone Dao, FlashInfer or vendored FLA paths. The unidentified HF implementation leaves projection, state ownership and mutation contracts unresolved.", + "revisit_when": "Pin the caller and compare initial, resumed, mixed, reordered, padded and speculative cases that the eventual binding supports, including state and output parity.", "keywords": [ - "HF caller", - "binding", - "unverified installation", - "numeric correctness" - ] + "HF binding", + "causal conv", + "GDN", + "in-tree import", + "Dao", + "state ownership" + ], + "scope": "regime_bound" } - ], - "limitations": [ - "Native kernel/model/device execution not_run", - "Representative agent evaluation not_evaluated", - "No concrete HF binding, external implementation equivalence, hardware compatibility or performance claim", - "Physical state allocation, release and recycling outside the readset" - ], - "attribution": "vLLM contributors; Apache-2.0 root and in-file notices retained in the source reference. The in-tree conv file additionally credits Tri Dao; vendored FLA carries its separate MIT notice.", - "required_resources": [ - "vllm-gdn-source-contract.json", - "vllm-gdn-causal-conv-contract.md" ] } diff --git a/src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json b/src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json index 7757866c3..f6679f5cd 100644 --- a/src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json +++ b/src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json @@ -28,80 +28,88 @@ "Source-supported hypothesis selection when observed long prefills interfere with ongoing decodes", "Chunked-prefill token-budget tradeoffs under specified latency, workload, hardware and parallelism conditions" ], + "limitations": [ + "Paper-author observations and proposed adaptation checks are not local measurements or current-engine support.", + "The combined design does not beat hybrid-only P50 TTFT in either Table 4 trace.", + "No universal budget, fairness guarantee, quantitative disaggregation comparison or empirical skill efficacy is established.", + "Raw source redistribution and adaptation permission remain UNKNOWN.", + "The same case identity and central mechanism appeared in closed-unmerged PR 156; this independently written fixed-edition contribution does not recover the missing historical archive or establish novelty over its unavailable synthesis." + ], + "attribution": "Original attributed explanation of Agrawal et al., OSDI 2024. The paper's reported results remain distinct from InferRSI's proposed checks; no raw source asset or prior proposal prose is reproduced.", "findings": [ { - "direction": "Split a long prefill into chunks and combine them with ongoing decodes", - "outcome": "tradeoff", - "scope": "general", - "reason": "Source mechanism for long prefills interrupting token generation; chunking limits prefill work per iteration but does not make that work free. Figure 9 still shows added batch latency relative to decode-only execution", - "revisit_when": "Diagnose queueing, memory shortage or suspended requests first; only attribute the delay to prefill interference after that", + "direction": "Subtract active decode tokens before filling a mixed batch with prefill", + "outcome": "caveat", + "reason": "Sarathi-Serve budgets one token per ongoing decode first, then accounts for partial prefills and admits new prompt chunks with the remaining budget and allocation capacity. The extra work can still extend a decode iteration.", + "revisit_when": "Confirm trace evidence of prefill interference and verify that the target engine’s mixed batch implements the same ordering before adapting the policy.", "keywords": [ + "Sarathi-Serve", + "mixed token budget", + "ongoing decode", "chunked prefill", - "long prefill interrupts decode", "TBT", - "time between tokens", - "prefill interference" - ] + "admission" + ], + "scope": "regime_bound" }, { - "direction": "Tighten the token budget with smaller chunks", + "direction": "Compare the token-gap gain from smaller chunks with repeated launch and KV costs", "outcome": "tradeoff", - "scope": "general", - "reason": "Smaller chunks reduce prefill work per iteration but can lower compute utilization and repeat fixed launch costs", - "revisit_when": "Profile the actual utilization and launch overhead for the target hardware before choosing a budget", - "keywords": [ - "token budget", - "chunk size", - "compute utilization", - "launch cost" - ] - }, - { - "direction": "Accept the KV re-read cost of later chunks", - "outcome": "caveat", - "scope": "general", - "reason": "Each later chunk attends to earlier chunks of the same prompt, rereading their KV cache. The source distinguishes these additional memory reads from unchanged attention computation", - "revisit_when": "Account for the extra memory reads separately from attention compute when estimating cost", + "reason": "Smaller chunks bound prefill work per iteration but can reduce utilization and repeat fixed overhead. Later chunks attend back over earlier prompt chunks, causing extra KV-cache reads even though total attention computation is unchanged.", + "revisit_when": "Profile candidate sizes on the actual device and report prefill time, sustained throughput and token gaps together.", "keywords": [ + "small chunks", "KV cache reread", - "memory reads", - "attention computation", - "chunk attention" - ] + "compute utilization", + "launch overhead", + "token gaps" + ], + "scope": "regime_bound" }, { - "direction": "Pick a budget without accounting for tile boundaries or pipeline parallelism", - "outcome": "caveat", - "scope": "general", - "reason": "Matrix tile boundaries affect execution cost and neighboring token counts need not have neighboring runtimes; with pipeline parallelism, batch runtime differences can leave stages idle", - "revisit_when": "Measure runtime across neighboring token counts and check stage idling before fixing a budget", + "direction": "Sample tile boundaries and stage imbalance when tuning chunk length", + "outcome": "tradeoff", + "reason": "The paper identifies tile boundaries and uneven pipeline batch runtimes as sources of nonlinear cost. Larger chunks can leave stages idle, while very small chunks also lose efficiency.", + "revisit_when": "Under the intended model and parallelism, profile nearby lengths on both sides of tile boundaries and include pipeline idle time when applicable.", "keywords": [ - "matrix tile boundary", + "tile granularity", "pipeline parallelism", "stage idle", - "batch runtime variation" - ] + "chunk size", + "Vidur" + ], + "scope": "regime_bound" }, { - "direction": "Read Table 4 as an all-round improvement", - "outcome": "caveat", - "scope": "regime_bound", - "reason": "The combined design has lower P99 TBT than either component alone, but its P50 TTFT is higher than hybrid-batching-only. It trades first-token latency against tighter token gaps", - "revisit_when": "State which metric the deployment optimizes; the combined design does not improve every metric over each alternative", + "direction": "Read the reported TTFT and TBT comparison as a workload-conditioned trade", + "outcome": "tradeoff", + "reason": "For both Table 4 traces, the combined design lowered P99 TBT versus either component alone. Median TTFT was below chunked-only and above hybrid-only in that setup.", + "revisit_when": "Preserve separate percentile and load definitions before borrowing this comparison for a new service configuration.", "keywords": [ "P99 TBT", "P50 TTFT", "hybrid batching", - "first token latency" - ] + "chunked prefills", + "Table 4", + "Yi-34B" + ], + "scope": "regime_bound", + "bounds": "128 requests, Yi-34B on two A100s, 1024-token budget, separately for OpenChat and arXiv summarization." + }, + { + "direction": "Freeze matched arrival, queueing and latency objectives before changing a serving budget", + "outcome": "caveat", + "reason": "The reported capacity work combined a model-specific token-gap objective with a scheduling-delay limit and defined request length filters. An improvement to one latency number alone does not settle the deployment tradeoff.", + "revisit_when": "Hold workload, model, hardware, runtime and parallelism constant. Measure TTFT, TBT, scheduling delay and sustainable throughput and keep the incumbent when the required evidence is missing.", + "keywords": [ + "capacity tuning", + "Poisson arrivals", + "scheduling delay", + "TTFT TBT SLO", + "sustainable throughput", + "length filters" + ], + "scope": "general" } - ], - "limitations": [ - "Paper-author observations and proposed adaptation checks are not local measurements or current-engine support.", - "The combined design does not beat hybrid-only P50 TTFT in either Table 4 trace.", - "No universal budget, fairness guarantee, quantitative disaggregation comparison or empirical skill efficacy is established.", - "Raw source redistribution and adaptation permission remain UNKNOWN.", - "The same case identity and central mechanism appeared in closed-unmerged PR 156; this independently written fixed-edition contribution does not recover the missing historical archive or establish novelty over its unavailable synthesis." - ], - "attribution": "Original attributed explanation of Agrawal et al., OSDI 2024. The paper's reported results remain distinct from InferRSI's proposed checks; no raw source asset or prior proposal prose is reproduced." + ] } diff --git a/tests/test_knowledge_find.py b/tests/test_knowledge_find.py index 170241472..b109cc851 100644 --- a/tests/test_knowledge_find.py +++ b/tests/test_knowledge_find.py @@ -137,8 +137,9 @@ def test_ties_break_towards_stronger_evidence(): def test_findings_are_ordered_by_query_overlap_and_truncated(corpus): result = find("cache per-step masks ran out of device memory", corpus, limit=5) item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") - assert item["findings"][0]["direction"] == "Cache per-step masks and prompt tensors" - assert item["findings"][0]["outcome"] == "rejected" + top = item["findings"][0] + assert "per-step masks" in top["direction"] and "prompt tensors" in top["direction"] + assert top["outcome"] == "rejected" assert len(item["findings"]) == 3 assert item["findings_total"] == 6 assert item["findings_withheld"] == 3 @@ -218,11 +219,11 @@ def test_finding_order_is_not_decided_by_corpus_common_terms(corpus): corpus, limit=5, ) item = next(i for i in result["items"] if i["id"] == "case.qwen-ascend-rejected-directions") - shown = [f["direction"] for f in item["findings"]] + shown = item["findings"] # Both memory failures the question distinguishes between must be present. - assert any("RoPE + VAE fastpath" in d for d in shown), shown - assert any("later busy host" in d for d in shown), shown - assert not any("TASK_QUEUE_ENABLE" in d for d in shown), shown + assert any("initialization" in f["direction"] and f["outcome"] == "rejected" for f in shown) + assert any("reservation" in f["direction"] and f["outcome"] == "inconclusive" for f in shown) + assert all("TASK_QUEUE_ENABLE" not in f["reason"] for f in shown) def _plant_case(root: Path, owner: str, *, registered: bool = True) -> None: @@ -395,9 +396,9 @@ def test_finding_order_holds_when_the_library_has_one_entry(tmp_path): "did the Qwen Ascend memory failure happen at startup or on later requests", corpus, limit=1, ) - shown = [f["direction"] for f in result["items"][0]["findings"]] - assert any("RoPE + VAE fastpath" in d for d in shown), shown - assert any("later busy host" in d for d in shown), shown + shown = result["items"][0]["findings"] + assert any("initialization" in f["direction"] and f["outcome"] == "rejected" for f in shown) + assert any("reservation" in f["direction"] and f["outcome"] == "inconclusive" for f in shown) def test_english_synonyms_bridge_a_spelling_the_corpus_does_not_use(corpus): From 791cf4d00d416357acc9a3095e2813a88e0f36a7 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Mon, 16 Nov 2026 17:38:16 +0800 Subject: [PATCH 6/8] Refresh skill projections for the rewritten findings Agent-assisted-by: Codex / gpt-6.1-sol [effort=max] (author) Signed-off-by: Zhou ziheng --- .../data/repo_packs/inferrsi/knowledge/_provenance.json | 2 +- .../common/skills/inference-optimization-casebook.json | 8 ++++---- .../knowledge/common/skills/kernel-writing-guide.json | 4 ++-- .../common/skills/optimization-search-policy.json | 2 +- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json index 11d279ad7..bb2588278 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json @@ -1,7 +1,7 @@ { "schema_version": 1, "repository": "https://github.com/InferMatrixLab/InferRSI", - "source_revision": "dc8b55f32f05937a37cc7afbaf63e5a0b35da5f4", + "source_revision": "1431037fe138679a0b225e1501dbbc43f97a2150", "registry": { "path": "src/inferrsi/knowledge/data/skill_library/skill-index.json", "sha256": "7904275c3383380db179287171e1430617aaa9ac4f439291eff8a71426987fbb" diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json index 62535c9ac..e2a3332c6 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/inference-optimization-casebook.json @@ -25,13 +25,13 @@ "checkout_ref": "checkout:inference-optimization-casebook", "source_files": { "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/SKILL.md": "89a6a82c692f7f5192f4d7f4c3a5ff08a4df3b1c4bdaa0aafb7e04292d8aa5b4", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json": "77bcd2a5dcdf7892877114c993358234ad009dbde417b284c877cf8c1667389f", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.json": "8cd0042edde7b1d74edec463cb901f689c3327fac0c34f70692ca10679dc8a1d", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/admission-drain-occupancy.md": "e88bbac5d253a8e7e3a3d5e21047f23f2970c11ac23887d4303ab94fca52dd27", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json": "059f16c5a9dcd70532e1b7035bcf80f505086f665ca63407ba4bb6f4174c0db2", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.json": "eba990c460265bab19a74093ea5a65196d7df1b17d99939e1b3fe28e5a066c28", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/drama3-rejected-directions.md": "6f143bc363a3290fc008c08e57cb075f1776d1c8cca8a6a59297d5b1fbbf61bf", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json": "49cc9bb498a0e05515e902b2e549f6548259a828dca70f0856a2c7eced3a60d9", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.json": "7f0ca00b2e5d56f363a2eb8611d7e68778755aa133fa6f951151dc6e9b40b0a9", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-ascend-rejected-directions.md": "be396983812262515899a3e2ba4f01b23ff92e1b836881882c7580372b8ae552", - "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json": "d463625c52c903e8b821772d8c05bddd0304f6cdb8b2454e73737f69f2bc0fb7", + "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.json": "52b5dd233f869e8836b6d2e3cf9bda1b0877a7bf72c1ab8ca3132cf1adc06d23", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/cases/qwen-b300-rejected-directions.md": "cce2930f544a0f78f21797fba4d8ded298e4a81f76c34d7977b0e31d361bc738", "src/inferrsi/knowledge/data/skill_library/methods/inference-optimization-casebook/metadata.json": "963488658f327d2379183d70357eabf832d70a6a70db2aa67f548493f4088d97" }, diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json index a5e7624fd..a92e4a3d3 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/kernel-writing-guide.json @@ -45,10 +45,10 @@ "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/SKILL.md": "360361f592ec3a24834198372cbf92aaedc1720404d182df94315eb0e01d58dd", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/causal-conv1d-dispatch-state.md": "2eb67dd522021bb2234616d7f660e6ba99acb43676f48cb23141bc69bf3e0029", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/hf-qwen35-gdn-binding.md": "b3a55af3f45f01c48e1cb18c8b029e5949a494c3e51dc444a5cccbad9acbba0a", - "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json": "7eb01ffeb40cdd555a2b0012e863f13a09df4a13a00d0ba1fab555645b245a42", + "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.json": "5237ac437eb76e910bb1d08bfcc364b87dea1b947de5e67eebce511ea1ef1d23", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/sglang-gdn-source-contract.md": "8b093faf0ab775d5605ecdd9653fb3689c2b839cba2641c5d5cda17c721bb2ac", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-causal-conv-contract.md": "2f3dcee66e2dfe7a98bee21a36b892538998de5de3caf1bc63257137f9984236", - "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json": "68f1cb0dc99bc3eb11692fa030d5f785622d0ea1733a084a76d286c9540890bb", + "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.json": "56af6b07bbb32cd6aad44e831404ec041e0450db9623fbe6b048fcbbca32e893", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/cases/vllm-gdn-source-contract.md": "7f9ebe8de7e55d3db2a119b6a56e15c37448fd7d4e022a47122a143206d31687", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/metadata.json": "766aa688d573c790789e2836f952315e91c4e06806fca3d04523a8a748bf9c32", "src/inferrsi/knowledge/data/skill_library/methods/kernel-writing-guide/references/akg-affine-tail-guards.md": "113deb2db10b171db3425bb3f1792bee763706b794c8cc2e222b4cc0d0114118", diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/optimization-search-policy.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/optimization-search-policy.json index 28d246f8e..91e68b62b 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/optimization-search-policy.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/common/skills/optimization-search-policy.json @@ -25,7 +25,7 @@ "checkout_ref": "checkout:optimization-search-policy", "source_files": { "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/SKILL.md": "493001e1266b91434d873d688c978486bbac4a4b86fb657492746877b652e4fb", - "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json": "59f00bc62d0dfa19328a2e1c6f2a261af155c25f67fc185e94254a479bc66d17", + "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.json": "681871983ec4033ef9861ad270bc0d312f9134a76933b8f7ce740659ef1e4dd5", "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/cases/sarathi-chunk-budget.md": "2c2a8147001d24d1abc8672f15cf0ca759ae461b734413c8dcb7497634bb6821", "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/metadata.json": "43e6378cdc801a71fe510e6adc17079016220c94bd6debfa8366b3cc7117d493", "src/inferrsi/knowledge/data/skill_library/methods/optimization-search-policy/references/distserve-phase-placement/README.md": "56e16efce447dfd4990a024ff1d6f7cb428f8e3e1f7b32a7e4d05f57139b836a", From 866fd2aa2816652976c107d6ab09eb80d0343ef1 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Mon, 16 Nov 2026 18:11:21 +0800 Subject: [PATCH 7/8] docs: explain the packaged skill fallback without PyYAML Use the packaged projection and its revision to resolve a Skill that the base installation cannot open from find. Agent-assisted-by: Codex (author) Signed-off-by: Zhou ziheng --- docs/operations.md | 5 +++++ plugins/inferrsi/skills/inferrsi/SKILL.md | 10 ++++++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/operations.md b/docs/operations.md index 025f0b9cc..ed78bb5f5 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -243,6 +243,11 @@ did. entrypoint, provenance and revision with a smaller response. Remove `--omit-resources` to list the bundle's resources. Hits that fail the reader's material check carry `unreadable` with the reason. +- **Base installation fallback.** When a Skill hit requires PyYAML, use + `inferrsi collect --relevance "" --limit 1 --material --json` + to look up the packaged skill. Fetch its `skill_documents.original` with + `inferrsi resource --revision --output ` using + the returned `ref` and `revision`. - **Side effects:** none. No revision is pinned; use `read`/`resource` when a capture or evidence record needs exact bytes. - **Example:** `inferrsi find "NPU OOM after caching per-step masks" --limit 3` returns the casebook entry recording that attempt first. A vaguer phrasing ranks the procedural skills above it, which is the first limit below rather than a defect. diff --git a/plugins/inferrsi/skills/inferrsi/SKILL.md b/plugins/inferrsi/skills/inferrsi/SKILL.md index cd18d471c..a665421a2 100644 --- a/plugins/inferrsi/skills/inferrsi/SKILL.md +++ b/plugins/inferrsi/skills/inferrsi/SKILL.md @@ -124,8 +124,14 @@ repository or user instead of guessing a model architecture. Use `find ""` to rank skills, imported records and casebook entries. Add `--kinds case` when you specifically need a past observation. Open a promising hit using its displayed `open` command or the JSON -`open_argv` argument list. If the result says `cannot open`, check the reason -and consult another hit when appropriate. +`open_argv` argument list. + +When a Skill hit is unreadable because PyYAML is missing, use +`collect --relevance "" --limit 1 --material --json` to look up +the packaged skill. Save the JSON to a file and use the hit's `ref`, +`revision` and `skill_documents.original` +to fetch the body with `resource --revision --output `. +For other `cannot open` results, check the reason before choosing another hit. `read` returns the material, its source, applicability, limitations and revision. To cite or reopen exact evidence, add `--revision ` to the From 8c0a7bfc39a9f95d610f6bd0b9f3682200ad0606 Mon Sep 17 00:00:00 2001 From: Zhou ziheng Date: Mon, 16 Nov 2026 22:40:01 +0800 Subject: [PATCH 8/8] Regenerate combined knowledge provenance and finalize the install regression Agent-assisted-by: Codex (author) Signed-off-by: Zhou ziheng --- .../data/repo_packs/inferrsi/knowledge/_provenance.json | 2 +- tests/test_knowledge_install.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json index 1b9f6d8e4..eb8a4af19 100644 --- a/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json +++ b/src/inferrsi/knowledge/data/repo_packs/inferrsi/knowledge/_provenance.json @@ -1,7 +1,7 @@ { "schema_version": 1, "repository": "https://github.com/InferMatrixLab/InferRSI", - "source_revision": "0143bc4e0228866a07cdc6c94e11fe0d76b93ab2", + "source_revision": "8288a004c333b0e338b3ad31fe823052a142b90a", "registry": { "path": "src/inferrsi/knowledge/data/skill_library/skill-index.json", "sha256": "7904275c3383380db179287171e1430617aaa9ac4f439291eff8a71426987fbb" diff --git a/tests/test_knowledge_install.py b/tests/test_knowledge_install.py index c38e12209..fb672c112 100644 --- a/tests/test_knowledge_install.py +++ b/tests/test_knowledge_install.py @@ -80,7 +80,8 @@ def test_packaged_skill_search_and_resource_work_without_pyyaml(without_yaml, tm "--output", str(output), ], env) assert fetched.returncode == 0, fetched.stderr - assert output.read_bytes() == (SKILLS / Path(original).relative_to("data/skill_library")).read_bytes() + expected = SKILLS / Path(original).relative_to("data/skill_library") + assert output.read_bytes() == expected.read_bytes() def test_skill_root_without_pyyaml_fails_with_install_hint(without_yaml):