fix bugs

Author: zhongkaifu

InferenceWeb.Tests/BatchedExecutorTests.cs

@@ -180,6 +180,43 @@ public void BatchExecutor_BatchedPath_RoutesPerSeqLogitsCorrectly()
         }
     }
 
+    [Fact]
+    public async Task InferenceEngine_ExecutorStepException_CompletesErroredRequestAndContinuesWaiting()
+    {
+        var model = new ThrowingBatchedStubModel("fp-step-error", badRequestId: "bad", peakToken: 7);
+        var cfg = new SchedulerConfig
+        {
+            MaxNumBatchedTokens = 256,
+            MaxNumRunningSequences = 1,
+            MaxPrefillChunkSize = 64,
+            NumBlocks = 4,
+            BlockSize = BlockSize,
+            EnablePrefixCaching = false,
+            DecodeQuantumTokens = 1,
+        };
+        using var engine = new InferenceEngine(model, cfg, NullLogger.Instance);
+
+        var badSeq = new SequenceState("bad", Enumerable.Range(1, 4).ToList(),
+            maxNewTokens: 2, BlockSize, SamplingConfig.Default);
+        var goodSeq = new SequenceState("good", Enumerable.Range(1, 4).ToList(),
+            maxNewTokens: 2, BlockSize, SamplingConfig.Default);
+
+        var bad = engine.SubmitRequest(badSeq);
+        var good = engine.SubmitRequest(goodSeq);
+
+        var ex = await Assert.ThrowsAsync<InvalidOperationException>(
+            () => bad.Completion.WaitAsync(TimeSpan.FromSeconds(5)));
+        Assert.Contains("bad", ex.Message);
+
+        var goodCompletion = await good.Completion.WaitAsync(TimeSpan.FromSeconds(5));
+        Assert.Equal(SequenceStatus.FinishedLengthCapped, goodCompletion.Status);
+        Assert.Equal(0, badSeq.BlockTable.NumBlocks);
+        Assert.Equal(SequenceStatus.FinishedError, badSeq.Status);
+        Assert.Contains("bad", model.ReleasedRequestIds);
+        Assert.Contains("good", model.ReleasedRequestIds);
+        Assert.Equal(cfg.NumBlocks, engine.PoolStats.freeBlocks);
+    }
+
     // ----- helpers -----
 
     private static SchedulerConfig SmallConfig() => new()
@@ -301,6 +338,61 @@ public IReadOnlyList<float[]> ForwardBatch(BatchedForwardContext ctx)
         }
     }
 
+    private sealed class ThrowingBatchedStubModel : IModelArchitecture, IBatchedPagedModel
+    {
+        private readonly string _fp;
+        private readonly string _badRequestId;
+        private readonly int _peak;
+
+        public ThrowingBatchedStubModel(string fp, string badRequestId, int peakToken)
+        {
+            _fp = fp;
+            _badRequestId = badRequestId;
+            _peak = peakToken;
+            Tokenizer = new StubTokenizer(VocabSize);
+        }
+
+        public List<string> ReleasedRequestIds { get; } = new();
+
+        public ModelConfig Config { get; } = new ModelConfig { VocabSize = VocabSize };
+        public ITokenizer Tokenizer { get; }
+        public IMultimodalInjector MultimodalInjector => null;
+        public IBackendExecutionPlan ExecutionPlan => null;
+        public bool SupportsKVCacheTruncation => true;
+        public bool SupportsKVStateSnapshot => true;
+        public string KVStateFingerprint => _fp;
+        public long ComputeKVBlockByteSize(int n) => 2L * NumLayers * NumKVHeads * n * HeadDim * sizeof(float);
+        public float[] Forward(int[] tokens) => new float[VocabSize];
+        public void ResetKVCache() { }
+        public void TruncateKVCache(int n) { }
+        public bool TryExtractKVBlock(int s, int n, Span<byte> dst) => true;
+        public bool TryInjectKVBlock(int s, int n, ReadOnlySpan<byte> src) => true;
+        public void Dispose() { }
+
+        public IReadOnlyList<float[]> ForwardBatch(BatchedForwardContext ctx)
+        {
+            for (int i = 0; i < ctx.Sequences.Count; i++)
+            {
+                if (ctx.Sequences[i].RequestId == _badRequestId)
+                    throw new InvalidOperationException($"boom for {_badRequestId}");
+            }
+
+            var result = new float[ctx.Sequences.Count][];
+            for (int i = 0; i < result.Length; i++)
+            {
+                var logits = new float[VocabSize];
+                logits[_peak] = 10f;
+                result[i] = logits;
+            }
+            return result;
+        }
+
+        public void OnSequenceReleased(string requestId)
+        {
+            ReleasedRequestIds.Add(requestId);
+        }
+    }
+
     private sealed class StubTokenizer : ITokenizer
     {
         public StubTokenizer(int vocab)

InferenceWeb.Tests/ChatSessionTests.cs

@@ -11,9 +11,8 @@
 namespace InferenceWeb.Tests;
 
 /// <summary>
-/// Unit tests for <see cref="ChatSession"/>: ensures each session owns an isolated
-/// <see cref="KVCache"/> and tracked-history buffer, and that disposing the session
-/// fully clears that state.
+/// Unit tests for <see cref="ChatSession"/>: ensures each session owns an
+/// isolated tracked-history buffer, and that disposing the session clears it.
 /// </summary>
 public class ChatSessionTests
 {
@@ -26,8 +25,6 @@ public void NewSession_HasUniqueIdAndEmptyState()
         Assert.False(string.IsNullOrEmpty(a.Id));
         Assert.False(string.IsNullOrEmpty(b.Id));
         Assert.NotEqual(a.Id, b.Id);
-        Assert.True(a.KVCache.IsEmpty);
-        Assert.True(b.KVCache.IsEmpty);
         Assert.Empty(a.TrackedHistory);
         Assert.Empty(b.TrackedHistory);
         Assert.False(a.IsDisposed);
@@ -36,30 +33,25 @@ public void NewSession_HasUniqueIdAndEmptyState()
     [Fact]
     public void SessionsHaveIndependentState()
     {
-        // Mutating one session's cache / history must not leak into another.
+        // Mutating one session's history must not leak into another.
         var a = new ChatSession();
         var b = new ChatSession();
 
-        a.KVCache.RecordAppend(new[] { 1, 2, 3 }, new float[] { 0.1f });
         a.TrackedHistory.Add(new ChatMessage { Role = "user", Content = "hello-a" });
 
-        Assert.Equal(3, a.KVCache.Count);
         Assert.Single(a.TrackedHistory);
-        Assert.True(b.KVCache.IsEmpty);
         Assert.Empty(b.TrackedHistory);
     }
 
     [Fact]
-    public void Dispose_ClearsTokensAndHistory()
+    public void Dispose_ClearsHistory()
     {
         var session = new ChatSession();
-        session.KVCache.RecordAppend(new[] { 42, 43 }, new float[] { 0.5f });
         session.TrackedHistory.Add(new ChatMessage { Role = "user", Content = "hi" });
 
         session.Dispose();
 
         Assert.True(session.IsDisposed);
-        Assert.True(session.KVCache.IsEmpty);
         Assert.Empty(session.TrackedHistory);
     }
 

…nceWeb.Tests/ModelServiceKVCacheTests.cs …ests/ModelServiceRawTokenHistoryTests.csInferenceWeb.Tests/ModelServiceKVCacheTests.cs renamed to InferenceWeb.Tests/ModelServiceRawTokenHistoryTests.cs InferenceWeb.Tests/ModelServiceKVCacheTests.cs renamed to InferenceWeb.Tests/ModelServiceRawTokenHistoryTests.cs

@@ -13,9 +13,9 @@ namespace InferenceWeb.Tests;
 /// <summary>
 /// Tests for the ModelService-level conversation tracking that keeps raw output tokens
 /// associated with assistant messages across HTTP requests, enabling the next turn's
-/// prompt render to splice raw tokens in for cached KV state.
+/// prompt render to preserve the exact tokenized conversation prefix.
 /// </summary>
-public class ModelServiceKVCacheTests
+public class ModelServiceRawTokenHistoryTests
 {
     [Fact]
     public void ResolvePrefillChunkSize_CudaLongPrompt_UsesSafeChunkSize()
@@ -82,13 +82,13 @@ public void AugmentWithCachedRawTokens_PreservesIncomingRawTokensIfAlreadySet()
     }
 
     /// <summary>
-    /// Reproduces the Qwen 3.5 / 3.6 WebUI cache-reset bug:
+    /// Reproduces the Qwen 3.5 / 3.6 WebUI raw-token history bug:
     /// the streaming output parser strips &lt;think&gt;...&lt;/think&gt; framing from the
     /// assistant text before the WebUI accumulates it, so on the next chat request the
     /// WebUI sends back an assistant message whose Content is a STRIPPED subset of what
     /// our tracked history stored. The augmenter MUST still recognise this as the same
     /// turn and splice the cached raw output tokens, otherwise every multi-turn chat with
-    /// a thinking model degenerates to a full prompt re-prefill.
+    /// a thinking model changes the rendered token prefix.
     /// </summary>
     [Fact]
     public void AugmentWithCachedRawTokens_WebUIParsedContentMismatch_StillSplicesRawTokens()

InferenceWeb.Tests/ModelServiceSessionTests.cs

@@ -11,24 +11,20 @@
 namespace InferenceWeb.Tests;
 
 /// <summary>
-/// Unit tests for <see cref="ModelService"/> session-lifecycle operations. These do
-/// not load a real model (they exercise only the cache bookkeeping), so they are
-/// safe to run on any machine. The expensive K/V tensor reset is a no-op when no
-/// model is loaded.
+/// Unit tests for <see cref="ModelService"/> session-lifecycle operations. These
+/// do not load a real model; server sessions now only own tracked history.
 /// </summary>
 public class ModelServiceSessionTests
 {
     [Fact]
-    public void ResetSession_ClearsCacheAndHistoryForTheGivenSession()
+    public void ResetSession_ClearsHistoryForTheGivenSession()
     {
         var svc = new ModelService();
         var session = new ChatSession();
-        session.KVCache.RecordAppend(new[] { 1, 2, 3 }, new float[] { 0.5f });
         session.TrackedHistory.Add(new ChatMessage { Role = "user", Content = "hi" });
 
         svc.ResetSession(session);
 
-        Assert.True(session.KVCache.IsEmpty);
         Assert.Empty(session.TrackedHistory);
         Assert.False(session.IsDisposed);
     }
@@ -40,16 +36,12 @@ public void ResetSession_LeavesOtherSessionsUntouched()
         var sessA = new ChatSession();
         var sessB = new ChatSession();
 
-        sessA.KVCache.RecordAppend(new[] { 1, 2 }, new float[] { 0.1f });
         sessA.TrackedHistory.Add(new ChatMessage { Role = "user", Content = "a" });
-        sessB.KVCache.RecordAppend(new[] { 9, 8 }, new float[] { 0.2f });
         sessB.TrackedHistory.Add(new ChatMessage { Role = "user", Content = "b" });
 
         svc.ResetSession(sessA);
 
-        Assert.True(sessA.KVCache.IsEmpty);
         Assert.Empty(sessA.TrackedHistory);
-        Assert.Equal(2, sessB.KVCache.Count);
         Assert.Single(sessB.TrackedHistory);
     }
 
@@ -65,13 +57,11 @@ public void DisposeSession_MarksSessionDisposedAndFreesState()
     {
         var svc = new ModelService();
         var session = new ChatSession();
-        session.KVCache.RecordAppend(new[] { 10, 20 }, new float[] { 0.3f });
         session.TrackedHistory.Add(new ChatMessage { Role = "user", Content = "x" });
 
         svc.DisposeSession(session);
 
         Assert.True(session.IsDisposed);
-        Assert.True(session.KVCache.IsEmpty);
         Assert.Empty(session.TrackedHistory);
     }
 
@@ -83,13 +73,11 @@ public void DisposeSession_DoesNotAffectOtherSessions()
         var sessA = new ChatSession();
         var sessB = new ChatSession();
 
-        sessB.KVCache.RecordAppend(new[] { 77 }, new float[] { 0.9f });
         sessB.TrackedHistory.Add(new ChatMessage { Role = "user", Content = "keep" });
 
         svc.DisposeSession(sessA);
 
         Assert.False(sessB.IsDisposed);
-        Assert.Equal(1, sessB.KVCache.Count);
         Assert.Single(sessB.TrackedHistory);
     }
 
@@ -120,6 +108,16 @@ public void InvalidateKVCache_DoesNotThrowWhenNoModelLoaded()
         Assert.False(svc.IsLoaded);
     }
 
+    [Fact]
+    public void KVCache_ReturnsIsolatedCompatibilityShim()
+    {
+        var svc = new ModelService();
+        var legacyView = svc.KVCache;
+        legacyView.RecordAppend(new[] { 1, 2, 3 }, new float[] { 0.5f });
+
+        Assert.True(svc.KVCache.IsEmpty);
+    }
+
     [Fact]
     public void ActiveSession_StartsNullUntilInferenceActivatesOne()
     {

InferenceWeb.Tests/SessionManagerTests.cs

@@ -87,8 +87,8 @@ public void TryRemove_NonexistentIdReturnsNull()
     [Fact]
     public void TryRemove_DefaultSessionIdReturnsNullAndKeepsSession()
     {
-        // The default session must survive for the lifetime of the server since the
-        // Ollama / OpenAI endpoints rely on it for cache reuse.
+        // The default session must survive for the lifetime of the server since
+        // the Ollama / OpenAI endpoints rely on it for tracked-history reuse.
         var mgr = new SessionManager();
         var before = mgr.DefaultSession;
 
@@ -106,28 +106,25 @@ public void CreatedSessions_HaveIsolatedState()
         var a = mgr.CreateSession();
         var b = mgr.CreateSession();
 
-        a.KVCache.RecordAppend(new[] { 10, 20, 30 }, new float[] { 1.0f });
         a.TrackedHistory.Add(new ChatMessage { Role = "user", Content = "A's secret" });
 
         // Session B must not see any of A's state.
-        Assert.True(b.KVCache.IsEmpty);
         Assert.Empty(b.TrackedHistory);
-        Assert.Equal(3, a.KVCache.Count);
+        Assert.Single(a.TrackedHistory);
     }
 
     [Fact]
     public void TryRemove_SessionIsNotAutoDisposed()
     {
-        // TryRemove only unregisters; disposal is the caller's responsibility so that
-        // ModelService can reset the model's K/V tensors before the bookkeeping is
-        // torn down. The returned session should still be usable for inspection.
+        // TryRemove only unregisters; disposal is the caller's responsibility.
+        // The returned session should still be usable for inspection.
         var mgr = new SessionManager();
         var created = mgr.CreateSession();
-        created.KVCache.RecordAppend(7, new float[] { 0.1f });
+        created.TrackedHistory.Add(new ChatMessage { Role = "user", Content = "keep" });
 
         var removed = mgr.TryRemove(created.Id);
 
         Assert.False(removed!.IsDisposed);
-        Assert.Equal(1, removed.KVCache.Count);
+        Assert.Single(removed.TrackedHistory);
     }
 }