From c4205991a4676fbb86aba562e132faed8cdea917 Mon Sep 17 00:00:00 2001 From: bompus Date: Sat, 31 Oct 2026 20:39:02 -0600 Subject: [PATCH 1/3] perf(index): skip the duplicate edge-index rebuild in the parse window A fresh full index drops the edge indexes in beginBulkParseLoad, then endBulkParseLoad rebuilt them, and the batched resolver immediately dropped them again for its own bulk-edge window. When the resolver's window will engage (pending refs >= minRefsForBulkLoad), defer the edge rebuild to the resolver's endBulkEdgeLoad. Small indexes that stay below the window rebuild in the parse phase as before. n8n: parse-index-rebuild 4.47 -> 4.02 s; the resolver's edge recreate still runs; total 52.64 -> 51.27 s across two runs per arm. Edge index count (5) and edge rows (1,236,342) are unchanged. --- src/codegraph.ts | 6 +++++- src/db/index.ts | 7 +++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/codegraph.ts b/src/codegraph.ts index b6208a150..c52e6f876 100644 --- a/src/codegraph.ts +++ b/src/codegraph.ts @@ -70,6 +70,7 @@ import { seedLiteralsInQuery, seedRouteNamesInQuery } from './extraction/literal import { createYielder } from './resolution/cooperative-yield'; import { createImportReader, reopenNameMatchedImports, reopenReferencesThroughAddedFiles, reopenReferencesThroughBarrels, reopenReferencesThroughChangedDefaults, reopenReferencesThroughDeclaredPackages, reopenReferencesThroughRetargetedModules } from './resolution/module-importers'; import { ESM_IMPORT_LANGUAGES } from './resolution/import-resolver'; +import { minRefsForBulkLoad } from './resolution/resolver-pool'; import { minRefsForPool } from './resolution/resolver-pool'; /** The extraction, resolution and watcher modules, loaded on first use. */ @@ -738,7 +739,10 @@ export class CodeGraph { } finally { if (freshDb) { const tIdx = Date.now(); - await this.db.endBulkParseLoad(); + // The batched resolver immediately re-drops and then rebuilds the + // edge indexes when its own bulk window engages; skip the first + // rebuild in that case. + await this.db.endBulkParseLoad(this.queries.getUnresolvedReferencesCount() >= minRefsForBulkLoad()); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] parse-index-rebuild: ${Date.now() - tIdx}ms`); } const tFts = Date.now(); diff --git a/src/db/index.ts b/src/db/index.ts index 3610572f6..ce21b4fe0 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -349,7 +349,7 @@ export class DatabaseConnection { * (small runs) are left with a complete schema; the batched resolver's * beginBulkEdgeLoad simply re-drops them (DROP IF EXISTS — idempotent). */ - async endBulkParseLoad(): Promise { + async endBulkParseLoad(deferEdgeIndexes = false): Promise { const schemaPath = path.join(__dirname, 'schema.sql'); const schema = fs.readFileSync(schemaPath, 'utf-8'); for (const idx of DatabaseConnection.BULK_PARSE_INDEX_NAMES) { @@ -358,7 +358,10 @@ export class DatabaseConnection { this.db.exec(m[0]); await new Promise((resolve) => setImmediate(resolve)); } - await this.endBulkEdgeLoad(); + // A full index that will run the resolver's own bulk-edge window re-drops + // these immediately; rebuilding them here is wasted. The resolver's + // endBulkEdgeLoad rebuilds them after resolution. + if (!deferEdgeIndexes) await this.endBulkEdgeLoad(); } /** From 6fe30fa865a8558bcb8d660ec28853ac833da3c2 Mon Sep 17 00:00:00 2001 From: bompus Date: Sat, 31 Oct 2026 21:03:35 -0600 Subject: [PATCH 2/3] fix(index): restore deferred edge indexes when a fresh index aborts Review of #611: the parse finally deferred the edge rebuild whenever the pending count cleared the bulk threshold, even when extraction returned success: false and the resolver (the only other rebuilder) was skipped. Defer only after successful parsing, and track the deferral through the outer finally so an aborted or short-circuited run rebuilds the indexes. Rebuilding after the resolver already did is a CREATE IF NOT EXISTS no-op. Test pins the defer/restore pair. --- __tests__/wal-deferral.test.ts | 21 +++++++++++++++++++++ src/codegraph.ts | 20 ++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/__tests__/wal-deferral.test.ts b/__tests__/wal-deferral.test.ts index 6af86af70..768a949e3 100644 --- a/__tests__/wal-deferral.test.ts +++ b/__tests__/wal-deferral.test.ts @@ -54,6 +54,27 @@ describe('resolveWalValveMb', () => { }); }); +describe('bulk parse/edge index windows', () => { + it('defers the edge indexes on endBulkParseLoad(true) and restores them on endBulkEdgeLoad', async () => { + const db = openDb(); + const edgeIndexes = () => (db.getDb().prepare( + "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='edges'" + ).all() as Array<{ name: string }>).length; + try { + const before = edgeIndexes(); + expect(before).toBeGreaterThan(1); + db.beginBulkParseLoad(); // drops the parse indexes and the four edge indexes + expect(edgeIndexes()).toBeLessThan(before); + await db.endBulkParseLoad(true); // deferred: the resolver's window owns the rebuild + expect(edgeIndexes()).toBeLessThan(before); + await db.endBulkEdgeLoad(); + expect(edgeIndexes()).toBe(before); + } finally { + db.close(); + } + }); +}); + describe('DatabaseConnection WAL helpers', () => { it('reads and writes the wal_autocheckpoint interval', () => { const db = openDb(); diff --git a/src/codegraph.ts b/src/codegraph.ts index c52e6f876..2d0a408a9 100644 --- a/src/codegraph.ts +++ b/src/codegraph.ts @@ -691,6 +691,10 @@ export class CodeGraph { // Set when the fastInit+pool path below defers autocheckpointing, so the // finally knows to restore the interval on that path too. let restoreAutocheckpoint = false; + // Set when the parse phase deferred the edge-index rebuild for the + // resolver's bulk window; the outer finally restores them if the run + // aborts before that window completes. + let deferredEdgeIndexes = false; if (deferWal) { priorAutocheckpoint = this.db.getWalAutocheckpoint(); this.db.setWalAutocheckpoint(0); @@ -725,6 +729,7 @@ export class CodeGraph { // — they delete per-file rows mid-phase through the file_path indexes. if (freshDb) this.db.beginBulkParseLoad(); let result: IndexResult; + let extractionOk = false; try { result = await this.orchestrator.indexAll( options.onProgress, @@ -736,13 +741,16 @@ export class CodeGraph { // edge snapshots) and delete, which belongs on one thread. freshDb ? { dbPath: this.db.getPath(), fastInit } : null ); + extractionOk = true; } finally { if (freshDb) { const tIdx = Date.now(); // The batched resolver immediately re-drops and then rebuilds the // edge indexes when its own bulk window engages; skip the first - // rebuild in that case. - await this.db.endBulkParseLoad(this.queries.getUnresolvedReferencesCount() >= minRefsForBulkLoad()); + // rebuild only when parsing succeeded and the resolver will run. + // The outer finally restores them if it never does. + deferredEdgeIndexes = extractionOk && this.queries.getUnresolvedReferencesCount() >= minRefsForBulkLoad(); + await this.db.endBulkParseLoad(deferredEdgeIndexes); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] parse-index-rebuild: ${Date.now() - tIdx}ms`); } const tFts = Date.now(); @@ -957,6 +965,14 @@ export class CodeGraph { // (SQLite replays the WAL on the next open) and the follow-up write // that folds it is the known cost of a failed run. if (walValve) { walValve.stop(); await walValve.drain(); } + // The parse phase may have deferred the edge indexes for the resolver's + // bulk window. If this run aborted before the resolver rebuilt them + // (or reduced the pending set below the window), restore them here so + // the live database never keeps them dropped. Rebuilding when the + // resolver already did is a CREATE IF NOT EXISTS no-op. + if (deferredEdgeIndexes) { + try { await this.db.endBulkEdgeLoad(); } catch { /* schema heals on the next open */ } + } // Deterministic kernel-conn teardown while the valve is stopped and // the pool is gone — a GC-timed rusqlite close could otherwise fire // mid-shm-I/O in a LATER run (cross-build wal-index locks can't see From b2db4798ca3e36dcdd2a72fd91ce878803aee214 Mon Sep 17 00:00:00 2001 From: bompus Date: Sat, 31 Oct 2026 21:21:02 -0600 Subject: [PATCH 3/3] fix(index): recover deferred edge indexes on the live connection Review of #611: a failed abort-path rebuild was swallowed, and the existing connection's off-thread recoverer did not know the indexes were missing. Register them with deferIndexRecovery() and run recoverIndexesOffThread() (its own connection, 30s busy timeout); a failed recovery leaves them registered for the next open and logs the failure. Test covers the deferred-register/recover path. --- __tests__/wal-deferral.test.ts | 9 +++++++++ src/codegraph.ts | 16 +++++++++++++++- src/db/index.ts | 9 +++++++++ 3 files changed, 33 insertions(+), 1 deletion(-) diff --git a/__tests__/wal-deferral.test.ts b/__tests__/wal-deferral.test.ts index 768a949e3..8275d7a08 100644 --- a/__tests__/wal-deferral.test.ts +++ b/__tests__/wal-deferral.test.ts @@ -69,6 +69,15 @@ describe('bulk parse/edge index windows', () => { expect(edgeIndexes()).toBeLessThan(before); await db.endBulkEdgeLoad(); expect(edgeIndexes()).toBe(before); + + // The abort path registers a failed rebuild for the live connection's + // off-thread recoverer instead of waiting for the next open. + db.beginBulkParseLoad(); + await db.endBulkParseLoad(true); + expect(edgeIndexes()).toBeLessThan(before); + db.deferIndexRecovery(); + await db.recoverIndexesOffThread(); + expect(edgeIndexes()).toBe(before); } finally { db.close(); } diff --git a/src/codegraph.ts b/src/codegraph.ts index 2d0a408a9..d951cb3c6 100644 --- a/src/codegraph.ts +++ b/src/codegraph.ts @@ -971,7 +971,21 @@ export class CodeGraph { // the live database never keeps them dropped. Rebuilding when the // resolver already did is a CREATE IF NOT EXISTS no-op. if (deferredEdgeIndexes) { - try { await this.db.endBulkEdgeLoad(); } catch { /* schema heals on the next open */ } + try { + await this.db.endBulkEdgeLoad(); + } catch (err) { + // Another connection may hold the write lock. Register the missing + // indexes for the live connection's off-thread recoverer (its own + // connection, 30s busy timeout) rather than returning with the + // edge indexes dropped; a failed recovery leaves them registered + // for the next open. + this.db.deferIndexRecovery(); + await this.db.recoverIndexesOffThread().catch((recErr) => { + console.error( + `[index] edge indexes remain dropped after an aborted index (${recErr instanceof Error ? recErr.message : String(recErr)}); they will rebuild on the next open` + ); + }); + } } // Deterministic kernel-conn teardown while the valve is stopped and // the pool is gone — a GC-timed rusqlite close could otherwise fire diff --git a/src/db/index.ts b/src/db/index.ts index ce21b4fe0..f2a758a5b 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -513,6 +513,15 @@ export class DatabaseConnection { return this.deferredIndexDdls.length > 0; } + /** Register the bulk indexes a failed window left missing so the live + * connection's off-thread recoverer rebuilds them (30s busy timeout on its + * own connection) instead of waiting for the next open. */ + deferIndexRecovery(): void { + if (this.deferredIndexDdls.length === 0) { + this.deferredIndexDdls = this.missingBulkSecondaryIndexDdls(); + } + } + /** * Recreate the secondary indexes {@link open} deferred, on a worker thread * with its own connection, so this thread's event loop, and the #850