diff --git a/__tests__/wal-deferral.test.ts b/__tests__/wal-deferral.test.ts index 6af86af70..8275d7a08 100644 --- a/__tests__/wal-deferral.test.ts +++ b/__tests__/wal-deferral.test.ts @@ -54,6 +54,36 @@ describe('resolveWalValveMb', () => { }); }); +describe('bulk parse/edge index windows', () => { + it('defers the edge indexes on endBulkParseLoad(true) and restores them on endBulkEdgeLoad', async () => { + const db = openDb(); + const edgeIndexes = () => (db.getDb().prepare( + "SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='edges'" + ).all() as Array<{ name: string }>).length; + try { + const before = edgeIndexes(); + expect(before).toBeGreaterThan(1); + db.beginBulkParseLoad(); // drops the parse indexes and the four edge indexes + expect(edgeIndexes()).toBeLessThan(before); + await db.endBulkParseLoad(true); // deferred: the resolver's window owns the rebuild + expect(edgeIndexes()).toBeLessThan(before); + await db.endBulkEdgeLoad(); + expect(edgeIndexes()).toBe(before); + + // The abort path registers a failed rebuild for the live connection's + // off-thread recoverer instead of waiting for the next open. + db.beginBulkParseLoad(); + await db.endBulkParseLoad(true); + expect(edgeIndexes()).toBeLessThan(before); + db.deferIndexRecovery(); + await db.recoverIndexesOffThread(); + expect(edgeIndexes()).toBe(before); + } finally { + db.close(); + } + }); +}); + describe('DatabaseConnection WAL helpers', () => { it('reads and writes the wal_autocheckpoint interval', () => { const db = openDb(); diff --git a/src/codegraph.ts b/src/codegraph.ts index b6208a150..d951cb3c6 100644 --- a/src/codegraph.ts +++ b/src/codegraph.ts @@ -70,6 +70,7 @@ import { seedLiteralsInQuery, seedRouteNamesInQuery } from './extraction/literal import { createYielder } from './resolution/cooperative-yield'; import { createImportReader, reopenNameMatchedImports, reopenReferencesThroughAddedFiles, reopenReferencesThroughBarrels, reopenReferencesThroughChangedDefaults, reopenReferencesThroughDeclaredPackages, reopenReferencesThroughRetargetedModules } from './resolution/module-importers'; import { ESM_IMPORT_LANGUAGES } from './resolution/import-resolver'; +import { minRefsForBulkLoad } from './resolution/resolver-pool'; import { minRefsForPool } from './resolution/resolver-pool'; /** The extraction, resolution and watcher modules, loaded on first use. */ @@ -690,6 +691,10 @@ export class CodeGraph { // Set when the fastInit+pool path below defers autocheckpointing, so the // finally knows to restore the interval on that path too. let restoreAutocheckpoint = false; + // Set when the parse phase deferred the edge-index rebuild for the + // resolver's bulk window; the outer finally restores them if the run + // aborts before that window completes. + let deferredEdgeIndexes = false; if (deferWal) { priorAutocheckpoint = this.db.getWalAutocheckpoint(); this.db.setWalAutocheckpoint(0); @@ -724,6 +729,7 @@ export class CodeGraph { // — they delete per-file rows mid-phase through the file_path indexes. if (freshDb) this.db.beginBulkParseLoad(); let result: IndexResult; + let extractionOk = false; try { result = await this.orchestrator.indexAll( options.onProgress, @@ -735,10 +741,16 @@ export class CodeGraph { // edge snapshots) and delete, which belongs on one thread. freshDb ? { dbPath: this.db.getPath(), fastInit } : null ); + extractionOk = true; } finally { if (freshDb) { const tIdx = Date.now(); - await this.db.endBulkParseLoad(); + // The batched resolver immediately re-drops and then rebuilds the + // edge indexes when its own bulk window engages; skip the first + // rebuild only when parsing succeeded and the resolver will run. + // The outer finally restores them if it never does. + deferredEdgeIndexes = extractionOk && this.queries.getUnresolvedReferencesCount() >= minRefsForBulkLoad(); + await this.db.endBulkParseLoad(deferredEdgeIndexes); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] parse-index-rebuild: ${Date.now() - tIdx}ms`); } const tFts = Date.now(); @@ -953,6 +965,28 @@ export class CodeGraph { // (SQLite replays the WAL on the next open) and the follow-up write // that folds it is the known cost of a failed run. if (walValve) { walValve.stop(); await walValve.drain(); } + // The parse phase may have deferred the edge indexes for the resolver's + // bulk window. If this run aborted before the resolver rebuilt them + // (or reduced the pending set below the window), restore them here so + // the live database never keeps them dropped. Rebuilding when the + // resolver already did is a CREATE IF NOT EXISTS no-op. + if (deferredEdgeIndexes) { + try { + await this.db.endBulkEdgeLoad(); + } catch (err) { + // Another connection may hold the write lock. Register the missing + // indexes for the live connection's off-thread recoverer (its own + // connection, 30s busy timeout) rather than returning with the + // edge indexes dropped; a failed recovery leaves them registered + // for the next open. + this.db.deferIndexRecovery(); + await this.db.recoverIndexesOffThread().catch((recErr) => { + console.error( + `[index] edge indexes remain dropped after an aborted index (${recErr instanceof Error ? recErr.message : String(recErr)}); they will rebuild on the next open` + ); + }); + } + } // Deterministic kernel-conn teardown while the valve is stopped and // the pool is gone — a GC-timed rusqlite close could otherwise fire // mid-shm-I/O in a LATER run (cross-build wal-index locks can't see diff --git a/src/db/index.ts b/src/db/index.ts index 3610572f6..f2a758a5b 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -349,7 +349,7 @@ export class DatabaseConnection { * (small runs) are left with a complete schema; the batched resolver's * beginBulkEdgeLoad simply re-drops them (DROP IF EXISTS — idempotent). */ - async endBulkParseLoad(): Promise { + async endBulkParseLoad(deferEdgeIndexes = false): Promise { const schemaPath = path.join(__dirname, 'schema.sql'); const schema = fs.readFileSync(schemaPath, 'utf-8'); for (const idx of DatabaseConnection.BULK_PARSE_INDEX_NAMES) { @@ -358,7 +358,10 @@ export class DatabaseConnection { this.db.exec(m[0]); await new Promise((resolve) => setImmediate(resolve)); } - await this.endBulkEdgeLoad(); + // A full index that will run the resolver's own bulk-edge window re-drops + // these immediately; rebuilding them here is wasted. The resolver's + // endBulkEdgeLoad rebuilds them after resolution. + if (!deferEdgeIndexes) await this.endBulkEdgeLoad(); } /** @@ -510,6 +513,15 @@ export class DatabaseConnection { return this.deferredIndexDdls.length > 0; } + /** Register the bulk indexes a failed window left missing so the live + * connection's off-thread recoverer rebuilds them (30s busy timeout on its + * own connection) instead of waiting for the next open. */ + deferIndexRecovery(): void { + if (this.deferredIndexDdls.length === 0) { + this.deferredIndexDdls = this.missingBulkSecondaryIndexDdls(); + } + } + /** * Recreate the secondary indexes {@link open} deferred, on a worker thread * with its own connection, so this thread's event loop, and the #850