diff --git a/CHANGELOG.md b/CHANGELOG.md index 0e26e3970c..548e1c9f3c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -100,6 +100,8 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- **A documentation link to a file that is not there no longer gets its own graph node.** A Markdown link or path mention whose target basename no indexed file has minted an import node and a `contains` edge, even though the resolver's file-path arm could not match it. On supabase a full index now skips 10,394 of 14,038 Markdown import nodes and their `contains` edges. The reference is always kept, so the resolver's route, module and table arms still match one (a relative `api/users.json` link included), `codegraph sync` can resolve a target added later from the parked failed reference, and a skipped target stays in a separate `links:` part of the file digest so it remains searchable past the 12-item display digest. Resolved Markdown edges are byte-identical on supabase (3,058), n8n, pretix and discourse, and the failed Markdown reference count is unchanged (11,368 on supabase). A full supabase index is a median 16.176 s to 15.953 s (Hyperfine 2.0.0, 8 runs per build, two ABBA rounds, 0.223 s/1.4%, interquartile ranges not overlapping). Re-index Markdown-heavy projects after upgrading. + - **`codegraph sessions` and `codegraph_sessions` can search history before a date.** A search could only look back a number of days (`--since`, `sinceDays`), so its best 100 hits always came from the newest matching sessions and older context could not be reached. `--until ` and `until` keep only docs before a date (read as UTC) or a timestamp that names its zone; together with `--since` they bound a window. - A background sync after editing a widely imported file no longer leaves the MCP daemon holding about 19 GB (n8n, one new export in a barrel imported from 335 files). Re-checking the importers built a native resolver per binding and the process kept about 2.5 MB of each; they now share one. That check went from 38.7 s to 0.5 s, and idle memory after the edit from 19.4 GB to 1.4 GB. diff --git a/__tests__/fixtures/golden/payroll-go.dump b/__tests__/fixtures/golden/payroll-go.dump index 7012d043da..cd7381ff6a 100644 --- a/__tests__/fixtures/golden/payroll-go.dump +++ b/__tests__/fixtures/golden/payroll-go.dump @@ -1,4 +1,4 @@ -== nodes (313) +== nodes (309) {"id":"constant:03cf22f9cca66ee928171cceccbf44b1","kind":"constant","name":"LineEarning","qualified_name":"LineEarning","file_path":"internal/domain/payroll/payslip.go","language":"go","start_line":25,"end_line":25,"start_column":1,"end_column":35,"docstring":null,"signature":"= \"earning\"","visibility":null,"is_exported":1,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"constant:49f451a6950b1d7e90eb6cfe2776192b","kind":"constant","name":"LineDeduction","qualified_name":"LineDeduction","file_path":"internal/domain/payroll/payslip.go","language":"go","start_line":26,"end_line":26,"start_column":1,"end_column":37,"docstring":null,"signature":"= \"deduction\"","visibility":null,"is_exported":1,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"constant:56977e604248e3674f9296a2f68154bb","kind":"constant","name":"PayrollService_GetPayrollCycle_FullMethodName","qualified_name":"PayrollService_GetPayrollCycle_FullMethodName","file_path":"internal/gen/payrollpb/payroll_grpc.pb.go","language":"go","start_line":17,"end_line":17,"start_column":1,"end_column":93,"docstring":null,"signature":"= \"/payroll.v1.PayrollService/GetPayrollCycle\"","visibility":null,"is_exported":1,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} @@ -8,7 +8,7 @@ {"id":"constant:c62a1da458c5f390b2ac8d4662f81801","kind":"constant","name":"PayrollService_ListPayslips_FullMethodName","qualified_name":"PayrollService_ListPayslips_FullMethodName","file_path":"internal/gen/payrollpb/payroll_grpc.pb.go","language":"go","start_line":18,"end_line":18,"start_column":1,"end_column":90,"docstring":null,"signature":"= \"/payroll.v1.PayrollService/ListPayslips\"","visibility":null,"is_exported":1,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"constant:ccc3400ce237825d2c42ef5fe177fe35","kind":"constant","name":"CycleOpen","qualified_name":"CycleOpen","file_path":"internal/domain/payroll/payslip.go","language":"go","start_line":9,"end_line":9,"start_column":1,"end_column":33,"docstring":null,"signature":"= \"open\"","visibility":null,"is_exported":1,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"constant:e8d10fecacdc9d2de71a3e3f7cd39d77","kind":"constant","name":"CycleClosed","qualified_name":"CycleClosed","file_path":"internal/domain/payroll/payslip.go","language":"go","start_line":10,"end_line":10,"start_column":1,"end_column":35,"docstring":null,"signature":"= \"closed\"","visibility":null,"is_exported":1,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} -{"id":"file:69f63e0581b8138f256b63d2aac60e8a","kind":"file","name":"README.md","qualified_name":"README.md","file_path":"README.md","language":"markdown","start_line":1,"end_line":96,"start_column":0,"end_column":0,"docstring":"A synthetic Go service reproducing the repo shape from issue #1500: · refs: payslip.go, store.go, .pb.go, cycle.go, explore-allocation-1500.test.ts, payslip_builder.go, probe-allocation.mjs, allocation-fixtures.js","signature":null,"visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} +{"id":"file:69f63e0581b8138f256b63d2aac60e8a","kind":"file","name":"README.md","qualified_name":"README.md","file_path":"README.md","language":"markdown","start_line":1,"end_line":96,"start_column":0,"end_column":0,"docstring":"A synthetic Go service reproducing the repo shape from issue #1500: · refs: payslip.go, store.go, .pb.go, cycle.go, explore-allocation-1500.test.ts, payslip_builder.go, probe-allocation.mjs, allocation-fixtures.js · links: .pb.go, explore-allocation-1500.test.ts, allocation-fixtures.js, explore-budget-allocation.md","signature":null,"visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"file:cmd/payrolld/main.go","kind":"file","name":"main.go","qualified_name":"cmd/payrolld/main.go","file_path":"cmd/payrolld/main.go","language":"go","start_line":1,"end_line":36,"start_column":0,"end_column":0,"docstring":null,"signature":null,"visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"file:internal/domain/payroll/payslip.go","kind":"file","name":"payslip.go","qualified_name":"internal/domain/payroll/payslip.go","file_path":"internal/domain/payroll/payslip.go","language":"go","start_line":1,"end_line":160,"start_column":0,"end_column":0,"docstring":null,"signature":null,"visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"file:internal/gen/fkit/employee/contract.go","kind":"file","name":"contract.go","qualified_name":"internal/gen/fkit/employee/contract.go","file_path":"internal/gen/fkit/employee/contract.go","language":"go","start_line":1,"end_line":85,"start_column":0,"end_column":0,"docstring":null,"signature":null,"visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} @@ -116,7 +116,6 @@ {"id":"import:0bd6a21b585b4de298027fb9855ca26a","kind":"import","name":"github.com/example/payroll-svc/internal/domain/payroll","qualified_name":"github.com/example/payroll-svc/internal/domain/payroll","file_path":"internal/usecase/payroll/prorate.go","language":"go","start_line":6,"end_line":6,"start_column":1,"end_column":57,"docstring":null,"signature":"\"github.com/example/payroll-svc/internal/domain/payroll\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:1150c8c775775f9f8560aa873017d153","kind":"import","name":"sort","qualified_name":"sort","file_path":"internal/usecase/payroll/cycle.go","language":"go","start_line":7,"end_line":7,"start_column":1,"end_column":7,"docstring":null,"signature":"\"sort\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:1ef8f8dc72d476dd57190a572c1f19e8","kind":"import","name":"time","qualified_name":"time","file_path":"internal/gen/fkit/timesheet/timesheet.go","language":"go","start_line":11,"end_line":11,"start_column":1,"end_column":7,"docstring":null,"signature":"\"time\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} -{"id":"import:20c50882bdafb89a0b54258fa936c5b2","kind":"import","name":"allocation-fixtures.js","qualified_name":"README.md::path:scripts/agent-eval/allocation-fixtures.js:79:49","file_path":"README.md","language":"markdown","start_line":79,"end_line":79,"start_column":49,"end_column":90,"docstring":null,"signature":"scripts/agent-eval/allocation-fixtures.js","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:2341305377f0bafe152c137e9466d659","kind":"import","name":"log","qualified_name":"log","file_path":"cmd/payrolld/main.go","language":"go","start_line":4,"end_line":4,"start_column":1,"end_column":6,"docstring":null,"signature":"\"log\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:2b0654627b2a83f4fa3e7b86866491c0","kind":"import","name":"context","qualified_name":"context","file_path":"internal/gen/fkit/employee/contract.go","language":"go","start_line":9,"end_line":9,"start_column":1,"end_column":10,"docstring":null,"signature":"\"context\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:3d7629cf02a4de5b41d0223adb5f3070","kind":"import","name":"time","qualified_name":"time","file_path":"internal/gen/fkit/employee/employee.go","language":"go","start_line":11,"end_line":11,"start_column":1,"end_column":7,"docstring":null,"signature":"\"time\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} @@ -139,13 +138,11 @@ {"id":"import:65db04d07bd3a1a85e6702c070edbcf7","kind":"import","name":"context","qualified_name":"context","file_path":"internal/gen/fkit/employee/employee.go","language":"go","start_line":9,"end_line":9,"start_column":1,"end_column":10,"docstring":null,"signature":"\"context\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:66bc065f81b0782b430037fc979102cc","kind":"import","name":"time","qualified_name":"time","file_path":"internal/gen/fkit/payroll/payslip.go","language":"go","start_line":11,"end_line":11,"start_column":1,"end_column":7,"docstring":null,"signature":"\"time\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:68f937efbcbc4ebd082c1c11e72df0ab","kind":"import","name":"cycle.go","qualified_name":"README.md::path:cycle.go:49:50","file_path":"README.md","language":"markdown","start_line":49,"end_line":49,"start_column":50,"end_column":58,"docstring":null,"signature":"cycle.go","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} -{"id":"import:6a7e6476a09ea4596d045f1ae552a8f2","kind":"import","name":"explore-allocation-1500.test.ts","qualified_name":"README.md::path:__tests__/explore-allocation-1500.test.ts:52:15","file_path":"README.md","language":"markdown","start_line":52,"end_line":52,"start_column":15,"end_column":56,"docstring":null,"signature":"__tests__/explore-allocation-1500.test.ts","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:7147cac4b1745ea38b0bfa164a3a5468","kind":"import","name":"time","qualified_name":"time","file_path":"internal/gen/fkit/payroll/dto.go","language":"go","start_line":8,"end_line":8,"start_column":7,"end_column":13,"docstring":null,"signature":"\"time\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:7222e16172e240000e603eceb98f94f9","kind":"import","name":"database/sql","qualified_name":"database/sql","file_path":"internal/gen/fkit/payroll/payslip.go","language":"go","start_line":10,"end_line":10,"start_column":1,"end_column":15,"docstring":null,"signature":"\"database/sql\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:76a964b6134242af279f95c049ac84c5","kind":"import","name":"store.go","qualified_name":"README.md::path:internal/gen/fkit/payroll/store.go:87:56","file_path":"README.md","language":"markdown","start_line":87,"end_line":87,"start_column":56,"end_column":90,"docstring":null,"signature":"internal/gen/fkit/payroll/store.go","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:7afb422fcf48196342c2d44ba9df65b4","kind":"import","name":"context","qualified_name":"context","file_path":"internal/gen/fkit/payroll/store.go","language":"go","start_line":9,"end_line":9,"start_column":1,"end_column":10,"docstring":null,"signature":"\"context\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:85d6602214f6381a034f6034da409655","kind":"import","name":"database/sql","qualified_name":"database/sql","file_path":"internal/gen/fkit/employee/contract.go","language":"go","start_line":10,"end_line":10,"start_column":1,"end_column":15,"docstring":null,"signature":"\"database/sql\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} -{"id":"import:8808ae056445d4f7fae22fa62685c9a4","kind":"import","name":".pb.go","qualified_name":"README.md::path:.pb.go:41:15","file_path":"README.md","language":"markdown","start_line":41,"end_line":41,"start_column":15,"end_column":21,"docstring":null,"signature":".pb.go","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:8a55768bc75a7d6e26bd621aa388263c","kind":"import","name":"time","qualified_name":"time","file_path":"internal/gen/fkit/payroll/payroll_cycle.go","language":"go","start_line":11,"end_line":11,"start_column":1,"end_column":7,"docstring":null,"signature":"\"time\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:8ca292ed8417acda5c41ff30467e3188","kind":"import","name":"google.golang.org/protobuf/runtime/protoimpl","qualified_name":"google.golang.org/protobuf/runtime/protoimpl","file_path":"internal/gen/payrollpb/payroll.pb.go","language":"go","start_line":11,"end_line":11,"start_column":1,"end_column":57,"docstring":null,"signature":"protoimpl \"google.golang.org/protobuf/runtime/protoimpl\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:94d6b9da5b862536535fdb747dd8ef41","kind":"import","name":"google.golang.org/protobuf/reflect/protoreflect","qualified_name":"google.golang.org/protobuf/reflect/protoreflect","file_path":"internal/gen/payrollpb/payroll.pb.go","language":"go","start_line":10,"end_line":10,"start_column":1,"end_column":63,"docstring":null,"signature":"protoreflect \"google.golang.org/protobuf/reflect/protoreflect\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} @@ -171,7 +168,6 @@ {"id":"import:e715312ec518c465211ed82ddce82285","kind":"import","name":"fmt","qualified_name":"fmt","file_path":"internal/usecase/payroll/payslip_builder.go","language":"go","start_line":5,"end_line":5,"start_column":1,"end_column":6,"docstring":null,"signature":"\"fmt\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:e9cab8103939c5ea907234285e597639","kind":"import","name":"time","qualified_name":"time","file_path":"internal/usecase/payroll/cycle.go","language":"go","start_line":8,"end_line":8,"start_column":1,"end_column":7,"docstring":null,"signature":"\"time\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:ee0a17b804d64444eaa8643521a20d48","kind":"import","name":"net/http","qualified_name":"net/http","file_path":"internal/transport/httpapi/router.go","language":"go","start_line":3,"end_line":3,"start_column":7,"end_column":17,"docstring":null,"signature":"\"net/http\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} -{"id":"import:f46fb6a4705f81701f5054f85a18d37f","kind":"import","name":"explore-budget-allocation.md","qualified_name":"README.md::path:docs/design/explore-budget-allocation.md:82:1","file_path":"README.md","language":"markdown","start_line":82,"end_line":82,"start_column":1,"end_column":41,"docstring":null,"signature":"docs/design/explore-budget-allocation.md","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:f779e8d1ccdf9fbef128be48f1cabe14","kind":"import","name":"errors","qualified_name":"errors","file_path":"internal/usecase/payroll/cycle.go","language":"go","start_line":5,"end_line":5,"start_column":1,"end_column":9,"docstring":null,"signature":"\"errors\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:fa23f47e6bf27f154f8044ad15346d18","kind":"import","name":"store.go","qualified_name":"README.md::path:store.go:38:45","file_path":"README.md","language":"markdown","start_line":38,"end_line":38,"start_column":45,"end_column":53,"docstring":null,"signature":"store.go","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"import:fe70ccaba78866b57f5239f961d3ea11","kind":"import","name":"fmt","qualified_name":"fmt","file_path":"internal/usecase/payroll/cycle.go","language":"go","start_line":6,"end_line":6,"start_column":1,"end_column":6,"docstring":null,"signature":"\"fmt\"","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} @@ -312,7 +308,7 @@ {"id":"variable:6b7bef84822056845bdabeb41a821f0a","kind":"variable","name":"ErrCycleClosed","qualified_name":"ErrCycleClosed","file_path":"internal/usecase/payroll/cycle.go","language":"go","start_line":16,"end_line":16,"start_column":4,"end_column":58,"docstring":"ErrCycleClosed is returned when a cycle has already been finalized.","signature":"= errors.New(\"payroll cycle is closed\")","visibility":null,"is_exported":1,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"variable:9c3a0c9681a69504be5d3af6ddb5378c","kind":"variable","name":"file_payroll_v1_payroll_proto_goTypes","qualified_name":"file_payroll_v1_payroll_proto_goTypes","file_path":"internal/gen/payrollpb/payroll.pb.go","language":"go","start_line":132,"end_line":137,"start_column":4,"end_column":1,"docstring":null,"signature":"= []any{\n\t(*RunPayrollCycleRequest)(nil),\n\t(*RunPayrollCycleResponse)(nil),\n\t(*Payslip)(nil),\n\t(*Payro...","visibility":null,"is_exported":0,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} {"id":"variable:baa4aee814b380ae99f1b1829407b106","kind":"variable","name":"ErrNoEmployees","qualified_name":"ErrNoEmployees","file_path":"internal/usecase/payroll/cycle.go","language":"go","start_line":19,"end_line":19,"start_column":4,"end_column":72,"docstring":"ErrNoEmployees is returned when a cycle resolves to an empty roster.","signature":"= errors.New(\"payroll cycle has no active employees\")","visibility":null,"is_exported":1,"is_async":0,"is_static":0,"is_abstract":0,"decorators":null,"type_parameters":null,"return_type":null} -== edges (657) +== edges (653) {"source":"file:69f63e0581b8138f256b63d2aac60e8a","target":"module:1be454da0953954aef45c525ee8f8b51","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"file:cmd/payrolld/main.go","target":"function:90eb51f51a3d9a397d4a53f26f2019aa","kind":"contains","metadata":null,"line":null,"col":null,"provenance":null} {"source":"file:cmd/payrolld/main.go","target":"import:0a9ee69fa4c64208200c3bebcbbf0b75","kind":"contains","metadata":null,"line":null,"col":null,"provenance":null} @@ -847,8 +843,6 @@ {"source":"module:1be454da0953954aef45c525ee8f8b51","target":"module:f8d63ed69e573ade27e2d1d4fc0ae8b4","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"module:39730d6601ee76ddcd5e78d50abfe5a2","target":"import:526c9118c077229729a7b9846c16882a","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"module:39730d6601ee76ddcd5e78d50abfe5a2","target":"import:68f937efbcbc4ebd082c1c11e72df0ab","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} -{"source":"module:39730d6601ee76ddcd5e78d50abfe5a2","target":"import:6a7e6476a09ea4596d045f1ae552a8f2","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} -{"source":"module:39730d6601ee76ddcd5e78d50abfe5a2","target":"import:8808ae056445d4f7fae22fa62685c9a4","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"module:39730d6601ee76ddcd5e78d50abfe5a2","target":"import:fa23f47e6bf27f154f8044ad15346d18","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"module:900ad55709598429050c36805696291a","target":"import:44b0fa4765bb8c262fb8b290698b9c05","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"module:900ad55709598429050c36805696291a","target":"import:e2b363aa11b1fe2c65cb016ea68a0948","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} @@ -872,8 +866,6 @@ {"source":"module:a883d05f3bb8edb262fd18c8c2849417","target":"function:4827032314863eb2b99f791098f8668c","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"module:a883d05f3bb8edb262fd18c8c2849417","target":"function:8e554156c22d75e629b346ea4ed215d1","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"module:a883d05f3bb8edb262fd18c8c2849417","target":"function:c2b743a41c78c6d39eb292c4ecd1138d","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} -{"source":"module:a883d05f3bb8edb262fd18c8c2849417","target":"import:20c50882bdafb89a0b54258fa936c5b2","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} -{"source":"module:a883d05f3bb8edb262fd18c8c2849417","target":"import:f46fb6a4705f81701f5054f85a18d37f","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"module:f8d63ed69e573ade27e2d1d4fc0ae8b4","target":"import:76a964b6134242af279f95c049ac84c5","kind":"contains","metadata":null,"line":null,"col":null,"provenance":"heuristic"} {"source":"route:internal/transport/httpapi/router.go:10:GET:/v1/payroll/cycles/{cycleID}","target":"method:8baae5c5fb59db7a88c19bb295635d18","kind":"references","metadata":"{\"confidence\":0.9,\"resolvedBy\":\"exact-match\",\"refName\":\"GetCycle\"}","line":10,"col":0,"provenance":null} {"source":"route:internal/transport/httpapi/router.go:11:GET:/v1/payroll/cycles/{cycleID}/payslips","target":"method:6e0fe8d0272ee955601347cf37b5a383","kind":"references","metadata":"{\"confidence\":0.7,\"resolvedBy\":\"exact-match\",\"refName\":\"ListPayslips\"}","line":11,"col":0,"provenance":null} @@ -2282,7 +2274,7 @@ {"file_path":"internal/usecase/payroll/prorate.go","name":"to","kind":"param","node_id":null,"target_spec":null,"target_name":null,"exported_as":null,"export_form":null,"scope_start":34,"scope_end":39,"storage":null,"line":34} {"file_path":"internal/usecase/payroll/prorate.go","name":"window","kind":"local","node_id":null,"target_spec":null,"target_name":null,"exported_as":null,"export_form":null,"scope_start":11,"scope_end":24,"storage":null,"line":12} == files (21) -{"path":"README.md","language":"markdown","node_count":37} +{"path":"README.md","language":"markdown","node_count":33} {"path":"cmd/payrolld/main.go","language":"go","node_count":10} {"path":"internal/domain/payroll/payslip.go","language":"go","node_count":25} {"path":"internal/gen/fkit/employee/contract.go","language":"go","node_count":11} diff --git a/__tests__/integration/full-pipeline.test.ts b/__tests__/integration/full-pipeline.test.ts index 6a2fca222f..6d32099019 100644 --- a/__tests__/integration/full-pipeline.test.ts +++ b/__tests__/integration/full-pipeline.test.ts @@ -131,6 +131,45 @@ node scripts/release.mjs } }); + it('resolves a full-index Markdown link after sync adds its target', async () => { + fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'README.md'), + `# Guide + +## Links + +See [Setup](docs/setup.md#install). +` + ); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + // The target is absent: the reference is kept (parked as failed), but no + // edge exists yet. + expect(cg.getNodesInFile('docs/setup.md')).toEqual([]); + const linksHeading = cg + .getNodesInFile('README.md') + .find((n) => n.kind === 'module' && n.name === 'Links')!; + expect( + cg.getOutgoingEdges(linksHeading.id) + .filter((e) => e.kind === 'imports') + .map((e) => cg.getNode(e.target)?.qualifiedName) + ).not.toContain('docs/setup.md#install'); + + fs.writeFileSync(path.join(tempDir, 'docs', 'setup.md'), '# Install\n\nInstall it.\n'); + await cg.sync(); + + const install = cg + .getNodesInFile('docs/setup.md') + .find((n) => n.kind === 'module' && n.name === 'Install')!; + expect(cg.getIncomingEdges(install.id, ['imports']).some((e) => e.source === linksHeading.id)).toBe(true); + } finally { + cg.destroy(); + } + }); + it('indexes Markdown template tables and resolves file-symbol references to implementation functions', async () => { fs.mkdirSync(path.join(tempDir, 'phases'), { recursive: true }); fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); diff --git a/__tests__/markdown-path-refs.test.ts b/__tests__/markdown-path-refs.test.ts new file mode 100644 index 0000000000..3a65d179cd --- /dev/null +++ b/__tests__/markdown-path-refs.test.ts @@ -0,0 +1,121 @@ +/** + * A Markdown path target whose basename no indexed file has cannot resolve to + * a file through the resolver's file-path arm. Its reference is still kept — + * the resolver's route/module/symbol arms may match it, and sync can retry it + * when a file appears later (#1240) — but its own import node only added a + * search child and a `contains` edge. A full index skips that node; sync keeps + * every node. Pruned targets stay in a separate, larger `links:` part of the + * file digest so a link beyond the 12-item display digest is still searchable. + */ +import { describe, it, expect } from 'vitest'; +import { extractFromSource } from '../src/extraction/tree-sitter'; +import type { MarkdownExtractorContext } from '../src/extraction/markdown-extractor'; + +function context(prune: boolean, ...files: string[]): MarkdownExtractorContext { + return { + knownFileNames: new Set(files.map((file) => file.split('/').pop()!)), + pruneMissingTargets: prune, + }; +} + +const FULL_INDEX = context(true, 'docs/guide.md', 'src/widget.ts', 'scripts/exists.sh'); +const SYNC = context(false, 'docs/guide.md', 'src/widget.ts', 'scripts/exists.sh'); + +const SOURCE = [ + '# Guide', + '', + 'Preamble about `createWidget`.', + '', + '## Links', + '', + '- [widget](../src/widget.ts)', + '- [missing](../src/missing.ts)', + '- [same name, other path](../apps/widget.ts#usage)', + '- [pricing](/pricing)', + '- [users route](/api/users/)', + '- [guide route](../guides/auth)', + '- [dotted route](api/users.json)', + '- [one](../src/one.ts)', + '- [two](../src/two.ts)', + '- [three](../src/three.ts)', + '- [four](../src/four.ts)', + '- [five](../src/five.ts)', + '- [six](../src/six.ts)', + '- [seven](../src/seven.ts)', + '- [eight](../src/eight.ts)', + '- [nine](../src/nine.ts)', + '- [ten](../src/ten.ts)', + '- [eleven](../src/eleven.ts)', + '- [twelve](../src/twelve.ts)', + '- [diagram](../assets/releaseblueprint.svg)', + '- [this section](#links)', + '', + 'Mention `scripts/run.sh`, `scripts/widget.ts` and `src/widget.ts::createWidget`.', + 'A missing symbol target: `src/missing.ts::createWidget`.', + '', + '```sh', + 'bash scripts/missing.sh', + 'bash scripts/exists.sh', + '```', +].join('\n'); + +function extract(withContext?: MarkdownExtractorContext) { + return extractFromSource('docs/guide.md', SOURCE, 'markdown', [], withContext); +} + +function refs(withContext?: MarkdownExtractorContext): string[] { + return extract(withContext).unresolvedReferences.map((ref) => ref.referenceName); +} + +function importTargets(withContext?: MarkdownExtractorContext): string[] { + return extract(withContext) + .nodes.filter((node) => node.kind === 'import') + .map((node) => node.qualifiedName); +} + +describe('Markdown link-node pruning', () => { + it('keeps every reference and link node without a context', () => { + const names = refs(); + expect(names).toContain('src/widget.ts'); + expect(names).toContain('src/missing.ts'); + expect(names).toContain('docs/scripts/run.sh'); + expect(importTargets().some((qn) => qn.includes('missing.ts'))).toBe(true); + }); + + it('keeps the reference but skips the link node on a full index', () => { + const names = refs(FULL_INDEX); + // The reference survives so routes, modules and later syncs can match it. + expect(names).toContain('src/missing.ts'); + expect(names).toContain('docs/scripts/run.sh'); + expect(names).toContain('docs/scripts/missing.sh'); + // The dead target's import node does not exist. + expect(importTargets(FULL_INDEX).some((qn) => qn.includes('missing.ts'))).toBe(false); + // A target whose basename is known keeps its node. + expect(importTargets(FULL_INDEX).some((qn) => qn.includes('src/widget.ts'))).toBe(true); + }); + + it('keeps every link node on an incremental sync', () => { + expect(importTargets(SYNC).some((qn) => qn.includes('missing.ts'))).toBe(true); + }); + + it('keeps routes and targets the resolver reaches by name', () => { + const names = refs(FULL_INDEX); + expect(names).toContain('apps/widget.ts#usage'); + expect(names).toContain('docs/scripts/widget.ts'); + expect(names).toContain('pricing'); + expect(names).toContain('api/users/'); + expect(names).toContain('guides/auth'); + // A relative route with a dotted final segment is kept for name matching. + expect(names).toContain('docs/api/users.json'); + expect(names).toContain('docs/src/widget.ts::createWidget'); + expect(names).toContain('docs/guide.md#links'); + expect(names).toContain('docs/scripts/exists.sh'); + }); + + it('keeps a pruned target searchable beyond the display digest budget', () => { + const fileNode = extract(FULL_INDEX).nodes.find((node) => node.kind === 'file'); + // The target is the 13th link, past the 12-item `refs:` budget. + expect(fileNode?.docstring).toContain('links:'); + expect(fileNode?.docstring).toContain('releaseblueprint.svg'); + }); +}); diff --git a/src/extraction/index.ts b/src/extraction/index.ts index ce05f8e208..640219bd6a 100644 --- a/src/extraction/index.ts +++ b/src/extraction/index.ts @@ -2402,6 +2402,24 @@ export class ExtractionOrchestrator { }, skipStats); this.prepareLiquidThemeRoots(files); if (process.env.CODEGRAPH_SYNTH_TIMINGS) console.error(`[phase-timing] scan: ${Date.now() - tScan}ms (${files.length} files)`); + + // Markdown link/path targets resolve through the file-path arm, which + // collects candidates by basename. Hand extraction that set so a path + // mention whose target file is absent creates no import node and no + // failed reference. Existing DB rows are included on a re-index so a ref + // whose target is still tracked (but gone from disk) resolves as before. + const knownFileNames = new Set(); + const rememberBasename = (file: string): void => { + const cut = Math.max(file.lastIndexOf('/'), file.lastIndexOf('\\')); + knownFileNames.add(cut >= 0 ? file.slice(cut + 1) : file); + }; + for (const file of files) rememberBasename(file); + for (const file of this.queries.getAllFilePaths()) rememberBasename(file); + // A full index has the whole project in hand, so a relative file target + // with no matching basename is dead. An incremental sync re-extracts one + // file at a time and must keep the reference for a target added later + // (#1240), so indexFile() passes no context and keeps every target. + const extractionContext = { knownFileNames, pruneMissingTargets: true }; /** Only meaningful when nothing was indexable — see IndexResult (#1502). */ const skipSummary = (): Pick => { let total = 0; @@ -2498,6 +2516,7 @@ export class ExtractionOrchestrator { recycleInterval: WORKER_RECYCLE_INTERVAL, parseTimeoutMs: PARSE_TIMEOUT_MS, log, + sharedContext: extractionContext, }); log(`Parse worker pool: ${poolSize} worker(s)`); // Bulk index: every core will be needed — spawn the whole pool now so @@ -2538,7 +2557,7 @@ export class ExtractionOrchestrator { const parseFile = (filePath: string, content: string): Promise => { const language = detectLanguage(filePath, content, overrides, this.rootDir); const names = this.frameworksForFile(filePath, frameworkNames); - if (!pool) return Promise.resolve(extractFromSource(filePath, content, language, names)); + if (!pool) return Promise.resolve(extractFromSource(filePath, content, language, names, extractionContext)); return pool.requestParse({ filePath, content, language, frameworkNames: names }); }; diff --git a/src/extraction/markdown-extractor.ts b/src/extraction/markdown-extractor.ts index 4695ea4be4..7037e14ec8 100644 --- a/src/extraction/markdown-extractor.ts +++ b/src/extraction/markdown-extractor.ts @@ -19,6 +19,27 @@ interface FenceState { lines: Array<{ text: string; line: number }>; } +/** Extraction inputs the Markdown extractor needs beyond the file itself. */ +export interface MarkdownExtractorContext { + /** + * Basenames of every file the index contains. Markdown path mentions and + * links resolve through the file-path arm, which looks candidates up by + * basename; a target whose basename is absent cannot resolve to a file. + * Omitted by callers without an index (unit tests, single-file tooling), + * which keeps the old permissive behaviour. + */ + knownFileNames?: ReadonlySet; + /** + * Skip the per-link import node for a target whose basename no indexed file + * has. Set only for a full project index. The reference is always kept: the + * resolver still tries the target's route/module/symbol arms and sync can + * retry a failed reference when a file appears later (#1240). The link node + * is decoration (a search child and a `contains` edge), so dropping it + * loses no edge. Without this flag every target gets its node, as before. + */ + pruneMissingTargets?: boolean; +} + /** * Lightweight Markdown extractor. * @@ -35,10 +56,20 @@ export class MarkdownExtractor { private errors: ExtractionError[] = []; private headings: HeadingInfo[] = []; private referenceKeys = new Set(); + private knownFileNames?: ReadonlySet; + private pruneMissingTargets: boolean; + /** + * Targets whose import node was skipped. Their references are still stored; + * this list feeds a separate, larger `links:` part of the file digest so a + * link beyond the display digest's 12-item budget still stays searchable. + */ + private prunedTargets: string[] = []; - constructor(filePath: string, source: string) { + constructor(filePath: string, source: string, context: MarkdownExtractorContext = {}) { this.filePath = normalizeRelativePath(filePath); this.lines = source.split('\n'); + this.knownFileNames = context.knownFileNames; + this.pruneMissingTargets = context.pruneMissingTargets === true; } extract(): ExtractionResult { @@ -430,27 +461,31 @@ export class MarkdownExtractor { if (!normalizedTarget) continue; const column = match.index; - const label = stripInlineMarkdown(match[1] || target) || target; - const displayName = path.posix.basename(stripAnchor(normalizedTarget)) || label; - const nodeId = generateNodeId(this.filePath, 'import', `${normalizedTarget}:${lineNumber}:${column}`, lineNumber); - const node: Node = { - id: nodeId, - kind: 'import', - name: displayName, - qualifiedName: `${this.filePath}::link:${normalizedTarget}:${lineNumber}:${column}`, - filePath: this.filePath, - language: 'markdown', - signature: match[0], - docstring: label, - startLine: lineNumber, - endLine: lineNumber, - startColumn: column, - endColumn: column + match[0].length, - updatedAt: Date.now(), - }; + if (this.shouldCreateLinkNode(normalizedTarget)) { + const label = stripInlineMarkdown(match[1] || target) || target; + const displayName = path.posix.basename(stripAnchor(normalizedTarget)) || label; + const nodeId = generateNodeId(this.filePath, 'import', `${normalizedTarget}:${lineNumber}:${column}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'import', + name: displayName, + qualifiedName: `${this.filePath}::link:${normalizedTarget}:${lineNumber}:${column}`, + filePath: this.filePath, + language: 'markdown', + signature: match[0], + docstring: label, + startLine: lineNumber, + endLine: lineNumber, + startColumn: column, + endColumn: column + match[0].length, + updatedAt: Date.now(), + }; - this.nodes.push(node); - this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + } else { + this.prunedTargets.push(normalizedTarget); + } this.addReference(owner.id, normalizedTarget, 'imports', lineNumber, column); } } @@ -466,24 +501,28 @@ export class MarkdownExtractor { const normalizedTarget = this.normalizeReference(raw); if (!normalizedTarget) continue; - const nodeId = generateNodeId(this.filePath, 'import', `${normalizedTarget}:${lineNumber}:${match.index}`, lineNumber); - const node: Node = { - id: nodeId, - kind: 'import', - name: path.posix.basename(stripAnchor(normalizedTarget)) || normalizedTarget, - qualifiedName: `${this.filePath}::path:${normalizedTarget}:${lineNumber}:${match.index}`, - filePath: this.filePath, - language: 'markdown', - signature: raw, - startLine: lineNumber, - endLine: lineNumber, - startColumn: match.index, - endColumn: match.index + raw.length, - updatedAt: Date.now(), - }; + if (this.shouldCreateLinkNode(normalizedTarget)) { + const nodeId = generateNodeId(this.filePath, 'import', `${normalizedTarget}:${lineNumber}:${match.index}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'import', + name: path.posix.basename(stripAnchor(normalizedTarget)) || normalizedTarget, + qualifiedName: `${this.filePath}::path:${normalizedTarget}:${lineNumber}:${match.index}`, + filePath: this.filePath, + language: 'markdown', + signature: raw, + startLine: lineNumber, + endLine: lineNumber, + startColumn: match.index, + endColumn: match.index + raw.length, + updatedAt: Date.now(), + }; - this.nodes.push(node); - this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + } else { + this.prunedTargets.push(normalizedTarget); + } this.addReference(owner.id, normalizedTarget, 'references', lineNumber, match.index); } } @@ -500,8 +539,10 @@ export class MarkdownExtractor { for (const symbol of symbols) { const normalizedTarget = this.normalizeReference(`${filePart}::${symbol}`); if (!normalizedTarget) continue; + const createNode = createImportNode && this.shouldCreateLinkNode(normalizedTarget); + if (createImportNode && !createNode) this.prunedTargets.push(normalizedTarget); - if (createImportNode) { + if (createNode) { const nodeId = generateNodeId(this.filePath, 'import', `${normalizedTarget}:${lineNumber}:${match.index}`, lineNumber); const node: Node = { id: nodeId, @@ -554,9 +595,8 @@ export class MarkdownExtractor { const raw = match[0]!; if (text.slice(match.index + raw.length, match.index + raw.length + 2) === '::') continue; const normalizedTarget = this.normalizeReference(raw); - if (normalizedTarget) { - this.addReference(owner.id, normalizedTarget, 'references', lineNumber, match.index); - } + if (!normalizedTarget) continue; + this.addReference(owner.id, normalizedTarget, 'references', lineNumber, match.index); } } @@ -622,6 +662,37 @@ export class MarkdownExtractor { return `${normalized}${symbolPart ? `::${symbolPart}` : ''}${anchor}`; } + /** + * Whether a target should get its own import node on a full index. The + * resolver's file-path arm collects candidates by basename, so a target + * whose basename no file node has cannot resolve to a file. Its reference + * is still recorded (the resolver's route/module/symbol arms may match it, + * and sync can retry it when a file appears), but the link node only added a + * search child and a `contains` edge. Without a known-file set, or outside a + * full index, every target keeps its node. + */ + private shouldCreateLinkNode(normalizedTarget: string): boolean { + if (!this.pruneMissingTargets || !this.knownFileNames) return true; + const [pathAndSymbol] = splitAnchor(normalizedTarget); + const [pathPart] = splitFileSymbol(pathAndSymbol); + if (!pathPart || pathPart === this.filePath) return true; + return this.knownFileNames.has(path.posix.basename(pathPart)); + } + + /** Pruned link targets for the file digest's separate `links:` part. */ + private summarizePrunedTargets(): string[] { + const seen = new Set(); + const out: string[] = []; + for (const target of this.prunedTargets) { + const display = compactRefDisplay(target); + if (!display || seen.has(display)) continue; + seen.add(display); + out.push(display); + if (out.length >= 64) break; + } + return out; + } + private buildDocstring(startLine: number, endLine: number): string | undefined { const text = this.lines .slice(Math.max(0, startLine - 1), Math.max(0, endLine)) @@ -659,9 +730,15 @@ export class MarkdownExtractor { private finalizeFileDigest(fileNode: Node): void { const intro = this.buildIntro(); const refs = this.collectKeyReferences(); + const links = this.summarizePrunedTargets(); const parts: string[] = []; if (intro) parts.push(intro); if (refs.length > 0) parts.push(`refs: ${refs.join(', ')}`); + if (links.length > 0) { + let text = links.join(', '); + if (text.length > 800) text = `${text.slice(0, 797).trimEnd()}...`; + parts.push(`links: ${text}`); + } const digest = parts.join(' · ').trim(); fileNode.docstring = digest || this.buildDocstring(1, Math.min(this.lines.length, 40)); } @@ -698,18 +775,20 @@ export class MarkdownExtractor { /** * The documents/symbols this file points at, de-duplicated and compacted * (basename + `::symbol`/`#anchor`) in document order. Self-anchors and pure - * in-page links are dropped. Bounded so the whole digest stays short enough - * to display. + * in-page links are dropped. Includes targets pruned from the graph, so a + * link to an unindexed asset still makes the file findable. Bounded so the + * whole digest stays short enough to display. */ private collectKeyReferences(maxItems = 12, maxChars = 150): string[] { const seen = new Set(); const out: string[] = []; let budget = maxChars; for (const ref of this.unresolvedReferences) { - const [pathSym] = splitAnchor(ref.referenceName); + const referenceName = ref.referenceName; + const [pathSym] = splitAnchor(referenceName); const [pathPart] = splitFileSymbol(pathSym); if (!pathPart || pathPart === this.filePath) continue; // pure/self anchor - const display = compactRefDisplay(ref.referenceName); + const display = compactRefDisplay(referenceName); if (!display || seen.has(display)) continue; if (out.length >= maxItems) break; if (out.length > 0 && budget - display.length - 2 < 0) break; diff --git a/src/extraction/parse-pool.ts b/src/extraction/parse-pool.ts index 383666ae58..05b3ae8091 100644 --- a/src/extraction/parse-pool.ts +++ b/src/extraction/parse-pool.ts @@ -22,6 +22,7 @@ import { Worker } from 'worker_threads'; import type { Language, ExtractionResult } from '../types'; +import type { MarkdownExtractorContext } from './markdown-extractor'; /** * Minimal worker surface the pool drives — satisfied by a real `worker_threads` @@ -210,6 +211,11 @@ export interface ParseWorkerPoolOptions { loadSettleMs?: number; /** Optional verbose logger (the orchestrator's `[worker] …` logger). */ log?: (msg: string) => void; + /** + * Extraction inputs every worker needs before its first parse (the + * index's known file basenames). Sent once per spawn, not per parse. + */ + sharedContext?: MarkdownExtractorContext; } export class ParseWorkerPool { @@ -236,6 +242,7 @@ export class ParseWorkerPool { private readonly createWorker: () => ParsePoolWorker; private readonly loadSettleMs: number; private readonly log: (msg: string) => void; + private readonly sharedContext?: MarkdownExtractorContext; constructor(opts: ParseWorkerPoolOptions) { this.languages = opts.languages; @@ -244,6 +251,7 @@ export class ParseWorkerPool { this.parseTimeoutMs = opts.parseTimeoutMs ?? DEFAULT_PARSE_TIMEOUT_MS; this.loadSettleMs = opts.loadSettleMs ?? GRAMMAR_LOAD_SETTLE_MS; this.log = opts.log ?? (() => {}); + this.sharedContext = opts.sharedContext; if (opts.createWorker) { this.createWorker = opts.createWorker; } else if (opts.workerScriptPath) { @@ -320,7 +328,16 @@ export class ParseWorkerPool { w.on('error', (e) => { this.loadSettled(w); this.onWorkerGone(w, `${WORKER_DIED}: ${e?.message ?? 'unknown error'}`); }); w.on('exit', (code) => { this.loadSettled(w); if (code !== 0) this.onWorkerGone(w, `${WORKER_DIED}: exited with code ${code}`); }); // Readiness handshake; the worker replies 'grammars-loaded' and only then is idle. - w.postMessage({ type: 'load-grammars', languages: this.languages }); + // The shared extraction context rides along, so a recycled worker gets it + // again without a per-parse clone. + w.postMessage({ + type: 'load-grammars', + languages: this.languages, + knownFileNames: this.sharedContext?.knownFileNames + ? [...this.sharedContext.knownFileNames] + : undefined, + pruneMissingTargets: this.sharedContext?.pruneMissingTargets, + }); } private onMessage(w: ParsePoolWorker, m: ParseWorkerMessage): void { diff --git a/src/extraction/parse-worker.ts b/src/extraction/parse-worker.ts index 4e30c35928..76bf578200 100644 --- a/src/extraction/parse-worker.ts +++ b/src/extraction/parse-worker.ts @@ -22,11 +22,20 @@ import { applyFrameworkExtracts } from './framework-extracts'; import { mayDefineCppMacroMethods } from './languages/cpp-macro-methods'; import type { Language, ExtractionResult } from '../types'; -parentPort!.on('message', async (msg: { type: string; id?: number; filePath?: string; content?: string; languages?: Language[]; frameworkNames?: string[]; language?: Language }) => { +/** + * The index's known file basenames, for the Markdown extractor's path check. + * Sent once per worker spawn (the pool re-sends it to a recycled worker). + */ +let knownFileNames: Set | undefined; +let pruneMissingTargets: boolean | undefined; + +parentPort!.on('message', async (msg: { type: string; id?: number; filePath?: string; content?: string; languages?: Language[]; frameworkNames?: string[]; language?: Language; knownFileNames?: string[]; pruneMissingTargets?: boolean }) => { if (msg.type === 'load-grammars') { // Nothing to load since the wasm path was removed: grammars live in the // kernel binary. The handshake stays so the pool's readiness model // (spawned → ready → idle) is unchanged. + knownFileNames = msg.knownFileNames ? new Set(msg.knownFileNames) : undefined; + pruneMissingTargets = msg.pruneMissingTargets; parentPort!.postMessage({ type: 'grammars-loaded' }); } else if (msg.type === 'parse') { const { id, filePath, content, frameworkNames } = msg; @@ -67,7 +76,15 @@ parentPort!.on('message', async (msg: { type: string; id?: number; filePath?: st applyFrameworkExtracts(result, filePath!, content!, language, frameworkNames); } } - result ??= extractFromSource(filePath!, content!, language, frameworkNames); + result ??= extractFromSource( + filePath!, + content!, + language, + frameworkNames, + knownFileNames || pruneMissingTargets + ? { knownFileNames, pruneMissingTargets } + : undefined + ); parentPort!.postMessage({ type: 'parse-result', id, result, parseMs: performance.now() - t0 }); } catch (err) { diff --git a/src/extraction/tree-sitter.ts b/src/extraction/tree-sitter.ts index e4c043e633..1924540b6f 100644 --- a/src/extraction/tree-sitter.ts +++ b/src/extraction/tree-sitter.ts @@ -39,7 +39,7 @@ import { DfmExtractor } from './dfm-extractor'; import { PASCAL_SYSTEM_NAMES } from './languages/pascal-system'; import { VueExtractor } from './vue-extractor'; import { MyBatisExtractor } from './mybatis-extractor'; -import { MarkdownExtractor } from './markdown-extractor'; +import { MarkdownExtractor, type MarkdownExtractorContext } from './markdown-extractor'; import { CfmlExtractor } from './cfml-extractor'; import { tryKernelExtract, takeDeferredPreParse, tryKernelBindings, attachBindingNodeIds } from './kernel'; import { captureLiterals } from './literal-capture'; @@ -9857,7 +9857,8 @@ export function extractFromSource( filePath: string, source: string, language?: Language, - frameworkNames?: string[] + frameworkNames?: string[], + context?: MarkdownExtractorContext ): ExtractionResult { const detectedLanguage = language || detectLanguage(filePath, source); const fileExtension = path.extname(filePath).toLowerCase(); @@ -9891,7 +9892,7 @@ export function extractFromSource( result = extractor.extract(); } else if (detectedLanguage === 'markdown') { // Use custom extractor for Markdown documentation structure and references - const extractor = new MarkdownExtractor(filePath, source); + const extractor = new MarkdownExtractor(filePath, source, context); result = extractor.extract(); } else if (detectedLanguage === 'cfml' || detectedLanguage === 'cfscript') { // Custom extractor for CFML (.cfc/.cfm) — dialect-switches between the