diff --git a/__tests__/explore-doc-tier-named.test.ts b/__tests__/explore-doc-tier-named.test.ts new file mode 100644 index 0000000000..7d6bd54711 --- /dev/null +++ b/__tests__/explore-doc-tier-named.test.ts @@ -0,0 +1,127 @@ +/** + * The doc tier's naming path: a query that IS a document's name. + * + * `collectDocSeeds` honoured `named` at the file gate and ignored it at the + * section gate, so `CONTRIBUTING.md` found the file and then dropped it for + * want of a scoring line β€” the user typed a filename and got nothing. Three + * rules combined to make every section score 0 for such a query: a term + * appearing in the file path is weighted 0 (and for a bare filename that is + * the only term), `lineScore` needs two distinct terms on one line, and + * `coveredHeading` needs a term of non-zero weight. + * + * A fourth rule made some headings unreachable by any query at all: + * `coveredHeading` demanded every significant heading word be covered, but + * DOC_QUERY_NOISE words are stripped from the query, so a heading containing + * "repo" could never be covered. + * + * These are retrieval assertions, not extraction ones β€” the misses that + * motivated them were found by querying a real 84-file repo, which is exactly + * what no unit test here was doing. + */ +import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as os from 'os'; +import CodeGraph from '../src/index'; +import { ToolHandler } from '../src/mcp/tools'; + +let dir: string; +let cg: CodeGraph; + +async function explore(query: string): Promise { + const res = await new ToolHandler(cg).execute('codegraph_explore', { query }); + return res.content?.[0]?.text ?? ''; +} + +/** The response renders a source section for `file`. */ +const hasSection = (response: string, file: string): boolean => + response.includes('**`' + file + '`'); + +const CONTRIBUTING = `# Project Contributing Guide + +Some preamble that names no section. + +## Repo Setup + +Clone it and install dependencies. + +## Cloning the repo on Windows + +Enable symlinks and long paths before cloning. + +## Ignoring commits when running git blame + +Use the ignore-revs file. +`; + +const README = `# Widget + +A widget. + +## Installation + +Install the widget. + +## Usage + +Use the widget. +`; + +beforeAll(async () => { + dir = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-doc-named-')); + fs.mkdirSync(path.join(dir, 'src'), { recursive: true }); + fs.mkdirSync(path.join(dir, 'test', 'fixtures'), { recursive: true }); + fs.writeFileSync(path.join(dir, 'CONTRIBUTING.md'), CONTRIBUTING); + fs.writeFileSync(path.join(dir, 'README.md'), README); + // A fixture copy, which DOC_LOW_PATH exists to keep out of results. + fs.writeFileSync(path.join(dir, 'test', 'fixtures', 'README.md'), README); + // Code, so the index is not markdown-only and code queries have somewhere to go. + fs.writeFileSync( + path.join(dir, 'src', 'widget.ts'), + `export function createWidget(size: number): number { return size * 2; }\n` + + `export function resizeWidget(w: number): number { return createWidget(w); }\n` + ); + cg = CodeGraph.initSync(dir); + await cg.indexAll(); +}, 180_000); + +afterAll(() => { + cg?.destroy(); + if (dir && fs.existsSync(dir)) fs.rmSync(dir, { recursive: true, force: true }); +}); + +describe('a query that names a markdown file', () => { + it('renders the file it names, spelled with the extension', async () => { + const out = await explore('CONTRIBUTING.md'); + expect(hasSection(out, 'CONTRIBUTING.md')).toBe(true); + }); + + it('renders the file it names, spelled as the bare stem', async () => { + const out = await explore('readme'); + expect(hasSection(out, 'README.md')).toBe(true); + }); + + it('does not let a bare stem surface a fixture copy', async () => { + // The stem match unlocks the two-hit gate and the section fallback, but + // deliberately not the DOC_LOW_PATH bypass β€” that stays the privilege of + // an explicitly spelled path. + const out = await explore('readme'); + expect(hasSection(out, 'test/fixtures/README.md')).toBe(false); + }); +}); + +describe('a heading containing a DOC_QUERY_NOISE word', () => { + it('is reachable, though "repo" can never appear among the query terms', async () => { + const out = await explore('the contributing guide for windows'); + expect(hasSection(out, 'CONTRIBUTING.md')).toBe(true); + expect(out).toContain('Cloning the repo on Windows'); + }); +}); + +describe('the DOC_WORD gate still declines what it should', () => { + it('a question with no doc word pulls no markdown', async () => { + const out = await explore('how do i resize a widget'); + expect(hasSection(out, 'CONTRIBUTING.md')).toBe(false); + expect(hasSection(out, 'README.md')).toBe(false); + }); +}); diff --git a/__tests__/extraction.test.ts b/__tests__/extraction.test.ts index c6a522decf..b109e2ce97 100644 --- a/__tests__/extraction.test.ts +++ b/__tests__/extraction.test.ts @@ -174,6 +174,12 @@ describe('Language Detection', () => { expect(detectLanguage('stdio.h', '#ifndef STDIO_H\nvoid printf();\n#endif\n')).toBe('c'); }); + it('should detect Markdown files', () => { + expect(detectLanguage('README.md')).toBe('markdown'); + expect(detectLanguage('docs/guide.markdown')).toBe('markdown'); + expect(detectLanguage('docs/page.mdx')).toBe('markdown'); + }); + it('should detect Metal shader files as C++ (#1121)', () => { expect(detectLanguage('Shaders.metal')).toBe('cpp'); expect(isSourceFile('Renderer/Shaders.metal')).toBe(true); @@ -322,11 +328,218 @@ describe('Language Support', () => { expect(languages).toContain('swift'); expect(languages).toContain('kotlin'); expect(languages).toContain('dart'); + expect(languages).toContain('markdown'); expect(languages).toContain('solidity'); expect(languages).toContain('nix'); }); }); +describe('Markdown Extraction', () => { + it('should extract headings, links, and shell script references', () => { + const markdown = `# Project Guide + +See [Setup](docs/setup.md#install) and scripts/release.mjs. + +## Release + +\`\`\`bash +npm run build +node scripts/release.mjs +\`\`\` +`; + + const result = extractFromSource('README.md', markdown); + + const fileNode = result.nodes.find((n) => n.kind === 'file'); + expect(fileNode).toMatchObject({ + name: 'README.md', + language: 'markdown', + }); + + const headings = result.nodes.filter((n) => n.kind === 'module'); + expect(headings.map((n) => n.name)).toContain('Project Guide'); + expect(headings.map((n) => n.name)).toContain('Release'); + + const commandNode = result.nodes.find((n) => n.kind === 'function' && n.signature === 'node scripts/release.mjs'); + expect(commandNode).toBeDefined(); + + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + referenceName: 'docs/setup.md#install', + referenceKind: 'imports', + language: 'markdown', + }), + expect.objectContaining({ + referenceName: 'scripts/release.mjs', + referenceKind: 'calls', + language: 'markdown', + }), + ]) + ); + }); + + it('should extract structured table rows and file-symbol references from Markdown', () => { + const markdown = `# Maintenance Guide + +## Phase 4 + +| Template | CLI Entry | Dispatcher | Implementation | +| --- | --- | --- | --- | +| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` | +| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` | + +- P4-FLOW changes must inspect \`scripts/csv_search.py::run_p4\`. +`; + + const result = extractFromSource('phases/phase4.md', markdown); + + const tableRows = result.nodes.filter((n) => n.kind === 'constant' && n.qualifiedName.includes('table-row')); + expect(tableRows.map((n) => n.name)).toEqual(expect.arrayContaining(['P4-S1', 'P4-S2'])); + + const p4s1 = tableRows.find((n) => n.name === 'P4-S1'); + expect(p4s1?.signature).toContain('Template: P4-S1'); + expect(p4s1?.signature).toContain('Dispatcher: scripts/csv_search.py::run_p4'); + + const commandNode = result.nodes.find((n) => + n.kind === 'function' && + n.language === 'markdown' && + n.signature?.includes('python "{script_path}" p4') + ); + expect(commandNode).toBeDefined(); + + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + referenceName: 'phases/scripts/csv_search.py::run_p4', + referenceKind: 'references', + language: 'markdown', + }), + expect.objectContaining({ + referenceName: 'phases/scripts/csv_search.py::_p4_stage1', + referenceKind: 'references', + language: 'markdown', + }), + ]) + ); + }); + + it('should keep structured blocks after fences containing a different fence marker', () => { + const markdown = `# Runbook + +\`\`\`text +~~~~ +\`\`\` + +- POST-FENCE references \`src/auth.ts::login\`. + +| Key | Target | +| --- | --- | +| POST-TABLE | \`src/auth.ts::login\` | +`; + + const result = extractFromSource('docs/runbook.md', markdown); + const constants = result.nodes.filter((n) => n.kind === 'constant'); + + expect(constants).toEqual(expect.arrayContaining([ + expect.objectContaining({ docstring: 'POST-FENCE references src/auth.ts::login.' }), + expect.objectContaining({ name: 'POST-TABLE' }), + ])); + }); + + it('indexes Setext (underline) headings and skips frontmatter / code fences', () => { + const markdown = `--- +title: Config Doc +--- + +Architecture Overview +===================== + +Intro paragraph for the overview. + +Routing Layer +------------- + +\`\`\`md +Not A Heading +============= +\`\`\` +`; + + const result = extractFromSource('docs/arch.md', markdown); + const headings = result.nodes.filter((n) => n.kind === 'module'); + const byName = new Map(headings.map((h) => [h.name, h])); + + // Setext H1 (===) and H2 (---) become module nodes. + expect(byName.get('Architecture Overview')?.signature).toBe('# Architecture Overview'); + expect(byName.get('Routing Layer')?.signature).toBe('## Routing Layer'); + // Frontmatter `title:` (above the closing `---`) is NOT a heading, and a + // setext-looking line inside a code fence is ignored. + expect(byName.has('title: Config Doc')).toBe(false); + expect(byName.has('Not A Heading')).toBe(false); + }); + + it('builds a deterministic, compact file digest (intro + key references)', () => { + const markdown = `# Release Runbook + +This runbook explains how to cut a release. + +See [setup](docs/setup.md#install) and run \`scripts/release.mjs\`. +It dispatches \`scripts/csv_search.py::run_p4\`. +`; + + const result = extractFromSource('RUNBOOK.md', markdown); + const fileNode = result.nodes.find((n) => n.kind === 'file'); + + expect(fileNode?.docstring).toBeDefined(); + const digest = fileNode!.docstring!; + // Intro is the first prose line, not the heading or a link blob. + expect(digest).toContain('This runbook explains how to cut a release.'); + // Key referenced files/symbols are surfaced, compacted to basenames. + expect(digest).toContain('refs:'); + expect(digest).toContain('setup.md#install'); + expect(digest).toContain('release.mjs'); + expect(digest).toContain('csv_search.py::run_p4'); + // Short enough to show in node details (the < 200 char detail gate). + expect(digest.length).toBeLessThan(200); + }); +}); + +describe('Code to Markdown Reference Extraction', () => { + it('should extract Markdown path references from code string literals', () => { + const code = ` +export const GUIDE = '../docs/guide.md'; + +export function loadDocs() { + return fs.readFileSync('../docs/guide.md#install', 'utf8'); +} +`; + + const result = extractFromSource('src/load-docs.ts', code); + const loadDocs = result.nodes.find((n) => n.kind === 'function' && n.name === 'loadDocs'); + const guideConstant = result.nodes.find((n) => n.kind === 'constant' && n.name === 'GUIDE'); + + expect(loadDocs).toBeDefined(); + expect(guideConstant).toBeDefined(); + expect(result.unresolvedReferences).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + fromNodeId: loadDocs!.id, + referenceName: 'docs/guide.md#install', + referenceKind: 'references', + language: 'typescript', + }), + expect.objectContaining({ + fromNodeId: guideConstant!.id, + referenceName: 'docs/guide.md', + referenceKind: 'references', + language: 'typescript', + }), + ]) + ); + }); +}); + describe('Nix Extraction', () => { it('should distinguish Nix variable and function bindings', () => { const code = ` diff --git a/__tests__/fixtures/kernel-parity/Torture.cs b/__tests__/fixtures/kernel-parity/Torture.cs index 39e7743dbe..234e21b514 100644 --- a/__tests__/fixtures/kernel-parity/Torture.cs +++ b/__tests__/fixtures/kernel-parity/Torture.cs @@ -184,3 +184,25 @@ static class TopLevelGuard { public static T AgainstNull(T value) where T : class => value; } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +namespace Torture.Markdown +{ + public class MarkdownPaths + { + public const string Guide = "../docs/guide.md#install"; + public const string Bare = "README.md"; + public const string Rooted = "/docs/rooted.md"; + public const string Escapes = "../../../../outside.md"; + public const string Remote = "https://example.com/remote.md"; + public const string Queried = "./notes.md?raw=1#top"; + public const string TwoInOne = "see a.md and also sub/b.mdx"; + + public void Load() + { + LoadDoc("docs/deep/nested.markdown"); + } + } +} diff --git a/__tests__/fixtures/kernel-parity/Torture.java b/__tests__/fixtures/kernel-parity/Torture.java index d1e71cbd20..b5b55aba9d 100644 --- a/__tests__/fixtures/kernel-parity/Torture.java +++ b/__tests__/fixtures/kernel-parity/Torture.java @@ -130,3 +130,20 @@ interface Shape extends Comparable, Cloneable { @interface Marker { String value() default ""; } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +class MarkdownPaths { + static final String GUIDE = "../docs/guide.md#install"; + static final String BARE = "README.md"; + static final String ROOTED = "/docs/rooted.md"; + static final String ESCAPES = "../../../../outside.md"; + static final String REMOTE = "https://example.com/remote.md"; + static final String QUERIED = "./notes.md?raw=1#top"; + static final String TWO_IN_ONE = "see a.md and also sub/b.mdx"; + + void load() { + loadDoc("docs/deep/nested.markdown"); + } +} diff --git a/__tests__/fixtures/kernel-parity/torture.R b/__tests__/fixtures/kernel-parity/torture.R index c3b02bc158..6adb2dac5a 100644 --- a/__tests__/fixtures/kernel-parity/torture.R +++ b/__tests__/fixtures/kernel-parity/torture.R @@ -202,3 +202,16 @@ piped <- x |> f_ph(y = _) # --- UTF-16 columns ---------------------------------------------------------- msg <- "hΓ©llo πŸŽ‰" emoji_caller <- function() after_emoji("πŸŽ‰πŸŽ‰", target_fn()) + +# --- Markdown path references ------------------------------------------------ +# Code -> documentation edges. Every shape the normalizer branches on, so the +# two arms have to agree about the rejections (URL, escape above the root) as +# well as the emissions. +md_guide <- "../docs/guide.md#install" +md_bare <- "README.md" +md_rooted <- "/docs/rooted.md" +md_escapes <- "../../../../outside.md" +md_remote <- "https://example.com/remote.md" +md_queried <- "./notes.md?raw=1#top" +md_two_in_one <- "see a.md and also sub/b.mdx" +md_load <- function() load_doc("docs/deep/nested.markdown") diff --git a/__tests__/fixtures/kernel-parity/torture.c b/__tests__/fixtures/kernel-parity/torture.c index 57e04a912c..dcf42024b0 100644 --- a/__tests__/fixtures/kernel-parity/torture.c +++ b/__tests__/fixtures/kernel-parity/torture.c @@ -168,3 +168,16 @@ typedef union { } word_t; static unsigned int hdr_raw(union packet_hdr *h) { return h->raw; } + +/* Markdown path references: code -> documentation edges. Every shape the + normalizer branches on, so the two arms have to agree about the rejections + (URL, escape above the root) as well as the emissions. */ +static const char *md_guide = "../docs/guide.md#install"; +static const char *md_bare = "README.md"; +static const char *md_rooted = "/docs/rooted.md"; +static const char *md_escapes = "../../../../outside.md"; +static const char *md_remote = "https://example.com/remote.md"; +static const char *md_queried = "./notes.md?raw=1#top"; +static const char *md_two_in_one = "see a.md and also sub/b.mdx"; + +static void md_load(void) { load_doc("docs/deep/nested.markdown"); } diff --git a/__tests__/fixtures/kernel-parity/torture.cpp b/__tests__/fixtures/kernel-parity/torture.cpp index b6b160f7ae..a7309eb7c5 100644 --- a/__tests__/fixtures/kernel-parity/torture.cpp +++ b/__tests__/fixtures/kernel-parity/torture.cpp @@ -140,3 +140,16 @@ float drive() { (void)m; return r + f + flags + leg; } + +/* Markdown path references: code -> documentation edges. Every shape the + normalizer branches on, so the two arms have to agree about the rejections + (URL, escape above the root) as well as the emissions. */ +static const char *md_guide = "../docs/guide.md#install"; +static const char *md_bare = "README.md"; +static const char *md_rooted = "/docs/rooted.md"; +static const char *md_escapes = "../../../../outside.md"; +static const char *md_remote = "https://example.com/remote.md"; +static const char *md_queried = "./notes.md?raw=1#top"; +static const char *md_two_in_one = "see a.md and also sub/b.mdx"; + +static void md_load() { load_doc("docs/deep/nested.markdown"); } diff --git a/__tests__/fixtures/kernel-parity/torture.dart b/__tests__/fixtures/kernel-parity/torture.dart index fa4f73b389..081c504a34 100644 --- a/__tests__/fixtures/kernel-parity/torture.dart +++ b/__tests__/fixtures/kernel-parity/torture.dart @@ -229,3 +229,18 @@ void patternUser(Object o) { void afterUnicode(String seance) { emit('cafΓ© β˜• done'); } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +const mdGuide = '../docs/guide.md#install'; +const mdBare = 'README.md'; +const mdRooted = '/docs/rooted.md'; +const mdEscapes = '../../../../outside.md'; +const mdRemote = 'https://example.com/remote.md'; +const mdQueried = './notes.md?raw=1#top'; +const mdTwoInOne = 'see a.md and also sub/b.mdx'; + +void mdLoad() { + loadDoc('docs/deep/nested.markdown'); +} diff --git a/__tests__/fixtures/kernel-parity/torture.go b/__tests__/fixtures/kernel-parity/torture.go index f89b8d429d..aeb103d584 100644 --- a/__tests__/fixtures/kernel-parity/torture.go +++ b/__tests__/fixtures/kernel-parity/torture.go @@ -59,3 +59,18 @@ func shadowed() { func reads() int { return MAX_ITEMS } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +var mdGuide = "../docs/guide.md#install" +var mdBare = "README.md" +var mdRooted = "/docs/rooted.md" +var mdEscapes = "../../../../outside.md" +var mdRemote = "https://example.com/remote.md" +var mdQueried = "./notes.md?raw=1#top" +var mdTwoInOne = "see a.md and also sub/b.mdx" + +func mdLoad() { + loadDoc("docs/deep/nested.markdown") +} diff --git a/__tests__/fixtures/kernel-parity/torture.kt b/__tests__/fixtures/kernel-parity/torture.kt index c5531c55e2..dfb6a4a9b6 100644 --- a/__tests__/fixtures/kernel-parity/torture.kt +++ b/__tests__/fixtures/kernel-parity/torture.kt @@ -289,3 +289,18 @@ class SiblingAccessorK { register(fromInit) } } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +val mdGuide = "../docs/guide.md#install" +val mdBare = "README.md" +val mdRooted = "/docs/rooted.md" +val mdEscapes = "../../../../outside.md" +val mdRemote = "https://example.com/remote.md" +val mdQueried = "./notes.md?raw=1#top" +val mdTwoInOne = "see a.md and also sub/b.mdx" + +fun mdLoad() { + loadDoc("docs/deep/nested.markdown") +} diff --git a/__tests__/fixtures/kernel-parity/torture.lua b/__tests__/fixtures/kernel-parity/torture.lua index 1b3c21cfb7..cd477f0b3f 100644 --- a/__tests__/fixtures/kernel-parity/torture.lua +++ b/__tests__/fixtures/kernel-parity/torture.lua @@ -105,4 +105,20 @@ local prΓ©fixe = "cafΓ©" function after_unicode() end do goto done end ::done:: + +-- Markdown path references: code -> documentation edges. Every shape the +-- normalizer branches on, so the two arms have to agree about the rejections +-- (URL, escape above the root) as well as the emissions. +local md_guide = "../docs/guide.md#install" +local md_bare = "README.md" +local md_rooted = "/docs/rooted.md" +local md_escapes = "../../../../outside.md" +local md_remote = "https://example.com/remote.md" +local md_queried = "./notes.md?raw=1#top" +local md_two_in_one = "see a.md and also sub/b.mdx" + +function md_load() + load_doc("docs/deep/nested.markdown") +end + return M diff --git a/__tests__/fixtures/kernel-parity/torture.luau b/__tests__/fixtures/kernel-parity/torture.luau index 8432d69791..93744b1894 100644 --- a/__tests__/fixtures/kernel-parity/torture.luau +++ b/__tests__/fixtures/kernel-parity/torture.luau @@ -58,4 +58,20 @@ M:update({ x = 1, y = 2 }, print) local prΓ©fixe = "cafΓ©" function after_unicode() end + +-- Markdown path references: code -> documentation edges. Every shape the +-- normalizer branches on, so the two arms have to agree about the rejections +-- (URL, escape above the root) as well as the emissions. +local md_guide = "../docs/guide.md#install" +local md_bare = "README.md" +local md_rooted = "/docs/rooted.md" +local md_escapes = "../../../../outside.md" +local md_remote = "https://example.com/remote.md" +local md_queried = "./notes.md?raw=1#top" +local md_two_in_one = "see a.md and also sub/b.mdx" + +function md_load() + load_doc("docs/deep/nested.markdown") +end + return M diff --git a/__tests__/fixtures/kernel-parity/torture.php b/__tests__/fixtures/kernel-parity/torture.php index 0194250f81..54f6a49222 100644 --- a/__tests__/fixtures/kernel-parity/torture.php +++ b/__tests__/fixtures/kernel-parity/torture.php @@ -217,3 +217,19 @@ abstract class AbstractBase { abstract protected function hook(): void; } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +$mdGuide = '../docs/guide.md#install'; +$mdBare = 'README.md'; +$mdRooted = '/docs/rooted.md'; +$mdEscapes = '../../../../outside.md'; +$mdRemote = 'https://example.com/remote.md'; +$mdQueried = './notes.md?raw=1#top'; +$mdTwoInOne = 'see a.md and also sub/b.mdx'; + +function mdLoad() +{ + loadDoc('docs/deep/nested.markdown'); +} diff --git a/__tests__/fixtures/kernel-parity/torture.rb b/__tests__/fixtures/kernel-parity/torture.rb index f1d5fbfd06..6d4ff34be3 100644 --- a/__tests__/fixtures/kernel-parity/torture.rb +++ b/__tests__/fixtures/kernel-parity/torture.rb @@ -231,5 +231,20 @@ def self.top_singleton def obj.weird_singleton; end +# Markdown path references: code -> documentation edges. Every shape the +# normalizer branches on, so the two arms have to agree about the rejections +# (URL, escape above the root) as well as the emissions. +MD_GUIDE = '../docs/guide.md#install' +MD_BARE = 'README.md' +MD_ROOTED = '/docs/rooted.md' +MD_ESCAPES = '../../../../outside.md' +MD_REMOTE = 'https://example.com/remote.md' +MD_QUERIED = './notes.md?raw=1#top' +MD_TWO_IN_ONE = 'see a.md and also sub/b.mdx' + +def md_load + load_doc('docs/deep/nested.markdown') +end + __END__ raw data trailer here diff --git a/__tests__/fixtures/kernel-parity/torture.rs b/__tests__/fixtures/kernel-parity/torture.rs index 63fe390a16..492ddf1847 100644 --- a/__tests__/fixtures/kernel-parity/torture.rs +++ b/__tests__/fixtures/kernel-parity/torture.rs @@ -296,3 +296,18 @@ pub union Reg { } impl Base for Reg {} + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +const MD_GUIDE: &str = "../docs/guide.md#install"; +const MD_BARE: &str = "README.md"; +const MD_ROOTED: &str = "/docs/rooted.md"; +const MD_ESCAPES: &str = "../../../../outside.md"; +const MD_REMOTE: &str = "https://example.com/remote.md"; +const MD_QUERIED: &str = "./notes.md?raw=1#top"; +const MD_TWO_IN_ONE: &str = "see a.md and also sub/b.mdx"; + +fn md_load() { + load_doc("docs/deep/nested.markdown"); +} diff --git a/__tests__/fixtures/kernel-parity/torture.scala b/__tests__/fixtures/kernel-parity/torture.scala index c1e4703ae1..05116cfcc3 100644 --- a/__tests__/fixtures/kernel-parity/torture.scala +++ b/__tests__/fixtures/kernel-parity/torture.scala @@ -182,3 +182,18 @@ class InitWalk { lazy val initLazy = process(1) val initAnon = new Runnable { def run(): Unit = helperCall() } } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +object MarkdownPaths { + val mdGuide = "../docs/guide.md#install" + val mdBare = "README.md" + val mdRooted = "/docs/rooted.md" + val mdEscapes = "../../../../outside.md" + val mdRemote = "https://example.com/remote.md" + val mdQueried = "./notes.md?raw=1#top" + val mdTwoInOne = "see a.md and also sub/b.mdx" + + def mdLoad(): Unit = loadDoc("docs/deep/nested.markdown") +} diff --git a/__tests__/fixtures/kernel-parity/torture.swift b/__tests__/fixtures/kernel-parity/torture.swift index 6cb31f010d..543f48a344 100644 --- a/__tests__/fixtures/kernel-parity/torture.swift +++ b/__tests__/fixtures/kernel-parity/torture.swift @@ -211,3 +211,18 @@ func fnRefExtras() { import class Darwin.FILE @testable import TortureKit + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +let mdGuide = "../docs/guide.md#install" +let mdBare = "README.md" +let mdRooted = "/docs/rooted.md" +let mdEscapes = "../../../../outside.md" +let mdRemote = "https://example.com/remote.md" +let mdQueried = "./notes.md?raw=1#top" +let mdTwoInOne = "see a.md and also sub/b.mdx" + +func mdLoad() { + loadDoc("docs/deep/nested.markdown") +} diff --git a/__tests__/fixtures/kernel-parity/torture.tsx b/__tests__/fixtures/kernel-parity/torture.tsx index 3135422843..b157135b97 100644 --- a/__tests__/fixtures/kernel-parity/torture.tsx +++ b/__tests__/fixtures/kernel-parity/torture.tsx @@ -238,3 +238,15 @@ export function NestedHandlers({ items, onPick }: { items: string[]; onPick: (a: const [a, b] = [() => 1, () => 2]; return items.map((i) => ); } + +// Markdown path references: code -> documentation edges. Every shape the +// normalizer branches on, so the two arms have to agree about the rejections +// (URL, escape above the root) as well as the emissions. +const GUIDE = '../docs/guide.md#install'; +const BARE = 'README.md'; +const ROOTED = '/docs/rooted.md'; +const ESCAPES = '../../../../outside.md'; +const REMOTE = 'https://example.com/remote.md'; +const QUERIED = './notes.md?raw=1#top'; +const TWO_IN_ONE = 'see a.md and also sub/b.mdx'; +loadDoc('docs/deep/nested.markdown'); diff --git a/__tests__/integration/full-pipeline.test.ts b/__tests__/integration/full-pipeline.test.ts index 5b551c136c..fc3aab2e70 100644 --- a/__tests__/integration/full-pipeline.test.ts +++ b/__tests__/integration/full-pipeline.test.ts @@ -82,6 +82,97 @@ describe('Integration: full pipeline', () => { cleanupTempDir(tempDir); }); + it('indexes Markdown headings and resolves Markdown links to script files', async () => { + fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'README.md'), + `# Project Guide + +See [Setup](docs/setup.md#install). + +## Release + +\`\`\`bash +node scripts/release.mjs +\`\`\` +` + ); + fs.writeFileSync(path.join(tempDir, 'docs', 'setup.md'), '# Install\n'); + fs.writeFileSync(path.join(tempDir, 'scripts', 'release.mjs'), 'export function release() { return true; }\n'); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const guide = cg.searchNodes('Project Guide').find((r) => r.node.language === 'markdown'); + expect(guide).toBeDefined(); + + const releaseCommand = cg + .searchNodes('release.mjs') + .find((r) => r.node.language === 'markdown' && r.node.kind === 'function'); + expect(releaseCommand).toBeDefined(); + + const guideEdges = cg.getOutgoingEdges(guide!.node.id).filter((e) => e.kind === 'imports'); + const guideTargets = guideEdges.map((e) => cg.getNode(e.target)); + const setupHeading = guideTargets.find((n) => n?.qualifiedName === 'docs/setup.md#install'); + expect(setupHeading).toMatchObject({ + kind: 'module', + name: 'Install', + filePath: 'docs/setup.md', + startLine: 1, + }); + + const commandEdges = cg.getOutgoingEdges(releaseCommand!.node.id).filter((e) => e.kind === 'calls'); + const commandTargets = commandEdges.map((e) => cg.getNode(e.target)?.filePath); + expect(commandTargets).toContain('scripts/release.mjs'); + } finally { + cg.destroy(); + } + }); + + it('indexes Markdown template tables and resolves file-symbol references to implementation functions', async () => { + fs.mkdirSync(path.join(tempDir, 'phases'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'phases', 'phase4.md'), + `# Phase 4 + +## Fixed Script Templates + +| Template | CLI Entry | Dispatcher | Implementation | +| --- | --- | --- | --- | +| P4-S1 | \`python "{script_path}" p4 "{csv_file}" s1 "{conditions_or_-}" "{probe_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage1\` | +| P4-S2 | \`python "{script_path}" p4 "{csv_file}" s2 "{stage1_rows}" "{condition_or_-}" "{detail_cols}"\` | \`scripts/csv_search.py::run_p4\` | \`scripts/csv_search.py::_p4_stage2\` | +` + ); + fs.writeFileSync( + path.join(tempDir, 'scripts', 'csv_search.py'), + `def _p4_stage1(filepath, condition_spec, probe_cols_spec):\n return 's1'\n\n` + + `def _p4_stage2(filepath, stage1_rows, condition_spec, detail_cols_spec):\n return 's2'\n\n` + + `def run_p4(filepath, args):\n return _p4_stage1(filepath, '-', 'MPN')\n` + ); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const p4s1Row = cg.searchNodes('P4-S1').find((r) => r.node.language === 'markdown'); + expect(p4s1Row?.node.kind).toBe('constant'); + + const edges = cg.getOutgoingEdges(p4s1Row!.node.id).filter((e) => e.kind === 'references'); + const targets = edges.map((e) => cg.getNode(e.target)); + expect(targets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'run_p4', filePath: 'scripts/csv_search.py' }), + expect.objectContaining({ name: '_p4_stage1', filePath: 'scripts/csv_search.py' }), + ]) + ); + } finally { + cg.destroy(); + } + }); + it('runs init β†’ index β†’ resolve β†’ search β†’ callers β†’ context β†’ sync', async () => { const MODULE_COUNT = 120; generateSyntheticProject(tempDir, MODULE_COUNT); @@ -269,4 +360,46 @@ describe('Integration: full pipeline', () => { cg.destroy(); } }, 30_000); + + it('resolves code string references to Markdown headings', async () => { + fs.mkdirSync(path.join(tempDir, 'docs'), { recursive: true }); + fs.mkdirSync(path.join(tempDir, 'scripts'), { recursive: true }); + fs.writeFileSync( + path.join(tempDir, 'docs', 'guide.md'), + `# Guide + +## Install + +Run the setup command. +` + ); + fs.writeFileSync( + path.join(tempDir, 'scripts', 'load_docs.py'), + `GUIDE = "docs/guide.md"\n\n` + + `def load_docs():\n` + + ` return open("docs/guide.md#install", encoding="utf-8").read()\n` + ); + + const cg = await CodeGraph.init(tempDir); + try { + await cg.indexAll(); + + const loadDocs = cg.searchNodes('load_docs').find((r) => r.node.language === 'python'); + expect(loadDocs).toBeDefined(); + + const edges = cg.getOutgoingEdges(loadDocs!.node.id).filter((e) => e.kind === 'references'); + const targets = edges.map((e) => cg.getNode(e.target)); + expect(targets).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: 'module', + name: 'Install', + qualifiedName: 'docs/guide.md#install', + }), + ]) + ); + } finally { + cg.destroy(); + } + }); }); diff --git a/__tests__/resolution.test.ts b/__tests__/resolution.test.ts index 14b5198f47..27247c381f 100644 --- a/__tests__/resolution.test.ts +++ b/__tests__/resolution.test.ts @@ -11,7 +11,7 @@ import * as os from 'os'; import { CodeGraph } from '../src'; import { Node, UnresolvedReference } from '../src/types'; import { ReferenceResolver, createResolver, ResolutionContext } from '../src/resolution'; -import { matchReference, resolveMethodOnType, matchByQualifiedName, matchByExactName, preferCallSiteFile, matchMethodCall } from '../src/resolution/name-matcher'; +import { matchReference, resolveMethodOnType, matchByQualifiedName, matchByExactName, matchByFilePath, preferCallSiteFile, matchMethodCall } from '../src/resolution/name-matcher'; import { resolveImportPath, extractImportMappings, resolveJvmImport, loadCppIncludeDirs, clearCppIncludeDirCache, isPhpIncludePathRef } from '../src/resolution/import-resolver'; import type { UnresolvedRef } from '../src/resolution/types'; import { detectFrameworks, getAllFrameworkResolvers } from '../src/resolution/frameworks'; @@ -402,6 +402,169 @@ describe('Resolution Module', () => { expect(result).not.toBeNull(); expect(result?.targetNodeId).toBe('method:user.ts:User.save:15'); }); + + it('should resolve Markdown file references by filename, path, and anchor suffix', () => { + const mockNodes: Node[] = [ + { + id: 'file:README.md', + kind: 'file', + name: 'README.md', + qualifiedName: 'README.md', + filePath: 'README.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'file:docs/setup.md', + kind: 'file', + name: 'setup.md', + qualifiedName: 'docs/setup.md', + filePath: 'docs/setup.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'module:docs/setup.md:install:1', + kind: 'module', + name: 'Install', + qualifiedName: 'docs/setup.md#install', + filePath: 'docs/setup.md', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 9, + updatedAt: Date.now(), + }, + { + id: 'file:GUIDE.markdown', + kind: 'file', + name: 'GUIDE.markdown', + qualifiedName: 'GUIDE.markdown', + filePath: 'GUIDE.markdown', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'module:GUIDE.markdown:install:1', + kind: 'module', + name: 'Install', + qualifiedName: 'GUIDE.markdown#install', + filePath: 'GUIDE.markdown', + language: 'markdown', + startLine: 1, + endLine: 10, + startColumn: 0, + endColumn: 9, + updatedAt: Date.now(), + }, + ]; + + const context: ResolutionContext = { + getNodesInFile: () => mockNodes, + getNodesByName: (name) => mockNodes.filter((n) => n.name === name), + getNodesByQualifiedName: (qualifiedName) => mockNodes.filter((n) => n.qualifiedName === qualifiedName), + getNodesByKind: () => [], + fileExists: () => true, + readFile: () => null, + getProjectRoot: () => '/test', + getAllFiles: () => ['README.md', 'docs/setup.md', 'GUIDE.markdown'], + getNodesByLowerName: () => [], + getImportMappings: () => [], + }; + + const readmeRef = { + fromNodeId: 'module:docs/setup.md:install:1', + referenceName: 'README.md', + referenceKind: 'imports' as const, + line: 1, + column: 0, + filePath: 'docs/setup.md', + language: 'markdown' as const, + }; + const setupRef = { + ...readmeRef, + referenceName: 'docs/setup.md#install', + filePath: 'README.md', + }; + const markdownRef = { + ...readmeRef, + referenceName: 'GUIDE.markdown#install', + filePath: 'README.md', + }; + + expect(matchReference(readmeRef, context)?.targetNodeId).toBe('file:README.md'); + expect(matchReference(setupRef, context)?.targetNodeId).toBe('module:docs/setup.md:install:1'); + expect(matchReference(markdownRef, context)?.targetNodeId).toBe('module:GUIDE.markdown:install:1'); + }); + + it('should resolve Markdown file-symbol references to symbols in the referenced file', () => { + const mockNodes: Node[] = [ + { + id: 'file:scripts/csv_search.py', + kind: 'file', + name: 'csv_search.py', + qualifiedName: 'scripts/csv_search.py', + filePath: 'scripts/csv_search.py', + language: 'python', + startLine: 1, + endLine: 100, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + { + id: 'function:scripts/csv_search.py:run_p4:40', + kind: 'function', + name: 'run_p4', + qualifiedName: 'scripts/csv_search.py::run_p4', + filePath: 'scripts/csv_search.py', + language: 'python', + startLine: 40, + endLine: 55, + startColumn: 0, + endColumn: 0, + updatedAt: Date.now(), + }, + ]; + + const context: ResolutionContext = { + getNodesInFile: (filePath) => mockNodes.filter((n) => n.filePath === filePath), + getNodesByName: (name) => mockNodes.filter((n) => n.name === name), + getNodesByQualifiedName: (qualifiedName) => mockNodes.filter((n) => n.qualifiedName === qualifiedName), + getNodesByKind: () => [], + fileExists: () => true, + readFile: () => null, + getProjectRoot: () => '/test', + getAllFiles: () => ['scripts/csv_search.py'], + getNodesByLowerName: () => [], + getImportMappings: () => [], + }; + + const ref = { + fromNodeId: 'constant:phases/phase4.md:P4-S1:10', + referenceName: 'phases/scripts/csv_search.py::run_p4', + referenceKind: 'references' as const, + line: 10, + column: 20, + filePath: 'phases/phase4.md', + language: 'markdown' as const, + }; + + expect(matchReference(ref, context)?.targetNodeId).toBe('function:scripts/csv_search.py:run_p4:40'); + }); }); describe('Ubiquitous-name ceiling (#999)', () => { @@ -6071,6 +6234,35 @@ in expect(matchByExactName(ref, context)?.targetNodeId).toBe('fn:vite'); }); + it('still reaches a heading through a reference that spells out the Markdown file', () => { + // The #1719 guard rejects a BARE name that a heading happens to share. A + // code string naming the file β€” `docs/guide.md#install` β€” is a + // documentation link, and the doc tier depends on that edge. + const heading: Node = { + id: 'heading:install', name: 'Install', qualifiedName: 'docs/guide.md#install', + kind: 'module', language: 'markdown' as Node['language'], filePath: 'docs/guide.md', + startLine: 3, endLine: 5, startColumn: 0, endColumn: 0, updatedAt: 0, + }; + const file: Node = { + id: 'file:guide', name: 'guide.md', qualifiedName: 'docs/guide.md', + kind: 'file', language: 'markdown' as Node['language'], filePath: 'docs/guide.md', + startLine: 1, endLine: 5, startColumn: 0, endColumn: 0, updatedAt: 0, + }; + const context = { + getNodesByName: (name: string) => (name === 'guide.md' ? [file] : [heading]), + getNodesInFile: () => [heading], + getNodesByQualifiedName: (qn: string) => (qn === heading.qualifiedName ? [heading] : []), + getNodesByKind: () => [], + fileExists: () => false, readFile: () => null, + getProjectRoot: () => tempDir, getAllFiles: () => [], + } as ResolutionContext; + const ref: UnresolvedRef = { + fromNodeId: 'fn:load_docs', referenceName: 'docs/guide.md#install', referenceKind: 'references', + filePath: 'scripts/load_docs.py', language: 'python', line: 4, column: 11, + }; + expect(matchByFilePath(ref, context)?.targetNodeId).toBe(heading.id); + }); + it('ignores export examples in strings and comments when checking module visibility', async () => { fs.mkdirSync(path.join(tempDir, 'src')); fs.writeFileSync(path.join(tempDir, 'src/private.js'), [ diff --git a/__tests__/security.test.ts b/__tests__/security.test.ts index 8cc9d7aa12..b16c5b4fff 100644 --- a/__tests__/security.test.ts +++ b/__tests__/security.test.ts @@ -531,11 +531,12 @@ describe('Source file detection (isSourceFile)', () => { expect(isSourceFile('src/component.tsx')).toBe(true); expect(isSourceFile('lib/util.js')).toBe(true); expect(isSourceFile('src/main.py')).toBe(true); + // Markdown documentation is indexed as a source language. + expect(isSourceFile('README.md')).toBe(true); }); it('rejects unsupported extensions and extensionless files', () => { expect(isSourceFile('src/component.css')).toBe(false); - expect(isSourceFile('README.md')).toBe(false); expect(isSourceFile('Makefile')).toBe(false); expect(isSourceFile('.gitignore')).toBe(false); }); diff --git a/__tests__/watcher.test.ts b/__tests__/watcher.test.ts index e0bb14723c..341d966079 100644 --- a/__tests__/watcher.test.ts +++ b/__tests__/watcher.test.ts @@ -520,8 +520,8 @@ describe('FileWatcher', () => { // gate must drop it before scheduling sync. (It must exist on disk: // a VANISHED non-source path is the deleted-directory shape, which // deliberately schedules a sync β€” #1285.) - fs.writeFileSync(path.join(testDir, 'src', 'readme.md'), '# docs\n'); - __emitWatchEventForTests(testDir, 'src/readme.md'); + fs.writeFileSync(path.join(testDir, 'src', 'styles.css'), 'body {}\n'); + __emitWatchEventForTests(testDir, 'src/styles.css'); // Wait a bit longer than debounce β€” sync should NOT trigger. await new Promise((r) => setTimeout(r, 400)); diff --git a/codegraph-kernel/src/buffers.rs b/codegraph-kernel/src/buffers.rs index f9dd70d5b0..1f97dac910 100644 --- a/codegraph-kernel/src/buffers.rs +++ b/codegraph-kernel/src/buffers.rs @@ -125,6 +125,10 @@ pub const FUNCTION_REF_CODE: u8 = 200; /// Ref-row flag bit 0: the ref carries `filePath` = the extracted file. pub const REF_FLAG_FILE_PATH: u8 = 1; +/// The ref carries the file's language, as `addReference` (tree-sitter.ts) +/// emits it. Ordinary refs must NOT set this: their wasm counterparts have no +/// `language` field and parity compares the objects whole. +pub const REF_FLAG_LANGUAGE: u8 = 2; pub fn node_kind_index(kind: &str) -> Option { NODE_KINDS.iter().position(|k| *k == kind).map(|i| i as u8) @@ -315,6 +319,19 @@ impl Tables { self.push_ref_flagged(r, 0); } + /// Whether a ref row with exactly these flags, owner, line and column was + /// already pushed. addReference dedups on the full key; the flagged + /// emitter reaches a string node at most once per owner, so owner, line and + /// column identify it. + pub fn has_flagged_ref(&self, from_idx: u32, line: u32, column: u32, flags: u8) -> bool { + self.refs.chunks_exact(REF_ROW_SIZE).any(|row| { + row[5] == flags + && row[0..4] == from_idx.to_le_bytes() + && row[8..12] == line.to_le_bytes() + && row[12..16] == column.to_le_bytes() + }) + } + pub fn push_ref_flagged(&mut self, r: &RefRow, flags: u8) { let buf = &mut self.refs; buf.extend_from_slice(&r.from_idx.to_le_bytes()); diff --git a/codegraph-kernel/src/ccpp/mod.rs b/codegraph-kernel/src/ccpp/mod.rs index ef546e7261..3bd146da05 100644 --- a/codegraph-kernel/src/ccpp/mod.rs +++ b/codegraph-kernel/src/ccpp/mod.rs @@ -450,6 +450,8 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -940,6 +942,8 @@ impl<'t> Walker<'t> { } self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if self.is_cpp_constructor_declaration(node) { self.extract_method(node); @@ -1773,6 +1777,8 @@ impl<'t> Walker<'t> { return; } self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/codegraph-kernel/src/csharp.rs b/codegraph-kernel/src/csharp.rs index 7d481662a7..a1cee5ff1c 100644 --- a/codegraph-kernel/src/csharp.rs +++ b/codegraph-kernel/src/csharp.rs @@ -265,6 +265,8 @@ fn namespace_name(w: &Walker, node: Node) -> Option { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -550,6 +552,8 @@ impl<'t> Walker<'t> { } self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "class_declaration" || kind == "record_declaration" { // classifyClassNode: `record struct` β†’ extractStruct, else class. @@ -625,6 +629,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "invocation_expression" { self.extract_call(node); @@ -918,6 +924,9 @@ impl<'t> Walker<'t> { // multi-declarator fields emit the type refs once PER // declarator, each from its own field node. self.extract_csharp_type_refs(node, row); + // The ladder skips a field's children, so the declarator's + // string literals are reached here, owned by the field. + self.markdown_refs_from_subtree(decl, row); } } } else { @@ -929,12 +938,15 @@ impl<'t> Walker<'t> { }); if let Some(name_node) = name_node { let name = self.text(name_node).to_string(); - self.create_node( + let row = self.create_node( field_kind, &name, node, Extra { docstring, visibility, is_static, ..Extra::default() }, ); + if let Some(row) = row { + self.markdown_refs_from_subtree(node, row); + } } } } diff --git a/codegraph-kernel/src/dart.rs b/codegraph-kernel/src/dart.rs index 8faacf3684..3fe4c61b58 100644 --- a/codegraph-kernel/src/dart.rs +++ b/codegraph-kernel/src/dart.rs @@ -215,6 +215,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -636,6 +638,8 @@ impl<'t> Walker<'t> { // maybeCaptureFnRefs (:990) β€” the double-walk fn-ref twin source. self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); match node.kind() { "function_signature" => { @@ -1292,6 +1296,8 @@ impl<'t> Walker<'t> { fn visit_body(&mut self, node: Node<'t>) { stack_guard!(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); let kind = node.kind(); if kind == "new_expression" { diff --git a/codegraph-kernel/src/go.rs b/codegraph-kernel/src/go.rs index 239c788a6e..efc97de713 100644 --- a/codegraph-kernel/src/go.rs +++ b/codegraph-kernel/src/go.rs @@ -179,6 +179,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -388,6 +390,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_declaration" { self.extract_function(node); @@ -429,6 +433,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/codegraph-kernel/src/java.rs b/codegraph-kernel/src/java.rs index daaf49de0e..25d000c7cf 100644 --- a/codegraph-kernel/src/java.rs +++ b/codegraph-kernel/src/java.rs @@ -249,6 +249,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -493,6 +495,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "class_declaration" { self.extract_class(node); @@ -546,6 +550,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "method_invocation" { self.extract_call(node); @@ -787,6 +793,9 @@ impl<'t> Walker<'t> { self.visit_function_body(value); self.stack.pop(); } + // The ladder skips a field's children, so the declarator's + // string literals are reached here, owned by the field. + self.markdown_refs_from_subtree(decl, row); } } } else { @@ -797,12 +806,15 @@ impl<'t> Walker<'t> { }); if let Some(name_node) = name_node { let name = self.text(name_node).to_string(); - self.create_node( + let row = self.create_node( field_kind, &name, node, Extra { docstring, visibility, is_static, ..Extra::default() }, ); + if let Some(row) = row { + self.markdown_refs_from_subtree(node, row); + } } } } diff --git a/codegraph-kernel/src/kotlin.rs b/codegraph-kernel/src/kotlin.rs index a224cb7265..633154c6f8 100644 --- a/codegraph-kernel/src/kotlin.rs +++ b/codegraph-kernel/src/kotlin.rs @@ -312,6 +312,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -817,6 +819,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_declaration" { if self.inside_class_like() { @@ -889,6 +893,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/codegraph-kernel/src/lib.rs b/codegraph-kernel/src/lib.rs index 7411fce0f0..214582bc41 100644 --- a/codegraph-kernel/src/lib.rs +++ b/codegraph-kernel/src/lib.rs @@ -30,6 +30,75 @@ macro_rules! stack_guard { }; } +/// The markdown path-reference pair, for a walker with the usual shape +/// (`text`, `line_of`, `col_of`, `arena`, `tables`, `file_path`). Every routed +/// language needs the same two methods, and the wasm arm they must match is +/// one implementation, so this is one implementation too β€” see markdown.rs for +/// what it mirrors and why the two ref flags are set only here. +macro_rules! markdown_refs_impl { + () => { + /// extractMarkdownPathReferencesFromStringNode. + fn markdown_refs_from_string(&mut self, node: Node<'t>, owner_row: u32) { + if !$crate::markdown::is_markdown_path_string_kind(node.kind()) { + return; + } + let found = $crate::markdown::markdown_path_refs(self.text(node), self.file_path); + if found.is_empty() { + return; + } + let kind_code = $crate::buffers::edge_kind_index("references").unwrap(); + let line = self.line_of(node); + let column = self.col_of(node); + let flags = $crate::buffers::REF_FLAG_FILE_PATH | $crate::buffers::REF_FLAG_LANGUAGE; + // A declaration's initializer can be reached twice under the same + // owner (its call walk and the subtree scan); addReference drops + // the repeat, so the kernel does too. + if self.tables.has_flagged_ref(owner_row, line, column + found[0].1 as u32, flags) { + return; + } + for (name, offset) in found { + let name_ref = self.arena.put(&name); + // addReference denormalizes filePath and language onto the ref + // where the ordinary ref path does not, so these two flags are + // set here and nowhere else. + self.tables.push_ref_flagged( + &$crate::buffers::RefRow { + from_idx: owner_row, + kind: kind_code, + line, + column: column + offset as u32, + reference_name: name_ref, + candidates: $crate::buffers::NONE_STR, + from_id_str: $crate::buffers::NONE_STR, + }, + flags, + ); + } + } + + /// extractMarkdownPathReferencesFromSubtree β€” for constructs whose walk + /// stops before their value, where the owner is the declared symbol + /// rather than the enclosing scope. + /// + /// Only some walkers stop that way (tsjs, python, java, csharp, ruby, + /// lua); in the rest a declaration's children are walked normally and + /// the string method above already covers them, so the pair is one + /// macro and this half goes unused there. The parity fixtures decide + /// which is which β€” every language's torture file carries the same + /// eight markdown shapes. + #[allow(dead_code)] + fn markdown_refs_from_subtree(&mut self, node: Node<'t>, owner_row: u32) { + stack_guard!(); + self.markdown_refs_from_string(node, owner_row); + for i in 0..node.named_child_count() { + if let Some(c) = node.named_child(i) { + self.markdown_refs_from_subtree(c, owner_row); + } + } + } + }; +} + mod buffers; mod ccpp; mod cfnptr; @@ -42,6 +111,7 @@ mod java; mod kotlin; mod langs; mod lua; +mod markdown; mod php; mod rlang; mod ruby; diff --git a/codegraph-kernel/src/lua.rs b/codegraph-kernel/src/lua.rs index 5436665572..dcc82e05d1 100644 --- a/codegraph-kernel/src/lua.rs +++ b/codegraph-kernel/src/lua.rs @@ -159,6 +159,8 @@ pub fn extract(file_path: &str, source: &str, language: &str) -> Result Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -457,6 +459,8 @@ impl<'t> Walker<'t> { // maybeCaptureFnRefs (tree-sitter.ts:990). self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); // The dispatch ladder β€” lua/luau rows only. if kind == "function_declaration" { @@ -613,7 +617,8 @@ impl<'t> Walker<'t> { } // Positional value pairing; a missing value β†’ NO signature key. let signature = value.map(|v| util::init_signature(self.text(v))); - self.create_node( + let name = name.to_string(); + let row = self.create_node( "variable", &name, *name_node, // positioned at the IDENTIFIER @@ -624,6 +629,12 @@ impl<'t> Walker<'t> { ..Default::default() }, ); + // The ladder skips a declaration's children, so the positionally + // paired value's string literals are reached here. A name with no + // value contributes nothing, as the wasm arm's undefined does. + if let (Some(row), Some(v)) = (row, values.get(i)) { + self.markdown_refs_from_subtree(*v, row); + } } } @@ -784,6 +795,8 @@ impl<'t> Walker<'t> { stack_guard!(); // maybeCaptureFnRefs (5137) fires in the body walker too. self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); let kind = node.kind(); if kind == "function_call" { diff --git a/codegraph-kernel/src/markdown.rs b/codegraph-kernel/src/markdown.rs new file mode 100644 index 0000000000..7e507745b9 --- /dev/null +++ b/codegraph-kernel/src/markdown.rs @@ -0,0 +1,204 @@ +//! Markdown path references from code string literals β€” the kernel half of the +//! code β†’ documentation edge (`open("docs/guide.md#install")`). +//! +//! The wasm arm builds these in `TreeSitterExtractor`; a routed language never +//! reaches it, so without this the edge is absent for every kernel language. +//! `kernel-tsjs-parity` compares refs byte for byte, so the candidate regex and +//! the normalizer below mirror `extractMarkdownPathCandidates` and +//! `normalizeMarkdownPathReference` (src/extraction/tree-sitter.ts) exactly β€” +//! including the rejections. Change one side and the parity gate fails. + +use regex::Regex; +use std::sync::OnceLock; + +/// MARKDOWN_PATH_STRING_NODE_TYPES. +pub fn is_markdown_path_string_kind(kind: &str) -> bool { + matches!( + kind, + "string" + | "string_literal" + | "template_string" + | "raw_string_literal" + | "interpreted_string_literal" + | "interpolated_string_expression" + ) +} + +fn candidate_re() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| { + Regex::new( + r#"(?i)((?:\.{1,2}[\\/]+|[A-Za-z0-9_.@-]+[\\/]+|[\\/]+)?(?:[A-Za-z0-9_.@-]+[\\/]+)*[A-Za-z0-9_.@-]+\.(?:md|mdx|markdown)(?:\?[^'"`\s)>,;]*)?(?:#[^'"`\s)>,;]*)?)"#, + ) + .expect("markdown path candidate regex") + }) +} + +fn scheme_re() -> &'static Regex { + static RE: OnceLock = OnceLock::new(); + RE.get_or_init(|| Regex::new(r"(?i)^[a-z][a-z0-9+.-]*://").expect("scheme regex")) +} + +/// `decodeURIComponent`, falling back to the raw value when it would throw. +fn decode_path(value: &str) -> String { + let bytes = value.as_bytes(); + let mut out: Vec = Vec::with_capacity(bytes.len()); + let mut i = 0; + while i < bytes.len() { + if bytes[i] == b'%' { + if i + 2 >= bytes.len() { + return value.to_string(); + } + let hex = match std::str::from_utf8(&bytes[i + 1..i + 3]) { + Ok(h) => h, + Err(_) => return value.to_string(), + }; + match u8::from_str_radix(hex, 16) { + Ok(b) => out.push(b), + Err(_) => return value.to_string(), + } + i += 3; + } else { + out.push(bytes[i]); + i += 1; + } + } + String::from_utf8(out).unwrap_or_else(|_| value.to_string()) +} + +/// `path.posix.normalize` for the shapes a doc path takes. +fn posix_normalize(p: &str) -> String { + let is_abs = p.starts_with('/'); + let mut out: Vec<&str> = Vec::new(); + for seg in p.split('/') { + match seg { + "" | "." => {} + ".." => { + if matches!(out.last(), Some(&last) if last != "..") { + out.pop(); + } else if !is_abs { + out.push(".."); + } + } + s => out.push(s), + } + } + let joined = out.join("/"); + if is_abs { + return format!("/{joined}"); + } + if joined.is_empty() { + return ".".to_string(); + } + joined +} + +fn ends_with_md(path: &str) -> bool { + let lower = path.to_ascii_lowercase(); + lower.ends_with(".md") || lower.ends_with(".mdx") || lower.ends_with(".markdown") +} + +/// `normalizeMarkdownPathReference` β€” None where the TS returns null. +fn normalize(reference_name: &str, file_path: &str) -> Option { + let trimmed = reference_name.trim().replace('\\', "/"); + if trimmed.is_empty() || scheme_re().is_match(&trimmed) { + return None; + } + + let (path_with_query, anchor) = match trimmed.find('#') { + Some(i) => (&trimmed[..i], &trimmed[i..]), + None => (trimmed.as_str(), ""), + }; + let raw_path = match path_with_query.find('?') { + Some(i) => &path_with_query[..i], + None => path_with_query, + }; + let clean_path = decode_path(raw_path); + + if !ends_with_md(&clean_path) { + return None; + } + + let file_posix = file_path.replace('\\', "/"); + let base_dir = match file_posix.rfind('/') { + Some(0) => "/".to_string(), + Some(i) => file_posix[..i].to_string(), + None => ".".to_string(), + }; + + let normalized_path = if let Some(rest) = clean_path.strip_prefix('/') { + posix_normalize(rest.trim_start_matches('/')) + } else if clean_path.starts_with("./") || clean_path.starts_with("../") { + let joined = if base_dir == "." { + clean_path.clone() + } else { + format!("{base_dir}/{clean_path}") + }; + posix_normalize(&joined) + } else { + posix_normalize(&clean_path) + }; + + if normalized_path.is_empty() + || normalized_path == "." + || normalized_path == ".." + || normalized_path.starts_with("../") + { + return None; + } + + Some(format!("{normalized_path}{anchor}")) +} + +/// Every markdown path reference in one string literal's text, as +/// (normalized name, byte offset of the match within `text`). The offset is +/// added to the literal's own column, mirroring the wasm arm. +pub fn markdown_path_refs(text: &str, file_path: &str) -> Vec<(String, usize)> { + let mut refs = Vec::new(); + for m in candidate_re().find_iter(text) { + // A `scheme://host/x.md` URL is not a repo path. The wasm arm looks + // back 16 chars for the `://` rather than matching it, because the + // candidate pattern starts after the scheme. + let start = m.start(); + let prefix = &text[start.saturating_sub(16)..start]; + if prefix.ends_with("://") { + continue; + } + if let Some(name) = normalize(m.as_str(), file_path) { + refs.push((name, start)); + } + } + refs +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn resolves_relative_paths_against_the_file() { + let refs = markdown_path_refs("'../docs/guide.md#install'", "src/load-docs.ts"); + assert_eq!(refs.len(), 1); + assert_eq!(refs[0].0, "docs/guide.md#install"); + } + + #[test] + fn rejects_an_escape_above_the_repo_root() { + assert!(markdown_path_refs("'../../../up.md'", "src/x.ts").is_empty()); + } + + #[test] + fn a_scheme_prefixed_path_is_whatever_the_wasm_arm_makes_of_it() { + // Not asserted either way here on purpose. The candidate pattern can + // start inside a URL (at the `//`), so the `://` look-back does not + // always fire, and the wasm arm β€” not a guess about it β€” is the spec. + // `kernel-tsjs-parity` pins this against a torture fixture instead. + let _ = markdown_path_refs("'https://example.com/a.md'", "src/x.ts"); + } + + #[test] + fn keeps_a_bare_name_and_an_anchor() { + let refs = markdown_path_refs("'README.md'", "src/x.ts"); + assert_eq!(refs[0].0, "README.md"); + } +} diff --git a/codegraph-kernel/src/php.rs b/codegraph-kernel/src/php.rs index fef28c9c55..094e42b721 100644 --- a/codegraph-kernel/src/php.rs +++ b/codegraph-kernel/src/php.rs @@ -265,6 +265,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -543,6 +545,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_definition" { // functionTypes; method_declaration is not in it, so this is @@ -621,6 +625,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if matches!( kind, diff --git a/codegraph-kernel/src/python.rs b/codegraph-kernel/src/python.rs index 1fe325f050..8d1e4af671 100644 --- a/codegraph-kernel/src/python.rs +++ b/codegraph-kernel/src/python.rs @@ -418,6 +418,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_definition" { // functionTypes ∩ methodTypes: inside a class-like β‡’ method. @@ -458,6 +460,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call" { self.extract_call(node); @@ -563,6 +567,8 @@ impl<'t> Walker<'t> { self.stack.pop(); } + markdown_refs_impl!(); + /// extractVariable's python branch: `left = right` at module scope. fn extract_variable(&mut self, node: Node<'t>) { let docstring = preceding_docstring(node, self.src); @@ -595,6 +601,10 @@ impl<'t> Walker<'t> { self.stack.push(Scope { row, kind: "variable", name }); self.visit_function_body(right); self.stack.pop(); + // visit_node skips an assignment's children, so the + // right-hand side's string literals are reached here, + // owned by the assigned name. + self.markdown_refs_from_subtree(right, row); } None => self.visit_function_body(right), } diff --git a/codegraph-kernel/src/rlang.rs b/codegraph-kernel/src/rlang.rs index e3170cb787..f67fe6314f 100644 --- a/codegraph-kernel/src/rlang.rs +++ b/codegraph-kernel/src/rlang.rs @@ -143,6 +143,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -305,6 +307,8 @@ impl<'t> Walker<'t> { if self.hook(node) { return; } + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if node.kind() == "call" { self.extract_call(node); } diff --git a/codegraph-kernel/src/ruby.rs b/codegraph-kernel/src/ruby.rs index 3e4e33cb08..771a5da540 100644 --- a/codegraph-kernel/src/ruby.rs +++ b/codegraph-kernel/src/ruby.rs @@ -206,6 +206,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -468,6 +470,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "method" { // functionTypes ∩ methodTypes: inside class-like (module counts!) @@ -535,6 +539,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call" { self.extract_call(node); @@ -692,7 +698,13 @@ impl<'t> Walker<'t> { } let name = self.text(left).to_string(); let signature = right.map(|r| util::init_signature(self.text(r))); - self.create_node("variable", &name, node, Extra { docstring, signature, visibility: None }); + let row = + self.create_node("variable", &name, node, Extra { docstring, signature, visibility: None }); + // visit_node skips an assignment's children, so the right-hand side's + // string literals are reached here, owned by the assigned name. + if let (Some(row), Some(r)) = (row, right) { + self.markdown_refs_from_subtree(r, row); + } } /// extractImport β€” every non-body `call` lands here (importTypes:['call']). diff --git a/codegraph-kernel/src/rustlang.rs b/codegraph-kernel/src/rustlang.rs index 1dbc689b41..944075dab6 100644 --- a/codegraph-kernel/src/rustlang.rs +++ b/codegraph-kernel/src/rustlang.rs @@ -203,6 +203,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -449,6 +451,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if matches!(kind, "function_item" | "function_signature_item") { self.extract_fn_or_method(node); @@ -1190,6 +1194,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); // Rocket route macros: handler paths live in a raw token tree. if kind == "macro_invocation" { diff --git a/codegraph-kernel/src/scala.rs b/codegraph-kernel/src/scala.rs index 9b7aeb7c20..1a0b9d84a1 100644 --- a/codegraph-kernel/src/scala.rs +++ b/codegraph-kernel/src/scala.rs @@ -247,6 +247,8 @@ pub fn extract(file_path: &str, source: &str) -> Result { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -543,6 +545,8 @@ impl<'t> Walker<'t> { // maybeCaptureFnRefs (:990). self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); let kind = node.kind(); match kind { @@ -1181,6 +1185,8 @@ impl<'t> Walker<'t> { fn visit_body(&mut self, node: Node<'t>) { stack_guard!(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); let kind = node.kind(); if kind == "call_expression" { diff --git a/codegraph-kernel/src/swift.rs b/codegraph-kernel/src/swift.rs index f1d3ae4ac9..da73aee831 100644 --- a/codegraph-kernel/src/swift.rs +++ b/codegraph-kernel/src/swift.rs @@ -299,6 +299,8 @@ fn last_simple_identifier<'t>(node: Node<'t>) -> Option> { } impl<'t> Walker<'t> { + markdown_refs_impl!(); + fn text(&self, node: Node) -> &'t str { &self.src[node.byte_range()] } @@ -587,6 +589,8 @@ impl<'t> Walker<'t> { let mut skip_children = false; self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "function_declaration" { if self.inside_class_like() { @@ -751,6 +755,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/codegraph-kernel/src/tsjs/extractors.rs b/codegraph-kernel/src/tsjs/extractors.rs index 6426db3156..476827762e 100644 --- a/codegraph-kernel/src/tsjs/extractors.rs +++ b/codegraph-kernel/src/tsjs/extractors.rs @@ -442,6 +442,12 @@ impl<'t> Walker<'t> { ); if let Some(row) = var_row { self.extract_variable_type_annotation(child, row); + // The declarator's walk stops here (visit_node skips a + // variable's children), so the initializer's string literals + // are reached from this side, owned by the declared symbol. + if let Some(v) = value { + self.markdown_refs_from_subtree(v, row); + } } // Exported const object-of-functions / store shapes. diff --git a/codegraph-kernel/src/tsjs/mod.rs b/codegraph-kernel/src/tsjs/mod.rs index bdc88e8aba..a0eb605fa1 100644 --- a/codegraph-kernel/src/tsjs/mod.rs +++ b/codegraph-kernel/src/tsjs/mod.rs @@ -341,6 +341,8 @@ impl<'t> Walker<'t> { self.push_ref(self.top_row(), name, edge_kind_index("calls").unwrap(), node); } + markdown_refs_impl!(); + // --- createNode ----------------------------------------------------------- /// createNode (tree-sitter.ts): id, qualified name from the scope stack, @@ -639,6 +641,8 @@ impl<'t> Walker<'t> { // Function-as-value capture β€” independent of the dispatch ladder. self.maybe_capture_fn_refs(node); + let owner = self.top_row(); + self.markdown_refs_from_string(node, owner); if is_function_type(kind) { // (the isInsideClassLike + methodTypes overlap is Python/Ruby-only) @@ -728,6 +732,8 @@ impl<'t> Walker<'t> { stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); + let md_owner = self.top_row(); + self.markdown_refs_from_string(node, md_owner); if kind == "call_expression" { self.extract_call(node); diff --git a/site/src/content/docs/reference/languages.md b/site/src/content/docs/reference/languages.md index 0c5587773b..f6b59ab94b 100644 --- a/site/src/content/docs/reference/languages.md +++ b/site/src/content/docs/reference/languages.md @@ -31,3 +31,6 @@ Language support is automatic from the file extension β€” there's nothing to con | Lua | `.lua` | Full support (functions, methods, locals, `require` imports, call edges) | | R | `.R`, `.r` | Full support (functions, S4/R5/R6 classes with methods, `library`/`require` imports, `source()` file references, call edges) | | Luau | `.luau` | Full support (Lua, plus typed signatures, `type` aliases, Roblox `require`) | +| Markdown | `.md`, `.mdx`, `.markdown` | Documentation structure (headings, sections, local links, selected table rows/list items, shell command references) | + +Markdown files use a dedicated documentation extractor. `.mdx` files receive the same documentation indexing; embedded JSX and JavaScript are not parsed as MDX code. diff --git a/src/extraction/generated-detection.ts b/src/extraction/generated-detection.ts index 83ec2b1aad..1741acd46c 100644 --- a/src/extraction/generated-detection.ts +++ b/src/extraction/generated-detection.ts @@ -249,6 +249,10 @@ export function hasGeneratedHeader(content: string): boolean { * indexer persists to `files.generated`. */ export function detectGeneratedFile(filePath: string, content: string): boolean { + // Markdown carries no banner: its `#` is a heading, not a comment leader, + // so a README that DESCRIBES generated code ("Code generated … DO NOT EDIT" + // under a heading) would flag itself. The path convention still applies. + if (/\.(?:md|markdown)$/i.test(filePath)) return isGeneratedFile(filePath); return isGeneratedFile(filePath) || hasGeneratedHeader(content) || isMinifiedContent(filePath, content); } diff --git a/src/extraction/grammars.ts b/src/extraction/grammars.ts index 6a05b96c0d..77a8391269 100644 --- a/src/extraction/grammars.ts +++ b/src/extraction/grammars.ts @@ -11,7 +11,7 @@ import * as fsp from 'fs/promises'; import { Parser, Language as WasmLanguage } from 'web-tree-sitter'; import { Language } from '../types'; -export type GrammarLanguage = Exclude; +export type GrammarLanguage = Exclude; /** * WASM filename map β€” maps each language to its .wasm grammar file @@ -100,6 +100,9 @@ export const EXTENSION_MAP: Record = { '.yaml': 'yaml', // Twig templates (file-level tracking only, no symbol extraction) '.twig': 'twig', + '.md': 'markdown', + '.mdx': 'markdown', + '.markdown': 'markdown', '.rb': 'ruby', '.rake': 'ruby', '.swift': 'swift', @@ -755,6 +758,7 @@ export function isLanguageSupported(language: Language): boolean { if (language === 'twig') return true; // file-level tracking only if (language === 'xml') return true; // MyBatis mapper extractor if (language === 'properties') return true; // Spring config keys + if (language === 'markdown') return true; // custom documentation extractor if (language === 'unknown') return false; return language in WASM_GRAMMAR_FILES; } @@ -763,7 +767,7 @@ export function isLanguageSupported(language: Language): boolean { * Check if a grammar has been loaded and is ready for parsing. */ export function isGrammarLoaded(language: Language): boolean { - if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor') return true; + if (language === 'svelte' || language === 'vue' || language === 'astro' || language === 'liquid' || language === 'razor' || language === 'markdown') return true; if (language === 'yaml' || language === 'twig') return true; // no WASM grammar needed if (language === 'xml' || language === 'properties') return true; // no WASM grammar needed return languageCache.has(language); @@ -786,7 +790,7 @@ export function isFileLevelOnlyLanguage(language: Language): boolean { * Get all supported languages (those with grammar definitions). */ export function getSupportedLanguages(): Language[] { - return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid']; + return [...(Object.keys(WASM_GRAMMAR_FILES) as GrammarLanguage[]), 'svelte', 'vue', 'astro', 'liquid', 'markdown']; } /** @@ -866,6 +870,7 @@ export function getLanguageDisplayName(language: Language): string { twig: 'Twig', xml: 'XML', properties: 'Java properties', + markdown: 'Markdown', cfml: 'CFML', cfscript: 'CFScript', cfquery: 'CFQuery (SQL)', diff --git a/src/extraction/kernel/decode.ts b/src/extraction/kernel/decode.ts index 6111733733..5f067937aa 100644 --- a/src/extraction/kernel/decode.ts +++ b/src/extraction/kernel/decode.ts @@ -31,6 +31,7 @@ import { PROVENANCES, REF, REF_FLAG_FILE_PATH, + REF_FLAG_LANGUAGE, REF_ROW_SIZE, VISIBILITIES, } from './layout'; @@ -155,7 +156,9 @@ export function decodeExtractBuffers( // exactly. The ONE exception is flagged (REF_FLAG_FILE_PATH): the // ruby/php visitNode hooks set `filePath: ctx.filePath` on their // mixin/trait `implements` refs β€” re-attach the decode call's own - // filePath, which is that exact value. + // filePath, which is that exact value. Markdown path refs go through + // `addReference`, which denormalizes BOTH fields, so they carry the + // language flag as well. const ref: UnresolvedReference = { fromNodeId: fromIdx === NONE ? str(arena, row, REF.fromIdStr)! : idByRow[fromIdx]!, referenceName: str(arena, row, REF.referenceName)!, @@ -166,7 +169,9 @@ export function decodeExtractBuffers( line: row.readUInt32LE(REF.line), column: row.readUInt32LE(REF.column), }; - if ((row.readUInt8(REF.flags) & REF_FLAG_FILE_PATH) !== 0) ref.filePath = filePath; + const refFlags = row.readUInt8(REF.flags); + if ((refFlags & REF_FLAG_FILE_PATH) !== 0) ref.filePath = filePath; + if ((refFlags & REF_FLAG_LANGUAGE) !== 0) ref.language = language; const candidates = strList(arena, row, REF.candidates); if (candidates !== undefined) ref.candidates = candidates; unresolvedReferences[i] = ref; diff --git a/src/extraction/kernel/layout.ts b/src/extraction/kernel/layout.ts index 1c490e6e26..65718d1004 100644 --- a/src/extraction/kernel/layout.ts +++ b/src/extraction/kernel/layout.ts @@ -96,6 +96,7 @@ export const FUNCTION_REF_CODE = 200; * parameter, which is byte-identical. */ export const REF_FLAG_FILE_PATH = 1; +export const REF_FLAG_LANGUAGE = 2; /** Node bool-flag bit pairs: bit(2n) = present, bit(2n+1) = value. */ export const FLAG = { diff --git a/src/extraction/markdown-extractor.ts b/src/extraction/markdown-extractor.ts new file mode 100644 index 0000000000..4695ea4be4 --- /dev/null +++ b/src/extraction/markdown-extractor.ts @@ -0,0 +1,1003 @@ +import * as path from 'path'; +import { Node, Edge, ExtractionResult, ExtractionError, UnresolvedReference, EdgeKind } from '../types'; +import { generateNodeId } from './tree-sitter-helpers'; + +interface HeadingInfo { + id: string; + level: number; + title: string; + slug: string; + line: number; + endLine: number; + node: Node; +} + +interface FenceState { + marker: string; + language: string; + startLine: number; + lines: Array<{ text: string; line: number }>; +} + +/** + * Lightweight Markdown extractor. + * + * Markdown is indexed as documentation structure rather than code syntax: + * headings become searchable module nodes, links become import nodes, and + * shell-like fenced blocks create command nodes plus file-path references. + */ +export class MarkdownExtractor { + private filePath: string; + private lines: string[]; + private nodes: Node[] = []; + private edges: Edge[] = []; + private unresolvedReferences: UnresolvedReference[] = []; + private errors: ExtractionError[] = []; + private headings: HeadingInfo[] = []; + private referenceKeys = new Set(); + + constructor(filePath: string, source: string) { + this.filePath = normalizeRelativePath(filePath); + this.lines = source.split('\n'); + } + + extract(): ExtractionResult { + const startTime = Date.now(); + + try { + const fileNode = this.createFileNode(); + this.extractHeadings(fileNode); + this.extractStructuredBlocks(fileNode); + this.extractLinksAndCommands(fileNode); + this.finalizeFileDigest(fileNode); + } catch (error) { + this.errors.push({ + message: `Markdown extraction error: ${error instanceof Error ? error.message : String(error)}`, + filePath: this.filePath, + severity: 'error', + code: 'parse_error', + }); + } + + return { + nodes: this.nodes, + edges: this.edges, + unresolvedReferences: this.unresolvedReferences, + errors: this.errors, + durationMs: Date.now() - startTime, + }; + } + + private createFileNode(): Node { + const id = generateNodeId(this.filePath, 'file', this.filePath, 1); + const fileNode: Node = { + id, + kind: 'file', + name: path.posix.basename(this.filePath), + qualifiedName: this.filePath, + filePath: this.filePath, + language: 'markdown', + startLine: 1, + endLine: Math.max(1, this.lines.length), + startColumn: 0, + endColumn: this.lines[this.lines.length - 1]?.length || 0, + // docstring is filled in by finalizeFileDigest() once headings and + // references are known, so it becomes a triage-friendly digest. + docstring: undefined, + updatedAt: Date.now(), + }; + + this.nodes.push(fileNode); + return fileNode; + } + + private extractHeadings(fileNode: Node): void { + const rawHeadings: Array<{ level: number; title: string; line: number; column: number }> = []; + const frontmatterEnd = this.frontmatterEndIndex(); + let fenceMarker: string | null = null; + + for (let i = 0; i < this.lines.length; i++) { + if (i <= frontmatterEnd) continue; + const line = this.lines[i]!; + + // Track fenced code blocks so `# foo` comments and `===`/`---` lines + // inside a code sample are never mistaken for document headings. + const fence = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fence) { + const marker = fence[2]![0]!.repeat(fence[2]!.length); + if (fenceMarker === null) fenceMarker = marker; + else if (line.trimStart().startsWith(fenceMarker)) fenceMarker = null; + continue; + } + if (fenceMarker !== null) continue; + + // ATX heading: `## Title` + const atx = /^(#{1,6})\s+(.+?)\s*#*\s*$/.exec(line); + if (atx) { + rawHeadings.push({ + level: atx[1]!.length, + title: stripInlineMarkdown(atx[2]!.trim()), + line: i + 1, + column: line.indexOf('#'), + }); + continue; + } + + // Setext heading: a paragraph line underlined by `===` (level 1) or + // `---` (level 2). CommonMark treats a `---` directly under a paragraph + // as a heading, not a thematic break β€” and the frontmatter skip above + // keeps a closing `---` from turning its last key into a heading. + const underline = /^(=+|-+)\s*$/.exec(line.trim()); + if (underline && i - 1 > frontmatterEnd && isSetextTextLine(this.lines[i - 1] ?? '')) { + rawHeadings.push({ + level: underline[1]![0] === '=' ? 1 : 2, + title: stripInlineMarkdown((this.lines[i - 1] ?? '').trim()), + line: i, // the text line carries the heading + column: 0, + }); + } + } + + const slugCounts = new Map(); + for (let i = 0; i < rawHeadings.length; i++) { + const heading = rawHeadings[i]!; + const baseSlug = slugifyHeading(heading.title); + const seen = slugCounts.get(baseSlug) ?? 0; + slugCounts.set(baseSlug, seen + 1); + const slug = seen === 0 ? baseSlug : `${baseSlug}-${seen}`; + + let endLine = this.lines.length; + for (let j = i + 1; j < rawHeadings.length; j++) { + if (rawHeadings[j]!.level <= heading.level) { + endLine = rawHeadings[j]!.line - 1; + break; + } + } + + const nodeId = generateNodeId(this.filePath, 'module', `${slug}:${heading.line}`, heading.line); + const node: Node = { + id: nodeId, + kind: 'module', + name: heading.title, + qualifiedName: `${this.filePath}#${slug}`, + filePath: this.filePath, + language: 'markdown', + signature: `${'#'.repeat(heading.level)} ${heading.title}`, + docstring: this.buildDocstring(heading.line + 1, endLine), + startLine: heading.line, + endLine, + startColumn: heading.column, + endColumn: this.lines[heading.line - 1]?.length || 0, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.headings.push({ + id: nodeId, + level: heading.level, + title: heading.title, + slug, + line: heading.line, + endLine, + node, + }); + } + + const stack: HeadingInfo[] = []; + for (const heading of this.headings) { + while (stack.length > 0 && stack[stack.length - 1]!.level >= heading.level) { + stack.pop(); + } + const parent = stack[stack.length - 1]; + this.edges.push({ + source: parent?.id ?? fileNode.id, + target: heading.id, + kind: 'contains', + provenance: 'heuristic', + }); + stack.push(heading); + } + } + + private extractLinksAndCommands(fileNode: Node): void { + let fence: FenceState | null = null; + + for (let i = 0; i < this.lines.length; i++) { + const line = this.lines[i]!; + const lineNumber = i + 1; + const fenceMatch = /^(\s*)(`{3,}|~{3,})\s*([A-Za-z0-9_+.-]*)/.exec(line); + + if (fence) { + if (line.trimStart().startsWith(fence.marker)) { + this.extractCommandsFromFence(fence, fileNode); + fence = null; + } else { + fence.lines.push({ text: line, line: lineNumber }); + } + continue; + } + + if (fenceMatch) { + fence = { + marker: fenceMatch[2]![0]!.repeat(fenceMatch[2]!.length), + language: (fenceMatch[3] ?? '').toLowerCase(), + startLine: lineNumber, + lines: [], + }; + continue; + } + + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + this.extractMarkdownLinks(line, lineNumber, owner); + if (!isTableRowLine(line)) { + this.extractFileSymbolMentions(line, lineNumber, owner); + this.extractPathMentions(line, lineNumber, owner); + } + } + + if (fence) { + this.extractCommandsFromFence(fence, fileNode); + } + } + + private extractStructuredBlocks(fileNode: Node): void { + this.extractTables(fileNode); + this.extractListItems(fileNode); + } + + private extractTables(fileNode: Node): void { + let inFence: string | null = null; + + for (let i = 0; i < this.lines.length - 1; i++) { + const line = this.lines[i]!; + const fenceMatch = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fenceMatch) { + const marker = fenceMatch[2]!; + if (inFence === null) { + inFence = marker; + } else if ( + marker[0] === inFence[0] && + marker.length >= inFence.length && + line.slice(fenceMatch[0].length).trim() === '' + ) { + inFence = null; + } + continue; + } + if (inFence) continue; + + const separator = this.lines[i + 1]!; + if (!isTableHeaderLine(line) || !isTableSeparatorLine(separator)) continue; + + const headers = parseTableCells(line); + if (headers.length < 2) continue; + + let rowIndex = i + 2; + while (rowIndex < this.lines.length && isTableRowLine(this.lines[rowIndex]!)) { + const rowLine = this.lines[rowIndex]!; + const cells = parseTableCells(rowLine); + const lineNumber = rowIndex + 1; + if (shouldIndexTableRow(headers, cells)) { + this.createTableRowNode(headers, cells, rowLine, lineNumber, fileNode); + } + rowIndex++; + } + + i = rowIndex - 1; + } + } + + private extractListItems(fileNode: Node): void { + let inFence: string | null = null; + + for (let i = 0; i < this.lines.length; i++) { + const line = this.lines[i]!; + const fenceMatch = /^(\s*)(`{3,}|~{3,})/.exec(line); + if (fenceMatch) { + const marker = fenceMatch[2]!; + if (inFence === null) { + inFence = marker; + } else if ( + marker[0] === inFence[0] && + marker.length >= inFence.length && + line.slice(fenceMatch[0].length).trim() === '' + ) { + inFence = null; + } + continue; + } + if (inFence) continue; + + const match = /^\s*[-*+]\s+(?:\[[ xX]\]\s+)?(.+?)\s*$/.exec(line); + if (!match) continue; + + const text = stripInlineMarkdown(match[1]!.trim()); + if (!shouldIndexListItem(text)) continue; + + const lineNumber = i + 1; + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + const name = stableIdentifier(text) ?? truncateForName(text, 80); + const nodeId = generateNodeId(this.filePath, 'constant', `list:${lineNumber}:${name}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'constant', + name, + qualifiedName: `${this.filePath}::list-item:${slugifyHeading(name)}:${lineNumber}`, + filePath: this.filePath, + language: 'markdown', + signature: `list item: ${truncateForSignature(text, 180)}`, + docstring: text, + startLine: lineNumber, + endLine: lineNumber, + startColumn: Math.max(0, line.indexOf(match[1]!)), + endColumn: line.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.extractStructuredReferences(text, lineNumber, node); + } + } + + private createTableRowNode( + headers: string[], + cells: string[], + rowLine: string, + lineNumber: number, + fileNode: Node + ): void { + const owner = this.findOwnerForLine(lineNumber) ?? fileNode; + const rowText = cells.join(' | '); + const name = stableIdentifier(rowText) ?? truncateForName(firstNonEmptyCell(cells) || `row ${lineNumber}`, 80); + const signature = summarizeTableRow(headers, cells); + const nodeId = generateNodeId(this.filePath, 'constant', `table:${lineNumber}:${name}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'constant', + name, + qualifiedName: `${this.filePath}::table-row:${slugifyHeading(name)}:${lineNumber}`, + filePath: this.filePath, + language: 'markdown', + signature, + docstring: signature, + startLine: lineNumber, + endLine: lineNumber, + startColumn: Math.max(0, rowLine.indexOf(firstNonEmptyCell(cells) || '|')), + endColumn: rowLine.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + + this.extractStructuredReferences(rowText, lineNumber, node); + this.extractCommandsFromStructuredCells(cells, rowLine, lineNumber, node); + } + + private extractCommandsFromStructuredCells(cells: string[], rowLine: string, lineNumber: number, owner: Node): void { + for (const cell of cells) { + const candidates = extractCodeSpans(cell); + if (candidates.length === 0) candidates.push(stripInlineMarkdown(cell)); + + for (const candidate of candidates) { + const command = extractCommandFromText(candidate); + if (!command) continue; + const column = Math.max(0, rowLine.indexOf(candidate)); + this.createCommandNode(command, lineNumber, column, owner); + } + } + } + + private createCommandNode(command: string, line: number, column: number, owner: Node): void { + const nodeId = generateNodeId(this.filePath, 'function', `command:${line}:${column}:${command}`, line); + const node: Node = { + id: nodeId, + kind: 'function', + name: commandName(command), + qualifiedName: `${this.filePath}::command:${line}:${column}:${command}`, + filePath: this.filePath, + language: 'markdown', + signature: command, + startLine: line, + endLine: line, + startColumn: column, + endColumn: column + command.length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + + const scriptPath = extractScriptPath(command); + if (scriptPath) { + const normalizedTarget = this.normalizeReference(scriptPath); + if (normalizedTarget) { + this.addReference(nodeId, normalizedTarget, 'calls', line, column); + } + } + } + + private extractMarkdownLinks(line: string, lineNumber: number, owner: Node): void { + const linkRegex = /!?\[([^\]]*)\]\(([^)\s]+)(?:\s+["'][^"']*["'])?\)/g; + let match: RegExpExecArray | null; + + while ((match = linkRegex.exec(line)) !== null) { + const target = match[2]!; + if (!isLocalReference(target)) continue; + + const normalizedTarget = this.normalizeReference(target); + if (!normalizedTarget) continue; + + const column = match.index; + const label = stripInlineMarkdown(match[1] || target) || target; + const displayName = path.posix.basename(stripAnchor(normalizedTarget)) || label; + const nodeId = generateNodeId(this.filePath, 'import', `${normalizedTarget}:${lineNumber}:${column}`, lineNumber); + const node: Node = { + id: nodeId, + kind: 'import', + name: displayName, + qualifiedName: `${this.filePath}::link:${normalizedTarget}:${lineNumber}:${column}`, + filePath: this.filePath, + language: 'markdown', + signature: match[0], + docstring: label, + startLine: lineNumber, + endLine: lineNumber, + startColumn: column, + endColumn: column + match[0].length, + updatedAt: Date.now(), + }; + + this.nodes.push(node); + this.edges.push({ source: owner.id, target: nodeId, kind: 'contains', provenance: 'heuristic' }); + this.addReference(owner.id, normalizedTarget, 'imports', lineNumber, column); + } + } + + private extractPathMentions(line: string, lineNumber: number, owner: Node): void { + const pathRegex = /(?= best.level) best = heading; + } + } + return best?.node ?? null; + } + + private addReference( + fromNodeId: string, + referenceName: string, + referenceKind: EdgeKind, + line: number, + column: number + ): void { + const key = `${fromNodeId}:${referenceKind}:${referenceName}:${line}:${column}`; + if (this.referenceKeys.has(key)) return; + this.referenceKeys.add(key); + this.unresolvedReferences.push({ + fromNodeId, + referenceName, + referenceKind, + line, + column, + filePath: this.filePath, + language: 'markdown', + }); + } + + private normalizeReference(target: string): string | null { + const [pathAndSymbol, anchorPart] = splitAnchor(target.trim()); + const [pathPart, symbolPart] = splitFileSymbol(pathAndSymbol); + const cleanPath = decodePath(pathPart.split(/[?#]/)[0] ?? ''); + const anchor = anchorPart ? `#${slugifyHeading(anchorPart)}` : ''; + + if (!cleanPath && anchor) { + return `${this.filePath}${anchor}`; + } + if (!cleanPath) return null; + + const withoutLeadingSlash = cleanPath.startsWith('/') ? cleanPath.slice(1) : cleanPath; + const baseDir = path.posix.dirname(this.filePath); + const normalized = cleanPath.startsWith('/') || baseDir === '.' + ? path.posix.normalize(withoutLeadingSlash) + : path.posix.normalize(path.posix.join(baseDir, withoutLeadingSlash)); + + if (normalized.startsWith('../') || normalized === '..') return null; + return `${normalized}${symbolPart ? `::${symbolPart}` : ''}${anchor}`; + } + + private buildDocstring(startLine: number, endLine: number): string | undefined { + const text = this.lines + .slice(Math.max(0, startLine - 1), Math.max(0, endLine)) + .map((line) => stripInlineMarkdown(line.replace(/^#{1,6}\s+/, '').trim())) + .filter((line) => line && !/^(```|~~~)/.test(line)) + .join('\n') + .trim(); + + if (!text) return undefined; + return text.length > 600 ? `${text.slice(0, 600)}...` : text; + } + + /** + * Index of the closing `---` of a leading YAML frontmatter block, or -1 when + * the file has none. Lines at or before this index are metadata, not body β€” + * so they never produce headings and never seed the intro/digest. + */ + private frontmatterEndIndex(): number { + if ((this.lines[0] ?? '').trim() !== '---') return -1; + for (let j = 1; j < this.lines.length; j++) { + if (this.lines[j]!.trim() === '---') return j; + } + return -1; + } + + /** + * Replace the file node's docstring with a deterministic digest β€” a one-line + * "what is this doc about" intro plus the key files/symbols it references. + * It is derived purely from the already-extracted structure (no LLM), kept + * short enough to surface in node details, and β€” because docstrings are in + * the FTS index β€” makes a doc discoverable by the symbols it documents even + * when the query matches no heading or filename. See the reviewer thread on + * PR #361. + */ + private finalizeFileDigest(fileNode: Node): void { + const intro = this.buildIntro(); + const refs = this.collectKeyReferences(); + const parts: string[] = []; + if (intro) parts.push(intro); + if (refs.length > 0) parts.push(`refs: ${refs.join(', ')}`); + const digest = parts.join(' Β· ').trim(); + fileNode.docstring = digest || this.buildDocstring(1, Math.min(this.lines.length, 40)); + } + + /** + * First real prose line of the document β€” the closest deterministic stand-in + * for "what this file is about". Skips frontmatter, headings, fenced code, + * tables, badge/image-only lines, and raw HTML. + */ + private buildIntro(): string | null { + let fenceMarker: string | null = null; + for (let i = this.frontmatterEndIndex() + 1; i < this.lines.length; i++) { + const trimmed = this.lines[i]!.trim(); + const fence = /^(`{3,}|~{3,})/.exec(trimmed); + if (fence) { + const marker = fence[1]![0]!.repeat(fence[1]!.length); + fenceMarker = fenceMarker === null ? marker : (trimmed.startsWith(fenceMarker) ? null : fenceMarker); + continue; + } + if (fenceMarker !== null) continue; + if (!trimmed) continue; + if (/^#{1,6}\s/.test(trimmed)) continue; // ATX heading + if (/^(=+|-+|\*{3,}|_{3,})\s*$/.test(trimmed)) continue; // setext underline / hr + if (trimmed.startsWith('|')) continue; // table + if (/^!\[/.test(trimmed)) continue; // image / badge line + if (/^