From 09e80a27c6050746021915117312e5c84189264b Mon Sep 17 00:00:00 2001 From: Oliver Georgi Date: Thu, 12 Nov 2026 09:33:47 +0100 Subject: [PATCH] fix: index .cc and .cxx sources and classify .h headers - Add .cc, .cxx, .hh and .hxx to the walker source extensions, so those files reach the parser and the language counts - Label a .h header cpp when its text uses a C++ construct, else c - Label a .h header cpp when a .cc, .cpp or .cxx file shares its directory, preferring the same base name - Resolve header language in the graph extractor and the parser pool, and bump the cache format so stale C labels are rebuilt once --- CHANGELOG.md | 4 ++ internal/cache/cache.go | 6 +- internal/graph/cpp_lang_test.go | 51 +++++++++++++ internal/graph/extract.go | 6 +- internal/graph/ts_dispatch.go | 21 ++++-- internal/parser/generic.go | 13 ++-- internal/parser/header_lang.go | 95 ++++++++++++++++++++++++ internal/parser/header_lang_test.go | 108 ++++++++++++++++++++++++++++ internal/parser/langs_wave2_test.go | 2 +- internal/parser/pool.go | 14 ++++ internal/parser/ts_c.go | 14 ++-- internal/walker/walker.go | 4 ++ internal/walker/walker_test.go | 24 +++++++ 13 files changed, 342 insertions(+), 20 deletions(-) create mode 100644 internal/graph/cpp_lang_test.go create mode 100644 internal/parser/header_lang.go create mode 100644 internal/parser/header_lang_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index be520a9..ff55ae0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ Follows the first real-agent A/B (bench/agent-ab, pull request #77), which found - `path` adds advice after the weakest hop score: sure (0.9 or more) needs no confirming grep, otherwise grep the weak hop. - A pre-registered second A/B round in `bench/agent-ab/tasks-round2/`, with a `stacklit-e` arm in the harness. It has not been run. +### Fixed +- `.cc` and `.cxx` sources were never indexed because the walker did not list those extensions, and `.hh` and `.hxx` were missing too. A C++ project whose sources use `.cc` was reported as C. +- A `.h` header was always labelled C. It is now C++ when its own text uses C++ constructs or a C++ translation unit sits in the same directory, so a header dominated project no longer reports the wrong primary language. Cached parse results are rebuilt once for this. + ### Changed - The MCP handshake instructions and `stacklit guide` use the routed wording instead of "source of truth, do not crawl" (about 100 tokens, was about 75). `STACKLIT_STEERING=d` keeps the earlier text for comparison. diff --git a/internal/cache/cache.go b/internal/cache/cache.go index 7911da0..5d3109d 100644 --- a/internal/cache/cache.go +++ b/internal/cache/cache.go @@ -14,8 +14,10 @@ import ( "github.com/glincker/stacklit/internal/parser" ) -// formatVersion is bumped when the on-disk layout changes. -const formatVersion = 1 +// formatVersion is bumped when the on-disk layout changes, or when a stored +// parse result would be wrong under the current classification. Version 2 +// changed how .h headers and .cc files are labelled. +const formatVersion = 2 type fileFormat struct { Format int `json:"format"` diff --git a/internal/graph/cpp_lang_test.go b/internal/graph/cpp_lang_test.go new file mode 100644 index 0000000..a3dad7f --- /dev/null +++ b/internal/graph/cpp_lang_test.go @@ -0,0 +1,51 @@ +package graph + +import ( + "os" + "path/filepath" + "testing" +) + +func TestExtractPathClassifiesHeaders(t *testing.T) { + dir := t.TempDir() + write := func(name, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, name), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + + // Plain C header with no C++ neighbour. + write("plain.h", "int process(const char *msg);\n") + if fs := ExtractPath(dir, "plain.h", 0); fs == nil || fs.Lang != "c" { + t.Fatalf("plain.h = %+v, want Lang c", fs) + } + + // Same header once a C++ translation unit sits beside it. + write("plain.cc", "int process(const char *msg) { return 0; }\n") + if fs := ExtractPath(dir, "plain.h", 0); fs == nil || fs.Lang != "cpp" { + t.Fatalf("plain.h beside .cc = %+v, want Lang cpp", fs) + } + + // A header that is C++ on its own text. + write("acquirer.h", "class Acquirer {\npublic:\n void run();\n};\n") + if fs := ExtractPath(dir, "acquirer.h", 0); fs == nil || fs.Lang != "cpp" { + t.Fatalf("acquirer.h = %+v, want Lang cpp", fs) + } + + // A .cc file itself is C++. + if fs := ExtractPath(dir, "plain.cc", 0); fs == nil || fs.Lang != "cpp" { + t.Fatalf("plain.cc = %+v, want Lang cpp", fs) + } +} + +func TestExtractFileWithoutDiskPathUsesContent(t *testing.T) { + fs := extractFileWith("hdr.h", []byte("namespace n { struct S; }\n"), ExtractOptions{}, nil) + if fs == nil || fs.Lang != "cpp" { + t.Fatalf("header with C++ text = %+v, want Lang cpp", fs) + } + fs = extractFileWith("hdr.h", []byte("int f(void);\n"), ExtractOptions{}, nil) + if fs == nil || fs.Lang != "c" { + t.Fatalf("header with C text = %+v, want Lang c", fs) + } +} diff --git a/internal/graph/extract.go b/internal/graph/extract.go index 2308b57..d8d63b0 100644 --- a/internal/graph/extract.go +++ b/internal/graph/extract.go @@ -111,7 +111,9 @@ func isTestFile(path string) bool { // ExtractFile parses one file's content into symbols and references using the // default size limits. func ExtractFile(path string, src []byte) *FileSyms { - return extractFileWith(path, src, ExtractOptions{}, nil) + // No on-disk path, so a header is classified from its own text only and + // the result does not depend on the process working directory. + return extractFileAt(path, "", src, ExtractOptions{}, nil) } // Extract reads and parses the given root-relative files in parallel. @@ -169,7 +171,7 @@ func extractPath(root, rel string, o ExtractOptions, gate *parseGate) *FileSyms if err != nil { return nil } - return extractFileWith(filepath.ToSlash(rel), src, o, gate) + return extractFileAt(filepath.ToSlash(rel), filepath.Join(root, filepath.FromSlash(rel)), src, o, gate) } // stripNoise blanks comments and string contents with spaces, keeping newlines diff --git a/internal/graph/ts_dispatch.go b/internal/graph/ts_dispatch.go index 9f33c01..b1e1faa 100644 --- a/internal/graph/ts_dispatch.go +++ b/internal/graph/ts_dispatch.go @@ -1,13 +1,14 @@ package graph import ( - "bytes" "os" "path/filepath" "strconv" "strings" "sync" "time" + + "github.com/glincker/stacklit/internal/parser" ) // Parse cost model, matching the main scanner: the GLR parser holds roughly @@ -133,21 +134,27 @@ func grammarFor(lang, path string, src []byte) string { } case "csharp": return "c_sharp" - case "c": - if ext == ".h" && (bytes.Contains(src, []byte("class ")) || bytes.Contains(src, []byte("namespace ")) || bytes.Contains(src, []byte("template<")) || bytes.Contains(src, []byte("template <"))) { - return "cpp" - } } return lang } -// extractFileWith dispatches on language. Tree-sitter handles everything except -// Go; the line scanners stay as the fallback for oversize or unparsable files. +// extractFileWith dispatches on language when only the source is available. +// The neighbour check that can mark a header as C++ is skipped. func extractFileWith(path string, src []byte, o ExtractOptions, gate *parseGate) *FileSyms { + return extractFileAt(path, "", src, o, gate) +} + +// extractFileAt is extractFileWith with the header's on-disk path, so a .h +// header can be classified from the translation units next to it. +func extractFileAt(path, diskPath string, src []byte, o ExtractOptions, gate *parseGate) *FileSyms { lang := LangOf(path) if lang == "" { return nil } + // A .h header is C++ when its own text or its neighbours say so. + if lang == "c" && strings.EqualFold(filepath.Ext(path), ".h") { + lang = parser.HeaderLanguage(diskPath, src) + } if lang == "go" { return extractGo(path, src) } diff --git a/internal/parser/generic.go b/internal/parser/generic.go index f216ed1..d7566e9 100644 --- a/internal/parser/generic.go +++ b/internal/parser/generic.go @@ -24,6 +24,8 @@ var extensionLanguageMap = map[string]string{ ".cc": "cpp", ".cxx": "cpp", ".hpp": "cpp", + ".hh": "cpp", + ".hxx": "cpp", ".cs": "csharp", ".php": "php", ".sh": "shell", @@ -111,14 +113,14 @@ func (g *GenericParser) CanParse(_ string) bool { func (g *GenericParser) Parse(path string, content []byte) (*FileInfo, error) { return &FileInfo{ Path: path, - Language: detectLanguage(path), + Language: detectLanguage(path, content), LineCount: countLines(content), }, nil } -// detectLanguage maps the file extension to a language name. -// Returns "unknown" for unrecognized extensions. -func detectLanguage(filename string) string { +// detectLanguage maps the file extension to a language name. A .h header is +// classified from its content. Returns "unknown" for unrecognized extensions. +func detectLanguage(filename string, content []byte) string { lower := strings.ToLower(filename) base := lower if i := strings.LastIndexAny(base, "/\\"); i >= 0 { @@ -139,6 +141,9 @@ func detectLanguage(filename string) string { } ext := lower[idx:] + if ext == ".h" && HeaderLooksCpp(content) { + return "cpp" + } if lang, ok := extensionLanguageMap[ext]; ok { return lang } diff --git a/internal/parser/header_lang.go b/internal/parser/header_lang.go new file mode 100644 index 0000000..59458e5 --- /dev/null +++ b/internal/parser/header_lang.go @@ -0,0 +1,95 @@ +package parser + +import ( + "bytes" + "os" + "path/filepath" + "strings" +) + +// cppMarkers are constructs that do not compile as C. They are matched as +// plain byte sequences on the whole header, because a header can hold several +// preprocessor branches and any C++ branch makes the file C++. +var cppMarkers = [][]byte{ + []byte("class "), + []byte("namespace "), + []byte("template<"), + []byte("template <"), + []byte("public:"), + []byte("private:"), + []byte("protected:"), + []byte("std::"), + []byte("virtual "), + []byte("constexpr "), + []byte("nullptr"), + []byte("using namespace "), + []byte("extern \"C++\""), +} + +// cppUnitExts are translation unit extensions that mark a directory as C++. +var cppUnitExts = []string{".cc", ".cpp", ".cxx"} + +// HeaderLanguage decides whether a .h header belongs to C or C++. It returns +// "cpp" when the source shows a C++ construct, or when a C++ translation unit +// shares the header's directory, and "c" otherwise. fullPath may be empty or +// unresolvable; the neighbour check is then skipped. src may be nil. +func HeaderLanguage(fullPath string, src []byte) string { + if HeaderLooksCpp(src) { + return "cpp" + } + if SiblingIsCpp(fullPath) { + return "cpp" + } + return "c" +} + +// HeaderLooksCpp reports whether the header source uses C++ only constructs. +func HeaderLooksCpp(src []byte) bool { + for _, m := range cppMarkers { + if bytes.Contains(src, m) { + return true + } + } + return false +} + +// SiblingIsCpp reports whether a C++ translation unit sits next to fullPath. +// A unit with the same base name wins over an unrelated one in the same +// directory. An empty or unreadable path reports false. +func SiblingIsCpp(fullPath string) bool { + if fullPath == "" { + return false + } + dir := filepath.Dir(fullPath) + entries, err := os.ReadDir(dir) + if err != nil { + return false + } + base := strings.ToLower(strings.TrimSuffix(filepath.Base(fullPath), filepath.Ext(fullPath))) + sameName, anyName := false, false + for _, e := range entries { + if e.IsDir() { + continue + } + name := strings.ToLower(e.Name()) + ext := filepath.Ext(name) + if !containsString(cppUnitExts, ext) { + continue + } + anyName = true + if strings.TrimSuffix(name, ext) == base { + sameName = true + break + } + } + return sameName || anyName +} + +func containsString(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} diff --git a/internal/parser/header_lang_test.go b/internal/parser/header_lang_test.go new file mode 100644 index 0000000..044e3f6 --- /dev/null +++ b/internal/parser/header_lang_test.go @@ -0,0 +1,108 @@ +package parser + +import ( + "os" + "path/filepath" + "testing" +) + +func TestHeaderLooksCpp(t *testing.T) { + cppHeaders := [][]byte{ + []byte("class Acquirer {\npublic:\n void run();\n};\n"), + []byte("namespace iso { struct Msg; }\n"), + []byte("template \nT clamp(T v);\n"), + []byte("std::string name;\n"), + []byte("virtual void process() = 0;\n"), + } + for _, src := range cppHeaders { + if !HeaderLooksCpp(src) { + t.Errorf("HeaderLooksCpp(%q) = false, want true", src) + } + } + + cHeaders := [][]byte{ + []byte("#ifndef ACQUIRER_H\n#define ACQUIRER_H\nint process(const char *msg);\n#endif\n"), + []byte("typedef struct { int id; } acquirer;\n"), + []byte(""), + } + for _, src := range cHeaders { + if HeaderLooksCpp(src) { + t.Errorf("HeaderLooksCpp(%q) = true, want false", src) + } + } +} + +func TestHeaderLanguageContentOnly(t *testing.T) { + if got := HeaderLanguage("", []byte("class A {};\n")); got != "cpp" { + t.Errorf("C++ header = %q, want cpp", got) + } + if got := HeaderLanguage("", []byte("int f(void);\n")); got != "c" { + t.Errorf("C header = %q, want c", got) + } +} + +func TestHeaderLanguageNeighbour(t *testing.T) { + dir := t.TempDir() + header := filepath.Join(dir, "acquirer.h") + if err := os.WriteFile(header, []byte("int process(const char *msg);\n"), 0o644); err != nil { + t.Fatal(err) + } + + // A plain C header with no C++ neighbour stays C. + if got := HeaderLanguage(header, []byte("int process(const char *msg);\n")); got != "c" { + t.Errorf("header without neighbours = %q, want c", got) + } + + // Any translation unit in the directory marks the header as C++. + if err := os.WriteFile(filepath.Join(dir, "acquirer.cc"), []byte("int process(const char *msg);\n"), 0o644); err != nil { + t.Fatal(err) + } + if got := HeaderLanguage(header, []byte("int process(const char *msg);\n")); got != "cpp" { + t.Errorf("header beside .cc = %q, want cpp", got) + } +} + +func TestSiblingIsCppPrefersSameBaseName(t *testing.T) { + dir := t.TempDir() + header := filepath.Join(dir, "acquirer.h") + if err := os.WriteFile(header, nil, 0o644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "unrelated.c"), nil, 0o644); err != nil { + t.Fatal(err) + } + if SiblingIsCpp(header) { + t.Error("SiblingIsCpp = true with only a .c neighbour, want false") + } + if err := os.WriteFile(filepath.Join(dir, "acquirer.cxx"), nil, 0o644); err != nil { + t.Fatal(err) + } + if !SiblingIsCpp(header) { + t.Error("SiblingIsCpp = false with acquirer.cxx present, want true") + } + if SiblingIsCpp("") { + t.Error("SiblingIsCpp(\"\") = true, want false") + } +} + +func TestCLanguageLabelsCCAndHeaders(t *testing.T) { + cases := map[string]string{ + "a.cc": "cpp", + "a.cxx": "cpp", + "a.hh": "cpp", + "a.hxx": "cpp", + "a.cpp": "cpp", + "a.c": "c", + } + for path, want := range cases { + if got := cLanguage(path, nil); got != want { + t.Errorf("cLanguage(%q) = %q, want %q", path, got, want) + } + } + if got := cLanguage("a.h", []byte("class A {};\n")); got != "cpp" { + t.Errorf("cLanguage(cpp header) = %q, want cpp", got) + } + if got := cLanguage("a.h", []byte("int f(void);\n")); got != "c" { + t.Errorf("cLanguage(c header) = %q, want c", got) + } +} diff --git a/internal/parser/langs_wave2_test.go b/internal/parser/langs_wave2_test.go index 9a02409..76a5420 100644 --- a/internal/parser/langs_wave2_test.go +++ b/internal/parser/langs_wave2_test.go @@ -85,7 +85,7 @@ func writeTemp(t *testing.T, name, content string) string { func TestGenericDetectsBuildFilesByName(t *testing.T) { for path, want := range map[string]string{"Makefile": "make", "sub/Dockerfile": "dockerfile", "CMakeLists.txt": "cmake", "Dockerfile.prod": "dockerfile", "x.fs": "fsharp"} { - if got := detectLanguage(path); got != want { + if got := detectLanguage(path, nil); got != want { t.Errorf("detectLanguage(%q) = %q, want %q", path, got, want) } } diff --git a/internal/parser/pool.go b/internal/parser/pool.go index 9959233..601b4d4 100644 --- a/internal/parser/pool.go +++ b/internal/parser/pool.go @@ -155,6 +155,7 @@ func scanOne(rel string, o PoolOptions) (res Scanned) { } if o.Cache != nil { if info, ok := o.Cache.Get(key); ok { + headerLangOverride(rel, o.Root, info) res.Info, res.Cached = info, true return res } @@ -173,6 +174,7 @@ func scanOne(rel string, o PoolOptions) (res Scanned) { _, res.Panicked = err.(*PanicError) return res } + headerLangOverride(rel, o.Root, info) res.Info = info if o.Cache != nil && info != nil { o.Cache.Put(key, info) @@ -180,6 +182,18 @@ func scanOne(rel string, o PoolOptions) (res Scanned) { return res } +// headerLangOverride upgrades a .h header from "c" to "cpp" when a C++ +// translation unit sits beside it. The parser already classifies a header from +// its own constructs; this covers headers that are C++ by context only. +func headerLangOverride(rel, root string, info *FileInfo) { + if info == nil || info.Language != "c" || !extIs(rel, ".h") { + return + } + if SiblingIsCpp(filepath.Join(root, filepath.FromSlash(rel))) { + info.Language = "cpp" + } +} + // parseCost estimates the peak bytes a tree-sitter parse of one file holds. // The GLR parser allocates roughly 6.5 KB per source byte plus a fixed floor // (measured on Python and TypeScript sources). Go uses the stdlib parser. diff --git a/internal/parser/ts_c.go b/internal/parser/ts_c.go index 6fe0d92..ae556e8 100644 --- a/internal/parser/ts_c.go +++ b/internal/parser/ts_c.go @@ -10,7 +10,7 @@ import ( // using the tree-sitter AST. The Language field is set based on the file extension. func extractC(root *gts.Node, lang *gts.Language, src []byte, path string) *FileInfo { info := &FileInfo{ - Language: cLanguage(path), + Language: cLanguage(path, src), TypeDefs: make(map[string]string), } @@ -143,13 +143,19 @@ func extractC(root *gts.Node, lang *gts.Language, src []byte, path string) *File return info } -// cLanguage maps file extension to a language label. -func cLanguage(filename string) string { +// cLanguage maps file extension to a language label. A .h header has no fixed +// language, so its source decides between C and C++. +func cLanguage(filename string, src []byte) string { switch { - case extIs(filename, ".cpp", ".cxx", ".cc", ".hpp", ".hxx"): + case extIs(filename, ".cpp", ".cxx", ".cc", ".hpp", ".hh", ".hxx"): return "cpp" case extIs(filename, ".m", ".mm"): return "objective-c" + case extIs(filename, ".h"): + if HeaderLooksCpp(src) { + return "cpp" + } + return "c" default: return "c" } diff --git a/internal/walker/walker.go b/internal/walker/walker.go index 6f00ae9..307b3cd 100644 --- a/internal/walker/walker.go +++ b/internal/walker/walker.go @@ -23,9 +23,13 @@ var sourceExtensions = map[string]bool{ ".kt": true, ".scala": true, ".c": true, + ".cc": true, ".cpp": true, + ".cxx": true, ".h": true, + ".hh": true, ".hpp": true, + ".hxx": true, ".vue": true, ".svelte": true, ".dart": true, diff --git a/internal/walker/walker_test.go b/internal/walker/walker_test.go index 610229e..ccc13fa 100644 --- a/internal/walker/walker_test.go +++ b/internal/walker/walker_test.go @@ -132,6 +132,30 @@ func TestWalkAlwaysIgnoresDirs(t *testing.T) { } } +func TestWalkIncludesCppExtensions(t *testing.T) { + dir := t.TempDir() + names := []string{"a.cc", "a.cxx", "a.cpp", "a.hh", "a.hpp", "a.hxx", "a.h", "a.c"} + for _, n := range names { + if err := os.WriteFile(filepath.Join(dir, n), []byte("int f(void);\n"), 0644); err != nil { + t.Fatal(err) + } + } + + files, err := walker.Walk(dir, nil) + if err != nil { + t.Fatalf("Walk returned error: %v", err) + } + got := make(map[string]bool, len(files)) + for _, f := range files { + got[f] = true + } + for _, n := range names { + if !got[n] { + t.Errorf("expected %q to be walked, got %v", n, sortedKeys(got)) + } + } +} + // sortedKeys returns sorted map keys for readable error messages. func sortedKeys(m map[string]bool) []string { keys := make([]string, 0, len(m))