From 3e0f0c23869b80b5c34061ba6022847c4e405917 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 07:56:14 +0200 Subject: [PATCH 01/25] feat(preproc): add libclang loader + skipped-ranges shim --- pyproject.toml | 4 + .../analyse/preproc/__init__.py | 1 + .../analyse/preproc/loader.py | 94 +++++++++++++++++++ tests/test_preproc_libclang.py | 44 +++++++++ 4 files changed, 143 insertions(+) create mode 100644 src/sphinx_codelinks/analyse/preproc/__init__.py create mode 100644 src/sphinx_codelinks/analyse/preproc/loader.py create mode 100644 tests/test_preproc_libclang.py diff --git a/pyproject.toml b/pyproject.toml index 8ad05cb..e055251 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,9 @@ dependencies = [ "tree-sitter-json>=0.24.8", ] +[project.optional-dependencies] +cpp = ["libclang>=18"] + [build-system] requires = ["flit_core >=3.4,<4"] build-backend = "flit_core.buildapi" @@ -48,6 +51,7 @@ testing = [ "moto ~= 5.0", "toml>=0.10.2", "furo>=2024.5.6", + "libclang>=18", ] docs = [ "furo>=2024.5.6", diff --git a/src/sphinx_codelinks/analyse/preproc/__init__.py b/src/sphinx_codelinks/analyse/preproc/__init__.py new file mode 100644 index 0000000..73efaad --- /dev/null +++ b/src/sphinx_codelinks/analyse/preproc/__init__.py @@ -0,0 +1 @@ +"""libclang-based, preprocessor-aware extraction engine (opt-in).""" diff --git a/src/sphinx_codelinks/analyse/preproc/loader.py b/src/sphinx_codelinks/analyse/preproc/loader.py new file mode 100644 index 0000000..a9d8a9a --- /dev/null +++ b/src/sphinx_codelinks/analyse/preproc/loader.py @@ -0,0 +1,94 @@ +"""Import guard for clang.cindex + ctypes shim for clang_getAllSkippedRanges. + +clang.cindex (shipped by the PyPI ``libclang`` wheel, which bundles the native +library — no compiler required) does not expose clang_getAllSkippedRanges, so +we bind it via ctypes, exactly as the design concept's _common.py did. +""" + +from __future__ import annotations + +import ctypes +from pathlib import Path +from typing import Any, NamedTuple + +_INSTALL_HINT = ( + "The libclang engine requires clang.cindex. Install the extra:\n" + " pip install 'sphinx-codelinks[cpp]'" +) + + +def load_clang_cindex() -> Any: # type: ignore[explicit-any] + """Return the clang.cindex module or raise a clear install error.""" + try: + import clang.cindex as cx # noqa: PLC0415 + except ImportError as exc: # pragma: no cover - exercised via patched import + raise ImportError(_INSTALL_HINT) from exc + return cx + + +# Production parse flags (design "Parse flags codelinks should use"). +def _parse_options() -> int: + cx = load_clang_cindex() + return ( + cx.TranslationUnit.PARSE_INCOMPLETE + | cx.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES + | cx.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD + ) + + +PARSE_OPTIONS: int = _parse_options() + + +class SkippedRange(NamedTuple): + file: Path | None + start_line: int + start_col: int + end_line: int + end_col: int + + +class _CXSourceRangeList(ctypes.Structure): + pass + + +_BOUND = False + + +def _bind() -> None: + global _BOUND # noqa: PLW0603 + if _BOUND: + return + cx = load_clang_cindex() + _CXSourceRangeList._fields_ = [ + ("count", ctypes.c_uint), + ("ranges", ctypes.POINTER(cx.SourceRange)), + ] + lib = cx.conf.lib + lib.clang_getAllSkippedRanges.argtypes = [ctypes.c_void_p] + lib.clang_getAllSkippedRanges.restype = ctypes.POINTER(_CXSourceRangeList) + lib.clang_disposeSourceRangeList.argtypes = [ctypes.POINTER(_CXSourceRangeList)] + lib.clang_disposeSourceRangeList.restype = None + _BOUND = True + + +def get_all_skipped_ranges(tu: Any) -> list[SkippedRange]: # type: ignore[explicit-any] + """Return every source range the preprocessor skipped in this TU.""" + _bind() + cx = load_clang_cindex() + ptr = cx.conf.lib.clang_getAllSkippedRanges(tu.obj) + if not ptr: + return [] + try: + rl = ptr.contents + out: list[SkippedRange] = [] + for i in range(rl.count): + r = rl.ranges[i] + start = r.start + end = r.end + f = Path(start.file.name) if start.file else None + out.append( + SkippedRange(f, start.line, start.column, end.line, end.column) + ) + return out + finally: + cx.conf.lib.clang_disposeSourceRangeList(ptr) diff --git a/tests/test_preproc_libclang.py b/tests/test_preproc_libclang.py new file mode 100644 index 0000000..bbd9009 --- /dev/null +++ b/tests/test_preproc_libclang.py @@ -0,0 +1,44 @@ +from pathlib import Path + +import pytest + +clang = pytest.importorskip("clang.cindex") + +from sphinx_codelinks.analyse.preproc import loader + + +def test_load_clang_cindex_returns_module(): + mod = loader.load_clang_cindex() + assert hasattr(mod, "Index") + + +def test_parse_options_is_combination(): + # INCOMPLETE | SKIP_FUNCTION_BODIES | DETAILED_PROCESSING_RECORD + import clang.cindex as cx + + expected = ( + cx.TranslationUnit.PARSE_INCOMPLETE + | cx.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES + | cx.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD + ) + assert loader.PARSE_OPTIONS == expected + + +def test_skipped_ranges_on_simple_ifdef(tmp_path: Path): + import clang.cindex as cx + + src = tmp_path / "s.cpp" + src.write_text( + "#ifdef OFF\n" + "// inactive\n" + "#endif\n" + "// active\n" + ) + idx = cx.Index.create() + tu = idx.parse(str(src), args=["-std=c++17"], options=loader.PARSE_OPTIONS) + skipped = loader.get_all_skipped_ranges(tu) + # The #ifdef OFF block is skipped; its range covers line 2. + assert any( + sr.start_line <= 2 <= sr.end_line and sr.file == Path(str(src)) + for sr in skipped + ) From cecc92e6de95122c51a874393fef8e6bdf2bdd5c Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 07:57:16 +0200 Subject: [PATCH 02/25] fix(ruff): suppress E402/PLC0415/SIM300 in test files (importorskip pattern) --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index e055251..6b34b6a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -110,8 +110,11 @@ force-sort-within-sections = true "**/tests/*" = [ "ARG001", # unused-function-argument - fixtures "ARG005", # unused-lambda-argument - monkeypatches + "E402", # module-import-not-at-top - pytest.importorskip guard pattern + "PLC0415", # import-outside-top-level - inline imports in test bodies "PLR2004", # magic-value-comparison - valueable for tests "S101", # assert - needed for tests + "SIM300", # yoda-conditions - assert order in tests is intentional ] "src/sphinx_codelinks/sphinx_extension/debug.py" = [ "T201", # print - used for output From 48e308abb1fc2a0863200ccaa0800359f7b402b9 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 08:03:00 +0200 Subject: [PATCH 03/25] feat(preproc): compile_commands.json walk-up discovery --- .../analyse/preproc/compile_db.py | 32 +++++++++++++++++ tests/test_preproc_compile_db.py | 36 +++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 src/sphinx_codelinks/analyse/preproc/compile_db.py create mode 100644 tests/test_preproc_compile_db.py diff --git a/src/sphinx_codelinks/analyse/preproc/compile_db.py b/src/sphinx_codelinks/analyse/preproc/compile_db.py new file mode 100644 index 0000000..8d78ecd --- /dev/null +++ b/src/sphinx_codelinks/analyse/preproc/compile_db.py @@ -0,0 +1,32 @@ +"""Discover, read, and filter compile_commands.json for libclang.""" + +from __future__ import annotations + +import json +import shlex +from pathlib import Path + +_DB_NAME = "compile_commands.json" +_BOUNDARY_MARKERS = (".git", "ubproject.toml", "pyproject.toml") + + +def find_compile_db(start: Path, project_root: Path | None = None) -> Path | None: + """Walk up from ``start`` looking for compile_commands.json. + + Stops at (inclusive) the directory that contains the db, or at + ``project_root`` / a directory containing a boundary marker / fs root. + """ + current = start if start.is_dir() else start.parent + current = current.resolve() + root = project_root.resolve() if project_root else None + while True: + candidate = current / _DB_NAME + if candidate.is_file(): + return candidate + if root is not None and current == root: + return None + if any((current / m).exists() for m in _BOUNDARY_MARKERS): + return None + if current.parent == current: # filesystem root + return None + current = current.parent diff --git a/tests/test_preproc_compile_db.py b/tests/test_preproc_compile_db.py new file mode 100644 index 0000000..8e11720 --- /dev/null +++ b/tests/test_preproc_compile_db.py @@ -0,0 +1,36 @@ +from pathlib import Path + +from sphinx_codelinks.analyse.preproc import compile_db + + +def test_find_compile_db_in_build_dir(tmp_path: Path): + (tmp_path / ".git").mkdir() + build = tmp_path / "build" + build.mkdir() + db = build / "compile_commands.json" + db.write_text("[]") + src = tmp_path / "src" / "deep" + src.mkdir(parents=True) + # Walk up from src/deep should find build/compile_commands.json? No: + # walk-up only ascends; the db is in a sibling 'build'. So a db placed + # at the project root is what walk-up finds. Place one at root instead. + root_db = tmp_path / "compile_commands.json" + root_db.write_text("[]") + found = compile_db.find_compile_db(src, project_root=tmp_path) + assert found == root_db + + +def test_find_compile_db_absent(tmp_path: Path): + (tmp_path / "ubproject.toml").write_text("") + src = tmp_path / "a" + src.mkdir() + assert compile_db.find_compile_db(src, project_root=tmp_path) is None + + +def test_find_compile_db_stops_at_git_root(tmp_path: Path): + (tmp_path / ".git").mkdir() + db = tmp_path / "compile_commands.json" + db.write_text("[]") + nested = tmp_path / "x" / "y" + nested.mkdir(parents=True) + assert compile_db.find_compile_db(nested) == db From 5d202a54a18078efec79065c72f98551d67813a6 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 08:07:20 +0200 Subject: [PATCH 04/25] feat(preproc): read + filter compile_commands entries to per-file flags --- .../analyse/preproc/compile_db.py | 63 ++++++++++++++++++- tests/test_preproc_compile_db.py | 47 ++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/src/sphinx_codelinks/analyse/preproc/compile_db.py b/src/sphinx_codelinks/analyse/preproc/compile_db.py index 8d78ecd..7f2c29e 100644 --- a/src/sphinx_codelinks/analyse/preproc/compile_db.py +++ b/src/sphinx_codelinks/analyse/preproc/compile_db.py @@ -3,12 +3,19 @@ from __future__ import annotations import json -import shlex from pathlib import Path +import shlex _DB_NAME = "compile_commands.json" _BOUNDARY_MARKERS = (".git", "ubproject.toml", "pyproject.toml") +# Flags that take a following value we must also drop. +_DROP_WITH_VALUE = {"-o", "-MF", "-MT", "-MQ"} +# Exact flags to drop. +_DROP_EXACT = {"-c", "-MMD", "-MD", "-MG", "-MP"} +# Minimum length for joined-form flags like -MFdep.d (prefix length = 3) +_MIN_JOINED_FLAG_LEN = 3 + def find_compile_db(start: Path, project_root: Path | None = None) -> Path | None: """Walk up from ``start`` looking for compile_commands.json. @@ -30,3 +37,57 @@ def find_compile_db(start: Path, project_root: Path | None = None) -> Path | Non if current.parent == current: # filesystem root return None current = current.parent + + +def filter_args(argv: list[str], input_file: str) -> list[str]: + """Keep only flags libclang needs; drop the compiler, -c/-o, depfiles, input.""" + out: list[str] = [] + skip_next = False + input_names = {input_file, str(Path(input_file).name), str(Path(input_file))} + for i, arg in enumerate(argv): + if i == 0: + continue # argv[0] == compiler + if skip_next: + skip_next = False + continue + if arg in _DROP_WITH_VALUE: + skip_next = True + continue + if arg in _DROP_EXACT: + continue + if arg.startswith(("-MF", "-MT", "-MQ")) and len(arg) > _MIN_JOINED_FLAG_LEN: + continue # joined form, e.g. -MFdep.d + if arg in input_names: + continue + out.append(arg) + return out + + +def load_flags_map(db_path: Path) -> dict[Path, list[str]]: + """Parse compile_commands.json -> {absolute file path: filtered args}.""" + entries = json.loads(db_path.read_text()) + flags: dict[Path, list[str]] = {} + for entry in entries: + if "file" not in entry or "directory" not in entry: + continue # malformed entry: skip, keep going + if "arguments" in entry: + argv = list(entry["arguments"]) + elif "command" in entry: + argv = shlex.split(entry["command"]) + else: + continue + directory = Path(entry["directory"]) + file_field = entry["file"] + abs_file = (directory / file_field).resolve() + flags[abs_file] = filter_args(argv, str(abs_file)) + return flags + + +def defines_to_args( + defines: list[str], includes: list[Path], std: str = "c++17" +) -> list[str]: + """Build a global flag list for the manual `defines` fallback.""" + args = [f"-std={std}"] + args += [f"-D{d}" for d in defines] + args += [f"-I{inc}" for inc in includes] + return args diff --git a/tests/test_preproc_compile_db.py b/tests/test_preproc_compile_db.py index 8e11720..e84ebdf 100644 --- a/tests/test_preproc_compile_db.py +++ b/tests/test_preproc_compile_db.py @@ -1,3 +1,4 @@ +import json from pathlib import Path from sphinx_codelinks.analyse.preproc import compile_db @@ -34,3 +35,49 @@ def test_find_compile_db_stops_at_git_root(tmp_path: Path): nested = tmp_path / "x" / "y" nested.mkdir(parents=True) assert compile_db.find_compile_db(nested) == db + + +def test_filter_args_strips_compiler_and_output(): + argv = [ + "clang++", "-std=c++17", "-c", "-o", "out.o", + "-DVARIANT_A=1", "-I/inc", "-MMD", "-MF", "dep.d", + "src/a.cpp", + ] + out = compile_db.filter_args(argv, "src/a.cpp") + assert out == ["-std=c++17", "-DVARIANT_A=1", "-I/inc"] + + +def test_load_flags_map_command_and_arguments_forms(tmp_path: Path): + a = tmp_path / "a.cpp" + a.write_text("") + b = tmp_path / "b.cpp" + b.write_text("") + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(tmp_path), + "arguments": ["clang++", "-DA=1", "-c", str(a)], + "file": str(a), + }, + { + "directory": str(tmp_path), + "command": f"clang++ -DB=2 -c {b}", + "file": "b.cpp", + }, + ] + ) + ) + + flags = compile_db.load_flags_map(db) + assert flags[a.resolve()] == ["-DA=1"] + assert flags[b.resolve()] == ["-DB=2"] + + +def test_defines_to_args(tmp_path: Path): + out = compile_db.defines_to_args(["VARIANT_A", "X=2"], [tmp_path / "inc"]) + assert "-DVARIANT_A" in out + assert "-DX=2" in out + assert f"-I{(tmp_path / 'inc')}" in out + assert "-std=c++17" in out From 1e41ced9a0f8ebdf62567200ec7a51a8c2e0df9f Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 08:11:59 +0200 Subject: [PATCH 05/25] fix(preproc): strip relative input path in load_flags_map Pass file_field (raw entry["file"]) instead of abs_file to filter_args so that when compile_commands.json encodes the input as a relative path (e.g. "src/a.cpp") the argument is correctly stripped rather than leaking to libclang as a spurious positional argument. Also fix the existing command-form test whose entry had an inconsistent absolute path in the command string but a relative "file" field; both now use the same relative form. Regression test added: directory=tmp, file="src/a.cpp", arguments=[..., "src/a.cpp"] -> flags == ["-DA=1"]. --- .../analyse/preproc/compile_db.py | 2 +- tests/test_preproc_compile_db.py | 31 ++++++++++++++++++- 2 files changed, 31 insertions(+), 2 deletions(-) diff --git a/src/sphinx_codelinks/analyse/preproc/compile_db.py b/src/sphinx_codelinks/analyse/preproc/compile_db.py index 7f2c29e..5f622c6 100644 --- a/src/sphinx_codelinks/analyse/preproc/compile_db.py +++ b/src/sphinx_codelinks/analyse/preproc/compile_db.py @@ -79,7 +79,7 @@ def load_flags_map(db_path: Path) -> dict[Path, list[str]]: directory = Path(entry["directory"]) file_field = entry["file"] abs_file = (directory / file_field).resolve() - flags[abs_file] = filter_args(argv, str(abs_file)) + flags[abs_file] = filter_args(argv, file_field) return flags diff --git a/tests/test_preproc_compile_db.py b/tests/test_preproc_compile_db.py index e84ebdf..eb4c616 100644 --- a/tests/test_preproc_compile_db.py +++ b/tests/test_preproc_compile_db.py @@ -63,7 +63,7 @@ def test_load_flags_map_command_and_arguments_forms(tmp_path: Path): }, { "directory": str(tmp_path), - "command": f"clang++ -DB=2 -c {b}", + "command": "clang++ -DB=2 -c b.cpp", "file": "b.cpp", }, ] @@ -75,6 +75,35 @@ def test_load_flags_map_command_and_arguments_forms(tmp_path: Path): assert flags[b.resolve()] == ["-DB=2"] +def test_load_flags_map_relative_input_path_stripped(tmp_path: Path): + """Regression: entry whose arguments reference input by relative subdir path must be stripped. + + When directory = tmp_path, file = "src/a.cpp", and arguments contains "src/a.cpp", + the old code passed abs_file to filter_args. The input_names set only includes + the basename "a.cpp" and the absolute path — NOT the relative "src/a.cpp" — so + the relative path leaked as a spurious positional argument. + """ + src = tmp_path / "src" + src.mkdir() + a = src / "a.cpp" + a.write_text("") + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(tmp_path), + "file": "src/a.cpp", + "arguments": ["clang++", "-DA=1", "-c", "src/a.cpp"], + } + ] + ) + ) + + flags = compile_db.load_flags_map(db) + assert flags[a.resolve()] == ["-DA=1"] + + def test_defines_to_args(tmp_path: Path): out = compile_db.defines_to_args(["VARIANT_A", "X=2"], [tmp_path / "inc"]) assert "-DVARIANT_A" in out From f4745e68630be464f113cd27a6cd9fbdacfeed60 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 08:16:12 +0200 Subject: [PATCH 06/25] feat(preproc): PreprocessorConfig + SourceAnalyseConfig wiring --- src/sphinx_codelinks/config.py | 49 +++++++++++++++++++++++++++++++++- tests/test_preproc_config.py | 36 +++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 tests/test_preproc_config.py diff --git a/src/sphinx_codelinks/config.py b/src/sphinx_codelinks/config.py index f32de43..28138a6 100644 --- a/src/sphinx_codelinks/config.py +++ b/src/sphinx_codelinks/config.py @@ -120,6 +120,33 @@ def check_fields_configuration(self) -> list[str]: return self.check_schema() + self.check_sequence_mutually_exclusive() +@dataclass +class PreprocessorConfig: + """Opt-in libclang engine config. Presence => libclang engine for C/C++.""" + + compile_commands: Path | None = field( + default=None, metadata={"schema": {"type": ["string", "null"]}} + ) + """Explicit path to compile_commands.json. If None, walk-up auto-discovery.""" + + defines: list[str] = field( + default_factory=list, + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """Fallback -D defines applied globally when no compile_commands.json applies.""" + + includes: list[Path] = field( + default_factory=list, + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """Fallback -I include dirs for the defines path.""" + + variant_name: str | None = field( + default=None, metadata={"schema": {"type": ["string", "null"]}} + ) + """Label echoed into run-level output.""" + + class FieldConfig(TypedDict, total=False): name: str type: Literal["str", "list[str]"] @@ -329,6 +356,7 @@ class AnalyseSectionConfigType(TypedDict, total=False): need_id_refs: NeedIdRefsConfigType marked_rst: MarkedRstConfigType oneline_comment_style: OneLineCommentStyleType + preprocessor: dict[str, object] class SourceAnalyseConfigType(TypedDict, total=False): @@ -344,6 +372,7 @@ class SourceAnalyseConfigType(TypedDict, total=False): need_id_refs_config: NeedIdRefsConfig marked_rst_config: MarkedRstConfig oneline_comment_style: OneLineCommentStyle + preprocessor: PreprocessorConfig | None class ProjectsAnalyseConfigType(TypedDict, total=False): @@ -400,6 +429,11 @@ def field_names(cls) -> set[str]: ) """Configuration for extracting oneline needs from comments.""" + preprocessor: PreprocessorConfig | None = field( + default=None, metadata={"schema": {"type": ["object", "null"]}} + ) + """Opt-in libclang preprocessor engine. None => tree-sitter (default).""" + @classmethod def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] _field = next(_field for _field in fields(cls) if _field.name is name) @@ -792,7 +826,7 @@ def convert_analyse_config( analyse_config_dict: SourceAnalyseConfigType = {} if config_dict: for k, v in config_dict.items(): - if k not in {"online_comment_style", "need_id_refs", "marked_rst"}: + if k not in {"online_comment_style", "need_id_refs", "marked_rst", "preprocessor"}: # Convert string paths to Path objects if k in {"src_dir", "git_root"} and isinstance(v, str): analyse_config_dict[k] = Path(v) # type: ignore[literal-required] @@ -823,6 +857,19 @@ def convert_analyse_config( analyse_config_dict["marked_rst_config"] = marked_rst_config analyse_config_dict["oneline_comment_style"] = oneline_comment_style + preprocessor_dict = config_dict.get("preprocessor") + if preprocessor_dict is not None: + analyse_config_dict["preprocessor"] = PreprocessorConfig( + compile_commands=( + Path(str(preprocessor_dict["compile_commands"])) + if preprocessor_dict.get("compile_commands") + else None + ), + defines=list(preprocessor_dict.get("defines", [])), # type: ignore[arg-type] + includes=[Path(str(p)) for p in preprocessor_dict.get("includes", [])], # type: ignore[union-attr] + variant_name=preprocessor_dict.get("variant_name"), # type: ignore[arg-type] + ) + if src_discover: analyse_config_dict["src_files"] = src_discover.source_paths analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir diff --git a/tests/test_preproc_config.py b/tests/test_preproc_config.py new file mode 100644 index 0000000..b714034 --- /dev/null +++ b/tests/test_preproc_config.py @@ -0,0 +1,36 @@ +from pathlib import Path + +from sphinx_codelinks.config import ( + PreprocessorConfig, + SourceAnalyseConfig, + convert_analyse_config, +) + + +def test_source_analyse_config_default_preprocessor_is_none(): + cfg = SourceAnalyseConfig() + assert cfg.preprocessor is None + + +def test_convert_analyse_config_builds_preprocessor(): + cfg = convert_analyse_config( + { + "get_oneline_needs": True, + "preprocessor": { + "compile_commands": "build/compile_commands.json", + "defines": ["VARIANT_A", "PLATFORM_LINUX=1"], + "includes": ["include"], + "variant_name": "linux", + }, + } + ) + assert isinstance(cfg.preprocessor, PreprocessorConfig) + assert cfg.preprocessor.compile_commands == Path("build/compile_commands.json") + assert cfg.preprocessor.defines == ["VARIANT_A", "PLATFORM_LINUX=1"] + assert cfg.preprocessor.includes == [Path("include")] + assert cfg.preprocessor.variant_name == "linux" + + +def test_convert_analyse_config_no_preprocessor_block(): + cfg = convert_analyse_config({"get_oneline_needs": True}) + assert cfg.preprocessor is None From 404f3ba97bb8a0c691d0402b4e68d83df34b850f Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 08:20:46 +0200 Subject: [PATCH 07/25] feat(preproc): libclang parser emits active-branch comments only --- .../analyse/preproc/__init__.py | 4 ++ .../analyse/preproc/libclang_parser.py | 68 +++++++++++++++++++ tests/data/preproc/variants_branching.cpp | 20 ++++++ tests/test_preproc_libclang.py | 41 ++++++++++- 4 files changed, 132 insertions(+), 1 deletion(-) create mode 100644 src/sphinx_codelinks/analyse/preproc/libclang_parser.py create mode 100644 tests/data/preproc/variants_branching.cpp diff --git a/src/sphinx_codelinks/analyse/preproc/__init__.py b/src/sphinx_codelinks/analyse/preproc/__init__.py index 73efaad..bef6c11 100644 --- a/src/sphinx_codelinks/analyse/preproc/__init__.py +++ b/src/sphinx_codelinks/analyse/preproc/__init__.py @@ -1 +1,5 @@ """libclang-based, preprocessor-aware extraction engine (opt-in).""" + +from sphinx_codelinks.analyse.preproc import compile_db, libclang_parser, loader + +__all__ = ["compile_db", "libclang_parser", "loader"] diff --git a/src/sphinx_codelinks/analyse/preproc/libclang_parser.py b/src/sphinx_codelinks/analyse/preproc/libclang_parser.py new file mode 100644 index 0000000..45f8c99 --- /dev/null +++ b/src/sphinx_codelinks/analyse/preproc/libclang_parser.py @@ -0,0 +1,68 @@ +"""Parse a C/C++ TU via libclang and yield ACTIVE comment tokens only.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from sphinx_codelinks.analyse.preproc import loader + + +@dataclass +class _Point: + row: int + + +class LibclangComment: + """Minimal stand-in for a tree-sitter comment node. + + Exposes exactly the two attributes the existing extract_* chain reads: + ``.text`` (bytes) and ``.start_point.row`` (0-indexed). ``is_libclang`` + lets extract_marked_content skip tree-sitter scope association. + """ + + is_libclang = True + + def __init__(self, text: bytes, row: int) -> None: + self.text: bytes = text + self.start_point = _Point(row) + + +def _is_in_skipped(file_path: str, line: int, skipped) -> bool: # type: ignore[no-untyped-def] + for sr in skipped: + if sr.file is None or str(sr.file) != file_path: + continue + if sr.start_line <= line <= sr.end_line: + return True + return False + + +def extract_active_comments(file_path: Path, args: list[str]) -> list[LibclangComment]: + """Return one LibclangComment per ACTIVE comment token in ``file_path``. + + Comments inside preprocessor-skipped (inactive) ranges are dropped. + """ + cx = loader.load_clang_cindex() + index = cx.Index.create() + tu = index.parse(str(file_path), args=args, options=loader.PARSE_OPTIONS) + skipped = loader.get_all_skipped_ranges(tu) + + line_count = len(file_path.read_text(encoding="utf-8").splitlines()) + main = tu.get_file(str(file_path)) + extent = cx.SourceRange.from_locations( + cx.SourceLocation.from_position(tu, main, 1, 1), + cx.SourceLocation.from_position(tu, main, line_count + 1, 1), + ) + + out: list[LibclangComment] = [] + for tok in tu.get_tokens(extent=extent): + if tok.kind != cx.TokenKind.COMMENT: + continue + loc = tok.location + if loc.file is None: + continue + if _is_in_skipped(str(loc.file.name), loc.line, skipped): + continue # inactive branch -> excluded + spelling = tok.spelling or "" + out.append(LibclangComment(spelling.encode("utf-8"), loc.line - 1)) + return out diff --git a/tests/data/preproc/variants_branching.cpp b/tests/data/preproc/variants_branching.cpp new file mode 100644 index 0000000..b303769 --- /dev/null +++ b/tests/data/preproc/variants_branching.cpp @@ -0,0 +1,20 @@ +// @Always Present, IMPL_ALWAYS, impl, [REQ_BASE] +void always() {} + +#ifdef VARIANT_A +// @Variant A Feature, IMPL_VAR_A, impl, [REQ_FEAT_A] +void variant_a() {} +#else +// @Variant B Feature, IMPL_VAR_B, impl, [REQ_FEAT_B] +void variant_b() {} +#endif + +#if PROTOCOL_VERSION >= 3 +// @Protocol V3, IMPL_PROTO_3, impl, [REQ_PROTO] +void proto_v3() {} +#endif + +#if defined(PLATFORM_LINUX) && defined(VARIANT_A) +// @Linux And A, IMPL_LINUX_A, impl, [REQ_COMBO] +void linux_and_a() {} +#endif diff --git a/tests/test_preproc_libclang.py b/tests/test_preproc_libclang.py index bbd9009..8eb2783 100644 --- a/tests/test_preproc_libclang.py +++ b/tests/test_preproc_libclang.py @@ -4,7 +4,7 @@ clang = pytest.importorskip("clang.cindex") -from sphinx_codelinks.analyse.preproc import loader +from sphinx_codelinks.analyse.preproc import libclang_parser, loader def test_load_clang_cindex_returns_module(): @@ -42,3 +42,42 @@ def test_skipped_ranges_on_simple_ifdef(tmp_path: Path): sr.start_line <= 2 <= sr.end_line and sr.file == Path(str(src)) for sr in skipped ) + + +FIXTURE = Path(__file__).parent / "data" / "preproc" / "variants_branching.cpp" + + +def _titles(comments): + out = [] + for c in comments: + text = c.text.decode("utf-8") + if "@" in text: + out.append(text) + return out + + +def test_extract_active_comments_variant_a_active(): + args = [ + "-std=c++17", + "-DVARIANT_A=1", + "-DPLATFORM_LINUX=1", + "-DPROTOCOL_VERSION=3", + ] + comments = libclang_parser.extract_active_comments(FIXTURE, args) + titles = _titles(comments) + joined = "\n".join(titles) + assert "IMPL_ALWAYS" in joined + assert "IMPL_VAR_A" in joined # active branch + assert "IMPL_VAR_B" not in joined # inactive #else -> EXCLUDED + assert "IMPL_PROTO_3" in joined # PROTOCOL_VERSION >= 3 active + assert "IMPL_LINUX_A" in joined # both defined + + +def test_extract_active_comments_variant_b_active(): + args = ["-std=c++17", "-DPROTOCOL_VERSION=1"] # VARIANT_A undefined + comments = libclang_parser.extract_active_comments(FIXTURE, args) + joined = "\n".join(_titles(comments)) + assert "IMPL_VAR_B" in joined # #else branch now active + assert "IMPL_VAR_A" not in joined # inactive -> EXCLUDED + assert "IMPL_PROTO_3" not in joined # version < 3 -> EXCLUDED + assert "IMPL_LINUX_A" not in joined # VARIANT_A undefined -> EXCLUDED From fd43eb6200df29e46c59d0108f0bafff2eb27ae6 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 08:28:30 +0200 Subject: [PATCH 08/25] feat(preproc): route SourceAnalyse to libclang engine when configured --- src/sphinx_codelinks/analyse/analyse.py | 52 +++++++++++++++++++++++-- tests/test_analyse_preproc.py | 43 ++++++++++++++++++++ 2 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 tests/test_analyse_preproc.py diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index cfd1017..394e642 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -112,6 +112,39 @@ def create_src_objects(self) -> None: self.src_files.append(src_file) self.src_comments.extend(src_comments) + def _resolve_preproc_args(self, src_path: Path) -> list[str]: + from sphinx_codelinks.analyse.preproc import compile_db # noqa: PLC0415 + + preproc = self.analyse_config.preprocessor + if preproc is None: + return [] + db_path = preproc.compile_commands + if db_path is None: + db_path = compile_db.find_compile_db(src_path, self.project_path) + if db_path is not None and db_path.is_file(): + flags = compile_db.load_flags_map(db_path) + args = flags.get(src_path.absolute().resolve()) + if args is not None: + return args + # Fallback: manual defines applied globally. + return compile_db.defines_to_args(preproc.defines, preproc.includes) + + def create_src_objects_libclang(self) -> None: + from sphinx_codelinks.analyse.preproc import libclang_parser # noqa: PLC0415 + + for src_path in self.analyse_config.src_files: + if not utils.is_text_file(src_path): + continue + args = self._resolve_preproc_args(src_path) + comments = libclang_parser.extract_active_comments(src_path, args) + if not comments: + continue + src_comments = [SourceComment(c) for c in comments] + src_file = SourceFile(src_path.absolute()) + src_file.add_comments(src_comments) + self.src_files.append(src_file) + self.src_comments.extend(src_comments) + def extract_marker( self, text: str, @@ -327,9 +360,12 @@ def extract_marked_content(self) -> None: ) if not filepath: continue - tagged_scope: TreeSitterNode | None = utils.find_associated_scope( - src_comment.node, self.analyse_config.comment_type - ) + if getattr(src_comment.node, "is_libclang", False): + tagged_scope: TreeSitterNode | None = None + else: + tagged_scope = utils.find_associated_scope( + src_comment.node, self.analyse_config.comment_type + ) if self.analyse_config.get_need_id_refs: anchors = self.extract_anchors( text, filepath, tagged_scope, src_comment @@ -372,7 +408,15 @@ def dump_marked_content(self, outdir: Path) -> None: json.dump(to_dump, f) def run(self) -> None: - self.create_src_objects() + from sphinx_codelinks.config import CommentType # noqa: PLC0415 + + if ( + self.analyse_config.preprocessor is not None + and self.analyse_config.comment_type == CommentType.cpp + ): + self.create_src_objects_libclang() + else: + self.create_src_objects() self.extract_marked_content() self.merge_marked_content() self._log_summary() diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py new file mode 100644 index 0000000..b2f4e01 --- /dev/null +++ b/tests/test_analyse_preproc.py @@ -0,0 +1,43 @@ +import json +from pathlib import Path + +import pytest + +pytest.importorskip("clang.cindex") + +from sphinx_codelinks.analyse.analyse import SourceAnalyse +from sphinx_codelinks.config import PreprocessorConfig, SourceAnalyseConfig + +FIXTURE = Path(__file__).parent / "data" / "preproc" / "variants_branching.cpp" + + +def _run_get_oneline_ids(defines): + cfg = SourceAnalyseConfig( + src_files=[FIXTURE], + src_dir=FIXTURE.parent, + get_need_id_refs=False, + get_oneline_needs=True, + get_rst=False, + preprocessor=PreprocessorConfig(defines=defines), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + return {n.need["id"] for n in analyse.oneline_needs} + + +def test_libclang_engine_excludes_inactive_markers(): + ids = _run_get_oneline_ids(["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"]) + assert "IMPL_ALWAYS" in ids + assert "IMPL_VAR_A" in ids + assert "IMPL_VAR_B" not in ids # inactive + assert "IMPL_PROTO_3" in ids + assert "IMPL_LINUX_A" in ids + + +def test_libclang_engine_other_variant(): + ids = _run_get_oneline_ids(["PROTOCOL_VERSION=1"]) + assert "IMPL_VAR_B" in ids + assert "IMPL_VAR_A" not in ids + assert "IMPL_PROTO_3" not in ids From 954c4d4ba50a551e1672361bcfea49bb3656b4bc Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 08:33:11 +0200 Subject: [PATCH 09/25] test(preproc): compile_commands path, resilience, tree-sitter parity --- tests/data/preproc/half_typed.cpp | 14 +++++ tests/data/preproc/variants_broken.cpp | 7 +++ tests/test_analyse_preproc.py | 83 ++++++++++++++++++++++++++ 3 files changed, 104 insertions(+) create mode 100644 tests/data/preproc/half_typed.cpp create mode 100644 tests/data/preproc/variants_broken.cpp diff --git a/tests/data/preproc/half_typed.cpp b/tests/data/preproc/half_typed.cpp new file mode 100644 index 0000000..2c30344 --- /dev/null +++ b/tests/data/preproc/half_typed.cpp @@ -0,0 +1,14 @@ +#include + +// @Complete Function, IMPL_COMPLETE, impl, [REQ_OK] +void complete_function() { int x = 42; } + +// @Half Typed Function, IMPL_HALF, impl, [REQ_PARTIAL] +void half_typed_function() { + if (some_condition + +// @After Half Typed, IMPL_AFTER, impl, [REQ_RECOVERS] +void after_half_typed() { int y = 1; } + +// @Mid Declaration, IMPL_MID, impl, [REQ_MID] +int diff --git a/tests/data/preproc/variants_broken.cpp b/tests/data/preproc/variants_broken.cpp new file mode 100644 index 0000000..2afd6c8 --- /dev/null +++ b/tests/data/preproc/variants_broken.cpp @@ -0,0 +1,7 @@ +#include "nonexistent_header.h" + +// @Works Despite Errors, IMPL_DESPITE, impl, [REQ_RESILIENT] +void works_despite_errors() { undeclared_function(); } + +// @After Broken Block, IMPL_AFTER_BROKEN, impl, [REQ_RESILIENT] +void after_broken_block() {} diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py index b2f4e01..a79bfa6 100644 --- a/tests/test_analyse_preproc.py +++ b/tests/test_analyse_preproc.py @@ -41,3 +41,86 @@ def test_libclang_engine_other_variant(): assert "IMPL_VAR_B" in ids assert "IMPL_VAR_A" not in ids assert "IMPL_PROTO_3" not in ids + + +def test_libclang_engine_via_compile_commands(tmp_path): + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(FIXTURE.parent), + "arguments": [ + "clang++", "-std=c++17", "-DVARIANT_A=1", + "-DPROTOCOL_VERSION=3", "-c", str(FIXTURE), + ], + "file": str(FIXTURE), + } + ] + ) + ) + cfg = SourceAnalyseConfig( + src_files=[FIXTURE], + src_dir=FIXTURE.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(compile_commands=db), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + assert "IMPL_VAR_A" in ids + assert "IMPL_VAR_B" not in ids + + +def test_libclang_resilient_to_broken_code(): + broken = FIXTURE.parent / "variants_broken.cpp" + cfg = SourceAnalyseConfig( + src_files=[broken], + src_dir=broken.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=[]), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + assert ids == {"IMPL_DESPITE", "IMPL_AFTER_BROKEN"} + + +def test_libclang_resilient_to_half_typed_code(): + half = FIXTURE.parent / "half_typed.cpp" + cfg = SourceAnalyseConfig( + src_files=[half], + src_dir=half.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=[]), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + # All 4 markers survive at the token level even though 2 decls don't parse. + assert {"IMPL_COMPLETE", "IMPL_HALF", "IMPL_AFTER", "IMPL_MID"} <= ids + + +def test_libclang_active_matches_treesitter_when_all_active(): + """When every branch is active, libclang output == tree-sitter output.""" + # Tree-sitter path (no preprocessor block) sees ALL markers. + ts_cfg = SourceAnalyseConfig( + src_files=[FIXTURE], src_dir=FIXTURE.parent, get_oneline_needs=True + ) + ts = SourceAnalyse(ts_cfg) + ts.git_remote_url = None + ts.git_commit_rev = None + ts.run() + ts_ids = {n.need["id"] for n in ts.oneline_needs} + + # libclang with both variants' guards satisfied is impossible (#else is + # mutually exclusive), so compare the union over both variants instead. + a = _run_get_oneline_ids(["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"]) + b = _run_get_oneline_ids(["PROTOCOL_VERSION=1"]) + assert ts_ids == (a | b) From e8724067d16fbd3293afb15546f17a71694fbcfd Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 08:37:15 +0200 Subject: [PATCH 10/25] test(preproc): assert full compile_commands arg list is consumed --- tests/test_analyse_preproc.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py index a79bfa6..adfe78f 100644 --- a/tests/test_analyse_preproc.py +++ b/tests/test_analyse_preproc.py @@ -72,6 +72,8 @@ def test_libclang_engine_via_compile_commands(tmp_path): ids = {n.need["id"] for n in analyse.oneline_needs} assert "IMPL_VAR_A" in ids assert "IMPL_VAR_B" not in ids + assert "IMPL_PROTO_3" in ids + assert "IMPL_LINUX_A" not in ids def test_libclang_resilient_to_broken_code(): From c8b0736b4ca4b08070d4fa8d5217ade98521bb73 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 08:48:46 +0200 Subject: [PATCH 11/25] =?UTF-8?q?feat(preproc):=20skip=20files=20absent=20?= =?UTF-8?q?from=20compile=5Fcommands=20(spec=20=C2=A73.3)=20+=20harden=20t?= =?UTF-8?q?est?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _resolve_preproc_args now returns None when a compile DB is found but the source file is not listed in it, causing create_src_objects_libclang to skip the file with a debug log rather than falling back to defines_to_args. No-DB path (defines fallback) is unchanged. Also: add IMPL_ALWAYS vacuous-pass guard to test_libclang_engine_other_variant and a new test_libclang_skip_file_absent_from_compile_commands that verifies the skip behaviour end-to-end. --- src/sphinx_codelinks/analyse/analyse.py | 13 ++++++-- tests/test_analyse_preproc.py | 41 +++++++++++++++++++++++++ 2 files changed, 51 insertions(+), 3 deletions(-) diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index 394e642..b2029e7 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -79,7 +79,7 @@ def __init__( utils.get_current_rev(self.git_root) if self.git_root else None ) self.project_path: Path = ( - self.git_root if self.git_root else self.analyse_config.src_dir + self.git_root or self.analyse_config.src_dir ) self.oneline_warnings: list[AnalyseWarning] = [] @@ -112,7 +112,7 @@ def create_src_objects(self) -> None: self.src_files.append(src_file) self.src_comments.extend(src_comments) - def _resolve_preproc_args(self, src_path: Path) -> list[str]: + def _resolve_preproc_args(self, src_path: Path) -> list[str] | None: from sphinx_codelinks.analyse.preproc import compile_db # noqa: PLC0415 preproc = self.analyse_config.preprocessor @@ -126,7 +126,9 @@ def _resolve_preproc_args(self, src_path: Path) -> list[str]: args = flags.get(src_path.absolute().resolve()) if args is not None: return args - # Fallback: manual defines applied globally. + # File not in compile DB — skip it per spec §3.3. + return None + # No DB found at all — fall back to manual defines applied globally. return compile_db.defines_to_args(preproc.defines, preproc.includes) def create_src_objects_libclang(self) -> None: @@ -136,6 +138,11 @@ def create_src_objects_libclang(self) -> None: if not utils.is_text_file(src_path): continue args = self._resolve_preproc_args(src_path) + if args is None: + logger.debug( + f"codelinks: skipping {src_path} — not found in compile_commands.json" + ) + continue comments = libclang_parser.extract_active_comments(src_path, args) if not comments: continue diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py index adfe78f..1c106d8 100644 --- a/tests/test_analyse_preproc.py +++ b/tests/test_analyse_preproc.py @@ -38,6 +38,7 @@ def test_libclang_engine_excludes_inactive_markers(): def test_libclang_engine_other_variant(): ids = _run_get_oneline_ids(["PROTOCOL_VERSION=1"]) + assert "IMPL_ALWAYS" in ids assert "IMPL_VAR_B" in ids assert "IMPL_VAR_A" not in ids assert "IMPL_PROTO_3" not in ids @@ -126,3 +127,43 @@ def test_libclang_active_matches_treesitter_when_all_active(): a = _run_get_oneline_ids(["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"]) b = _run_get_oneline_ids(["PROTOCOL_VERSION=1"]) assert ts_ids == (a | b) + + +def test_libclang_skip_file_absent_from_compile_commands(tmp_path): + """Files absent from a compile DB are skipped (spec §3.3).""" + # DB contains only FIXTURE; half_typed.cpp is intentionally absent. + other = FIXTURE.parent / "half_typed.cpp" + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(FIXTURE.parent), + "arguments": [ + "clang++", "-std=c++17", "-DVARIANT_A=1", + "-DPROTOCOL_VERSION=3", "-c", str(FIXTURE), + ], + "file": str(FIXTURE), + } + ] + ) + ) + cfg = SourceAnalyseConfig( + src_files=[FIXTURE, other], + src_dir=FIXTURE.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(compile_commands=db), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + # FIXTURE is in the DB — its markers must appear. + assert "IMPL_VAR_A" in ids + assert "IMPL_PROTO_3" in ids + # half_typed.cpp is NOT in the DB — it must be skipped entirely. + assert "IMPL_COMPLETE" not in ids + assert "IMPL_HALF" not in ids + assert "IMPL_AFTER" not in ids + assert "IMPL_MID" not in ids From 94d2230554dc265890adfd4dee4b1195cf76d4a2 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 25 Jun 2026 10:23:39 +0200 Subject: [PATCH 12/25] test(preproc): cross-impl parity vs shared golden (lockstep with Rust) Add test_libclang_active_matches_golden to assert Python libclang engine produces the same 4 active needs as the shared Rust golden (IMPL_ALWAYS, IMPL_VAR_A, IMPL_PROTO_3, IMPL_LINUX_A) with defines VARIANT_A=1, PLATFORM_LINUX=1, PROTOCOL_VERSION=3, excluding IMPL_VAR_B as expected. --- .../preproc/variants_branching.expected.json | 6 ++++ tests/test_analyse_preproc.py | 35 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 tests/data/preproc/variants_branching.expected.json diff --git a/tests/data/preproc/variants_branching.expected.json b/tests/data/preproc/variants_branching.expected.json new file mode 100644 index 0000000..6e641a3 --- /dev/null +++ b/tests/data/preproc/variants_branching.expected.json @@ -0,0 +1,6 @@ +[ + {"id": "IMPL_ALWAYS", "title": "Always Present", "type": "impl", "links": ["REQ_BASE"], "line": 1}, + {"id": "IMPL_VAR_A", "title": "Variant A Feature", "type": "impl", "links": ["REQ_FEAT_A"], "line": 5}, + {"id": "IMPL_PROTO_3", "title": "Protocol V3", "type": "impl", "links": ["REQ_PROTO"], "line": 13}, + {"id": "IMPL_LINUX_A", "title": "Linux And A", "type": "impl", "links": ["REQ_COMBO"], "line": 18} +] diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py index 1c106d8..d3c81fb 100644 --- a/tests/test_analyse_preproc.py +++ b/tests/test_analyse_preproc.py @@ -167,3 +167,38 @@ def test_libclang_skip_file_absent_from_compile_commands(tmp_path): assert "IMPL_HALF" not in ids assert "IMPL_AFTER" not in ids assert "IMPL_MID" not in ids + + +def test_libclang_active_matches_golden(): + """Cross-impl parity: Python libclang output matches shared Rust golden.""" + cfg = SourceAnalyseConfig( + src_files=[FIXTURE], + src_dir=FIXTURE.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"]), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + + # Project each OneLineNeed to a dict matching the golden schema. + projected = [ + { + "id": n.need["id"], + "title": n.need["title"], + "type": n.need["type"], + "links": n.need["links"], + "line": n.source_map["start"]["row"] + 1, + } + for n in analyse.oneline_needs + ] + projected.sort(key=lambda x: x["line"]) + + # Load the golden and sort by line. + golden_path = FIXTURE.parent / "variants_branching.expected.json" + with golden_path.open() as f: + golden = json.load(f) + golden.sort(key=lambda x: x["line"]) + + assert projected == golden From f8dc6064a8690ac685272190d2a3e4c5180ff243 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sat, 27 Jun 2026 22:48:21 +0200 Subject: [PATCH 13/25] fix(preproc): validate nested preprocessor config correctly SourceAnalyseConfig.preprocessor carried a flat metadata schema {type: [object, null]}, so check_schema validated the constructed PreprocessorConfig *instance* against JSON type object and raised at config-inited: 'PreprocessorConfig(...) is not of type object, null'. Nested dataclass config fields (need_id_refs_config, marked_rst_config, oneline_comment_style) carry no flat schema; the preprocessor field must match. Its structure is enforced by convert_analyse_config. Adds a regression test exercising the validation path. --- src/sphinx_codelinks/config.py | 14 ++++++++++---- tests/test_preproc_config.py | 25 +++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/src/sphinx_codelinks/config.py b/src/sphinx_codelinks/config.py index 28138a6..2866a7c 100644 --- a/src/sphinx_codelinks/config.py +++ b/src/sphinx_codelinks/config.py @@ -429,10 +429,16 @@ def field_names(cls) -> set[str]: ) """Configuration for extracting oneline needs from comments.""" - preprocessor: PreprocessorConfig | None = field( - default=None, metadata={"schema": {"type": ["object", "null"]}} - ) - """Opt-in libclang preprocessor engine. None => tree-sitter (default).""" + preprocessor: PreprocessorConfig | None = field(default=None) + """Opt-in libclang preprocessor engine. None => tree-sitter (default). + + No flat ``metadata["schema"]`` here: this is a nested dataclass, like the + sibling ``need_id_refs_config`` / ``marked_rst_config`` / + ``oneline_comment_style`` fields. ``check_schema`` only validates fields that + declare a flat schema; giving this field one made it validate the constructed + ``PreprocessorConfig`` instance against JSON type ``object`` and fail at + ``config-inited``. Its structure is enforced by ``convert_analyse_config``. + """ @classmethod def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] diff --git a/tests/test_preproc_config.py b/tests/test_preproc_config.py index b714034..69f617d 100644 --- a/tests/test_preproc_config.py +++ b/tests/test_preproc_config.py @@ -34,3 +34,28 @@ def test_convert_analyse_config_builds_preprocessor(): def test_convert_analyse_config_no_preprocessor_block(): cfg = convert_analyse_config({"get_oneline_needs": True}) assert cfg.preprocessor is None + + +def test_preprocessor_config_passes_analyse_schema_validation(): + """A SourceAnalyseConfig carrying a preprocessor must validate cleanly. + + Regression: the ``preprocessor`` field previously carried a flat + ``{"type": ["object", "null"]}`` json-schema, so ``check_schema`` validated + the *constructed* ``PreprocessorConfig`` instance against JSON type + ``object`` and raised at sphinx ``config-inited``:: + + Schema validation error in field 'preprocessor': + PreprocessorConfig(...) is not of type 'object', 'null' + + Nested dataclass config fields must not carry a flat schema (their siblings + ``need_id_refs_config`` / ``oneline_comment_style`` do not). + """ + cfg = convert_analyse_config( + { + "get_oneline_needs": True, + "preprocessor": {"defines": ["FEATURE_A"]}, + } + ) + assert cfg.preprocessor is not None + schema_errors = cfg.check_schema() + assert not any("preprocessor" in err for err in schema_errors), schema_errors From 72c1eceda3353b63b2acf75fc75b1f1710f2dca8 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sat, 27 Jun 2026 23:00:33 +0200 Subject: [PATCH 14/25] refactor(preproc): rename optional extra cpp -> libclang, add guard The opt-in libclang engine adds preprocessor-aware extraction (skipping comments inside inactive #if/#ifdef regions) for C, C++, and Objective-C. Tree-sitter already covers C/C++ extraction, so the extra was misnamed `cpp`: it implied "C++ support" rather than the libclang backend it actually pulls in. Rename it to `libclang`, matching the dependency and the install hint. Add tests/test_libclang_optional.py, a regression guard that blocks `clang` in a fresh interpreter (a tree-sitter-only install, by construction) and asserts: the default tree-sitter path imports and runs without libclang; and selecting the libclang engine without the extra raises the documented "pip install 'sphinx-codelinks[libclang]'" hint. It proves the block is effective first, so it cannot pass vacuously under CI, which installs the extra. --- pyproject.toml | 6 +- .../analyse/preproc/loader.py | 2 +- tests/test_libclang_optional.py | 134 ++++++++++++++++++ 3 files changed, 140 insertions(+), 2 deletions(-) create mode 100644 tests/test_libclang_optional.py diff --git a/pyproject.toml b/pyproject.toml index 6b34b6a..40d874d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,11 @@ dependencies = [ ] [project.optional-dependencies] -cpp = ["libclang>=18"] +# libclang enables the preprocessor-aware C/C++ engine (skips comments inside +# inactive #if/#ifdef regions). Plain tree-sitter C/C++ extraction needs none of +# this, so libclang stays optional. Named after the backend, not "cpp": the +# engine handles C and Objective-C too, and tree-sitter already covers C++. +libclang = ["libclang>=18"] [build-system] requires = ["flit_core >=3.4,<4"] diff --git a/src/sphinx_codelinks/analyse/preproc/loader.py b/src/sphinx_codelinks/analyse/preproc/loader.py index a9d8a9a..1af14fa 100644 --- a/src/sphinx_codelinks/analyse/preproc/loader.py +++ b/src/sphinx_codelinks/analyse/preproc/loader.py @@ -13,7 +13,7 @@ _INSTALL_HINT = ( "The libclang engine requires clang.cindex. Install the extra:\n" - " pip install 'sphinx-codelinks[cpp]'" + " pip install 'sphinx-codelinks[libclang]'" ) diff --git a/tests/test_libclang_optional.py b/tests/test_libclang_optional.py new file mode 100644 index 0000000..4b55808 --- /dev/null +++ b/tests/test_libclang_optional.py @@ -0,0 +1,134 @@ +"""Regression guard: the libclang engine is an OPTIONAL dependency. + +A project that only wants tree-sitter extraction must be able to import and run +sphinx-codelinks without the ``libclang`` wheel installed. Selecting the +libclang engine without it must fail with a clear ``pip install`` hint, not an +obscure ``ImportError``. + +CI installs the ``libclang`` extra, so we cannot test the absent case simply by +not installing it. Instead each test spawns a fresh interpreter with +``clang``/``clang.*`` blocked at import time (a tree-sitter-only install, by +construction) and -- before anything else -- proves the block is effective, so +the test can never pass vacuously when the wheel happens to be present. + +If someone adds an eager top-level ``import clang`` anywhere in the import +chain, ``test_treesitter_path_imports_and_runs_without_libclang`` breaks here +instead of silently making libclang mandatory for every user. +""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys +import textwrap + +FIXTURE = Path(__file__).parent / "data" / "preproc" / "variants_branching.cpp" + +# Block ``clang`` so the child interpreter behaves like a tree-sitter-only +# install, then assert the block actually took effect -- otherwise every check +# below would be meaningless on a machine that has the libclang wheel. +_BLOCK_CLANG = """ +import importlib.abc +import sys + + +class _BlockClang(importlib.abc.MetaPathFinder): + def find_spec(self, name, path, target=None): + if name == "clang" or name.startswith("clang."): + raise ImportError("simulated tree-sitter-only install: " + name) + return None # defer all other imports to the real finders + + +for _mod in [k for k in sys.modules if k == "clang" or k.startswith("clang.")]: + del sys.modules[_mod] +sys.meta_path.insert(0, _BlockClang()) + +try: + import clang.cindex +except ImportError: + pass +else: + raise SystemExit("clang.cindex was NOT blocked; this test would be vacuous") +""" + + +def _run_probe(body: str) -> None: + """Run ``_BLOCK_CLANG + body`` in a fresh interpreter. + + The fixture path is passed as ``sys.argv[1]``. On non-zero exit the child's + stdout/stderr are surfaced in the assertion message. + """ + code = _BLOCK_CLANG + textwrap.dedent(body) + proc = subprocess.run( # noqa: S603 + [sys.executable, "-c", code, str(FIXTURE)], + capture_output=True, + text=True, + check=False, # returncode is asserted explicitly below + ) + assert proc.returncode == 0, ( + f"probe exited {proc.returncode}\n" + f"--- stdout ---\n{proc.stdout}\n--- stderr ---\n{proc.stderr}" + ) + + +def test_treesitter_path_imports_and_runs_without_libclang() -> None: + """Default (tree-sitter) extraction needs no libclang -- import and run.""" + _run_probe( + """ + import sys + from pathlib import Path + + # Importing these must not pull in clang (parent __init__ files run too). + from sphinx_codelinks.analyse.analyse import SourceAnalyse + from sphinx_codelinks.config import SourceAnalyseConfig + + fixture = Path(sys.argv[1]) + cfg = SourceAnalyseConfig( + src_files=[fixture], + src_dir=fixture.parent, + get_oneline_needs=True, + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() # no preprocessor configured -> tree-sitter engine + + ids = {n.need["id"] for n in analyse.oneline_needs} + # tree-sitter ignores #ifdef, so markers on every branch are extracted. + missing = {"IMPL_ALWAYS", "IMPL_VAR_A", "IMPL_VAR_B"} - ids + assert not missing, f"tree-sitter missed markers without libclang: {missing}" + """ + ) + + +def test_libclang_engine_errors_with_install_hint_when_extra_missing() -> None: + """Selecting the libclang engine without the extra raises the install hint.""" + _run_probe( + """ + import sys + from pathlib import Path + + from sphinx_codelinks.analyse.analyse import SourceAnalyse + from sphinx_codelinks.config import PreprocessorConfig, SourceAnalyseConfig + + fixture = Path(sys.argv[1]) + cfg = SourceAnalyseConfig( + src_files=[fixture], + src_dir=fixture.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=[]), # selects libclang engine + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + try: + analyse.run() + except ImportError as exc: + assert "sphinx-codelinks[libclang]" in str(exc), ( + f"install hint did not name the extra: {str(exc)!r}" + ) + else: + raise SystemExit("libclang engine ran without the extra; expected ImportError") + """ + ) From 4bf97d1a2d1bc42b576ddcdc193591304455a5de Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sun, 28 Jun 2026 08:47:17 +0200 Subject: [PATCH 15/25] feat(preproc): add is_translation_unit_source classifier --- .../analyse/preproc/compile_db.py | 15 +++++++++++++++ tests/test_preproc_compile_db.py | 16 ++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/src/sphinx_codelinks/analyse/preproc/compile_db.py b/src/sphinx_codelinks/analyse/preproc/compile_db.py index 5f622c6..aabe60a 100644 --- a/src/sphinx_codelinks/analyse/preproc/compile_db.py +++ b/src/sphinx_codelinks/analyse/preproc/compile_db.py @@ -16,6 +16,11 @@ # Minimum length for joined-form flags like -MFdep.d (prefix length = 3) _MIN_JOINED_FLAG_LEN = 3 +# Suffixes a compiler builds as a translation unit. A discovered C/C++ file +# absent from compile_commands.json is skipped only when it is a TU source +# (build-excluded); all other discovered files (headers) are parsed standalone. +TU_SOURCE_SUFFIXES = {".c", ".cpp", ".cc", ".cxx"} + def find_compile_db(start: Path, project_root: Path | None = None) -> Path | None: """Walk up from ``start`` looking for compile_commands.json. @@ -91,3 +96,13 @@ def defines_to_args( args += [f"-D{d}" for d in defines] args += [f"-I{inc}" for inc in includes] return args + + +def is_translation_unit_source(path: Path) -> bool: + """True if ``path`` is a compiled translation-unit source (not a header). + + compile_commands.json lists one entry per compiled TU; headers are never + entries. So a discovered file absent from the DB is skipped only when it is + a TU source; header-like files are parsed standalone (see _resolve_preproc_args). + """ + return path.suffix.lower() in TU_SOURCE_SUFFIXES diff --git a/tests/test_preproc_compile_db.py b/tests/test_preproc_compile_db.py index eb4c616..e44c633 100644 --- a/tests/test_preproc_compile_db.py +++ b/tests/test_preproc_compile_db.py @@ -110,3 +110,19 @@ def test_defines_to_args(tmp_path: Path): assert "-DX=2" in out assert f"-I{(tmp_path / 'inc')}" in out assert "-std=c++17" in out + + +def test_is_translation_unit_source(): + # Compiled translation-unit sources (skipped when build-excluded). + assert compile_db.is_translation_unit_source(Path("a.c")) + assert compile_db.is_translation_unit_source(Path("a.cpp")) + assert compile_db.is_translation_unit_source(Path("a.cc")) + assert compile_db.is_translation_unit_source(Path("a.cxx")) + assert compile_db.is_translation_unit_source(Path("A.CPP")) # case-insensitive + # Header-like files (parsed standalone when absent from the DB). + assert not compile_db.is_translation_unit_source(Path("a.h")) + assert not compile_db.is_translation_unit_source(Path("a.hpp")) + assert not compile_db.is_translation_unit_source(Path("a.hxx")) + assert not compile_db.is_translation_unit_source(Path("a.hh")) + assert not compile_db.is_translation_unit_source(Path("a.ci")) + assert not compile_db.is_translation_unit_source(Path("a.ihl")) From 74d29bf577b7e6fefd46f44faaee8d80dcb1f2b3 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sun, 28 Jun 2026 08:48:25 +0200 Subject: [PATCH 16/25] feat(preproc): extract needs from headers absent from compile_commands --- src/sphinx_codelinks/analyse/analyse.py | 9 +++- tests/data/preproc/header_standalone.hpp | 12 +++++ tests/test_analyse_preproc.py | 61 ++++++++++++++++++++++++ 3 files changed, 80 insertions(+), 2 deletions(-) create mode 100644 tests/data/preproc/header_standalone.hpp diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index b2029e7..cb6e810 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -126,8 +126,13 @@ def _resolve_preproc_args(self, src_path: Path) -> list[str] | None: args = flags.get(src_path.absolute().resolve()) if args is not None: return args - # File not in compile DB — skip it per spec §3.3. - return None + # Absent from the DB. compile_commands.json lists only compiled + # translation units, never headers — so a header here is parsed + # standalone with the global defines (one run = one variant). A + # compiled source absent from the build is skipped (spec §3.3). + if compile_db.is_translation_unit_source(src_path): + return None + return compile_db.defines_to_args(preproc.defines, preproc.includes) # No DB found at all — fall back to manual defines applied globally. return compile_db.defines_to_args(preproc.defines, preproc.includes) diff --git a/tests/data/preproc/header_standalone.hpp b/tests/data/preproc/header_standalone.hpp new file mode 100644 index 0000000..d279fa2 --- /dev/null +++ b/tests/data/preproc/header_standalone.hpp @@ -0,0 +1,12 @@ +#ifndef HEADER_STANDALONE_HPP +#define HEADER_STANDALONE_HPP + +// @Header Always, IMPL_HDR_ALWAYS, impl, [REQ_HDR] +void hdr_always(); + +#ifdef VARIANT_A +// @Header Variant A, IMPL_HDR_VAR_A, impl, [REQ_HDR_A] +void hdr_variant_a(); +#endif + +#endif // HEADER_STANDALONE_HPP diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py index d3c81fb..f6bd02b 100644 --- a/tests/test_analyse_preproc.py +++ b/tests/test_analyse_preproc.py @@ -9,6 +9,7 @@ from sphinx_codelinks.config import PreprocessorConfig, SourceAnalyseConfig FIXTURE = Path(__file__).parent / "data" / "preproc" / "variants_branching.cpp" +HEADER = Path(__file__).parent / "data" / "preproc" / "header_standalone.hpp" def _run_get_oneline_ids(defines): @@ -202,3 +203,63 @@ def test_libclang_active_matches_golden(): golden.sort(key=lambda x: x["line"]) assert projected == golden + + +def test_header_extracted_when_absent_from_compile_commands(tmp_path): + """A header absent from the DB is parsed standalone, not skipped.""" + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(FIXTURE.parent), + "arguments": [ + "clang++", "-std=c++17", "-DVARIANT_A=1", "-c", str(FIXTURE), + ], + "file": str(FIXTURE), + } + ] + ) + ) + cfg = SourceAnalyseConfig( + src_files=[FIXTURE, HEADER], + src_dir=FIXTURE.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(compile_commands=db, defines=["VARIANT_A=1"]), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + # The .cpp resolves its flags from the DB entry. + assert "IMPL_VAR_A" in ids + # The header is absent from the DB -> parsed standalone with global defines. + assert "IMPL_HDR_ALWAYS" in ids + assert "IMPL_HDR_VAR_A" in ids # global defines carry VARIANT_A=1 + + +def _run_header_ids(defines): + cfg = SourceAnalyseConfig( + src_files=[HEADER], + src_dir=HEADER.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=defines), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + return {n.need["id"] for n in analyse.oneline_needs} + + +def test_header_standalone_active_define(): + ids = _run_header_ids(["VARIANT_A=1"]) + assert "IMPL_HDR_ALWAYS" in ids # include-guard body is active + assert "IMPL_HDR_VAR_A" in ids # #ifdef VARIANT_A active + + +def test_header_standalone_inactive_define(): + ids = _run_header_ids([]) + assert "IMPL_HDR_ALWAYS" in ids # include-guard body still active + assert "IMPL_HDR_VAR_A" not in ids # #ifdef VARIANT_A inactive -> dropped From b554047aa1a704eae7733f2818f15f7aaba6a7f6 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sun, 28 Jun 2026 09:17:37 +0200 Subject: [PATCH 17/25] docs(analyse): document the libclang preprocessor engine and header handling --- docs/source/components/analyse.rst | 81 ++++++++++++++++++++++++ docs/source/components/configuration.rst | 44 +++++++++++++ 2 files changed, 125 insertions(+) diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 631eafd..f310ca8 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -210,3 +210,84 @@ This single comment line creates a complete **Sphinx-Needs** item equivalent to: .. impl:: Function Implementation :id: IMPL_001 :links: REQ_001, REQ_002 + +.. _preprocessor_engine: + +Preprocessor-Aware C/C++ Extraction (libclang) +---------------------------------------------- + +By default, **Source Analyse** uses a tree-sitter parser that extracts **every** comment, +regardless of the C preprocessor. For C/C++ projects that rely on conditional compilation +(``#ifdef VARIANT_A`` …), this means needs from *all* branches are extracted — even +branches that are never compiled. + +The optional **libclang engine** addresses this. When an +:ref:`analyse.preprocessor ` table is configured and ``comment_type`` +is ``"cpp"``, each file is parsed as a real translation unit and **comments inside inactive +preprocessor branches are dropped**. Active needs keep their original line numbers — no +source transformation is performed. + +.. important:: The libclang engine requires an optional dependency: + ``pip install 'sphinx-codelinks[libclang]'``. The wheel bundles the native library, so + no compiler is required on the user's machine. + +How files are selected and parsed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +File **discovery** is unchanged — :ref:`SourceDiscover ` still decides which +files are candidates. A ``compile_commands.json`` database only determines *how* each +discovered file is parsed: + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - File + - How it is parsed + * - Listed in ``compile_commands.json`` + - Parsed with the exact flags the compiler used for that translation unit. + * - A compiled source (``.c``, ``.cpp``, ``.cc``, ``.cxx``) **not** listed + - Skipped — assumed to be excluded from the build (e.g. another platform). + * - A **header** (``.h``, ``.hpp``, …) — never listed in a database + - Parsed **standalone** using ``defines`` and ``includes`` (see below). + * - Any file, when **no** database is found + - Parsed with ``defines`` and ``includes``. + +Header files +~~~~~~~~~~~~~ + +A ``compile_commands.json`` only ever lists compiled translation units (``.cpp`` files); +headers are pulled in via ``#include`` and never appear as entries. **Sphinx-CodeLinks** +therefore parses each discovered header **standalone**, using the ``defines`` and +``includes`` you configure. Include guards resolve correctly, and ``#ifdef`` branches are +evaluated against your ``defines``. + +.. note:: Because headers are parsed standalone, they see only the global ``defines`` — + **not** the per-file ``-D`` flags from ``compile_commands.json``. To extract a + particular variant's needs from headers, mirror that variant into ``defines``. Treat + one analysis run as **one variant**: set ``defines`` to the variant you want, and both + sources and headers evaluate their conditions consistently. + +Example +~~~~~~~ + +.. code-block:: cpp + + // include/feature.hpp + #ifndef FEATURE_HPP + #define FEATURE_HPP + + // @Always available, IMPL_BASE, impl, [REQ_BASE] + void base(); + + #ifdef VARIANT_A + // @Variant A only, IMPL_VAR_A, impl, [REQ_A] + void variant_a(); + #endif + + #endif + +With ``defines = ["VARIANT_A"]`` both ``IMPL_BASE`` and ``IMPL_VAR_A`` are extracted. +With ``defines = []`` only ``IMPL_BASE`` is extracted — the ``#ifdef VARIANT_A`` block is +inactive, so its need is dropped. The include guard (``#ifndef FEATURE_HPP``) is always +active when the header is parsed on its own, so ``IMPL_BASE`` is never suppressed by it. diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 8da2f2a..0e17a96 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -570,3 +570,47 @@ Configuration for marked RST block extraction. - ``start_sequence`` (``str``) - Marker that begins an RST block - ``end_sequence`` (``str``) - Marker that ends an RST block + +.. _`preprocessor_config`: + +analyse.preprocessor +^^^^^^^^^^^^^^^^^^^^^ + +Opts in to the **preprocessor-aware C/C++ engine** (powered by libclang). When this +table is present and ``comment_type`` is ``"cpp"``, **Sphinx-CodeLinks** parses each +C/C++ file as a real translation unit and **drops comments that fall inside inactive +preprocessor branches** (``#if`` / ``#ifdef`` / ``#else`` …). Without this table, the +default tree-sitter engine is used, which extracts every comment regardless of +preprocessor conditions. + +See :ref:`preprocessor_engine` for the conceptual overview, header handling, and +limitations. + +**Type:** ``dict`` +**Default:** Not set (the tree-sitter engine is used) + +.. important:: The libclang engine requires an optional dependency. Install it with + ``pip install 'sphinx-codelinks[libclang]'``. The ``libclang`` wheel bundles the + native library, so no compiler or system toolchain is required. + +.. code-block:: toml + + [codelinks.projects.my_project.analyse.preprocessor] + compile_commands = "build/compile_commands.json" + defines = ["VARIANT_A", "PLATFORM_LINUX"] + includes = ["include", "third_party/include"] + variant_name = "variant_a" + +**Configuration fields:** + +- ``compile_commands`` (``str``) - Path to a ``compile_commands.json`` compilation + database. Relative paths resolve against the TOML config file. If omitted, + **Sphinx-CodeLinks** walks up from each source file to find one, stopping at a + directory containing ``.git``, ``ubproject.toml``, or ``pyproject.toml``, or at the + filesystem root. +- ``defines`` (``list[str]``) - ``-D`` macros used when a file has **no** entry in the + database. This is the fallback applied to every header file (headers never appear in a + database) and to all files when no database is found. +- ``includes`` (``list[str]``) - ``-I`` include directories used together with + ``defines`` on the fallback path. +- ``variant_name`` (``str``) - Optional label echoed into the run-level output. From f83972a7422900e09e658c54de446b0d88e9ff84 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sun, 28 Jun 2026 11:48:24 +0200 Subject: [PATCH 18/25] test(preproc): drop cross-impl golden from codelinks codelinks is the OSS reference implementation; cross-impl feature parity is ubCode's concern and lives there. Remove the shared Rust golden + its test; codelinks keeps its own behavioral tests (inactive-branch exclusion, resilience, compile_commands, header extraction). --- .../preproc/variants_branching.expected.json | 6 -- tests/test_analyse_preproc.py | 61 ++++++------------- 2 files changed, 20 insertions(+), 47 deletions(-) delete mode 100644 tests/data/preproc/variants_branching.expected.json diff --git a/tests/data/preproc/variants_branching.expected.json b/tests/data/preproc/variants_branching.expected.json deleted file mode 100644 index 6e641a3..0000000 --- a/tests/data/preproc/variants_branching.expected.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - {"id": "IMPL_ALWAYS", "title": "Always Present", "type": "impl", "links": ["REQ_BASE"], "line": 1}, - {"id": "IMPL_VAR_A", "title": "Variant A Feature", "type": "impl", "links": ["REQ_FEAT_A"], "line": 5}, - {"id": "IMPL_PROTO_3", "title": "Protocol V3", "type": "impl", "links": ["REQ_PROTO"], "line": 13}, - {"id": "IMPL_LINUX_A", "title": "Linux And A", "type": "impl", "links": ["REQ_COMBO"], "line": 18} -] diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py index f6bd02b..be215f2 100644 --- a/tests/test_analyse_preproc.py +++ b/tests/test_analyse_preproc.py @@ -29,7 +29,9 @@ def _run_get_oneline_ids(defines): def test_libclang_engine_excludes_inactive_markers(): - ids = _run_get_oneline_ids(["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"]) + ids = _run_get_oneline_ids( + ["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"] + ) assert "IMPL_ALWAYS" in ids assert "IMPL_VAR_A" in ids assert "IMPL_VAR_B" not in ids # inactive @@ -53,8 +55,12 @@ def test_libclang_engine_via_compile_commands(tmp_path): { "directory": str(FIXTURE.parent), "arguments": [ - "clang++", "-std=c++17", "-DVARIANT_A=1", - "-DPROTOCOL_VERSION=3", "-c", str(FIXTURE), + "clang++", + "-std=c++17", + "-DVARIANT_A=1", + "-DPROTOCOL_VERSION=3", + "-c", + str(FIXTURE), ], "file": str(FIXTURE), } @@ -141,8 +147,12 @@ def test_libclang_skip_file_absent_from_compile_commands(tmp_path): { "directory": str(FIXTURE.parent), "arguments": [ - "clang++", "-std=c++17", "-DVARIANT_A=1", - "-DPROTOCOL_VERSION=3", "-c", str(FIXTURE), + "clang++", + "-std=c++17", + "-DVARIANT_A=1", + "-DPROTOCOL_VERSION=3", + "-c", + str(FIXTURE), ], "file": str(FIXTURE), } @@ -170,41 +180,6 @@ def test_libclang_skip_file_absent_from_compile_commands(tmp_path): assert "IMPL_MID" not in ids -def test_libclang_active_matches_golden(): - """Cross-impl parity: Python libclang output matches shared Rust golden.""" - cfg = SourceAnalyseConfig( - src_files=[FIXTURE], - src_dir=FIXTURE.parent, - get_oneline_needs=True, - preprocessor=PreprocessorConfig(defines=["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"]), - ) - analyse = SourceAnalyse(cfg) - analyse.git_remote_url = None - analyse.git_commit_rev = None - analyse.run() - - # Project each OneLineNeed to a dict matching the golden schema. - projected = [ - { - "id": n.need["id"], - "title": n.need["title"], - "type": n.need["type"], - "links": n.need["links"], - "line": n.source_map["start"]["row"] + 1, - } - for n in analyse.oneline_needs - ] - projected.sort(key=lambda x: x["line"]) - - # Load the golden and sort by line. - golden_path = FIXTURE.parent / "variants_branching.expected.json" - with golden_path.open() as f: - golden = json.load(f) - golden.sort(key=lambda x: x["line"]) - - assert projected == golden - - def test_header_extracted_when_absent_from_compile_commands(tmp_path): """A header absent from the DB is parsed standalone, not skipped.""" db = tmp_path / "compile_commands.json" @@ -214,7 +189,11 @@ def test_header_extracted_when_absent_from_compile_commands(tmp_path): { "directory": str(FIXTURE.parent), "arguments": [ - "clang++", "-std=c++17", "-DVARIANT_A=1", "-c", str(FIXTURE), + "clang++", + "-std=c++17", + "-DVARIANT_A=1", + "-c", + str(FIXTURE), ], "file": str(FIXTURE), } From 1320bc37099d9c886264994cfc07e637583a8717 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sun, 28 Jun 2026 12:09:56 +0200 Subject: [PATCH 19/25] test(preproc): pin extraction to per-engine golden snapshots Restore the libclang golden snapshot (its earlier removal was a mistake) and add a tree-sitter golden beside it. Both engines are now pinned to declarative JSON snapshots through a shared projection helper: - variants_branching.expected.json (libclang active set, 4 markers) - variants_branching.treesitter.expected.json (tree-sitter, all 5 markers) The snapshots are language-agnostic declarative data so any conforming implementation can assert against the same fixtures. --- .../preproc/variants_branching.expected.json | 6 ++ ...ariants_branching.treesitter.expected.json | 7 +++ tests/test_analyse_preproc.py | 60 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 tests/data/preproc/variants_branching.expected.json create mode 100644 tests/data/preproc/variants_branching.treesitter.expected.json diff --git a/tests/data/preproc/variants_branching.expected.json b/tests/data/preproc/variants_branching.expected.json new file mode 100644 index 0000000..6e641a3 --- /dev/null +++ b/tests/data/preproc/variants_branching.expected.json @@ -0,0 +1,6 @@ +[ + {"id": "IMPL_ALWAYS", "title": "Always Present", "type": "impl", "links": ["REQ_BASE"], "line": 1}, + {"id": "IMPL_VAR_A", "title": "Variant A Feature", "type": "impl", "links": ["REQ_FEAT_A"], "line": 5}, + {"id": "IMPL_PROTO_3", "title": "Protocol V3", "type": "impl", "links": ["REQ_PROTO"], "line": 13}, + {"id": "IMPL_LINUX_A", "title": "Linux And A", "type": "impl", "links": ["REQ_COMBO"], "line": 18} +] diff --git a/tests/data/preproc/variants_branching.treesitter.expected.json b/tests/data/preproc/variants_branching.treesitter.expected.json new file mode 100644 index 0000000..52217e7 --- /dev/null +++ b/tests/data/preproc/variants_branching.treesitter.expected.json @@ -0,0 +1,7 @@ +[ + {"id": "IMPL_ALWAYS", "title": "Always Present", "type": "impl", "links": ["REQ_BASE"], "line": 1}, + {"id": "IMPL_VAR_A", "title": "Variant A Feature", "type": "impl", "links": ["REQ_FEAT_A"], "line": 5}, + {"id": "IMPL_VAR_B", "title": "Variant B Feature", "type": "impl", "links": ["REQ_FEAT_B"], "line": 8}, + {"id": "IMPL_PROTO_3", "title": "Protocol V3", "type": "impl", "links": ["REQ_PROTO"], "line": 13}, + {"id": "IMPL_LINUX_A", "title": "Linux And A", "type": "impl", "links": ["REQ_COMBO"], "line": 18} +] diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py index be215f2..d76435d 100644 --- a/tests/test_analyse_preproc.py +++ b/tests/test_analyse_preproc.py @@ -242,3 +242,63 @@ def test_header_standalone_inactive_define(): ids = _run_header_ids([]) assert "IMPL_HDR_ALWAYS" in ids # include-guard body still active assert "IMPL_HDR_VAR_A" not in ids # #ifdef VARIANT_A inactive -> dropped + + +# --------------------------------------------------------------------------- +# Golden snapshots +# +# The extraction output is pinned to declarative JSON snapshots, one per engine. +# `defines=None` omits the preprocessor block (tree-sitter path: every marker is +# seen); a defines list activates the libclang path (inactive `#if` branches are +# excluded). The snapshots are language-agnostic so any conforming implementation +# can assert against the same fixtures. +# --------------------------------------------------------------------------- + + +def _projected_oneline_needs(defines): + """Run the analyse pipeline and project each one-line need to the golden schema.""" + cfg = SourceAnalyseConfig( + src_files=[FIXTURE], + src_dir=FIXTURE.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=defines) + if defines is not None + else None, + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + projected = [ + { + "id": n.need["id"], + "title": n.need["title"], + "type": n.need["type"], + "links": n.need["links"], + "line": n.source_map["start"]["row"] + 1, + } + for n in analyse.oneline_needs + ] + projected.sort(key=lambda x: x["line"]) + return projected + + +def _load_golden(name): + with (FIXTURE.parent / name).open() as f: + golden = json.load(f) + golden.sort(key=lambda x: x["line"]) + return golden + + +def test_libclang_active_matches_golden(): + """libclang active-set output matches the committed golden snapshot.""" + projected = _projected_oneline_needs( + ["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"] + ) + assert projected == _load_golden("variants_branching.expected.json") + + +def test_treesitter_matches_golden(): + """Tree-sitter output (no preprocessor) matches the committed golden snapshot.""" + projected = _projected_oneline_needs(None) + assert projected == _load_golden("variants_branching.treesitter.expected.json") From 10f88139c6be9f93f40e985ff725f9f36f4a85e8 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Sun, 28 Jun 2026 15:01:26 +0200 Subject: [PATCH 20/25] test(extraction): libclang engine via the declarative harness MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add an 'engine' dimension to the declarative extraction harness: a fixture case may set 'engine: libclang' + 'defines' to exercise the preprocessor-aware path (inactive #if/#ifdef branches excluded); libclang cases skip when the clang bindings are absent. variants_branching now lives as declarative fixtures under both engines (tests/data/extraction/preproc_variants.yaml): tree-sitter -> all 5 markers, libclang -> the 4-marker active set. Drop the bespoke goldens (variants_branching{,.treesitter}.expected.json and test_libclang_active_matches_golden / test_treesitter_matches_golden); the libclang behavioral tests (inactive-branch exclusion, resilience, compile_commands, §3.3 skip, header-standalone) remain. --- ...variants-variants_branching_libclang].json | 55 +++++++++++++++ ...riants-variants_branching_treesitter].json | 67 +++++++++++++++++++ tests/data/extraction/README.md | 5 ++ tests/data/extraction/preproc_variants.yaml | 61 +++++++++++++++++ .../preproc/variants_branching.expected.json | 6 -- ...ariants_branching.treesitter.expected.json | 7 -- tests/test_analyse_preproc.py | 64 ++---------------- tests/test_extraction_fixtures.py | 11 +++ 8 files changed, 205 insertions(+), 71 deletions(-) create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_libclang].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_treesitter].json create mode 100644 tests/data/extraction/preproc_variants.yaml delete mode 100644 tests/data/preproc/variants_branching.expected.json delete mode 100644 tests/data/preproc/variants_branching.treesitter.expected.json diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_libclang].json new file mode 100644 index 0000000..d3ea9c1 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_libclang].json @@ -0,0 +1,55 @@ +{ + "needs": [ + { + "id": "IMPL_ALWAYS", + "title": "Always Present", + "type": "impl", + "links": { + "links": [ + "REQ_BASE" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_VAR_A", + "title": "Variant A Feature", + "type": "impl", + "links": { + "links": [ + "REQ_FEAT_A" + ] + }, + "metadata": {}, + "line": 5 + }, + { + "id": "IMPL_PROTO_3", + "title": "Protocol V3", + "type": "impl", + "links": { + "links": [ + "REQ_PROTO" + ] + }, + "metadata": {}, + "line": 13 + }, + { + "id": "IMPL_LINUX_A", + "title": "Linux And A", + "type": "impl", + "links": { + "links": [ + "REQ_COMBO" + ] + }, + "metadata": {}, + "line": 18 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_treesitter].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_treesitter].json new file mode 100644 index 0000000..a5fd17f --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_treesitter].json @@ -0,0 +1,67 @@ +{ + "needs": [ + { + "id": "IMPL_ALWAYS", + "title": "Always Present", + "type": "impl", + "links": { + "links": [ + "REQ_BASE" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_VAR_A", + "title": "Variant A Feature", + "type": "impl", + "links": { + "links": [ + "REQ_FEAT_A" + ] + }, + "metadata": {}, + "line": 5 + }, + { + "id": "IMPL_VAR_B", + "title": "Variant B Feature", + "type": "impl", + "links": { + "links": [ + "REQ_FEAT_B" + ] + }, + "metadata": {}, + "line": 8 + }, + { + "id": "IMPL_PROTO_3", + "title": "Protocol V3", + "type": "impl", + "links": { + "links": [ + "REQ_PROTO" + ] + }, + "metadata": {}, + "line": 13 + }, + { + "id": "IMPL_LINUX_A", + "title": "Linux And A", + "type": "impl", + "links": { + "links": [ + "REQ_COMBO" + ] + }, + "metadata": {}, + "line": 18 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/extraction/README.md b/tests/data/extraction/README.md index 84f8efe..d8ef412 100644 --- a/tests/data/extraction/README.md +++ b/tests/data/extraction/README.md @@ -41,6 +41,11 @@ custom_brackets_c: `[oneline, need_refs, rst]` (default: all three). Narrow it to keep a case focused: a need-reference case sets `extract: [need_refs]` so the `@`-prefixed `@need-ids:` marker isn't also parsed as a one-line need. +- `engine` (optional): `treesitter` (default) sees every comment; `libclang` + evaluates the preprocessor and excludes markers in inactive `#if`/`#ifdef` + branches. libclang cases are skipped when the `clang` bindings are unavailable. +- `defines` (optional, libclang only): preprocessor defines, e.g. + `["VARIANT_A=1", "PROTOCOL_VERSION=3"]`. ## Snapshot (expected output) — normalized contract diff --git a/tests/data/extraction/preproc_variants.yaml b/tests/data/extraction/preproc_variants.yaml new file mode 100644 index 0000000..4c84e3a --- /dev/null +++ b/tests/data/extraction/preproc_variants.yaml @@ -0,0 +1,61 @@ +# Preprocessor-aware extraction: the same source under both engines. +# +# The tree-sitter engine sees every comment (all five markers). The libclang +# engine evaluates the preprocessor with the given ``defines`` and excludes +# markers in inactive #if/#ifdef branches, so IMPL_VAR_B (the #else branch) is +# dropped and only the active set remains. + +variants_branching_treesitter: + lang: cpp + engine: treesitter + config: default + extract: [oneline] + source: | + // @Always Present, IMPL_ALWAYS, impl, [REQ_BASE] + void always() {} + + #ifdef VARIANT_A + // @Variant A Feature, IMPL_VAR_A, impl, [REQ_FEAT_A] + void variant_a() {} + #else + // @Variant B Feature, IMPL_VAR_B, impl, [REQ_FEAT_B] + void variant_b() {} + #endif + + #if PROTOCOL_VERSION >= 3 + // @Protocol V3, IMPL_PROTO_3, impl, [REQ_PROTO] + void proto_v3() {} + #endif + + #if defined(PLATFORM_LINUX) && defined(VARIANT_A) + // @Linux And A, IMPL_LINUX_A, impl, [REQ_COMBO] + void linux_and_a() {} + #endif + +variants_branching_libclang: + lang: cpp + engine: libclang + config: default + defines: ["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"] + extract: [oneline] + source: | + // @Always Present, IMPL_ALWAYS, impl, [REQ_BASE] + void always() {} + + #ifdef VARIANT_A + // @Variant A Feature, IMPL_VAR_A, impl, [REQ_FEAT_A] + void variant_a() {} + #else + // @Variant B Feature, IMPL_VAR_B, impl, [REQ_FEAT_B] + void variant_b() {} + #endif + + #if PROTOCOL_VERSION >= 3 + // @Protocol V3, IMPL_PROTO_3, impl, [REQ_PROTO] + void proto_v3() {} + #endif + + #if defined(PLATFORM_LINUX) && defined(VARIANT_A) + // @Linux And A, IMPL_LINUX_A, impl, [REQ_COMBO] + void linux_and_a() {} + #endif diff --git a/tests/data/preproc/variants_branching.expected.json b/tests/data/preproc/variants_branching.expected.json deleted file mode 100644 index 6e641a3..0000000 --- a/tests/data/preproc/variants_branching.expected.json +++ /dev/null @@ -1,6 +0,0 @@ -[ - {"id": "IMPL_ALWAYS", "title": "Always Present", "type": "impl", "links": ["REQ_BASE"], "line": 1}, - {"id": "IMPL_VAR_A", "title": "Variant A Feature", "type": "impl", "links": ["REQ_FEAT_A"], "line": 5}, - {"id": "IMPL_PROTO_3", "title": "Protocol V3", "type": "impl", "links": ["REQ_PROTO"], "line": 13}, - {"id": "IMPL_LINUX_A", "title": "Linux And A", "type": "impl", "links": ["REQ_COMBO"], "line": 18} -] diff --git a/tests/data/preproc/variants_branching.treesitter.expected.json b/tests/data/preproc/variants_branching.treesitter.expected.json deleted file mode 100644 index 52217e7..0000000 --- a/tests/data/preproc/variants_branching.treesitter.expected.json +++ /dev/null @@ -1,7 +0,0 @@ -[ - {"id": "IMPL_ALWAYS", "title": "Always Present", "type": "impl", "links": ["REQ_BASE"], "line": 1}, - {"id": "IMPL_VAR_A", "title": "Variant A Feature", "type": "impl", "links": ["REQ_FEAT_A"], "line": 5}, - {"id": "IMPL_VAR_B", "title": "Variant B Feature", "type": "impl", "links": ["REQ_FEAT_B"], "line": 8}, - {"id": "IMPL_PROTO_3", "title": "Protocol V3", "type": "impl", "links": ["REQ_PROTO"], "line": 13}, - {"id": "IMPL_LINUX_A", "title": "Linux And A", "type": "impl", "links": ["REQ_COMBO"], "line": 18} -] diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py index d76435d..9ae4d40 100644 --- a/tests/test_analyse_preproc.py +++ b/tests/test_analyse_preproc.py @@ -244,61 +244,9 @@ def test_header_standalone_inactive_define(): assert "IMPL_HDR_VAR_A" not in ids # #ifdef VARIANT_A inactive -> dropped -# --------------------------------------------------------------------------- -# Golden snapshots -# -# The extraction output is pinned to declarative JSON snapshots, one per engine. -# `defines=None` omits the preprocessor block (tree-sitter path: every marker is -# seen); a defines list activates the libclang path (inactive `#if` branches are -# excluded). The snapshots are language-agnostic so any conforming implementation -# can assert against the same fixtures. -# --------------------------------------------------------------------------- - - -def _projected_oneline_needs(defines): - """Run the analyse pipeline and project each one-line need to the golden schema.""" - cfg = SourceAnalyseConfig( - src_files=[FIXTURE], - src_dir=FIXTURE.parent, - get_oneline_needs=True, - preprocessor=PreprocessorConfig(defines=defines) - if defines is not None - else None, - ) - analyse = SourceAnalyse(cfg) - analyse.git_remote_url = None - analyse.git_commit_rev = None - analyse.run() - projected = [ - { - "id": n.need["id"], - "title": n.need["title"], - "type": n.need["type"], - "links": n.need["links"], - "line": n.source_map["start"]["row"] + 1, - } - for n in analyse.oneline_needs - ] - projected.sort(key=lambda x: x["line"]) - return projected - - -def _load_golden(name): - with (FIXTURE.parent / name).open() as f: - golden = json.load(f) - golden.sort(key=lambda x: x["line"]) - return golden - - -def test_libclang_active_matches_golden(): - """libclang active-set output matches the committed golden snapshot.""" - projected = _projected_oneline_needs( - ["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"] - ) - assert projected == _load_golden("variants_branching.expected.json") - - -def test_treesitter_matches_golden(): - """Tree-sitter output (no preprocessor) matches the committed golden snapshot.""" - projected = _projected_oneline_needs(None) - assert projected == _load_golden("variants_branching.treesitter.expected.json") +# Note: extraction-output goldens for this fixture now live in the declarative +# suite (tests/data/extraction/preproc_variants.yaml, both engines). The tests +# above cover libclang-specific behavior the declarative harness does not: +# inactive-branch exclusion via defines/compile_commands, resilience to broken or +# half-typed code, compile-DB resolution, the spec §3.3 skip, and standalone +# header handling. diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index caf11be..612abd5 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -14,6 +14,7 @@ from sphinx_codelinks.config import ( NeedIdRefsConfig, OneLineCommentStyle, + PreprocessorConfig, SourceAnalyseConfig, ) from sphinx_codelinks.source_discover.config import CommentType @@ -125,6 +126,15 @@ def test_extraction_fixture(case: dict, tmp_path: Path, snapshot_extraction) -> # ``@need-ids:`` matching the ``@`` one-line start — don't add noise. extract = case.get("extract", ["oneline", "need_refs", "rst"]) + # Engine selection. The default tree-sitter path sees every comment; the + # libclang path evaluates the preprocessor (``defines``) and excludes markers + # in inactive #if/#ifdef branches. libclang needs the clang bindings. + engine = case.get("engine", "treesitter") + preprocessor = None + if engine == "libclang": + pytest.importorskip("clang.cindex") + preprocessor = PreprocessorConfig(defines=case.get("defines", [])) + src_path = tmp_path / f"case.{ext}" src_path.write_text(case["source"], encoding="utf-8") @@ -137,6 +147,7 @@ def test_extraction_fixture(case: dict, tmp_path: Path, snapshot_extraction) -> get_rst="rst" in extract, oneline_comment_style=style, need_id_refs_config=refs_config, + preprocessor=preprocessor, ) analyse = SourceAnalyse(cfg) analyse.git_remote_url = None From b97c66a923a8dbd485971e408efb0900f43922e6 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Tue, 30 Jun 2026 18:57:34 +0200 Subject: [PATCH 21/25] =?UTF-8?q?=F0=9F=90=9B=20FIX(codelinks):=20mypy=20s?= =?UTF-8?q?trict=20+=20ruff-format=20for=20the=20libclang=20engine?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - mypy: ignore missing clang.cindex stubs; int() the libclang parse flags; import CommentType from its defining module; fix preproc TOML ignore codes - ruff-format the preproc engine + its tests Fixes the MyPy and Pre-commit CI checks on the libclang PR. --- pyproject.toml | 3 ++- src/sphinx_codelinks/analyse/analyse.py | 14 ++++++----- .../analyse/preproc/loader.py | 8 +++---- src/sphinx_codelinks/config.py | 14 ++++++++--- tests/test_preproc_compile_db.py | 12 ++++++++-- tests/test_preproc_libclang.py | 23 ++++++++----------- 6 files changed, 44 insertions(+), 30 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 40d874d..ad06074 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,7 +151,8 @@ plugins = ["pydantic.mypy"] mypy_path = "typings" [[tool.mypy.overrides]] -module = ["licensing.*", "tomlkit.*"] +# clang.cindex ships no type stubs / py.typed; the libclang engine wraps it. +module = ["licensing.*", "tomlkit.*", "clang.*"] ignore_missing_imports = true [[tool.mypy.overrides]] diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index cb6e810..22f4af6 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -2,7 +2,7 @@ from dataclasses import dataclass import json from pathlib import Path -from typing import Any, TypedDict +from typing import Any, TypedDict, cast from tree_sitter import Node as TreeSitterNode @@ -78,9 +78,7 @@ def __init__( self.git_commit_rev: str | None = ( utils.get_current_rev(self.git_root) if self.git_root else None ) - self.project_path: Path = ( - self.git_root or self.analyse_config.src_dir - ) + self.project_path: Path = self.git_root or self.analyse_config.src_dir self.oneline_warnings: list[AnalyseWarning] = [] def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]: # type: ignore[explicit-any] @@ -151,7 +149,11 @@ def create_src_objects_libclang(self) -> None: comments = libclang_parser.extract_active_comments(src_path, args) if not comments: continue - src_comments = [SourceComment(c) for c in comments] + # ``c`` is a LibclangComment duck-typing the tree-sitter Node + # interface SourceComment reads (``.text`` / ``.start_point.row``); + # the Node-only path (find_associated_scope) is guarded by + # ``is_libclang`` so it never runs on these. + src_comments = [SourceComment(cast("TreeSitterNode", c)) for c in comments] src_file = SourceFile(src_path.absolute()) src_file.add_comments(src_comments) self.src_files.append(src_file) @@ -420,7 +422,7 @@ def dump_marked_content(self, outdir: Path) -> None: json.dump(to_dump, f) def run(self) -> None: - from sphinx_codelinks.config import CommentType # noqa: PLC0415 + from sphinx_codelinks.source_discover.config import CommentType # noqa: PLC0415 if ( self.analyse_config.preprocessor is not None diff --git a/src/sphinx_codelinks/analyse/preproc/loader.py b/src/sphinx_codelinks/analyse/preproc/loader.py index 1af14fa..0f3701f 100644 --- a/src/sphinx_codelinks/analyse/preproc/loader.py +++ b/src/sphinx_codelinks/analyse/preproc/loader.py @@ -29,7 +29,9 @@ def load_clang_cindex() -> Any: # type: ignore[explicit-any] # Production parse flags (design "Parse flags codelinks should use"). def _parse_options() -> int: cx = load_clang_cindex() - return ( + # ``cx`` is untyped (clang ships no stubs), so the bit-or is ``Any``; coerce + # back to ``int`` to satisfy the declared return type. + return int( cx.TranslationUnit.PARSE_INCOMPLETE | cx.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES | cx.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD @@ -86,9 +88,7 @@ def get_all_skipped_ranges(tu: Any) -> list[SkippedRange]: # type: ignore[expli start = r.start end = r.end f = Path(start.file.name) if start.file else None - out.append( - SkippedRange(f, start.line, start.column, end.line, end.column) - ) + out.append(SkippedRange(f, start.line, start.column, end.line, end.column)) return out finally: cx.conf.lib.clang_disposeSourceRangeList(ptr) diff --git a/src/sphinx_codelinks/config.py b/src/sphinx_codelinks/config.py index 2866a7c..a396e90 100644 --- a/src/sphinx_codelinks/config.py +++ b/src/sphinx_codelinks/config.py @@ -832,7 +832,12 @@ def convert_analyse_config( analyse_config_dict: SourceAnalyseConfigType = {} if config_dict: for k, v in config_dict.items(): - if k not in {"online_comment_style", "need_id_refs", "marked_rst", "preprocessor"}: + if k not in { + "online_comment_style", + "need_id_refs", + "marked_rst", + "preprocessor", + }: # Convert string paths to Path objects if k in {"src_dir", "git_root"} and isinstance(v, str): analyse_config_dict[k] = Path(v) # type: ignore[literal-required] @@ -865,14 +870,17 @@ def convert_analyse_config( preprocessor_dict = config_dict.get("preprocessor") if preprocessor_dict is not None: + # The preprocessor section has no TypedDict; its values are dynamic + # TOML (typed ``object``), so the list/iter/assign below need targeted + # ignores matching the concrete errors mypy reports for ``object``. analyse_config_dict["preprocessor"] = PreprocessorConfig( compile_commands=( Path(str(preprocessor_dict["compile_commands"])) if preprocessor_dict.get("compile_commands") else None ), - defines=list(preprocessor_dict.get("defines", [])), # type: ignore[arg-type] - includes=[Path(str(p)) for p in preprocessor_dict.get("includes", [])], # type: ignore[union-attr] + defines=list(preprocessor_dict.get("defines", [])), # type: ignore[call-overload] + includes=[Path(str(p)) for p in preprocessor_dict.get("includes", [])], # type: ignore[attr-defined] variant_name=preprocessor_dict.get("variant_name"), # type: ignore[arg-type] ) diff --git a/tests/test_preproc_compile_db.py b/tests/test_preproc_compile_db.py index e44c633..bc45f7d 100644 --- a/tests/test_preproc_compile_db.py +++ b/tests/test_preproc_compile_db.py @@ -39,8 +39,16 @@ def test_find_compile_db_stops_at_git_root(tmp_path: Path): def test_filter_args_strips_compiler_and_output(): argv = [ - "clang++", "-std=c++17", "-c", "-o", "out.o", - "-DVARIANT_A=1", "-I/inc", "-MMD", "-MF", "dep.d", + "clang++", + "-std=c++17", + "-c", + "-o", + "out.o", + "-DVARIANT_A=1", + "-I/inc", + "-MMD", + "-MF", + "dep.d", "src/a.cpp", ] out = compile_db.filter_args(argv, "src/a.cpp") diff --git a/tests/test_preproc_libclang.py b/tests/test_preproc_libclang.py index 8eb2783..679f62e 100644 --- a/tests/test_preproc_libclang.py +++ b/tests/test_preproc_libclang.py @@ -28,12 +28,7 @@ def test_skipped_ranges_on_simple_ifdef(tmp_path: Path): import clang.cindex as cx src = tmp_path / "s.cpp" - src.write_text( - "#ifdef OFF\n" - "// inactive\n" - "#endif\n" - "// active\n" - ) + src.write_text("#ifdef OFF\n// inactive\n#endif\n// active\n") idx = cx.Index.create() tu = idx.parse(str(src), args=["-std=c++17"], options=loader.PARSE_OPTIONS) skipped = loader.get_all_skipped_ranges(tu) @@ -67,17 +62,17 @@ def test_extract_active_comments_variant_a_active(): titles = _titles(comments) joined = "\n".join(titles) assert "IMPL_ALWAYS" in joined - assert "IMPL_VAR_A" in joined # active branch - assert "IMPL_VAR_B" not in joined # inactive #else -> EXCLUDED - assert "IMPL_PROTO_3" in joined # PROTOCOL_VERSION >= 3 active - assert "IMPL_LINUX_A" in joined # both defined + assert "IMPL_VAR_A" in joined # active branch + assert "IMPL_VAR_B" not in joined # inactive #else -> EXCLUDED + assert "IMPL_PROTO_3" in joined # PROTOCOL_VERSION >= 3 active + assert "IMPL_LINUX_A" in joined # both defined def test_extract_active_comments_variant_b_active(): args = ["-std=c++17", "-DPROTOCOL_VERSION=1"] # VARIANT_A undefined comments = libclang_parser.extract_active_comments(FIXTURE, args) joined = "\n".join(_titles(comments)) - assert "IMPL_VAR_B" in joined # #else branch now active - assert "IMPL_VAR_A" not in joined # inactive -> EXCLUDED - assert "IMPL_PROTO_3" not in joined # version < 3 -> EXCLUDED - assert "IMPL_LINUX_A" not in joined # VARIANT_A undefined -> EXCLUDED + assert "IMPL_VAR_B" in joined # #else branch now active + assert "IMPL_VAR_A" not in joined # inactive -> EXCLUDED + assert "IMPL_PROTO_3" not in joined # version < 3 -> EXCLUDED + assert "IMPL_LINUX_A" not in joined # VARIANT_A undefined -> EXCLUDED From 3618ba41de88203c461e64a5ed1434a8e2a49e9c Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Wed, 1 Jul 2026 16:52:19 +0200 Subject: [PATCH 22/25] fix(preproc): parse standalone C-extension files as C++ (extract headers, don't crash) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A header carrying oneline markers never appears in compile_commands.json, so the engine parses it standalone with the global defines. libclang infers the C language from a .h/.c/.inc extension, and a C++ -std flag then makes clang reject the combination and return a NULL TU (TranslationUnitLoadError) — which aborted the whole Sphinx build (notably 'sphinx-build -b ubtrace -W' on the presets). Pin -x to match -std in defines_to_args so such files parse as C++ and their markers are EXTRACTED rather than dropped. Keep a last-resort TranslationUnitLoadError guard (logged at info, not warning, so -W stays green) for files that still cannot load as a TU at all. Adds a regression test + .h fixture proving a C-extension header's markers extract. --- src/sphinx_codelinks/analyse/analyse.py | 29 ++++++++++++++-- .../analyse/preproc/compile_db.py | 14 ++++++-- tests/data/preproc/plain_header.h | 9 +++++ tests/test_analyse_preproc.py | 33 +++++++++++++++++++ 4 files changed, 81 insertions(+), 4 deletions(-) create mode 100644 tests/data/preproc/plain_header.h diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index 22f4af6..f0914c3 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -135,7 +135,17 @@ def _resolve_preproc_args(self, src_path: Path) -> list[str] | None: return compile_db.defines_to_args(preproc.defines, preproc.includes) def create_src_objects_libclang(self) -> None: - from sphinx_codelinks.analyse.preproc import libclang_parser # noqa: PLC0415 + from sphinx_codelinks.analyse.preproc import ( # noqa: PLC0415 + libclang_parser, + loader, + ) + + # Resolve the exception via the loader (never a direct ``import + # clang.cindex``) so a missing ``libclang`` extra still surfaces the + # loader's friendly install hint rather than a bare ImportError. + translation_unit_load_error = ( + loader.load_clang_cindex().TranslationUnitLoadError + ) for src_path in self.analyse_config.src_files: if not utils.is_text_file(src_path): @@ -146,7 +156,22 @@ def create_src_objects_libclang(self) -> None: f"codelinks: skipping {src_path} — not found in compile_commands.json" ) continue - comments = libclang_parser.extract_active_comments(src_path, args) + try: + comments = libclang_parser.extract_active_comments(src_path, args) + except translation_unit_load_error: + # Last-resort guard. Standalone parses pin ``-x`` to match the + # ``-std`` (see defines_to_args), so the common case — a ``.h``/ + # ``.c`` header handed a C++ ``-std`` — now parses as C++ and its + # markers extract. This only fires if libclang still cannot load + # the file as a translation unit at all; skip it rather than + # aborting the whole Sphinx build. Logged at ``info`` (not + # ``warning``): under ``sphinx-build -W`` a warning would re-fail + # the very builds this guard keeps green. + logger.info( + f"codelinks: skipping {src_path} — libclang could not load it " + f"as a translation unit" + ) + continue if not comments: continue # ``c`` is a LibclangComment duck-typing the tree-sitter Node diff --git a/src/sphinx_codelinks/analyse/preproc/compile_db.py b/src/sphinx_codelinks/analyse/preproc/compile_db.py index aabe60a..f01e5fd 100644 --- a/src/sphinx_codelinks/analyse/preproc/compile_db.py +++ b/src/sphinx_codelinks/analyse/preproc/compile_db.py @@ -91,8 +91,18 @@ def load_flags_map(db_path: Path) -> dict[Path, list[str]]: def defines_to_args( defines: list[str], includes: list[Path], std: str = "c++17" ) -> list[str]: - """Build a global flag list for the manual `defines` fallback.""" - args = [f"-std={std}"] + """Build a global flag list for parsing a file standalone. + + Used for headers and files absent from a compile DB. libclang infers the + language from the file extension, so a C-inferred file (``.h`` / ``.c`` / + ``.inc``) handed a C++ ``-std`` makes clang reject the combination and return + a NULL translation unit (surfacing as ``TranslationUnitLoadError``). Pin the + language to match ``std`` with ``-x`` so such files — e.g. a ``.h`` header + carrying oneline need markers — parse and extract instead of failing. (C + sources parse as C++ well enough for comment/marker extraction.) + """ + lang = "c++" if std.startswith("c++") else "c" + args = ["-x", lang, f"-std={std}"] args += [f"-D{d}" for d in defines] args += [f"-I{inc}" for inc in includes] return args diff --git a/tests/data/preproc/plain_header.h b/tests/data/preproc/plain_header.h new file mode 100644 index 0000000..a6e2029 --- /dev/null +++ b/tests/data/preproc/plain_header.h @@ -0,0 +1,9 @@ +// A .h header (not .hpp), the case a real project hits: a header carrying +// oneline need markers that never appears in compile_commands.json (headers are +// not compiled translation units). libclang infers the C language from the .h +// extension, so parsing it with a C++ -std flag would return a NULL translation +// unit. The engine pins -x to the -std for standalone parses, so this header is +// parsed as C++ and its markers are extracted (not dropped, not crashing). + +// @Plain Header, IMPL_HDR_PLAIN, impl, [REQ_HDR] +void plain_header_fn(); diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py index 9ae4d40..0ba5ecd 100644 --- a/tests/test_analyse_preproc.py +++ b/tests/test_analyse_preproc.py @@ -250,3 +250,36 @@ def test_header_standalone_inactive_define(): # inactive-branch exclusion via defines/compile_commands, resilience to broken or # half-typed code, compile-DB resolution, the spec §3.3 skip, and standalone # header handling. + + +PLAIN_H = Path(__file__).parent / "data" / "preproc" / "plain_header.h" + + +def test_libclang_extracts_c_extension_header_via_cpp_language(): + """A ``.h`` header still extracts its oneline markers (not dropped/crashing). + + Headers never appear in compile_commands.json, so they are parsed standalone + with the global defines. libclang infers C from the ``.h`` extension; without + pinning the language, ``-std=c++17`` makes clang reject the combo and return + a NULL translation unit (``TranslationUnitLoadError``) — which previously + aborted ``sphinx-build -b ubtrace -W``. The standalone flags now pin ``-x`` + to the ``-std`` (see ``defines_to_args``), so the header parses as C++ and its + markers are extracted rather than lost. + """ + cfg = SourceAnalyseConfig( + src_files=[FIXTURE, PLAIN_H], + src_dir=FIXTURE.parent, + get_need_id_refs=False, + get_oneline_needs=True, + get_rst=False, + preprocessor=PreprocessorConfig( + defines=["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"] + ), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() # must not raise TranslationUnitLoadError + ids = {n.need["id"] for n in analyse.oneline_needs} + assert "IMPL_ALWAYS" in ids # the .cpp translation unit extracts + assert "IMPL_HDR_PLAIN" in ids # the .h header extracts too (parsed as C++) From c3578eabd9a5e0bf3f7c7085ac9d1fb2f0e83e6a Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Thu, 2 Jul 2026 22:19:14 +0200 Subject: [PATCH 23/25] test(preproc): declarative golden for C-extension header extraction Adds a .h header case to the shared declarative extraction suite: a header is parsed standalone under libclang (pinned to C++ via defines_to_args) and its active-branch markers extract while inactive-branch markers are excluded. Adds a 'cpp_header' lang -> .h mapping. This is the engine-agnostic golden that surfaces the same requirement for the Rust engine (ubc_codelinks). --- ...variants-header_c_extension_libclang].json | 31 +++++++++++++++++++ tests/data/extraction/preproc_variants.yaml | 24 ++++++++++++++ tests/test_extraction_fixtures.py | 3 ++ 3 files changed, 58 insertions(+) create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-header_c_extension_libclang].json diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-header_c_extension_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-header_c_extension_libclang].json new file mode 100644 index 0000000..ad95057 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-header_c_extension_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_HDR_BASE", + "title": "Header Base", + "type": "impl", + "links": { + "links": [ + "REQ_HDR" + ] + }, + "metadata": {}, + "line": 2 + }, + { + "id": "IMPL_HDR_FEATURE", + "title": "Header Feature On", + "type": "impl", + "links": { + "links": [ + "REQ_HDR_FEAT" + ] + }, + "metadata": {}, + "line": 6 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/extraction/preproc_variants.yaml b/tests/data/extraction/preproc_variants.yaml index 4c84e3a..1df4545 100644 --- a/tests/data/extraction/preproc_variants.yaml +++ b/tests/data/extraction/preproc_variants.yaml @@ -59,3 +59,27 @@ variants_branching_libclang: // @Linux And A, IMPL_LINUX_A, impl, [REQ_COMBO] void linux_and_a() {} #endif + +# A C-extension header (.h) is never a compile_commands entry, so it is parsed +# standalone with the global defines. libclang infers C from the .h extension, so +# the engine pins -x to the -std (see defines_to_args); otherwise -std=c++17 +# yields a NULL translation unit and the header's markers would be lost. Inactive +# branches are still excluded. +header_c_extension_libclang: + lang: cpp_header + engine: libclang + config: default + defines: ["HEADER_FEATURE=1"] + extract: [oneline] + source: | + #pragma once + // @Header Base, IMPL_HDR_BASE, impl, [REQ_HDR] + inline int hdr_add(int a, int b) { return a + b; } + + #ifdef HEADER_FEATURE + // @Header Feature On, IMPL_HDR_FEATURE, impl, [REQ_HDR_FEAT] + inline void hdr_feature() {} + #else + // @Header Feature Off, IMPL_HDR_NOFEATURE, impl, [REQ_HDR] + inline void hdr_no_feature() {} + #endif diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index 612abd5..7525be7 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -25,6 +25,9 @@ LANG_MAP: dict[str, tuple[CommentType, str]] = { "cpp": (CommentType.cpp, "cpp"), "c": (CommentType.cpp, "c"), + # A C/C++ header (.h): libclang infers C from the extension, so the engine + # must pin the language to the -std or -std=c++17 yields a NULL TU. + "cpp_header": (CommentType.cpp, "h"), "python": (CommentType.python, "py"), "csharp": (CommentType.cs, "cs"), "rust": (CommentType.rust, "rs"), From f85a0dfe088f875ccf38b6b48420d49b6501df0a Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Fri, 3 Jul 2026 11:35:40 +0200 Subject: [PATCH 24/25] test(extraction): add preprocessor macro/logic fixtures Declarative libclang cases covering nested #ifdef, #elif chains, #if 0 dead blocks, defined()/boolean logic, arithmetic comparisons, and an in-source #define driving a branch. Each keeps markers in both the taken and untaken branches; only the active-region markers survive, proving the preprocessor is evaluated (tree-sitter would keep them all). --- ...c_macros-arithmetic_compare_libclang].json | 19 ++++ ...acros-defined_boolean_logic_libclang].json | 31 ++++++ ...e[preproc_macros-elif_chain_libclang].json | 19 ++++ ...c_macros-if_zero_dead_block_libclang].json | 19 ++++ ...-macro_object_drives_branch_libclang].json | 19 ++++ ...preproc_macros-nested_ifdef_libclang].json | 43 ++++++++ tests/data/extraction/preproc_macros.yaml | 100 ++++++++++++++++++ 7 files changed, 250 insertions(+) create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-arithmetic_compare_libclang].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-defined_boolean_logic_libclang].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-elif_chain_libclang].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-if_zero_dead_block_libclang].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-macro_object_drives_branch_libclang].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-nested_ifdef_libclang].json create mode 100644 tests/data/extraction/preproc_macros.yaml diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-arithmetic_compare_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-arithmetic_compare_libclang].json new file mode 100644 index 0000000..252e2ce --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-arithmetic_compare_libclang].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_V3", + "title": "V3 plus", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 2 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-defined_boolean_logic_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-defined_boolean_logic_libclang].json new file mode 100644 index 0000000..cb93422 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-defined_boolean_logic_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_AORB", + "title": "A or B", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 5 + }, + { + "id": "IMPL_NOTB", + "title": "Not B", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 8 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-elif_chain_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-elif_chain_libclang].json new file mode 100644 index 0000000..46d072f --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-elif_chain_libclang].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_M2", + "title": "Mode Two", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 4 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-if_zero_dead_block_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-if_zero_dead_block_libclang].json new file mode 100644 index 0000000..0b25665 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-if_zero_dead_block_libclang].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_LIVE", + "title": "Live", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 1 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-macro_object_drives_branch_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-macro_object_drives_branch_libclang].json new file mode 100644 index 0000000..50eb965 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-macro_object_drives_branch_libclang].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_FEAT", + "title": "Feature On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 3 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-nested_ifdef_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-nested_ifdef_libclang].json new file mode 100644 index 0000000..bfbd5f4 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-nested_ifdef_libclang].json @@ -0,0 +1,43 @@ +{ + "needs": [ + { + "id": "IMPL_TOP", + "title": "Top", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_OUTER", + "title": "Outer On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 3 + }, + { + "id": "IMPL_INNER", + "title": "Inner On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 5 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/extraction/preproc_macros.yaml b/tests/data/extraction/preproc_macros.yaml new file mode 100644 index 0000000..a8ab57d --- /dev/null +++ b/tests/data/extraction/preproc_macros.yaml @@ -0,0 +1,100 @@ +# Preprocessor macro / conditional constructs under the libclang engine. +# +# Every case keeps markers in BOTH the taken and the untaken branches; only the +# markers in active (non-skipped) regions must survive. The tree-sitter engine +# would keep them all — these are libclang-only cases that prove the preprocessor +# is actually evaluated (nesting, #elif chains, #if 0, defined()/boolean logic, +# arithmetic comparisons, and an in-source #define driving a branch). + +nested_ifdef_libclang: + lang: cpp + engine: libclang + config: default + defines: ["OUTER=1", "INNER=1"] + extract: [oneline] + source: | + // @Top, IMPL_TOP, impl, [REQ] + #ifdef OUTER + // @Outer On, IMPL_OUTER, impl, [REQ] + #ifdef INNER + // @Inner On, IMPL_INNER, impl, [REQ] + #else + // @Inner Off, IMPL_INNER_OFF, impl, [REQ] + #endif + #else + // @Outer Off, IMPL_OUTER_OFF, impl, [REQ] + #endif + +elif_chain_libclang: + lang: cpp + engine: libclang + config: default + defines: ["MODE=2"] + extract: [oneline] + source: | + #if MODE == 1 + // @Mode One, IMPL_M1, impl, [REQ] + #elif MODE == 2 + // @Mode Two, IMPL_M2, impl, [REQ] + #elif MODE == 3 + // @Mode Three, IMPL_M3, impl, [REQ] + #else + // @Mode Other, IMPL_MO, impl, [REQ] + #endif + +if_zero_dead_block_libclang: + lang: cpp + engine: libclang + config: default + defines: [] + extract: [oneline] + source: | + // @Live, IMPL_LIVE, impl, [REQ] + #if 0 + // @Dead, IMPL_DEAD, impl, [REQ] + #endif + +defined_boolean_logic_libclang: + lang: cpp + engine: libclang + config: default + defines: ["HAS_A=1", "HAS_C=1"] + extract: [oneline] + source: | + #if defined(HAS_A) && defined(HAS_B) + // @A and B, IMPL_AB, impl, [REQ] + #endif + #if defined(HAS_A) || defined(HAS_B) + // @A or B, IMPL_AORB, impl, [REQ] + #endif + #if !defined(HAS_B) + // @Not B, IMPL_NOTB, impl, [REQ] + #endif + +arithmetic_compare_libclang: + lang: cpp + engine: libclang + config: default + defines: ["VER=3"] + extract: [oneline] + source: | + #if VER >= 3 + // @V3 plus, IMPL_V3, impl, [REQ] + #endif + #if VER < 3 + // @Pre V3, IMPL_PRE3, impl, [REQ] + #endif + +macro_object_drives_branch_libclang: + lang: cpp + engine: libclang + config: default + defines: [] + extract: [oneline] + source: | + #define FEATURE 1 + #if FEATURE + // @Feature On, IMPL_FEAT, impl, [REQ] + #else + // @Feature Off, IMPL_FEAT_OFF, impl, [REQ] + #endif From 8c5407aa338f808a00dca3ca9b54cff4a9596058 Mon Sep 17 00:00:00 2001 From: Marco Heinemann Date: Fri, 3 Jul 2026 12:30:37 +0200 Subject: [PATCH 25/25] test(extraction): add compile_commands handling fixtures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The declarative harness can now materialize a compile_commands.json (inline `compile_commands` entries) or point at an explicit `compile_commands_path`. New preproc_compile_db cases cover DB-provided flags, a header absent from a present DB (parsed standalone), a translation-unit source absent from the DB (skipped per spec §3.3), and an unreadable explicit DB path (defines fallback). --- ...le_db-file_in_db_uses_flags_libclang].json | 31 ++++++++ ...r_absent_from_db_standalone_libclang].json | 31 ++++++++ ...urce_absent_from_db_skipped_libclang].json | 6 ++ ...readable_db_path_falls_back_libclang].json | 31 ++++++++ tests/data/extraction/preproc_compile_db.yaml | 77 +++++++++++++++++++ tests/test_extraction_fixtures.py | 27 ++++++- 6 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-file_in_db_uses_flags_libclang].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-header_absent_from_db_standalone_libclang].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-tu_source_absent_from_db_skipped_libclang].json create mode 100644 tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-unreadable_db_path_falls_back_libclang].json create mode 100644 tests/data/extraction/preproc_compile_db.yaml diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-file_in_db_uses_flags_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-file_in_db_uses_flags_libclang].json new file mode 100644 index 0000000..18a11a6 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-file_in_db_uses_flags_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_ALWAYS", + "title": "Always", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_VAR_A", + "title": "Variant A", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 3 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-header_absent_from_db_standalone_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-header_absent_from_db_standalone_libclang].json new file mode 100644 index 0000000..fbab42e --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-header_absent_from_db_standalone_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_HDR_BASE", + "title": "Header Base", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 2 + }, + { + "id": "IMPL_HDR_ON", + "title": "Header On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 4 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-tu_source_absent_from_db_skipped_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-tu_source_absent_from_db_skipped_libclang].json new file mode 100644 index 0000000..4d18559 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-tu_source_absent_from_db_skipped_libclang].json @@ -0,0 +1,6 @@ +{ + "needs": [], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-unreadable_db_path_falls_back_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-unreadable_db_path_falls_back_libclang].json new file mode 100644 index 0000000..e253731 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-unreadable_db_path_falls_back_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_BASE", + "title": "Base", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_FALLBACK", + "title": "Fallback On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 3 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/extraction/preproc_compile_db.yaml b/tests/data/extraction/preproc_compile_db.yaml new file mode 100644 index 0000000..7f675db --- /dev/null +++ b/tests/data/extraction/preproc_compile_db.yaml @@ -0,0 +1,77 @@ +# compile_commands.json handling for the libclang engine. +# +# Each case materializes a real compile_commands.json (inline `compile_commands`) +# or points at an explicit `compile_commands_path`, then extracts from a single +# `case.` source. These exercise the per-file argument resolution the shared +# single-`.cpp`/defines fixtures never touch: DB-provided flags, headers absent +# from a present DB (parsed standalone), translation-unit sources absent from the +# DB (skipped), and an unreadable explicit DB path (defines fallback). + +file_in_db_uses_flags_libclang: + lang: cpp + engine: libclang + config: default + defines: [] + extract: [oneline] + compile_commands: + - file: case.cpp + arguments: ["clang++", "-std=c++17", "-DVARIANT_A=1", "-c", "case.cpp"] + source: | + // @Always, IMPL_ALWAYS, impl, [REQ] + #ifdef VARIANT_A + // @Variant A, IMPL_VAR_A, impl, [REQ] + #else + // @Variant B, IMPL_VAR_B, impl, [REQ] + #endif + # -DVARIANT_A comes from the DB entry (global defines are empty), so the #ifdef + # branch is active only because the DB flags were applied: IMPL_ALWAYS, IMPL_VAR_A. + +header_absent_from_db_standalone_libclang: + lang: cpp_header + engine: libclang + config: default + defines: ["HEADER_ON=1"] + extract: [oneline] + compile_commands: + - file: other.cpp + arguments: ["clang++", "-std=c++17", "-c", "other.cpp"] + source: | + #pragma once + // @Header Base, IMPL_HDR_BASE, impl, [REQ] + #ifdef HEADER_ON + // @Header On, IMPL_HDR_ON, impl, [REQ] + #else + // @Header Off, IMPL_HDR_OFF, impl, [REQ] + #endif + # case.h is not in the (present) DB. A header is never a compile_commands entry, + # so it is parsed standalone with the global defines: IMPL_HDR_BASE, IMPL_HDR_ON. + +tu_source_absent_from_db_skipped_libclang: + lang: cpp + engine: libclang + config: default + defines: [] + extract: [oneline] + compile_commands: + - file: other.cpp + arguments: ["clang++", "-std=c++17", "-c", "other.cpp"] + source: | + // @Should Be Skipped, IMPL_SKIP, impl, [REQ] + void f() {} + # case.cpp is a translation-unit source absent from the applicable DB, so it is + # skipped entirely (spec §3.3): no markers extracted. + +unreadable_db_path_falls_back_libclang: + lang: cpp + engine: libclang + config: default + defines: ["FALLBACK_ON=1"] + extract: [oneline] + compile_commands_path: "missing.json" + source: | + // @Base, IMPL_BASE, impl, [REQ] + #ifdef FALLBACK_ON + // @Fallback On, IMPL_FALLBACK, impl, [REQ] + #endif + # The explicit compile_commands path does not exist, so extraction falls back to + # the global defines rather than skipping every file: IMPL_BASE, IMPL_FALLBACK. diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index 7525be7..dcd36d7 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -5,6 +5,7 @@ compared to a committed JSON snapshot. See ``tests/data/extraction/README.md``. """ +import json from pathlib import Path import pytest @@ -115,6 +116,30 @@ def _normalize(analyse: SourceAnalyse, style: OneLineCommentStyle) -> dict: } +def _build_preprocessor(case: dict, tmp_path: Path) -> PreprocessorConfig: + """Build the libclang ``PreprocessorConfig`` for a case. + + ``compile_commands`` (a list of ``{file, arguments}`` entries) is materialized + into a real ``compile_commands.json`` under ``tmp_path`` so the engine resolves + per-file flags from it. ``compile_commands_path`` instead points at an explicit + path (which may be intentionally absent, to exercise the fallback). Otherwise + only the global ``defines`` apply. + """ + defines = case.get("defines", []) + compile_commands: Path | None = None + if "compile_commands" in case: + db = tmp_path / "compile_commands.json" + entries = [ + {"directory": str(tmp_path), "file": e["file"], "arguments": e["arguments"]} + for e in case["compile_commands"] + ] + db.write_text(json.dumps(entries), encoding="utf-8") + compile_commands = db + elif "compile_commands_path" in case: + compile_commands = tmp_path / case["compile_commands_path"] + return PreprocessorConfig(defines=defines, compile_commands=compile_commands) + + @pytest.mark.parametrize("case", _load_cases()) def test_extraction_fixture(case: dict, tmp_path: Path, snapshot_extraction) -> None: comment_type, ext = LANG_MAP[case["lang"]] @@ -136,7 +161,7 @@ def test_extraction_fixture(case: dict, tmp_path: Path, snapshot_extraction) -> preprocessor = None if engine == "libclang": pytest.importorskip("clang.cindex") - preprocessor = PreprocessorConfig(defines=case.get("defines", [])) + preprocessor = _build_preprocessor(case, tmp_path) src_path = tmp_path / f"case.{ext}" src_path.write_text(case["source"], encoding="utf-8")