diff --git a/docs/source/components/analyse.rst b/docs/source/components/analyse.rst index 631eafd..f310ca8 100644 --- a/docs/source/components/analyse.rst +++ b/docs/source/components/analyse.rst @@ -210,3 +210,84 @@ This single comment line creates a complete **Sphinx-Needs** item equivalent to: .. impl:: Function Implementation :id: IMPL_001 :links: REQ_001, REQ_002 + +.. _preprocessor_engine: + +Preprocessor-Aware C/C++ Extraction (libclang) +---------------------------------------------- + +By default, **Source Analyse** uses a tree-sitter parser that extracts **every** comment, +regardless of the C preprocessor. For C/C++ projects that rely on conditional compilation +(``#ifdef VARIANT_A`` …), this means needs from *all* branches are extracted — even +branches that are never compiled. + +The optional **libclang engine** addresses this. When an +:ref:`analyse.preprocessor ` table is configured and ``comment_type`` +is ``"cpp"``, each file is parsed as a real translation unit and **comments inside inactive +preprocessor branches are dropped**. Active needs keep their original line numbers — no +source transformation is performed. + +.. important:: The libclang engine requires an optional dependency: + ``pip install 'sphinx-codelinks[libclang]'``. The wheel bundles the native library, so + no compiler is required on the user's machine. + +How files are selected and parsed +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +File **discovery** is unchanged — :ref:`SourceDiscover ` still decides which +files are candidates. A ``compile_commands.json`` database only determines *how* each +discovered file is parsed: + +.. list-table:: + :header-rows: 1 + :widths: 45 55 + + * - File + - How it is parsed + * - Listed in ``compile_commands.json`` + - Parsed with the exact flags the compiler used for that translation unit. + * - A compiled source (``.c``, ``.cpp``, ``.cc``, ``.cxx``) **not** listed + - Skipped — assumed to be excluded from the build (e.g. another platform). + * - A **header** (``.h``, ``.hpp``, …) — never listed in a database + - Parsed **standalone** using ``defines`` and ``includes`` (see below). + * - Any file, when **no** database is found + - Parsed with ``defines`` and ``includes``. + +Header files +~~~~~~~~~~~~~ + +A ``compile_commands.json`` only ever lists compiled translation units (``.cpp`` files); +headers are pulled in via ``#include`` and never appear as entries. **Sphinx-CodeLinks** +therefore parses each discovered header **standalone**, using the ``defines`` and +``includes`` you configure. Include guards resolve correctly, and ``#ifdef`` branches are +evaluated against your ``defines``. + +.. note:: Because headers are parsed standalone, they see only the global ``defines`` — + **not** the per-file ``-D`` flags from ``compile_commands.json``. To extract a + particular variant's needs from headers, mirror that variant into ``defines``. Treat + one analysis run as **one variant**: set ``defines`` to the variant you want, and both + sources and headers evaluate their conditions consistently. + +Example +~~~~~~~ + +.. code-block:: cpp + + // include/feature.hpp + #ifndef FEATURE_HPP + #define FEATURE_HPP + + // @Always available, IMPL_BASE, impl, [REQ_BASE] + void base(); + + #ifdef VARIANT_A + // @Variant A only, IMPL_VAR_A, impl, [REQ_A] + void variant_a(); + #endif + + #endif + +With ``defines = ["VARIANT_A"]`` both ``IMPL_BASE`` and ``IMPL_VAR_A`` are extracted. +With ``defines = []`` only ``IMPL_BASE`` is extracted — the ``#ifdef VARIANT_A`` block is +inactive, so its need is dropped. The include guard (``#ifndef FEATURE_HPP``) is always +active when the header is parsed on its own, so ``IMPL_BASE`` is never suppressed by it. diff --git a/docs/source/components/configuration.rst b/docs/source/components/configuration.rst index 8da2f2a..0e17a96 100644 --- a/docs/source/components/configuration.rst +++ b/docs/source/components/configuration.rst @@ -570,3 +570,47 @@ Configuration for marked RST block extraction. - ``start_sequence`` (``str``) - Marker that begins an RST block - ``end_sequence`` (``str``) - Marker that ends an RST block + +.. _`preprocessor_config`: + +analyse.preprocessor +^^^^^^^^^^^^^^^^^^^^^ + +Opts in to the **preprocessor-aware C/C++ engine** (powered by libclang). When this +table is present and ``comment_type`` is ``"cpp"``, **Sphinx-CodeLinks** parses each +C/C++ file as a real translation unit and **drops comments that fall inside inactive +preprocessor branches** (``#if`` / ``#ifdef`` / ``#else`` …). Without this table, the +default tree-sitter engine is used, which extracts every comment regardless of +preprocessor conditions. + +See :ref:`preprocessor_engine` for the conceptual overview, header handling, and +limitations. + +**Type:** ``dict`` +**Default:** Not set (the tree-sitter engine is used) + +.. important:: The libclang engine requires an optional dependency. Install it with + ``pip install 'sphinx-codelinks[libclang]'``. The ``libclang`` wheel bundles the + native library, so no compiler or system toolchain is required. + +.. code-block:: toml + + [codelinks.projects.my_project.analyse.preprocessor] + compile_commands = "build/compile_commands.json" + defines = ["VARIANT_A", "PLATFORM_LINUX"] + includes = ["include", "third_party/include"] + variant_name = "variant_a" + +**Configuration fields:** + +- ``compile_commands`` (``str``) - Path to a ``compile_commands.json`` compilation + database. Relative paths resolve against the TOML config file. If omitted, + **Sphinx-CodeLinks** walks up from each source file to find one, stopping at a + directory containing ``.git``, ``ubproject.toml``, or ``pyproject.toml``, or at the + filesystem root. +- ``defines`` (``list[str]``) - ``-D`` macros used when a file has **no** entry in the + database. This is the fallback applied to every header file (headers never appear in a + database) and to all files when no database is found. +- ``includes`` (``list[str]``) - ``-I`` include directories used together with + ``defines`` on the fallback path. +- ``variant_name`` (``str``) - Optional label echoed into the run-level output. diff --git a/pyproject.toml b/pyproject.toml index 8ad05cb..ad06074 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,13 @@ dependencies = [ "tree-sitter-json>=0.24.8", ] +[project.optional-dependencies] +# libclang enables the preprocessor-aware C/C++ engine (skips comments inside +# inactive #if/#ifdef regions). Plain tree-sitter C/C++ extraction needs none of +# this, so libclang stays optional. Named after the backend, not "cpp": the +# engine handles C and Objective-C too, and tree-sitter already covers C++. +libclang = ["libclang>=18"] + [build-system] requires = ["flit_core >=3.4,<4"] build-backend = "flit_core.buildapi" @@ -48,6 +55,7 @@ testing = [ "moto ~= 5.0", "toml>=0.10.2", "furo>=2024.5.6", + "libclang>=18", ] docs = [ "furo>=2024.5.6", @@ -106,8 +114,11 @@ force-sort-within-sections = true "**/tests/*" = [ "ARG001", # unused-function-argument - fixtures "ARG005", # unused-lambda-argument - monkeypatches + "E402", # module-import-not-at-top - pytest.importorskip guard pattern + "PLC0415", # import-outside-top-level - inline imports in test bodies "PLR2004", # magic-value-comparison - valueable for tests "S101", # assert - needed for tests + "SIM300", # yoda-conditions - assert order in tests is intentional ] "src/sphinx_codelinks/sphinx_extension/debug.py" = [ "T201", # print - used for output @@ -140,7 +151,8 @@ plugins = ["pydantic.mypy"] mypy_path = "typings" [[tool.mypy.overrides]] -module = ["licensing.*", "tomlkit.*"] +# clang.cindex ships no type stubs / py.typed; the libclang engine wraps it. +module = ["licensing.*", "tomlkit.*", "clang.*"] ignore_missing_imports = true [[tool.mypy.overrides]] diff --git a/src/sphinx_codelinks/analyse/analyse.py b/src/sphinx_codelinks/analyse/analyse.py index cfd1017..f0914c3 100644 --- a/src/sphinx_codelinks/analyse/analyse.py +++ b/src/sphinx_codelinks/analyse/analyse.py @@ -2,7 +2,7 @@ from dataclasses import dataclass import json from pathlib import Path -from typing import Any, TypedDict +from typing import Any, TypedDict, cast from tree_sitter import Node as TreeSitterNode @@ -78,9 +78,7 @@ def __init__( self.git_commit_rev: str | None = ( utils.get_current_rev(self.git_root) if self.git_root else None ) - self.project_path: Path = ( - self.git_root if self.git_root else self.analyse_config.src_dir - ) + self.project_path: Path = self.git_root or self.analyse_config.src_dir self.oneline_warnings: list[AnalyseWarning] = [] def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]: # type: ignore[explicit-any] @@ -112,6 +110,80 @@ def create_src_objects(self) -> None: self.src_files.append(src_file) self.src_comments.extend(src_comments) + def _resolve_preproc_args(self, src_path: Path) -> list[str] | None: + from sphinx_codelinks.analyse.preproc import compile_db # noqa: PLC0415 + + preproc = self.analyse_config.preprocessor + if preproc is None: + return [] + db_path = preproc.compile_commands + if db_path is None: + db_path = compile_db.find_compile_db(src_path, self.project_path) + if db_path is not None and db_path.is_file(): + flags = compile_db.load_flags_map(db_path) + args = flags.get(src_path.absolute().resolve()) + if args is not None: + return args + # Absent from the DB. compile_commands.json lists only compiled + # translation units, never headers — so a header here is parsed + # standalone with the global defines (one run = one variant). A + # compiled source absent from the build is skipped (spec §3.3). + if compile_db.is_translation_unit_source(src_path): + return None + return compile_db.defines_to_args(preproc.defines, preproc.includes) + # No DB found at all — fall back to manual defines applied globally. + return compile_db.defines_to_args(preproc.defines, preproc.includes) + + def create_src_objects_libclang(self) -> None: + from sphinx_codelinks.analyse.preproc import ( # noqa: PLC0415 + libclang_parser, + loader, + ) + + # Resolve the exception via the loader (never a direct ``import + # clang.cindex``) so a missing ``libclang`` extra still surfaces the + # loader's friendly install hint rather than a bare ImportError. + translation_unit_load_error = ( + loader.load_clang_cindex().TranslationUnitLoadError + ) + + for src_path in self.analyse_config.src_files: + if not utils.is_text_file(src_path): + continue + args = self._resolve_preproc_args(src_path) + if args is None: + logger.debug( + f"codelinks: skipping {src_path} — not found in compile_commands.json" + ) + continue + try: + comments = libclang_parser.extract_active_comments(src_path, args) + except translation_unit_load_error: + # Last-resort guard. Standalone parses pin ``-x`` to match the + # ``-std`` (see defines_to_args), so the common case — a ``.h``/ + # ``.c`` header handed a C++ ``-std`` — now parses as C++ and its + # markers extract. This only fires if libclang still cannot load + # the file as a translation unit at all; skip it rather than + # aborting the whole Sphinx build. Logged at ``info`` (not + # ``warning``): under ``sphinx-build -W`` a warning would re-fail + # the very builds this guard keeps green. + logger.info( + f"codelinks: skipping {src_path} — libclang could not load it " + f"as a translation unit" + ) + continue + if not comments: + continue + # ``c`` is a LibclangComment duck-typing the tree-sitter Node + # interface SourceComment reads (``.text`` / ``.start_point.row``); + # the Node-only path (find_associated_scope) is guarded by + # ``is_libclang`` so it never runs on these. + src_comments = [SourceComment(cast("TreeSitterNode", c)) for c in comments] + src_file = SourceFile(src_path.absolute()) + src_file.add_comments(src_comments) + self.src_files.append(src_file) + self.src_comments.extend(src_comments) + def extract_marker( self, text: str, @@ -327,9 +399,12 @@ def extract_marked_content(self) -> None: ) if not filepath: continue - tagged_scope: TreeSitterNode | None = utils.find_associated_scope( - src_comment.node, self.analyse_config.comment_type - ) + if getattr(src_comment.node, "is_libclang", False): + tagged_scope: TreeSitterNode | None = None + else: + tagged_scope = utils.find_associated_scope( + src_comment.node, self.analyse_config.comment_type + ) if self.analyse_config.get_need_id_refs: anchors = self.extract_anchors( text, filepath, tagged_scope, src_comment @@ -372,7 +447,15 @@ def dump_marked_content(self, outdir: Path) -> None: json.dump(to_dump, f) def run(self) -> None: - self.create_src_objects() + from sphinx_codelinks.source_discover.config import CommentType # noqa: PLC0415 + + if ( + self.analyse_config.preprocessor is not None + and self.analyse_config.comment_type == CommentType.cpp + ): + self.create_src_objects_libclang() + else: + self.create_src_objects() self.extract_marked_content() self.merge_marked_content() self._log_summary() diff --git a/src/sphinx_codelinks/analyse/preproc/__init__.py b/src/sphinx_codelinks/analyse/preproc/__init__.py new file mode 100644 index 0000000..bef6c11 --- /dev/null +++ b/src/sphinx_codelinks/analyse/preproc/__init__.py @@ -0,0 +1,5 @@ +"""libclang-based, preprocessor-aware extraction engine (opt-in).""" + +from sphinx_codelinks.analyse.preproc import compile_db, libclang_parser, loader + +__all__ = ["compile_db", "libclang_parser", "loader"] diff --git a/src/sphinx_codelinks/analyse/preproc/compile_db.py b/src/sphinx_codelinks/analyse/preproc/compile_db.py new file mode 100644 index 0000000..f01e5fd --- /dev/null +++ b/src/sphinx_codelinks/analyse/preproc/compile_db.py @@ -0,0 +1,118 @@ +"""Discover, read, and filter compile_commands.json for libclang.""" + +from __future__ import annotations + +import json +from pathlib import Path +import shlex + +_DB_NAME = "compile_commands.json" +_BOUNDARY_MARKERS = (".git", "ubproject.toml", "pyproject.toml") + +# Flags that take a following value we must also drop. +_DROP_WITH_VALUE = {"-o", "-MF", "-MT", "-MQ"} +# Exact flags to drop. +_DROP_EXACT = {"-c", "-MMD", "-MD", "-MG", "-MP"} +# Minimum length for joined-form flags like -MFdep.d (prefix length = 3) +_MIN_JOINED_FLAG_LEN = 3 + +# Suffixes a compiler builds as a translation unit. A discovered C/C++ file +# absent from compile_commands.json is skipped only when it is a TU source +# (build-excluded); all other discovered files (headers) are parsed standalone. +TU_SOURCE_SUFFIXES = {".c", ".cpp", ".cc", ".cxx"} + + +def find_compile_db(start: Path, project_root: Path | None = None) -> Path | None: + """Walk up from ``start`` looking for compile_commands.json. + + Stops at (inclusive) the directory that contains the db, or at + ``project_root`` / a directory containing a boundary marker / fs root. + """ + current = start if start.is_dir() else start.parent + current = current.resolve() + root = project_root.resolve() if project_root else None + while True: + candidate = current / _DB_NAME + if candidate.is_file(): + return candidate + if root is not None and current == root: + return None + if any((current / m).exists() for m in _BOUNDARY_MARKERS): + return None + if current.parent == current: # filesystem root + return None + current = current.parent + + +def filter_args(argv: list[str], input_file: str) -> list[str]: + """Keep only flags libclang needs; drop the compiler, -c/-o, depfiles, input.""" + out: list[str] = [] + skip_next = False + input_names = {input_file, str(Path(input_file).name), str(Path(input_file))} + for i, arg in enumerate(argv): + if i == 0: + continue # argv[0] == compiler + if skip_next: + skip_next = False + continue + if arg in _DROP_WITH_VALUE: + skip_next = True + continue + if arg in _DROP_EXACT: + continue + if arg.startswith(("-MF", "-MT", "-MQ")) and len(arg) > _MIN_JOINED_FLAG_LEN: + continue # joined form, e.g. -MFdep.d + if arg in input_names: + continue + out.append(arg) + return out + + +def load_flags_map(db_path: Path) -> dict[Path, list[str]]: + """Parse compile_commands.json -> {absolute file path: filtered args}.""" + entries = json.loads(db_path.read_text()) + flags: dict[Path, list[str]] = {} + for entry in entries: + if "file" not in entry or "directory" not in entry: + continue # malformed entry: skip, keep going + if "arguments" in entry: + argv = list(entry["arguments"]) + elif "command" in entry: + argv = shlex.split(entry["command"]) + else: + continue + directory = Path(entry["directory"]) + file_field = entry["file"] + abs_file = (directory / file_field).resolve() + flags[abs_file] = filter_args(argv, file_field) + return flags + + +def defines_to_args( + defines: list[str], includes: list[Path], std: str = "c++17" +) -> list[str]: + """Build a global flag list for parsing a file standalone. + + Used for headers and files absent from a compile DB. libclang infers the + language from the file extension, so a C-inferred file (``.h`` / ``.c`` / + ``.inc``) handed a C++ ``-std`` makes clang reject the combination and return + a NULL translation unit (surfacing as ``TranslationUnitLoadError``). Pin the + language to match ``std`` with ``-x`` so such files — e.g. a ``.h`` header + carrying oneline need markers — parse and extract instead of failing. (C + sources parse as C++ well enough for comment/marker extraction.) + """ + lang = "c++" if std.startswith("c++") else "c" + args = ["-x", lang, f"-std={std}"] + args += [f"-D{d}" for d in defines] + args += [f"-I{inc}" for inc in includes] + return args + + +def is_translation_unit_source(path: Path) -> bool: + """True if ``path`` is a compiled translation-unit source (not a header). + + compile_commands.json lists one entry per compiled TU; headers are never + entries. So a discovered file absent from the DB is skipped only when it is + a TU source; header-like files are parsed standalone (see _resolve_preproc_args). + """ + return path.suffix.lower() in TU_SOURCE_SUFFIXES diff --git a/src/sphinx_codelinks/analyse/preproc/libclang_parser.py b/src/sphinx_codelinks/analyse/preproc/libclang_parser.py new file mode 100644 index 0000000..45f8c99 --- /dev/null +++ b/src/sphinx_codelinks/analyse/preproc/libclang_parser.py @@ -0,0 +1,68 @@ +"""Parse a C/C++ TU via libclang and yield ACTIVE comment tokens only.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from sphinx_codelinks.analyse.preproc import loader + + +@dataclass +class _Point: + row: int + + +class LibclangComment: + """Minimal stand-in for a tree-sitter comment node. + + Exposes exactly the two attributes the existing extract_* chain reads: + ``.text`` (bytes) and ``.start_point.row`` (0-indexed). ``is_libclang`` + lets extract_marked_content skip tree-sitter scope association. + """ + + is_libclang = True + + def __init__(self, text: bytes, row: int) -> None: + self.text: bytes = text + self.start_point = _Point(row) + + +def _is_in_skipped(file_path: str, line: int, skipped) -> bool: # type: ignore[no-untyped-def] + for sr in skipped: + if sr.file is None or str(sr.file) != file_path: + continue + if sr.start_line <= line <= sr.end_line: + return True + return False + + +def extract_active_comments(file_path: Path, args: list[str]) -> list[LibclangComment]: + """Return one LibclangComment per ACTIVE comment token in ``file_path``. + + Comments inside preprocessor-skipped (inactive) ranges are dropped. + """ + cx = loader.load_clang_cindex() + index = cx.Index.create() + tu = index.parse(str(file_path), args=args, options=loader.PARSE_OPTIONS) + skipped = loader.get_all_skipped_ranges(tu) + + line_count = len(file_path.read_text(encoding="utf-8").splitlines()) + main = tu.get_file(str(file_path)) + extent = cx.SourceRange.from_locations( + cx.SourceLocation.from_position(tu, main, 1, 1), + cx.SourceLocation.from_position(tu, main, line_count + 1, 1), + ) + + out: list[LibclangComment] = [] + for tok in tu.get_tokens(extent=extent): + if tok.kind != cx.TokenKind.COMMENT: + continue + loc = tok.location + if loc.file is None: + continue + if _is_in_skipped(str(loc.file.name), loc.line, skipped): + continue # inactive branch -> excluded + spelling = tok.spelling or "" + out.append(LibclangComment(spelling.encode("utf-8"), loc.line - 1)) + return out diff --git a/src/sphinx_codelinks/analyse/preproc/loader.py b/src/sphinx_codelinks/analyse/preproc/loader.py new file mode 100644 index 0000000..0f3701f --- /dev/null +++ b/src/sphinx_codelinks/analyse/preproc/loader.py @@ -0,0 +1,94 @@ +"""Import guard for clang.cindex + ctypes shim for clang_getAllSkippedRanges. + +clang.cindex (shipped by the PyPI ``libclang`` wheel, which bundles the native +library — no compiler required) does not expose clang_getAllSkippedRanges, so +we bind it via ctypes, exactly as the design concept's _common.py did. +""" + +from __future__ import annotations + +import ctypes +from pathlib import Path +from typing import Any, NamedTuple + +_INSTALL_HINT = ( + "The libclang engine requires clang.cindex. Install the extra:\n" + " pip install 'sphinx-codelinks[libclang]'" +) + + +def load_clang_cindex() -> Any: # type: ignore[explicit-any] + """Return the clang.cindex module or raise a clear install error.""" + try: + import clang.cindex as cx # noqa: PLC0415 + except ImportError as exc: # pragma: no cover - exercised via patched import + raise ImportError(_INSTALL_HINT) from exc + return cx + + +# Production parse flags (design "Parse flags codelinks should use"). +def _parse_options() -> int: + cx = load_clang_cindex() + # ``cx`` is untyped (clang ships no stubs), so the bit-or is ``Any``; coerce + # back to ``int`` to satisfy the declared return type. + return int( + cx.TranslationUnit.PARSE_INCOMPLETE + | cx.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES + | cx.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD + ) + + +PARSE_OPTIONS: int = _parse_options() + + +class SkippedRange(NamedTuple): + file: Path | None + start_line: int + start_col: int + end_line: int + end_col: int + + +class _CXSourceRangeList(ctypes.Structure): + pass + + +_BOUND = False + + +def _bind() -> None: + global _BOUND # noqa: PLW0603 + if _BOUND: + return + cx = load_clang_cindex() + _CXSourceRangeList._fields_ = [ + ("count", ctypes.c_uint), + ("ranges", ctypes.POINTER(cx.SourceRange)), + ] + lib = cx.conf.lib + lib.clang_getAllSkippedRanges.argtypes = [ctypes.c_void_p] + lib.clang_getAllSkippedRanges.restype = ctypes.POINTER(_CXSourceRangeList) + lib.clang_disposeSourceRangeList.argtypes = [ctypes.POINTER(_CXSourceRangeList)] + lib.clang_disposeSourceRangeList.restype = None + _BOUND = True + + +def get_all_skipped_ranges(tu: Any) -> list[SkippedRange]: # type: ignore[explicit-any] + """Return every source range the preprocessor skipped in this TU.""" + _bind() + cx = load_clang_cindex() + ptr = cx.conf.lib.clang_getAllSkippedRanges(tu.obj) + if not ptr: + return [] + try: + rl = ptr.contents + out: list[SkippedRange] = [] + for i in range(rl.count): + r = rl.ranges[i] + start = r.start + end = r.end + f = Path(start.file.name) if start.file else None + out.append(SkippedRange(f, start.line, start.column, end.line, end.column)) + return out + finally: + cx.conf.lib.clang_disposeSourceRangeList(ptr) diff --git a/src/sphinx_codelinks/config.py b/src/sphinx_codelinks/config.py index f32de43..a396e90 100644 --- a/src/sphinx_codelinks/config.py +++ b/src/sphinx_codelinks/config.py @@ -120,6 +120,33 @@ def check_fields_configuration(self) -> list[str]: return self.check_schema() + self.check_sequence_mutually_exclusive() +@dataclass +class PreprocessorConfig: + """Opt-in libclang engine config. Presence => libclang engine for C/C++.""" + + compile_commands: Path | None = field( + default=None, metadata={"schema": {"type": ["string", "null"]}} + ) + """Explicit path to compile_commands.json. If None, walk-up auto-discovery.""" + + defines: list[str] = field( + default_factory=list, + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """Fallback -D defines applied globally when no compile_commands.json applies.""" + + includes: list[Path] = field( + default_factory=list, + metadata={"schema": {"type": "array", "items": {"type": "string"}}}, + ) + """Fallback -I include dirs for the defines path.""" + + variant_name: str | None = field( + default=None, metadata={"schema": {"type": ["string", "null"]}} + ) + """Label echoed into run-level output.""" + + class FieldConfig(TypedDict, total=False): name: str type: Literal["str", "list[str]"] @@ -329,6 +356,7 @@ class AnalyseSectionConfigType(TypedDict, total=False): need_id_refs: NeedIdRefsConfigType marked_rst: MarkedRstConfigType oneline_comment_style: OneLineCommentStyleType + preprocessor: dict[str, object] class SourceAnalyseConfigType(TypedDict, total=False): @@ -344,6 +372,7 @@ class SourceAnalyseConfigType(TypedDict, total=False): need_id_refs_config: NeedIdRefsConfig marked_rst_config: MarkedRstConfig oneline_comment_style: OneLineCommentStyle + preprocessor: PreprocessorConfig | None class ProjectsAnalyseConfigType(TypedDict, total=False): @@ -400,6 +429,17 @@ def field_names(cls) -> set[str]: ) """Configuration for extracting oneline needs from comments.""" + preprocessor: PreprocessorConfig | None = field(default=None) + """Opt-in libclang preprocessor engine. None => tree-sitter (default). + + No flat ``metadata["schema"]`` here: this is a nested dataclass, like the + sibling ``need_id_refs_config`` / ``marked_rst_config`` / + ``oneline_comment_style`` fields. ``check_schema`` only validates fields that + declare a flat schema; giving this field one made it validate the constructed + ``PreprocessorConfig`` instance against JSON type ``object`` and fail at + ``config-inited``. Its structure is enforced by ``convert_analyse_config``. + """ + @classmethod def get_schema(cls, name: str) -> dict[str, Any] | None: # type: ignore[explicit-any] _field = next(_field for _field in fields(cls) if _field.name is name) @@ -792,7 +832,12 @@ def convert_analyse_config( analyse_config_dict: SourceAnalyseConfigType = {} if config_dict: for k, v in config_dict.items(): - if k not in {"online_comment_style", "need_id_refs", "marked_rst"}: + if k not in { + "online_comment_style", + "need_id_refs", + "marked_rst", + "preprocessor", + }: # Convert string paths to Path objects if k in {"src_dir", "git_root"} and isinstance(v, str): analyse_config_dict[k] = Path(v) # type: ignore[literal-required] @@ -823,6 +868,22 @@ def convert_analyse_config( analyse_config_dict["marked_rst_config"] = marked_rst_config analyse_config_dict["oneline_comment_style"] = oneline_comment_style + preprocessor_dict = config_dict.get("preprocessor") + if preprocessor_dict is not None: + # The preprocessor section has no TypedDict; its values are dynamic + # TOML (typed ``object``), so the list/iter/assign below need targeted + # ignores matching the concrete errors mypy reports for ``object``. + analyse_config_dict["preprocessor"] = PreprocessorConfig( + compile_commands=( + Path(str(preprocessor_dict["compile_commands"])) + if preprocessor_dict.get("compile_commands") + else None + ), + defines=list(preprocessor_dict.get("defines", [])), # type: ignore[call-overload] + includes=[Path(str(p)) for p in preprocessor_dict.get("includes", [])], # type: ignore[attr-defined] + variant_name=preprocessor_dict.get("variant_name"), # type: ignore[arg-type] + ) + if src_discover: analyse_config_dict["src_files"] = src_discover.source_paths analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-file_in_db_uses_flags_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-file_in_db_uses_flags_libclang].json new file mode 100644 index 0000000..18a11a6 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-file_in_db_uses_flags_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_ALWAYS", + "title": "Always", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_VAR_A", + "title": "Variant A", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 3 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-header_absent_from_db_standalone_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-header_absent_from_db_standalone_libclang].json new file mode 100644 index 0000000..fbab42e --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-header_absent_from_db_standalone_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_HDR_BASE", + "title": "Header Base", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 2 + }, + { + "id": "IMPL_HDR_ON", + "title": "Header On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 4 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-tu_source_absent_from_db_skipped_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-tu_source_absent_from_db_skipped_libclang].json new file mode 100644 index 0000000..4d18559 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-tu_source_absent_from_db_skipped_libclang].json @@ -0,0 +1,6 @@ +{ + "needs": [], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-unreadable_db_path_falls_back_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-unreadable_db_path_falls_back_libclang].json new file mode 100644 index 0000000..e253731 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_compile_db-unreadable_db_path_falls_back_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_BASE", + "title": "Base", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_FALLBACK", + "title": "Fallback On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 3 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-arithmetic_compare_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-arithmetic_compare_libclang].json new file mode 100644 index 0000000..252e2ce --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-arithmetic_compare_libclang].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_V3", + "title": "V3 plus", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 2 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-defined_boolean_logic_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-defined_boolean_logic_libclang].json new file mode 100644 index 0000000..cb93422 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-defined_boolean_logic_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_AORB", + "title": "A or B", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 5 + }, + { + "id": "IMPL_NOTB", + "title": "Not B", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 8 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-elif_chain_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-elif_chain_libclang].json new file mode 100644 index 0000000..46d072f --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-elif_chain_libclang].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_M2", + "title": "Mode Two", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 4 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-if_zero_dead_block_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-if_zero_dead_block_libclang].json new file mode 100644 index 0000000..0b25665 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-if_zero_dead_block_libclang].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_LIVE", + "title": "Live", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 1 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-macro_object_drives_branch_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-macro_object_drives_branch_libclang].json new file mode 100644 index 0000000..50eb965 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-macro_object_drives_branch_libclang].json @@ -0,0 +1,19 @@ +{ + "needs": [ + { + "id": "IMPL_FEAT", + "title": "Feature On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 3 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-nested_ifdef_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-nested_ifdef_libclang].json new file mode 100644 index 0000000..bfbd5f4 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_macros-nested_ifdef_libclang].json @@ -0,0 +1,43 @@ +{ + "needs": [ + { + "id": "IMPL_TOP", + "title": "Top", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_OUTER", + "title": "Outer On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 3 + }, + { + "id": "IMPL_INNER", + "title": "Inner On", + "type": "impl", + "links": { + "links": [ + "REQ" + ] + }, + "metadata": {}, + "line": 5 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-header_c_extension_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-header_c_extension_libclang].json new file mode 100644 index 0000000..ad95057 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-header_c_extension_libclang].json @@ -0,0 +1,31 @@ +{ + "needs": [ + { + "id": "IMPL_HDR_BASE", + "title": "Header Base", + "type": "impl", + "links": { + "links": [ + "REQ_HDR" + ] + }, + "metadata": {}, + "line": 2 + }, + { + "id": "IMPL_HDR_FEATURE", + "title": "Header Feature On", + "type": "impl", + "links": { + "links": [ + "REQ_HDR_FEAT" + ] + }, + "metadata": {}, + "line": 6 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_libclang].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_libclang].json new file mode 100644 index 0000000..d3ea9c1 --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_libclang].json @@ -0,0 +1,55 @@ +{ + "needs": [ + { + "id": "IMPL_ALWAYS", + "title": "Always Present", + "type": "impl", + "links": { + "links": [ + "REQ_BASE" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_VAR_A", + "title": "Variant A Feature", + "type": "impl", + "links": { + "links": [ + "REQ_FEAT_A" + ] + }, + "metadata": {}, + "line": 5 + }, + { + "id": "IMPL_PROTO_3", + "title": "Protocol V3", + "type": "impl", + "links": { + "links": [ + "REQ_PROTO" + ] + }, + "metadata": {}, + "line": 13 + }, + { + "id": "IMPL_LINUX_A", + "title": "Linux And A", + "type": "impl", + "links": { + "links": [ + "REQ_COMBO" + ] + }, + "metadata": {}, + "line": 18 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_treesitter].json b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_treesitter].json new file mode 100644 index 0000000..a5fd17f --- /dev/null +++ b/tests/__snapshots__/test_extraction_fixtures/test_extraction_fixture[preproc_variants-variants_branching_treesitter].json @@ -0,0 +1,67 @@ +{ + "needs": [ + { + "id": "IMPL_ALWAYS", + "title": "Always Present", + "type": "impl", + "links": { + "links": [ + "REQ_BASE" + ] + }, + "metadata": {}, + "line": 1 + }, + { + "id": "IMPL_VAR_A", + "title": "Variant A Feature", + "type": "impl", + "links": { + "links": [ + "REQ_FEAT_A" + ] + }, + "metadata": {}, + "line": 5 + }, + { + "id": "IMPL_VAR_B", + "title": "Variant B Feature", + "type": "impl", + "links": { + "links": [ + "REQ_FEAT_B" + ] + }, + "metadata": {}, + "line": 8 + }, + { + "id": "IMPL_PROTO_3", + "title": "Protocol V3", + "type": "impl", + "links": { + "links": [ + "REQ_PROTO" + ] + }, + "metadata": {}, + "line": 13 + }, + { + "id": "IMPL_LINUX_A", + "title": "Linux And A", + "type": "impl", + "links": { + "links": [ + "REQ_COMBO" + ] + }, + "metadata": {}, + "line": 18 + } + ], + "need_refs": [], + "marked_rst": [], + "warnings": [] +} \ No newline at end of file diff --git a/tests/data/extraction/README.md b/tests/data/extraction/README.md index 84f8efe..d8ef412 100644 --- a/tests/data/extraction/README.md +++ b/tests/data/extraction/README.md @@ -41,6 +41,11 @@ custom_brackets_c: `[oneline, need_refs, rst]` (default: all three). Narrow it to keep a case focused: a need-reference case sets `extract: [need_refs]` so the `@`-prefixed `@need-ids:` marker isn't also parsed as a one-line need. +- `engine` (optional): `treesitter` (default) sees every comment; `libclang` + evaluates the preprocessor and excludes markers in inactive `#if`/`#ifdef` + branches. libclang cases are skipped when the `clang` bindings are unavailable. +- `defines` (optional, libclang only): preprocessor defines, e.g. + `["VARIANT_A=1", "PROTOCOL_VERSION=3"]`. ## Snapshot (expected output) — normalized contract diff --git a/tests/data/extraction/preproc_compile_db.yaml b/tests/data/extraction/preproc_compile_db.yaml new file mode 100644 index 0000000..7f675db --- /dev/null +++ b/tests/data/extraction/preproc_compile_db.yaml @@ -0,0 +1,77 @@ +# compile_commands.json handling for the libclang engine. +# +# Each case materializes a real compile_commands.json (inline `compile_commands`) +# or points at an explicit `compile_commands_path`, then extracts from a single +# `case.` source. These exercise the per-file argument resolution the shared +# single-`.cpp`/defines fixtures never touch: DB-provided flags, headers absent +# from a present DB (parsed standalone), translation-unit sources absent from the +# DB (skipped), and an unreadable explicit DB path (defines fallback). + +file_in_db_uses_flags_libclang: + lang: cpp + engine: libclang + config: default + defines: [] + extract: [oneline] + compile_commands: + - file: case.cpp + arguments: ["clang++", "-std=c++17", "-DVARIANT_A=1", "-c", "case.cpp"] + source: | + // @Always, IMPL_ALWAYS, impl, [REQ] + #ifdef VARIANT_A + // @Variant A, IMPL_VAR_A, impl, [REQ] + #else + // @Variant B, IMPL_VAR_B, impl, [REQ] + #endif + # -DVARIANT_A comes from the DB entry (global defines are empty), so the #ifdef + # branch is active only because the DB flags were applied: IMPL_ALWAYS, IMPL_VAR_A. + +header_absent_from_db_standalone_libclang: + lang: cpp_header + engine: libclang + config: default + defines: ["HEADER_ON=1"] + extract: [oneline] + compile_commands: + - file: other.cpp + arguments: ["clang++", "-std=c++17", "-c", "other.cpp"] + source: | + #pragma once + // @Header Base, IMPL_HDR_BASE, impl, [REQ] + #ifdef HEADER_ON + // @Header On, IMPL_HDR_ON, impl, [REQ] + #else + // @Header Off, IMPL_HDR_OFF, impl, [REQ] + #endif + # case.h is not in the (present) DB. A header is never a compile_commands entry, + # so it is parsed standalone with the global defines: IMPL_HDR_BASE, IMPL_HDR_ON. + +tu_source_absent_from_db_skipped_libclang: + lang: cpp + engine: libclang + config: default + defines: [] + extract: [oneline] + compile_commands: + - file: other.cpp + arguments: ["clang++", "-std=c++17", "-c", "other.cpp"] + source: | + // @Should Be Skipped, IMPL_SKIP, impl, [REQ] + void f() {} + # case.cpp is a translation-unit source absent from the applicable DB, so it is + # skipped entirely (spec §3.3): no markers extracted. + +unreadable_db_path_falls_back_libclang: + lang: cpp + engine: libclang + config: default + defines: ["FALLBACK_ON=1"] + extract: [oneline] + compile_commands_path: "missing.json" + source: | + // @Base, IMPL_BASE, impl, [REQ] + #ifdef FALLBACK_ON + // @Fallback On, IMPL_FALLBACK, impl, [REQ] + #endif + # The explicit compile_commands path does not exist, so extraction falls back to + # the global defines rather than skipping every file: IMPL_BASE, IMPL_FALLBACK. diff --git a/tests/data/extraction/preproc_macros.yaml b/tests/data/extraction/preproc_macros.yaml new file mode 100644 index 0000000..a8ab57d --- /dev/null +++ b/tests/data/extraction/preproc_macros.yaml @@ -0,0 +1,100 @@ +# Preprocessor macro / conditional constructs under the libclang engine. +# +# Every case keeps markers in BOTH the taken and the untaken branches; only the +# markers in active (non-skipped) regions must survive. The tree-sitter engine +# would keep them all — these are libclang-only cases that prove the preprocessor +# is actually evaluated (nesting, #elif chains, #if 0, defined()/boolean logic, +# arithmetic comparisons, and an in-source #define driving a branch). + +nested_ifdef_libclang: + lang: cpp + engine: libclang + config: default + defines: ["OUTER=1", "INNER=1"] + extract: [oneline] + source: | + // @Top, IMPL_TOP, impl, [REQ] + #ifdef OUTER + // @Outer On, IMPL_OUTER, impl, [REQ] + #ifdef INNER + // @Inner On, IMPL_INNER, impl, [REQ] + #else + // @Inner Off, IMPL_INNER_OFF, impl, [REQ] + #endif + #else + // @Outer Off, IMPL_OUTER_OFF, impl, [REQ] + #endif + +elif_chain_libclang: + lang: cpp + engine: libclang + config: default + defines: ["MODE=2"] + extract: [oneline] + source: | + #if MODE == 1 + // @Mode One, IMPL_M1, impl, [REQ] + #elif MODE == 2 + // @Mode Two, IMPL_M2, impl, [REQ] + #elif MODE == 3 + // @Mode Three, IMPL_M3, impl, [REQ] + #else + // @Mode Other, IMPL_MO, impl, [REQ] + #endif + +if_zero_dead_block_libclang: + lang: cpp + engine: libclang + config: default + defines: [] + extract: [oneline] + source: | + // @Live, IMPL_LIVE, impl, [REQ] + #if 0 + // @Dead, IMPL_DEAD, impl, [REQ] + #endif + +defined_boolean_logic_libclang: + lang: cpp + engine: libclang + config: default + defines: ["HAS_A=1", "HAS_C=1"] + extract: [oneline] + source: | + #if defined(HAS_A) && defined(HAS_B) + // @A and B, IMPL_AB, impl, [REQ] + #endif + #if defined(HAS_A) || defined(HAS_B) + // @A or B, IMPL_AORB, impl, [REQ] + #endif + #if !defined(HAS_B) + // @Not B, IMPL_NOTB, impl, [REQ] + #endif + +arithmetic_compare_libclang: + lang: cpp + engine: libclang + config: default + defines: ["VER=3"] + extract: [oneline] + source: | + #if VER >= 3 + // @V3 plus, IMPL_V3, impl, [REQ] + #endif + #if VER < 3 + // @Pre V3, IMPL_PRE3, impl, [REQ] + #endif + +macro_object_drives_branch_libclang: + lang: cpp + engine: libclang + config: default + defines: [] + extract: [oneline] + source: | + #define FEATURE 1 + #if FEATURE + // @Feature On, IMPL_FEAT, impl, [REQ] + #else + // @Feature Off, IMPL_FEAT_OFF, impl, [REQ] + #endif diff --git a/tests/data/extraction/preproc_variants.yaml b/tests/data/extraction/preproc_variants.yaml new file mode 100644 index 0000000..1df4545 --- /dev/null +++ b/tests/data/extraction/preproc_variants.yaml @@ -0,0 +1,85 @@ +# Preprocessor-aware extraction: the same source under both engines. +# +# The tree-sitter engine sees every comment (all five markers). The libclang +# engine evaluates the preprocessor with the given ``defines`` and excludes +# markers in inactive #if/#ifdef branches, so IMPL_VAR_B (the #else branch) is +# dropped and only the active set remains. + +variants_branching_treesitter: + lang: cpp + engine: treesitter + config: default + extract: [oneline] + source: | + // @Always Present, IMPL_ALWAYS, impl, [REQ_BASE] + void always() {} + + #ifdef VARIANT_A + // @Variant A Feature, IMPL_VAR_A, impl, [REQ_FEAT_A] + void variant_a() {} + #else + // @Variant B Feature, IMPL_VAR_B, impl, [REQ_FEAT_B] + void variant_b() {} + #endif + + #if PROTOCOL_VERSION >= 3 + // @Protocol V3, IMPL_PROTO_3, impl, [REQ_PROTO] + void proto_v3() {} + #endif + + #if defined(PLATFORM_LINUX) && defined(VARIANT_A) + // @Linux And A, IMPL_LINUX_A, impl, [REQ_COMBO] + void linux_and_a() {} + #endif + +variants_branching_libclang: + lang: cpp + engine: libclang + config: default + defines: ["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"] + extract: [oneline] + source: | + // @Always Present, IMPL_ALWAYS, impl, [REQ_BASE] + void always() {} + + #ifdef VARIANT_A + // @Variant A Feature, IMPL_VAR_A, impl, [REQ_FEAT_A] + void variant_a() {} + #else + // @Variant B Feature, IMPL_VAR_B, impl, [REQ_FEAT_B] + void variant_b() {} + #endif + + #if PROTOCOL_VERSION >= 3 + // @Protocol V3, IMPL_PROTO_3, impl, [REQ_PROTO] + void proto_v3() {} + #endif + + #if defined(PLATFORM_LINUX) && defined(VARIANT_A) + // @Linux And A, IMPL_LINUX_A, impl, [REQ_COMBO] + void linux_and_a() {} + #endif + +# A C-extension header (.h) is never a compile_commands entry, so it is parsed +# standalone with the global defines. libclang infers C from the .h extension, so +# the engine pins -x to the -std (see defines_to_args); otherwise -std=c++17 +# yields a NULL translation unit and the header's markers would be lost. Inactive +# branches are still excluded. +header_c_extension_libclang: + lang: cpp_header + engine: libclang + config: default + defines: ["HEADER_FEATURE=1"] + extract: [oneline] + source: | + #pragma once + // @Header Base, IMPL_HDR_BASE, impl, [REQ_HDR] + inline int hdr_add(int a, int b) { return a + b; } + + #ifdef HEADER_FEATURE + // @Header Feature On, IMPL_HDR_FEATURE, impl, [REQ_HDR_FEAT] + inline void hdr_feature() {} + #else + // @Header Feature Off, IMPL_HDR_NOFEATURE, impl, [REQ_HDR] + inline void hdr_no_feature() {} + #endif diff --git a/tests/data/preproc/half_typed.cpp b/tests/data/preproc/half_typed.cpp new file mode 100644 index 0000000..2c30344 --- /dev/null +++ b/tests/data/preproc/half_typed.cpp @@ -0,0 +1,14 @@ +#include + +// @Complete Function, IMPL_COMPLETE, impl, [REQ_OK] +void complete_function() { int x = 42; } + +// @Half Typed Function, IMPL_HALF, impl, [REQ_PARTIAL] +void half_typed_function() { + if (some_condition + +// @After Half Typed, IMPL_AFTER, impl, [REQ_RECOVERS] +void after_half_typed() { int y = 1; } + +// @Mid Declaration, IMPL_MID, impl, [REQ_MID] +int diff --git a/tests/data/preproc/header_standalone.hpp b/tests/data/preproc/header_standalone.hpp new file mode 100644 index 0000000..d279fa2 --- /dev/null +++ b/tests/data/preproc/header_standalone.hpp @@ -0,0 +1,12 @@ +#ifndef HEADER_STANDALONE_HPP +#define HEADER_STANDALONE_HPP + +// @Header Always, IMPL_HDR_ALWAYS, impl, [REQ_HDR] +void hdr_always(); + +#ifdef VARIANT_A +// @Header Variant A, IMPL_HDR_VAR_A, impl, [REQ_HDR_A] +void hdr_variant_a(); +#endif + +#endif // HEADER_STANDALONE_HPP diff --git a/tests/data/preproc/plain_header.h b/tests/data/preproc/plain_header.h new file mode 100644 index 0000000..a6e2029 --- /dev/null +++ b/tests/data/preproc/plain_header.h @@ -0,0 +1,9 @@ +// A .h header (not .hpp), the case a real project hits: a header carrying +// oneline need markers that never appears in compile_commands.json (headers are +// not compiled translation units). libclang infers the C language from the .h +// extension, so parsing it with a C++ -std flag would return a NULL translation +// unit. The engine pins -x to the -std for standalone parses, so this header is +// parsed as C++ and its markers are extracted (not dropped, not crashing). + +// @Plain Header, IMPL_HDR_PLAIN, impl, [REQ_HDR] +void plain_header_fn(); diff --git a/tests/data/preproc/variants_branching.cpp b/tests/data/preproc/variants_branching.cpp new file mode 100644 index 0000000..b303769 --- /dev/null +++ b/tests/data/preproc/variants_branching.cpp @@ -0,0 +1,20 @@ +// @Always Present, IMPL_ALWAYS, impl, [REQ_BASE] +void always() {} + +#ifdef VARIANT_A +// @Variant A Feature, IMPL_VAR_A, impl, [REQ_FEAT_A] +void variant_a() {} +#else +// @Variant B Feature, IMPL_VAR_B, impl, [REQ_FEAT_B] +void variant_b() {} +#endif + +#if PROTOCOL_VERSION >= 3 +// @Protocol V3, IMPL_PROTO_3, impl, [REQ_PROTO] +void proto_v3() {} +#endif + +#if defined(PLATFORM_LINUX) && defined(VARIANT_A) +// @Linux And A, IMPL_LINUX_A, impl, [REQ_COMBO] +void linux_and_a() {} +#endif diff --git a/tests/data/preproc/variants_broken.cpp b/tests/data/preproc/variants_broken.cpp new file mode 100644 index 0000000..2afd6c8 --- /dev/null +++ b/tests/data/preproc/variants_broken.cpp @@ -0,0 +1,7 @@ +#include "nonexistent_header.h" + +// @Works Despite Errors, IMPL_DESPITE, impl, [REQ_RESILIENT] +void works_despite_errors() { undeclared_function(); } + +// @After Broken Block, IMPL_AFTER_BROKEN, impl, [REQ_RESILIENT] +void after_broken_block() {} diff --git a/tests/test_analyse_preproc.py b/tests/test_analyse_preproc.py new file mode 100644 index 0000000..0ba5ecd --- /dev/null +++ b/tests/test_analyse_preproc.py @@ -0,0 +1,285 @@ +import json +from pathlib import Path + +import pytest + +pytest.importorskip("clang.cindex") + +from sphinx_codelinks.analyse.analyse import SourceAnalyse +from sphinx_codelinks.config import PreprocessorConfig, SourceAnalyseConfig + +FIXTURE = Path(__file__).parent / "data" / "preproc" / "variants_branching.cpp" +HEADER = Path(__file__).parent / "data" / "preproc" / "header_standalone.hpp" + + +def _run_get_oneline_ids(defines): + cfg = SourceAnalyseConfig( + src_files=[FIXTURE], + src_dir=FIXTURE.parent, + get_need_id_refs=False, + get_oneline_needs=True, + get_rst=False, + preprocessor=PreprocessorConfig(defines=defines), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + return {n.need["id"] for n in analyse.oneline_needs} + + +def test_libclang_engine_excludes_inactive_markers(): + ids = _run_get_oneline_ids( + ["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"] + ) + assert "IMPL_ALWAYS" in ids + assert "IMPL_VAR_A" in ids + assert "IMPL_VAR_B" not in ids # inactive + assert "IMPL_PROTO_3" in ids + assert "IMPL_LINUX_A" in ids + + +def test_libclang_engine_other_variant(): + ids = _run_get_oneline_ids(["PROTOCOL_VERSION=1"]) + assert "IMPL_ALWAYS" in ids + assert "IMPL_VAR_B" in ids + assert "IMPL_VAR_A" not in ids + assert "IMPL_PROTO_3" not in ids + + +def test_libclang_engine_via_compile_commands(tmp_path): + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(FIXTURE.parent), + "arguments": [ + "clang++", + "-std=c++17", + "-DVARIANT_A=1", + "-DPROTOCOL_VERSION=3", + "-c", + str(FIXTURE), + ], + "file": str(FIXTURE), + } + ] + ) + ) + cfg = SourceAnalyseConfig( + src_files=[FIXTURE], + src_dir=FIXTURE.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(compile_commands=db), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + assert "IMPL_VAR_A" in ids + assert "IMPL_VAR_B" not in ids + assert "IMPL_PROTO_3" in ids + assert "IMPL_LINUX_A" not in ids + + +def test_libclang_resilient_to_broken_code(): + broken = FIXTURE.parent / "variants_broken.cpp" + cfg = SourceAnalyseConfig( + src_files=[broken], + src_dir=broken.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=[]), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + assert ids == {"IMPL_DESPITE", "IMPL_AFTER_BROKEN"} + + +def test_libclang_resilient_to_half_typed_code(): + half = FIXTURE.parent / "half_typed.cpp" + cfg = SourceAnalyseConfig( + src_files=[half], + src_dir=half.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=[]), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + # All 4 markers survive at the token level even though 2 decls don't parse. + assert {"IMPL_COMPLETE", "IMPL_HALF", "IMPL_AFTER", "IMPL_MID"} <= ids + + +def test_libclang_active_matches_treesitter_when_all_active(): + """When every branch is active, libclang output == tree-sitter output.""" + # Tree-sitter path (no preprocessor block) sees ALL markers. + ts_cfg = SourceAnalyseConfig( + src_files=[FIXTURE], src_dir=FIXTURE.parent, get_oneline_needs=True + ) + ts = SourceAnalyse(ts_cfg) + ts.git_remote_url = None + ts.git_commit_rev = None + ts.run() + ts_ids = {n.need["id"] for n in ts.oneline_needs} + + # libclang with both variants' guards satisfied is impossible (#else is + # mutually exclusive), so compare the union over both variants instead. + a = _run_get_oneline_ids(["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"]) + b = _run_get_oneline_ids(["PROTOCOL_VERSION=1"]) + assert ts_ids == (a | b) + + +def test_libclang_skip_file_absent_from_compile_commands(tmp_path): + """Files absent from a compile DB are skipped (spec §3.3).""" + # DB contains only FIXTURE; half_typed.cpp is intentionally absent. + other = FIXTURE.parent / "half_typed.cpp" + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(FIXTURE.parent), + "arguments": [ + "clang++", + "-std=c++17", + "-DVARIANT_A=1", + "-DPROTOCOL_VERSION=3", + "-c", + str(FIXTURE), + ], + "file": str(FIXTURE), + } + ] + ) + ) + cfg = SourceAnalyseConfig( + src_files=[FIXTURE, other], + src_dir=FIXTURE.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(compile_commands=db), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + # FIXTURE is in the DB — its markers must appear. + assert "IMPL_VAR_A" in ids + assert "IMPL_PROTO_3" in ids + # half_typed.cpp is NOT in the DB — it must be skipped entirely. + assert "IMPL_COMPLETE" not in ids + assert "IMPL_HALF" not in ids + assert "IMPL_AFTER" not in ids + assert "IMPL_MID" not in ids + + +def test_header_extracted_when_absent_from_compile_commands(tmp_path): + """A header absent from the DB is parsed standalone, not skipped.""" + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(FIXTURE.parent), + "arguments": [ + "clang++", + "-std=c++17", + "-DVARIANT_A=1", + "-c", + str(FIXTURE), + ], + "file": str(FIXTURE), + } + ] + ) + ) + cfg = SourceAnalyseConfig( + src_files=[FIXTURE, HEADER], + src_dir=FIXTURE.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(compile_commands=db, defines=["VARIANT_A=1"]), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + ids = {n.need["id"] for n in analyse.oneline_needs} + # The .cpp resolves its flags from the DB entry. + assert "IMPL_VAR_A" in ids + # The header is absent from the DB -> parsed standalone with global defines. + assert "IMPL_HDR_ALWAYS" in ids + assert "IMPL_HDR_VAR_A" in ids # global defines carry VARIANT_A=1 + + +def _run_header_ids(defines): + cfg = SourceAnalyseConfig( + src_files=[HEADER], + src_dir=HEADER.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=defines), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() + return {n.need["id"] for n in analyse.oneline_needs} + + +def test_header_standalone_active_define(): + ids = _run_header_ids(["VARIANT_A=1"]) + assert "IMPL_HDR_ALWAYS" in ids # include-guard body is active + assert "IMPL_HDR_VAR_A" in ids # #ifdef VARIANT_A active + + +def test_header_standalone_inactive_define(): + ids = _run_header_ids([]) + assert "IMPL_HDR_ALWAYS" in ids # include-guard body still active + assert "IMPL_HDR_VAR_A" not in ids # #ifdef VARIANT_A inactive -> dropped + + +# Note: extraction-output goldens for this fixture now live in the declarative +# suite (tests/data/extraction/preproc_variants.yaml, both engines). The tests +# above cover libclang-specific behavior the declarative harness does not: +# inactive-branch exclusion via defines/compile_commands, resilience to broken or +# half-typed code, compile-DB resolution, the spec §3.3 skip, and standalone +# header handling. + + +PLAIN_H = Path(__file__).parent / "data" / "preproc" / "plain_header.h" + + +def test_libclang_extracts_c_extension_header_via_cpp_language(): + """A ``.h`` header still extracts its oneline markers (not dropped/crashing). + + Headers never appear in compile_commands.json, so they are parsed standalone + with the global defines. libclang infers C from the ``.h`` extension; without + pinning the language, ``-std=c++17`` makes clang reject the combo and return + a NULL translation unit (``TranslationUnitLoadError``) — which previously + aborted ``sphinx-build -b ubtrace -W``. The standalone flags now pin ``-x`` + to the ``-std`` (see ``defines_to_args``), so the header parses as C++ and its + markers are extracted rather than lost. + """ + cfg = SourceAnalyseConfig( + src_files=[FIXTURE, PLAIN_H], + src_dir=FIXTURE.parent, + get_need_id_refs=False, + get_oneline_needs=True, + get_rst=False, + preprocessor=PreprocessorConfig( + defines=["VARIANT_A=1", "PLATFORM_LINUX=1", "PROTOCOL_VERSION=3"] + ), + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() # must not raise TranslationUnitLoadError + ids = {n.need["id"] for n in analyse.oneline_needs} + assert "IMPL_ALWAYS" in ids # the .cpp translation unit extracts + assert "IMPL_HDR_PLAIN" in ids # the .h header extracts too (parsed as C++) diff --git a/tests/test_extraction_fixtures.py b/tests/test_extraction_fixtures.py index caf11be..dcd36d7 100644 --- a/tests/test_extraction_fixtures.py +++ b/tests/test_extraction_fixtures.py @@ -5,6 +5,7 @@ compared to a committed JSON snapshot. See ``tests/data/extraction/README.md``. """ +import json from pathlib import Path import pytest @@ -14,6 +15,7 @@ from sphinx_codelinks.config import ( NeedIdRefsConfig, OneLineCommentStyle, + PreprocessorConfig, SourceAnalyseConfig, ) from sphinx_codelinks.source_discover.config import CommentType @@ -24,6 +26,9 @@ LANG_MAP: dict[str, tuple[CommentType, str]] = { "cpp": (CommentType.cpp, "cpp"), "c": (CommentType.cpp, "c"), + # A C/C++ header (.h): libclang infers C from the extension, so the engine + # must pin the language to the -std or -std=c++17 yields a NULL TU. + "cpp_header": (CommentType.cpp, "h"), "python": (CommentType.python, "py"), "csharp": (CommentType.cs, "cs"), "rust": (CommentType.rust, "rs"), @@ -111,6 +116,30 @@ def _normalize(analyse: SourceAnalyse, style: OneLineCommentStyle) -> dict: } +def _build_preprocessor(case: dict, tmp_path: Path) -> PreprocessorConfig: + """Build the libclang ``PreprocessorConfig`` for a case. + + ``compile_commands`` (a list of ``{file, arguments}`` entries) is materialized + into a real ``compile_commands.json`` under ``tmp_path`` so the engine resolves + per-file flags from it. ``compile_commands_path`` instead points at an explicit + path (which may be intentionally absent, to exercise the fallback). Otherwise + only the global ``defines`` apply. + """ + defines = case.get("defines", []) + compile_commands: Path | None = None + if "compile_commands" in case: + db = tmp_path / "compile_commands.json" + entries = [ + {"directory": str(tmp_path), "file": e["file"], "arguments": e["arguments"]} + for e in case["compile_commands"] + ] + db.write_text(json.dumps(entries), encoding="utf-8") + compile_commands = db + elif "compile_commands_path" in case: + compile_commands = tmp_path / case["compile_commands_path"] + return PreprocessorConfig(defines=defines, compile_commands=compile_commands) + + @pytest.mark.parametrize("case", _load_cases()) def test_extraction_fixture(case: dict, tmp_path: Path, snapshot_extraction) -> None: comment_type, ext = LANG_MAP[case["lang"]] @@ -125,6 +154,15 @@ def test_extraction_fixture(case: dict, tmp_path: Path, snapshot_extraction) -> # ``@need-ids:`` matching the ``@`` one-line start — don't add noise. extract = case.get("extract", ["oneline", "need_refs", "rst"]) + # Engine selection. The default tree-sitter path sees every comment; the + # libclang path evaluates the preprocessor (``defines``) and excludes markers + # in inactive #if/#ifdef branches. libclang needs the clang bindings. + engine = case.get("engine", "treesitter") + preprocessor = None + if engine == "libclang": + pytest.importorskip("clang.cindex") + preprocessor = _build_preprocessor(case, tmp_path) + src_path = tmp_path / f"case.{ext}" src_path.write_text(case["source"], encoding="utf-8") @@ -137,6 +175,7 @@ def test_extraction_fixture(case: dict, tmp_path: Path, snapshot_extraction) -> get_rst="rst" in extract, oneline_comment_style=style, need_id_refs_config=refs_config, + preprocessor=preprocessor, ) analyse = SourceAnalyse(cfg) analyse.git_remote_url = None diff --git a/tests/test_libclang_optional.py b/tests/test_libclang_optional.py new file mode 100644 index 0000000..4b55808 --- /dev/null +++ b/tests/test_libclang_optional.py @@ -0,0 +1,134 @@ +"""Regression guard: the libclang engine is an OPTIONAL dependency. + +A project that only wants tree-sitter extraction must be able to import and run +sphinx-codelinks without the ``libclang`` wheel installed. Selecting the +libclang engine without it must fail with a clear ``pip install`` hint, not an +obscure ``ImportError``. + +CI installs the ``libclang`` extra, so we cannot test the absent case simply by +not installing it. Instead each test spawns a fresh interpreter with +``clang``/``clang.*`` blocked at import time (a tree-sitter-only install, by +construction) and -- before anything else -- proves the block is effective, so +the test can never pass vacuously when the wheel happens to be present. + +If someone adds an eager top-level ``import clang`` anywhere in the import +chain, ``test_treesitter_path_imports_and_runs_without_libclang`` breaks here +instead of silently making libclang mandatory for every user. +""" + +from __future__ import annotations + +from pathlib import Path +import subprocess +import sys +import textwrap + +FIXTURE = Path(__file__).parent / "data" / "preproc" / "variants_branching.cpp" + +# Block ``clang`` so the child interpreter behaves like a tree-sitter-only +# install, then assert the block actually took effect -- otherwise every check +# below would be meaningless on a machine that has the libclang wheel. +_BLOCK_CLANG = """ +import importlib.abc +import sys + + +class _BlockClang(importlib.abc.MetaPathFinder): + def find_spec(self, name, path, target=None): + if name == "clang" or name.startswith("clang."): + raise ImportError("simulated tree-sitter-only install: " + name) + return None # defer all other imports to the real finders + + +for _mod in [k for k in sys.modules if k == "clang" or k.startswith("clang.")]: + del sys.modules[_mod] +sys.meta_path.insert(0, _BlockClang()) + +try: + import clang.cindex +except ImportError: + pass +else: + raise SystemExit("clang.cindex was NOT blocked; this test would be vacuous") +""" + + +def _run_probe(body: str) -> None: + """Run ``_BLOCK_CLANG + body`` in a fresh interpreter. + + The fixture path is passed as ``sys.argv[1]``. On non-zero exit the child's + stdout/stderr are surfaced in the assertion message. + """ + code = _BLOCK_CLANG + textwrap.dedent(body) + proc = subprocess.run( # noqa: S603 + [sys.executable, "-c", code, str(FIXTURE)], + capture_output=True, + text=True, + check=False, # returncode is asserted explicitly below + ) + assert proc.returncode == 0, ( + f"probe exited {proc.returncode}\n" + f"--- stdout ---\n{proc.stdout}\n--- stderr ---\n{proc.stderr}" + ) + + +def test_treesitter_path_imports_and_runs_without_libclang() -> None: + """Default (tree-sitter) extraction needs no libclang -- import and run.""" + _run_probe( + """ + import sys + from pathlib import Path + + # Importing these must not pull in clang (parent __init__ files run too). + from sphinx_codelinks.analyse.analyse import SourceAnalyse + from sphinx_codelinks.config import SourceAnalyseConfig + + fixture = Path(sys.argv[1]) + cfg = SourceAnalyseConfig( + src_files=[fixture], + src_dir=fixture.parent, + get_oneline_needs=True, + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + analyse.run() # no preprocessor configured -> tree-sitter engine + + ids = {n.need["id"] for n in analyse.oneline_needs} + # tree-sitter ignores #ifdef, so markers on every branch are extracted. + missing = {"IMPL_ALWAYS", "IMPL_VAR_A", "IMPL_VAR_B"} - ids + assert not missing, f"tree-sitter missed markers without libclang: {missing}" + """ + ) + + +def test_libclang_engine_errors_with_install_hint_when_extra_missing() -> None: + """Selecting the libclang engine without the extra raises the install hint.""" + _run_probe( + """ + import sys + from pathlib import Path + + from sphinx_codelinks.analyse.analyse import SourceAnalyse + from sphinx_codelinks.config import PreprocessorConfig, SourceAnalyseConfig + + fixture = Path(sys.argv[1]) + cfg = SourceAnalyseConfig( + src_files=[fixture], + src_dir=fixture.parent, + get_oneline_needs=True, + preprocessor=PreprocessorConfig(defines=[]), # selects libclang engine + ) + analyse = SourceAnalyse(cfg) + analyse.git_remote_url = None + analyse.git_commit_rev = None + try: + analyse.run() + except ImportError as exc: + assert "sphinx-codelinks[libclang]" in str(exc), ( + f"install hint did not name the extra: {str(exc)!r}" + ) + else: + raise SystemExit("libclang engine ran without the extra; expected ImportError") + """ + ) diff --git a/tests/test_preproc_compile_db.py b/tests/test_preproc_compile_db.py new file mode 100644 index 0000000..bc45f7d --- /dev/null +++ b/tests/test_preproc_compile_db.py @@ -0,0 +1,136 @@ +import json +from pathlib import Path + +from sphinx_codelinks.analyse.preproc import compile_db + + +def test_find_compile_db_in_build_dir(tmp_path: Path): + (tmp_path / ".git").mkdir() + build = tmp_path / "build" + build.mkdir() + db = build / "compile_commands.json" + db.write_text("[]") + src = tmp_path / "src" / "deep" + src.mkdir(parents=True) + # Walk up from src/deep should find build/compile_commands.json? No: + # walk-up only ascends; the db is in a sibling 'build'. So a db placed + # at the project root is what walk-up finds. Place one at root instead. + root_db = tmp_path / "compile_commands.json" + root_db.write_text("[]") + found = compile_db.find_compile_db(src, project_root=tmp_path) + assert found == root_db + + +def test_find_compile_db_absent(tmp_path: Path): + (tmp_path / "ubproject.toml").write_text("") + src = tmp_path / "a" + src.mkdir() + assert compile_db.find_compile_db(src, project_root=tmp_path) is None + + +def test_find_compile_db_stops_at_git_root(tmp_path: Path): + (tmp_path / ".git").mkdir() + db = tmp_path / "compile_commands.json" + db.write_text("[]") + nested = tmp_path / "x" / "y" + nested.mkdir(parents=True) + assert compile_db.find_compile_db(nested) == db + + +def test_filter_args_strips_compiler_and_output(): + argv = [ + "clang++", + "-std=c++17", + "-c", + "-o", + "out.o", + "-DVARIANT_A=1", + "-I/inc", + "-MMD", + "-MF", + "dep.d", + "src/a.cpp", + ] + out = compile_db.filter_args(argv, "src/a.cpp") + assert out == ["-std=c++17", "-DVARIANT_A=1", "-I/inc"] + + +def test_load_flags_map_command_and_arguments_forms(tmp_path: Path): + a = tmp_path / "a.cpp" + a.write_text("") + b = tmp_path / "b.cpp" + b.write_text("") + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(tmp_path), + "arguments": ["clang++", "-DA=1", "-c", str(a)], + "file": str(a), + }, + { + "directory": str(tmp_path), + "command": "clang++ -DB=2 -c b.cpp", + "file": "b.cpp", + }, + ] + ) + ) + + flags = compile_db.load_flags_map(db) + assert flags[a.resolve()] == ["-DA=1"] + assert flags[b.resolve()] == ["-DB=2"] + + +def test_load_flags_map_relative_input_path_stripped(tmp_path: Path): + """Regression: entry whose arguments reference input by relative subdir path must be stripped. + + When directory = tmp_path, file = "src/a.cpp", and arguments contains "src/a.cpp", + the old code passed abs_file to filter_args. The input_names set only includes + the basename "a.cpp" and the absolute path — NOT the relative "src/a.cpp" — so + the relative path leaked as a spurious positional argument. + """ + src = tmp_path / "src" + src.mkdir() + a = src / "a.cpp" + a.write_text("") + db = tmp_path / "compile_commands.json" + db.write_text( + json.dumps( + [ + { + "directory": str(tmp_path), + "file": "src/a.cpp", + "arguments": ["clang++", "-DA=1", "-c", "src/a.cpp"], + } + ] + ) + ) + + flags = compile_db.load_flags_map(db) + assert flags[a.resolve()] == ["-DA=1"] + + +def test_defines_to_args(tmp_path: Path): + out = compile_db.defines_to_args(["VARIANT_A", "X=2"], [tmp_path / "inc"]) + assert "-DVARIANT_A" in out + assert "-DX=2" in out + assert f"-I{(tmp_path / 'inc')}" in out + assert "-std=c++17" in out + + +def test_is_translation_unit_source(): + # Compiled translation-unit sources (skipped when build-excluded). + assert compile_db.is_translation_unit_source(Path("a.c")) + assert compile_db.is_translation_unit_source(Path("a.cpp")) + assert compile_db.is_translation_unit_source(Path("a.cc")) + assert compile_db.is_translation_unit_source(Path("a.cxx")) + assert compile_db.is_translation_unit_source(Path("A.CPP")) # case-insensitive + # Header-like files (parsed standalone when absent from the DB). + assert not compile_db.is_translation_unit_source(Path("a.h")) + assert not compile_db.is_translation_unit_source(Path("a.hpp")) + assert not compile_db.is_translation_unit_source(Path("a.hxx")) + assert not compile_db.is_translation_unit_source(Path("a.hh")) + assert not compile_db.is_translation_unit_source(Path("a.ci")) + assert not compile_db.is_translation_unit_source(Path("a.ihl")) diff --git a/tests/test_preproc_config.py b/tests/test_preproc_config.py new file mode 100644 index 0000000..69f617d --- /dev/null +++ b/tests/test_preproc_config.py @@ -0,0 +1,61 @@ +from pathlib import Path + +from sphinx_codelinks.config import ( + PreprocessorConfig, + SourceAnalyseConfig, + convert_analyse_config, +) + + +def test_source_analyse_config_default_preprocessor_is_none(): + cfg = SourceAnalyseConfig() + assert cfg.preprocessor is None + + +def test_convert_analyse_config_builds_preprocessor(): + cfg = convert_analyse_config( + { + "get_oneline_needs": True, + "preprocessor": { + "compile_commands": "build/compile_commands.json", + "defines": ["VARIANT_A", "PLATFORM_LINUX=1"], + "includes": ["include"], + "variant_name": "linux", + }, + } + ) + assert isinstance(cfg.preprocessor, PreprocessorConfig) + assert cfg.preprocessor.compile_commands == Path("build/compile_commands.json") + assert cfg.preprocessor.defines == ["VARIANT_A", "PLATFORM_LINUX=1"] + assert cfg.preprocessor.includes == [Path("include")] + assert cfg.preprocessor.variant_name == "linux" + + +def test_convert_analyse_config_no_preprocessor_block(): + cfg = convert_analyse_config({"get_oneline_needs": True}) + assert cfg.preprocessor is None + + +def test_preprocessor_config_passes_analyse_schema_validation(): + """A SourceAnalyseConfig carrying a preprocessor must validate cleanly. + + Regression: the ``preprocessor`` field previously carried a flat + ``{"type": ["object", "null"]}`` json-schema, so ``check_schema`` validated + the *constructed* ``PreprocessorConfig`` instance against JSON type + ``object`` and raised at sphinx ``config-inited``:: + + Schema validation error in field 'preprocessor': + PreprocessorConfig(...) is not of type 'object', 'null' + + Nested dataclass config fields must not carry a flat schema (their siblings + ``need_id_refs_config`` / ``oneline_comment_style`` do not). + """ + cfg = convert_analyse_config( + { + "get_oneline_needs": True, + "preprocessor": {"defines": ["FEATURE_A"]}, + } + ) + assert cfg.preprocessor is not None + schema_errors = cfg.check_schema() + assert not any("preprocessor" in err for err in schema_errors), schema_errors diff --git a/tests/test_preproc_libclang.py b/tests/test_preproc_libclang.py new file mode 100644 index 0000000..679f62e --- /dev/null +++ b/tests/test_preproc_libclang.py @@ -0,0 +1,78 @@ +from pathlib import Path + +import pytest + +clang = pytest.importorskip("clang.cindex") + +from sphinx_codelinks.analyse.preproc import libclang_parser, loader + + +def test_load_clang_cindex_returns_module(): + mod = loader.load_clang_cindex() + assert hasattr(mod, "Index") + + +def test_parse_options_is_combination(): + # INCOMPLETE | SKIP_FUNCTION_BODIES | DETAILED_PROCESSING_RECORD + import clang.cindex as cx + + expected = ( + cx.TranslationUnit.PARSE_INCOMPLETE + | cx.TranslationUnit.PARSE_SKIP_FUNCTION_BODIES + | cx.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD + ) + assert loader.PARSE_OPTIONS == expected + + +def test_skipped_ranges_on_simple_ifdef(tmp_path: Path): + import clang.cindex as cx + + src = tmp_path / "s.cpp" + src.write_text("#ifdef OFF\n// inactive\n#endif\n// active\n") + idx = cx.Index.create() + tu = idx.parse(str(src), args=["-std=c++17"], options=loader.PARSE_OPTIONS) + skipped = loader.get_all_skipped_ranges(tu) + # The #ifdef OFF block is skipped; its range covers line 2. + assert any( + sr.start_line <= 2 <= sr.end_line and sr.file == Path(str(src)) + for sr in skipped + ) + + +FIXTURE = Path(__file__).parent / "data" / "preproc" / "variants_branching.cpp" + + +def _titles(comments): + out = [] + for c in comments: + text = c.text.decode("utf-8") + if "@" in text: + out.append(text) + return out + + +def test_extract_active_comments_variant_a_active(): + args = [ + "-std=c++17", + "-DVARIANT_A=1", + "-DPLATFORM_LINUX=1", + "-DPROTOCOL_VERSION=3", + ] + comments = libclang_parser.extract_active_comments(FIXTURE, args) + titles = _titles(comments) + joined = "\n".join(titles) + assert "IMPL_ALWAYS" in joined + assert "IMPL_VAR_A" in joined # active branch + assert "IMPL_VAR_B" not in joined # inactive #else -> EXCLUDED + assert "IMPL_PROTO_3" in joined # PROTOCOL_VERSION >= 3 active + assert "IMPL_LINUX_A" in joined # both defined + + +def test_extract_active_comments_variant_b_active(): + args = ["-std=c++17", "-DPROTOCOL_VERSION=1"] # VARIANT_A undefined + comments = libclang_parser.extract_active_comments(FIXTURE, args) + joined = "\n".join(_titles(comments)) + assert "IMPL_VAR_B" in joined # #else branch now active + assert "IMPL_VAR_A" not in joined # inactive -> EXCLUDED + assert "IMPL_PROTO_3" not in joined # version < 3 -> EXCLUDED + assert "IMPL_LINUX_A" not in joined # VARIANT_A undefined -> EXCLUDED