1import json
  2from collections.abc import Generator
  3from dataclasses import dataclass
  4from pathlib import Path
  5from typing import Any, cast
  6
  7from tree_sitter import Node as TreeSitterNode
  8
  9from sphinx_codelinks.analyse import utils
 10from sphinx_codelinks.analyse.models import (
 11    MarkedContentType,
 12    MarkedRst,
 13    NeedIdRefs,
 14    OneLineNeed,
 15    SourceComment,
 16    SourceFile,
 17    SourceMap,
 18)
 19from sphinx_codelinks.analyse.oneline_parser import (
 20    OnelineParserInvalidWarning,
 21    oneline_parser,
 22)
 23from sphinx_codelinks.config import (
 24    UNIX_NEWLINE,
 25    OneLineCommentStyle,
 26    SourceAnalyseConfig,
 27)
 28from sphinx_codelinks.logger import get_logger
 29from sphinx_codelinks.source_discover.config import CommentType
 30
 31logger = get_logger(__name__)
 32
 33
 34def _count(n: int, noun: str) -> str:
 35    """Format ``n noun`` with a naive (append-s) plural for progress summaries."""
 36    return f"{n} {noun}" if n == 1 else f"{n} {noun}s"
 37
 38
 39@dataclass
 40class AnalyseWarning:
 41    file_path: str
 42    lineno: int
 43    msg: str
 44    type: str
 45    sub_type: str
 46
 47
 48class SourceAnalyse:
 49    def __init__(
 50        self,
 51        analyse_config: SourceAnalyseConfig,
 52        *,
 53        name: str = "",
 54    ) -> None:
 55        self.name = name
 56        self.analyse_config = analyse_config
 57        self.src_files: list[SourceFile] = []
 58        self.src_comments: list[SourceComment] = []
 59        self.need_id_refs: list[NeedIdRefs] = []
 60        self.oneline_needs: list[OneLineNeed] = []
 61        self.marked_rst: list[MarkedRst] = []
 62        self.all_marked_content: list[NeedIdRefs | OneLineNeed | MarkedRst] = []
 63        # Use explicitly configured git_root if provided, otherwise auto-detect
 64        if self.analyse_config.git_root is not None:
 65            self.git_root: Path | None = self.analyse_config.git_root.resolve()
 66        else:
 67            self.git_root = utils.locate_git_root(self.analyse_config.src_dir)
 68        self.git_remote_url: str | None = (
 69            utils.get_remote_url(self.git_root) if self.git_root else None
 70        )
 71        self.git_commit_rev: str | None = (
 72            utils.get_current_rev(self.git_root) if self.git_root else None
 73        )
 74        self.project_path: Path = self.git_root or self.analyse_config.src_dir
 75        self.oneline_warnings: list[AnalyseWarning] = []
 76        # Per-run memo of parsed compile_commands.json, keyed by DB path, so the
 77        # database is read once per run instead of once per source file.
 78        self._flags_map_cache: dict[Path, dict[Path, list[str]] | None] = {}
 79
 80    def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]:
 81        """Load source files and extract their content."""
 82        for src_path in self.analyse_config.src_files:
 83            if not utils.is_text_file(src_path):
 84                continue
 85            with src_path.open("r", encoding="utf-8", newline="") as f:
 86                # Normalize all line endings to Unix LF
 87                text = f.read()
 88            text = text.replace("\r\n", "\n").replace("\r", "\n")
 89            yield src_path, text.encode("utf-8")
 90
 91    def create_src_objects(self) -> None:
 92        parser, query = utils.init_tree_sitter(self.analyse_config.comment_type)
 93
 94        for src_path, src_string in self.get_src_strings():
 95            comments: list[TreeSitterNode] | None = utils.extract_comments(
 96                src_string, parser, query
 97            )
 98            if not comments:
 99                continue
100            src_comments: list[SourceComment] = [
101                SourceComment(node) for node in comments
102            ]
103
104            src_file = SourceFile(src_path.absolute())
105            src_file.add_comments(src_comments)
106            self.src_files.append(src_file)
107            self.src_comments.extend(src_comments)
108
109    def _flags_map_for(self, db_path: Path) -> dict[Path, list[str]] | None:
110        """Return the parsed compile_commands.json for ``db_path``, memoized.
111
112        The database is read and parsed once per run instead of once per source
113        file (O(files x entries) -> O(entries)). A present-but-malformed or
114        unreadable database is warned once and cached as ``None`` so callers fall
115        back to the configured defines without re-reading or re-warning per file.
116        """
117        from sphinx_codelinks.analyse.preproc import compile_db
118
119        if db_path in self._flags_map_cache:
120            return self._flags_map_cache[db_path]
121        try:
122            flags = compile_db.load_flags_map(db_path)
123        except (OSError, ValueError, TypeError) as exc:
124            logger.warning(
125                f"codelinks: failed to read {db_path} ({exc}); "
126                f"falling back to the configured defines"
127            )
128            self._flags_map_cache[db_path] = None
129            return None
130        self._flags_map_cache[db_path] = flags
131        return flags
132
133    def _resolve_preproc_args(self, src_path: Path) -> list[str] | None:
134        from sphinx_codelinks.analyse.preproc import compile_db
135
136        # `run()` calls this (via create_src_objects_libclang) only when
137        # `preprocessor is not None`, but keep the guard: it narrows the type for
138        # the checker and is a cheap defense (an `assert` would trip bandit S101).
139        preproc = self.analyse_config.preprocessor
140        if preproc is None:
141            return []
142        db_path = preproc.compile_commands
143        if db_path is None:
144            db_path = compile_db.find_compile_db(src_path, self.project_path)
145        if db_path is not None and db_path.is_file():
146            flags = self._flags_map_for(db_path)
147            if flags is None:
148                # Present-but-malformed/unreadable DB (warned once in the helper):
149                # fall back to the configured defines so extraction still runs.
150                return compile_db.defines_to_args(
151                    preproc.defines, preproc.includes, preproc.std
152                )
153            args = flags.get(src_path.absolute().resolve())
154            if args is not None:
155                return args
156            # Absent from the DB. compile_commands.json lists only compiled
157            # translation units, never headers — so a header here is parsed
158            # standalone with the global defines (one run = one variant). A
159            # compiled source absent from the build is skipped (spec §3.3).
160            if compile_db.is_translation_unit_source(src_path):
161                return None
162            return compile_db.defines_to_args(
163                preproc.defines, preproc.includes, preproc.std
164            )
165        # No readable DB. `find_compile_db` only ever returns a real file, so a
166        # non-None `db_path` that reaches here is an explicit `compile_commands`
167        # path that is not a readable file (typo/missing): warn and fall back
168        # rather than skip silently. A genuinely absent DB (db_path is None) just
169        # falls back to the manual defines applied globally.
170        if db_path is not None:
171            logger.warning(
172                f"codelinks: compile_commands path {db_path} is not a readable "
173                f"file; falling back to the configured defines"
174            )
175        return compile_db.defines_to_args(
176            preproc.defines, preproc.includes, preproc.std
177        )
178
[docs]179    # @Extract traceability objects with the preprocessor-aware libclang engine, IMPL_PREPROC_1, impl, [FE_PREPROC]
180    def create_src_objects_libclang(self) -> None:
181        from sphinx_codelinks.analyse.preproc import (
182            libclang_parser,
183            loader,
184        )
185
186        # Resolve the exception via the loader (never a direct ``import
187        # clang.cindex``) so a missing ``libclang`` extra still surfaces the
188        # loader's friendly install hint rather than a bare ImportError.
189        translation_unit_load_error = (
190            loader.load_clang_cindex().TranslationUnitLoadError
191        )
192
193        for src_path in self.analyse_config.src_files:
194            if not utils.is_text_file(src_path):
195                continue
196            args = self._resolve_preproc_args(src_path)
197            if args is None:
198                logger.debug(
199                    f"codelinks: skipping {src_path} — not found in compile_commands.json"
200                )
201                continue
202            try:
203                comments = libclang_parser.extract_active_comments(src_path, args)
204            except translation_unit_load_error:
205                # Last-resort guard. Standalone parses pin ``-x`` to match the
206                # ``-std`` (see defines_to_args), so the common case — a ``.h``/
207                # ``.c`` header handed a C++ ``-std`` — now parses as C++ and its
208                # markers extract. This only fires if libclang still cannot load
209                # the file as a translation unit at all; skip it rather than
210                # aborting the whole run. Surfaced as a ``warning`` so the silent
211                # data-loss (a file's markers dropped) is visible — accepting that
212                # this fails ``sphinx-build -W``.
213                logger.warning(
214                    f"codelinks: skipping {src_path} — libclang could not load it "
215                    f"as a translation unit"
216                )
217                continue
218            if not comments:
219                continue
220            # ``c`` is a LibclangComment duck-typing the tree-sitter Node
221            # interface SourceComment reads (``.text`` / ``.start_point.row``);
222            # the Node-only path (find_associated_scope) is guarded by
223            # ``is_libclang`` so it never runs on these.
224            src_comments = [SourceComment(cast("TreeSitterNode", c)) for c in comments]
225            src_file = SourceFile(src_path.absolute())
226            src_file.add_comments(src_comments)
227            self.src_files.append(src_file)
228            self.src_comments.extend(src_comments)
229
230    def extract_marker(
231        self,
232        text: str,
233    ) -> Generator[tuple[str, list[str], int, int, int], None, None]:
234        lines = text.splitlines()
235        row_offset = 0
236        for line in lines:
237            for marker in self.analyse_config.need_id_refs_config.markers:
238                marker_idx = line.find(marker)
239                if marker_idx == -1:
240                    continue
241                markered_text = line[marker_idx + len(marker) :].strip()
242                need_ids = markered_text.replace(",", " ").split()
243                start_column = marker_idx + len(marker)
244                end_column = start_column + len(markered_text)
245                yield marker, need_ids, row_offset, start_column, end_column
246            row_offset += 1
247
[docs]248    # @Extract need ID references from code comments, IMPL_LNK_1, impl, [FE_LNK]
249    def extract_anchors(
250        self,
251        text: str,
252        filepath: Path,
253        tagged_scope: TreeSitterNode | None,
254        src_comment: SourceComment,
255    ) -> list[NeedIdRefs]:
256        """Extract need-ids-refs from a comment."""
257        anchors: list[NeedIdRefs] = []
258        for (
259            marker,
260            need_ids,
261            row_offset,
262            start_column,
263            end_column,
264        ) in self.extract_marker(text):
265            lineno = src_comment.node.start_point.row + row_offset + 1
266            remote_url = self.git_remote_url
267            if self.git_remote_url and self.git_commit_rev:
268                remote_url = utils.form_https_url(
269                    self.git_remote_url,
270                    self.git_commit_rev,
271                    self.project_path,
272                    filepath,
273                    lineno,
274                )
275            source_map: SourceMap = {
276                "start": {
277                    "row": lineno - 1,
278                    "column": start_column,
279                },
280                "end": {
281                    "row": lineno - 1,
282                    "column": end_column,
283                },
284            }
285            anchors.append(
286                NeedIdRefs(
287                    filepath,
288                    remote_url,
289                    source_map,
290                    src_comment,
291                    tagged_scope,
292                    need_ids,
293                    marker,
294                )
295            )
296        return anchors
297
298    def _is_need_id_refs_line(self, line: str) -> bool:
299        """Whether a comment line is an ``@need-ids:`` reference rather than a need.
300
301        A line whose text, after comment decoration and whitespace, starts with a
302        configured need-id-refs marker is a reference, and never a one-line need --
303        ubCode's precedence. Without it the default one-line start sequence ``@`` also
304        matched ``// @need-ids: A, B`` and made a need titled ``need-ids: A`` with the id
305        ``B``. A line with anything alphanumeric before the marker is not a reference
306        line here (``// see @need-ids: X`` -- the one-line parser ignores it anyway, and
307        the references on it are still extracted), and a line holding a one-line need
308        before the marker (``[[…]] @need-ids: X``) keeps its need: only a line that
309        STARTS with the marker is withheld from the one-line parser.
310        """
311        for marker in self.analyse_config.need_id_refs_config.markers:
312            marker_idx = line.find(marker)
313            if marker_idx != -1 and not any(
314                char.isalnum() for char in line[:marker_idx]
315            ):
316                return True
317        return False
318
319    def extract_oneline_need(
320        self,
321        text: str,
322        src_comment: SourceComment,
323        oneline_comment_style: OneLineCommentStyle,
324    ) -> Generator[tuple[dict[str, str | list[str] | int], int]]:
325        lines = text.splitlines(keepends=True)
326        row_offset = 0
327        if len(lines) == 1:
328            # single line comment has no newline char in the extracted comment
329            lines[0] = f"{lines[0]}{UNIX_NEWLINE}"
330
331        for line in lines:
332            if self._is_need_id_refs_line(line):
333                row_offset += 1
334                continue
335            resolved = oneline_parser(line, oneline_comment_style)
336            if not resolved:
337                row_offset += 1
338                continue
339            if isinstance(resolved, OnelineParserInvalidWarning):
340                if not src_comment.source_file:
341                    row_offset += 1
342                    continue
343                lineno = src_comment.node.start_point.row + row_offset + 1
344                warning = AnalyseWarning(
345                    str(src_comment.source_file.filepath),
346                    lineno,
347                    resolved.msg,
348                    MarkedContentType.need,
349                    resolved.sub_type.value,
350                )
351                self.oneline_warnings.append(warning)
352                row_offset += 1
353                continue
354            yield resolved, row_offset
355            row_offset += 1
356
[docs]357    # @Extract one-line traceability needs from comments, IMPL_ONE_1, impl, [FE_DEF, FE_CMT]
358    def extract_oneline_needs(
359        self,
360        text: str,
361        filepath: Path,
362        tagged_scope: TreeSitterNode | None,
363        src_comment: SourceComment,
364        oneline_comment_style: OneLineCommentStyle,
365    ) -> list[OneLineNeed]:
366        row_offset = 0
367        oneline_needs = []
368        for resolved, row_offset in self.extract_oneline_need(
369            text, src_comment, oneline_comment_style
370        ):
371            lineno = src_comment.node.start_point.row + row_offset + 1
372            remote_url = self.git_remote_url
373            if self.git_remote_url and self.git_commit_rev:
374                remote_url = utils.form_https_url(
375                    self.git_remote_url,
376                    self.git_commit_rev,
377                    self.project_path,
378                    filepath,
379                    lineno,
380                )
381            source_map: SourceMap = {
382                "start": {
383                    "row": lineno - 1,
384                    "column": resolved[
385                        "start_column"
386                    ],  # dynamic keys  # ty: ignore[invalid-argument-type]
387                },
388                "end": {
389                    "row": lineno - 1,
390                    "column": resolved[
391                        "end_column"
392                    ],  # dynamic keys  # ty: ignore[invalid-argument-type]
393                },
394            }
395            del resolved["start_column"]
396            del resolved["end_column"]
397            oneline_needs.append(
398                OneLineNeed(
399                    filepath,
400                    remote_url,
401                    source_map,
402                    src_comment,
403                    tagged_scope,
404                    resolved,  # int arguments were deleted  # ty: ignore[invalid-argument-type]
405                )
406            )
407        return oneline_needs
408
[docs]409    # @Extract marked reStructuredText blocks from comments, IMPL_MRST_1, impl, [FE_RST_EXTRACTION]
410    def extract_marked_rst(
411        self,
412        text: str,
413        filepath: Path,
414        tagged_scope: TreeSitterNode | None,
415        src_comment: SourceComment,
416    ) -> MarkedRst | None:
417        """Extract marked rst from a comment.
418
419        Presumably, only one marked rst text in a comment.
420        """
421        extracted_rst = utils.extract_rst(
422            text,
423            self.analyse_config.marked_rst_config.start_sequence,
424            self.analyse_config.marked_rst_config.end_sequence,
425        )
426        if not extracted_rst:
427            return None
428        if UNIX_NEWLINE in extracted_rst["rst_text"]:
429            rst_text = utils.remove_leading_sequences(extracted_rst["rst_text"], ["*"])
430        else:
431            rst_text = extracted_rst["rst_text"]
432        lineno = src_comment.node.start_point.row + extracted_rst["row_offset"] + 1
433        remote_url = self.git_remote_url
434        if self.git_remote_url and self.git_commit_rev:
435            remote_url = utils.form_https_url(
436                self.git_remote_url,
437                self.git_commit_rev,
438                self.project_path,
439                filepath,
440                lineno,
441            )
442        source_map: SourceMap = {
443            "start": {
444                "row": lineno - 1,
445                "column": extracted_rst["start_idx"],
446            },
447            "end": {
448                "row": lineno - 1,
449                "column": extracted_rst["end_idx"],
450            },
451        }
452        return MarkedRst(
453            filepath,
454            remote_url,
455            source_map,
456            src_comment,
457            tagged_scope,
458            rst_text,
459        )
460
461    def extract_marked_content(self) -> None:
462        for src_comment in self.src_comments:
463            text = (
464                src_comment.node.text.decode("utf-8") if src_comment.node.text else None
465            )
466            if not text:
467                continue
468            filepath = (
469                src_comment.source_file.filepath if src_comment.source_file else None
470            )
471            if not filepath:
472                continue
473            if getattr(src_comment.node, "is_libclang", False):
474                tagged_scope: TreeSitterNode | None = None
475            else:
476                tagged_scope = utils.find_associated_scope(
477                    src_comment.node, self.analyse_config.comment_type
478                )
479            if self.analyse_config.get_need_id_refs:
480                anchors = self.extract_anchors(
481                    text, filepath, tagged_scope, src_comment
482                )
483                self.need_id_refs.extend(anchors)
484
485            if self.analyse_config.get_oneline_needs:
486                oneline_needs = self.extract_oneline_needs(
487                    text,
488                    filepath,
489                    tagged_scope,
490                    src_comment,
491                    self.analyse_config.oneline_comment_style,
492                )
493                self.oneline_needs.extend(oneline_needs)
494            if self.analyse_config.get_rst:
495                marked_rst = self.extract_marked_rst(
496                    text, filepath, tagged_scope, src_comment
497                )
498                if marked_rst:
499                    self.marked_rst.append(marked_rst)
500
501    def merge_marked_content(self) -> None:
502        self.all_marked_content.extend(self.need_id_refs)
503        self.oneline_needs.sort(key=lambda x: x.source_map["start"]["row"])
504        self.all_marked_content.extend(self.oneline_needs)
505        self.all_marked_content.extend(self.marked_rst)
506        self.all_marked_content.sort(
507            key=lambda x: (x.filepath, x.source_map["start"]["row"])
508        )
509
510    def dump_marked_content(self, outdir: Path) -> None:
511        output_path = outdir / "marked_content.json"
512        if not output_path.parent.exists():
513            output_path.parent.mkdir(parents=True)
514        to_dump = [
515            marked_content.to_dict() for marked_content in self.all_marked_content
516        ]
517        with output_path.open("w") as f:
518            json.dump(to_dump, f)
519
520    def run(self, *, log_summary: bool = True) -> None:
521        """Parse the source files and extract their markers.
522
523        :param log_summary: Whether to print the per-project summary line; a caller
524            that prints its own (the configuration pass of the Sphinx extension) passes
525            ``False``.
526        """
527        if (
528            self.analyse_config.preprocessor is not None
529            and self.analyse_config.comment_type == CommentType.cpp
530        ):
531            self.create_src_objects_libclang()
532        else:
533            self.create_src_objects()
534        self.extract_marked_content()
535        self.merge_marked_content()
536        if log_summary:
537            self.log_summary()
538
539    def log_summary(self, extra: str = "") -> None:
540        """Emit a per-project marker (default-visible) plus a -v breakdown.
541
542        :param extra: Appended to the default-visible line (the ``src-trace``
543            directive's count of the one-line needs it did not create).
544        """
545        label = f"codelinks [{self.name}]" if self.name else "codelinks"
546        logger.info(
547            f"{label}: {_count(len(self.src_files), 'file')}, "
548            f"{_count(len(self.all_marked_content), 'marker')}{extra}"
549        )
550        logger.debug(
551            f"{label}: {_count(len(self.src_comments), 'comment')}, "
552            f"{_count(len(self.oneline_needs), 'oneline need')}, "
553            f"{_count(len(self.need_id_refs), 'id-ref')}, "
554            f"{_count(len(self.marked_rst), 'marked-rst block')}"
555        )