1import json
2from collections.abc import Generator
3from dataclasses import dataclass
4from pathlib import Path
5from typing import Any, cast
6
7from tree_sitter import Node as TreeSitterNode
8
9from sphinx_codelinks.analyse import utils
10from sphinx_codelinks.analyse.models import (
11 MarkedContentType,
12 MarkedRst,
13 NeedIdRefs,
14 OneLineNeed,
15 SourceComment,
16 SourceFile,
17 SourceMap,
18)
19from sphinx_codelinks.analyse.oneline_parser import (
20 OnelineParserInvalidWarning,
21 oneline_parser,
22)
23from sphinx_codelinks.config import (
24 UNIX_NEWLINE,
25 OneLineCommentStyle,
26 SourceAnalyseConfig,
27)
28from sphinx_codelinks.logger import get_logger
29from sphinx_codelinks.source_discover.config import CommentType
30
31logger = get_logger(__name__)
32
33
34def _count(n: int, noun: str) -> str:
35 """Format ``n noun`` with a naive (append-s) plural for progress summaries."""
36 return f"{n} {noun}" if n == 1 else f"{n} {noun}s"
37
38
39@dataclass
40class AnalyseWarning:
41 file_path: str
42 lineno: int
43 msg: str
44 type: str
45 sub_type: str
46
47
48class SourceAnalyse:
49 def __init__(
50 self,
51 analyse_config: SourceAnalyseConfig,
52 *,
53 name: str = "",
54 ) -> None:
55 self.name = name
56 self.analyse_config = analyse_config
57 self.src_files: list[SourceFile] = []
58 self.src_comments: list[SourceComment] = []
59 self.need_id_refs: list[NeedIdRefs] = []
60 self.oneline_needs: list[OneLineNeed] = []
61 self.marked_rst: list[MarkedRst] = []
62 self.all_marked_content: list[NeedIdRefs | OneLineNeed | MarkedRst] = []
63 # Use explicitly configured git_root if provided, otherwise auto-detect
64 if self.analyse_config.git_root is not None:
65 self.git_root: Path | None = self.analyse_config.git_root.resolve()
66 else:
67 self.git_root = utils.locate_git_root(self.analyse_config.src_dir)
68 self.git_remote_url: str | None = (
69 utils.get_remote_url(self.git_root) if self.git_root else None
70 )
71 self.git_commit_rev: str | None = (
72 utils.get_current_rev(self.git_root) if self.git_root else None
73 )
74 self.project_path: Path = self.git_root or self.analyse_config.src_dir
75 self.oneline_warnings: list[AnalyseWarning] = []
76 # Per-run memo of parsed compile_commands.json, keyed by DB path, so the
77 # database is read once per run instead of once per source file.
78 self._flags_map_cache: dict[Path, dict[Path, list[str]] | None] = {}
79
80 def get_src_strings(self) -> Generator[tuple[Path, bytes], Any, None]:
81 """Load source files and extract their content."""
82 for src_path in self.analyse_config.src_files:
83 if not utils.is_text_file(src_path):
84 continue
85 with src_path.open("r", encoding="utf-8", newline="") as f:
86 # Normalize all line endings to Unix LF
87 text = f.read()
88 text = text.replace("\r\n", "\n").replace("\r", "\n")
89 yield src_path, text.encode("utf-8")
90
91 def create_src_objects(self) -> None:
92 parser, query = utils.init_tree_sitter(self.analyse_config.comment_type)
93
94 for src_path, src_string in self.get_src_strings():
95 comments: list[TreeSitterNode] | None = utils.extract_comments(
96 src_string, parser, query
97 )
98 if not comments:
99 continue
100 src_comments: list[SourceComment] = [
101 SourceComment(node) for node in comments
102 ]
103
104 src_file = SourceFile(src_path.absolute())
105 src_file.add_comments(src_comments)
106 self.src_files.append(src_file)
107 self.src_comments.extend(src_comments)
108
109 def _flags_map_for(self, db_path: Path) -> dict[Path, list[str]] | None:
110 """Return the parsed compile_commands.json for ``db_path``, memoized.
111
112 The database is read and parsed once per run instead of once per source
113 file (O(files x entries) -> O(entries)). A present-but-malformed or
114 unreadable database is warned once and cached as ``None`` so callers fall
115 back to the configured defines without re-reading or re-warning per file.
116 """
117 from sphinx_codelinks.analyse.preproc import compile_db
118
119 if db_path in self._flags_map_cache:
120 return self._flags_map_cache[db_path]
121 try:
122 flags = compile_db.load_flags_map(db_path)
123 except (OSError, ValueError, TypeError) as exc:
124 logger.warning(
125 f"codelinks: failed to read {db_path} ({exc}); "
126 f"falling back to the configured defines"
127 )
128 self._flags_map_cache[db_path] = None
129 return None
130 self._flags_map_cache[db_path] = flags
131 return flags
132
133 def _resolve_preproc_args(self, src_path: Path) -> list[str] | None:
134 from sphinx_codelinks.analyse.preproc import compile_db
135
136 # `run()` calls this (via create_src_objects_libclang) only when
137 # `preprocessor is not None`, but keep the guard: it narrows the type for
138 # the checker and is a cheap defense (an `assert` would trip bandit S101).
139 preproc = self.analyse_config.preprocessor
140 if preproc is None:
141 return []
142 db_path = preproc.compile_commands
143 if db_path is None:
144 db_path = compile_db.find_compile_db(src_path, self.project_path)
145 if db_path is not None and db_path.is_file():
146 flags = self._flags_map_for(db_path)
147 if flags is None:
148 # Present-but-malformed/unreadable DB (warned once in the helper):
149 # fall back to the configured defines so extraction still runs.
150 return compile_db.defines_to_args(
151 preproc.defines, preproc.includes, preproc.std
152 )
153 args = flags.get(src_path.absolute().resolve())
154 if args is not None:
155 return args
156 # Absent from the DB. compile_commands.json lists only compiled
157 # translation units, never headers — so a header here is parsed
158 # standalone with the global defines (one run = one variant). A
159 # compiled source absent from the build is skipped (spec §3.3).
160 if compile_db.is_translation_unit_source(src_path):
161 return None
162 return compile_db.defines_to_args(
163 preproc.defines, preproc.includes, preproc.std
164 )
165 # No readable DB. `find_compile_db` only ever returns a real file, so a
166 # non-None `db_path` that reaches here is an explicit `compile_commands`
167 # path that is not a readable file (typo/missing): warn and fall back
168 # rather than skip silently. A genuinely absent DB (db_path is None) just
169 # falls back to the manual defines applied globally.
170 if db_path is not None:
171 logger.warning(
172 f"codelinks: compile_commands path {db_path} is not a readable "
173 f"file; falling back to the configured defines"
174 )
175 return compile_db.defines_to_args(
176 preproc.defines, preproc.includes, preproc.std
177 )
178
[docs]179 # @Extract traceability objects with the preprocessor-aware libclang engine, IMPL_PREPROC_1, impl, [FE_PREPROC]
180 def create_src_objects_libclang(self) -> None:
181 from sphinx_codelinks.analyse.preproc import (
182 libclang_parser,
183 loader,
184 )
185
186 # Resolve the exception via the loader (never a direct ``import
187 # clang.cindex``) so a missing ``libclang`` extra still surfaces the
188 # loader's friendly install hint rather than a bare ImportError.
189 translation_unit_load_error = (
190 loader.load_clang_cindex().TranslationUnitLoadError
191 )
192
193 for src_path in self.analyse_config.src_files:
194 if not utils.is_text_file(src_path):
195 continue
196 args = self._resolve_preproc_args(src_path)
197 if args is None:
198 logger.debug(
199 f"codelinks: skipping {src_path} — not found in compile_commands.json"
200 )
201 continue
202 try:
203 comments = libclang_parser.extract_active_comments(src_path, args)
204 except translation_unit_load_error:
205 # Last-resort guard. Standalone parses pin ``-x`` to match the
206 # ``-std`` (see defines_to_args), so the common case — a ``.h``/
207 # ``.c`` header handed a C++ ``-std`` — now parses as C++ and its
208 # markers extract. This only fires if libclang still cannot load
209 # the file as a translation unit at all; skip it rather than
210 # aborting the whole run. Surfaced as a ``warning`` so the silent
211 # data-loss (a file's markers dropped) is visible — accepting that
212 # this fails ``sphinx-build -W``.
213 logger.warning(
214 f"codelinks: skipping {src_path} — libclang could not load it "
215 f"as a translation unit"
216 )
217 continue
218 if not comments:
219 continue
220 # ``c`` is a LibclangComment duck-typing the tree-sitter Node
221 # interface SourceComment reads (``.text`` / ``.start_point.row``);
222 # the Node-only path (find_associated_scope) is guarded by
223 # ``is_libclang`` so it never runs on these.
224 src_comments = [SourceComment(cast("TreeSitterNode", c)) for c in comments]
225 src_file = SourceFile(src_path.absolute())
226 src_file.add_comments(src_comments)
227 self.src_files.append(src_file)
228 self.src_comments.extend(src_comments)
229
230 def extract_marker(
231 self,
232 text: str,
233 ) -> Generator[tuple[str, list[str], int, int, int], None, None]:
234 lines = text.splitlines()
235 row_offset = 0
236 for line in lines:
237 for marker in self.analyse_config.need_id_refs_config.markers:
238 marker_idx = line.find(marker)
239 if marker_idx == -1:
240 continue
241 markered_text = line[marker_idx + len(marker) :].strip()
242 need_ids = markered_text.replace(",", " ").split()
243 start_column = marker_idx + len(marker)
244 end_column = start_column + len(markered_text)
245 yield marker, need_ids, row_offset, start_column, end_column
246 row_offset += 1
247
[docs]248 # @Extract need ID references from code comments, IMPL_LNK_1, impl, [FE_LNK]
249 def extract_anchors(
250 self,
251 text: str,
252 filepath: Path,
253 tagged_scope: TreeSitterNode | None,
254 src_comment: SourceComment,
255 ) -> list[NeedIdRefs]:
256 """Extract need-ids-refs from a comment."""
257 anchors: list[NeedIdRefs] = []
258 for (
259 marker,
260 need_ids,
261 row_offset,
262 start_column,
263 end_column,
264 ) in self.extract_marker(text):
265 lineno = src_comment.node.start_point.row + row_offset + 1
266 remote_url = self.git_remote_url
267 if self.git_remote_url and self.git_commit_rev:
268 remote_url = utils.form_https_url(
269 self.git_remote_url,
270 self.git_commit_rev,
271 self.project_path,
272 filepath,
273 lineno,
274 )
275 source_map: SourceMap = {
276 "start": {
277 "row": lineno - 1,
278 "column": start_column,
279 },
280 "end": {
281 "row": lineno - 1,
282 "column": end_column,
283 },
284 }
285 anchors.append(
286 NeedIdRefs(
287 filepath,
288 remote_url,
289 source_map,
290 src_comment,
291 tagged_scope,
292 need_ids,
293 marker,
294 )
295 )
296 return anchors
297
298 def _is_need_id_refs_line(self, line: str) -> bool:
299 """Whether a comment line is an ``@need-ids:`` reference rather than a need.
300
301 A line whose text, after comment decoration and whitespace, starts with a
302 configured need-id-refs marker is a reference, and never a one-line need --
303 ubCode's precedence. Without it the default one-line start sequence ``@`` also
304 matched ``// @need-ids: A, B`` and made a need titled ``need-ids: A`` with the id
305 ``B``. A line with anything alphanumeric before the marker is not a reference
306 line here (``// see @need-ids: X`` -- the one-line parser ignores it anyway, and
307 the references on it are still extracted), and a line holding a one-line need
308 before the marker (``[[…]] @need-ids: X``) keeps its need: only a line that
309 STARTS with the marker is withheld from the one-line parser.
310 """
311 for marker in self.analyse_config.need_id_refs_config.markers:
312 marker_idx = line.find(marker)
313 if marker_idx != -1 and not any(
314 char.isalnum() for char in line[:marker_idx]
315 ):
316 return True
317 return False
318
319 def extract_oneline_need(
320 self,
321 text: str,
322 src_comment: SourceComment,
323 oneline_comment_style: OneLineCommentStyle,
324 ) -> Generator[tuple[dict[str, str | list[str] | int], int]]:
325 lines = text.splitlines(keepends=True)
326 row_offset = 0
327 if len(lines) == 1:
328 # single line comment has no newline char in the extracted comment
329 lines[0] = f"{lines[0]}{UNIX_NEWLINE}"
330
331 for line in lines:
332 if self._is_need_id_refs_line(line):
333 row_offset += 1
334 continue
335 resolved = oneline_parser(line, oneline_comment_style)
336 if not resolved:
337 row_offset += 1
338 continue
339 if isinstance(resolved, OnelineParserInvalidWarning):
340 if not src_comment.source_file:
341 row_offset += 1
342 continue
343 lineno = src_comment.node.start_point.row + row_offset + 1
344 warning = AnalyseWarning(
345 str(src_comment.source_file.filepath),
346 lineno,
347 resolved.msg,
348 MarkedContentType.need,
349 resolved.sub_type.value,
350 )
351 self.oneline_warnings.append(warning)
352 row_offset += 1
353 continue
354 yield resolved, row_offset
355 row_offset += 1
356
[docs]357 # @Extract one-line traceability needs from comments, IMPL_ONE_1, impl, [FE_DEF, FE_CMT]
358 def extract_oneline_needs(
359 self,
360 text: str,
361 filepath: Path,
362 tagged_scope: TreeSitterNode | None,
363 src_comment: SourceComment,
364 oneline_comment_style: OneLineCommentStyle,
365 ) -> list[OneLineNeed]:
366 row_offset = 0
367 oneline_needs = []
368 for resolved, row_offset in self.extract_oneline_need(
369 text, src_comment, oneline_comment_style
370 ):
371 lineno = src_comment.node.start_point.row + row_offset + 1
372 remote_url = self.git_remote_url
373 if self.git_remote_url and self.git_commit_rev:
374 remote_url = utils.form_https_url(
375 self.git_remote_url,
376 self.git_commit_rev,
377 self.project_path,
378 filepath,
379 lineno,
380 )
381 source_map: SourceMap = {
382 "start": {
383 "row": lineno - 1,
384 "column": resolved[
385 "start_column"
386 ], # dynamic keys # ty: ignore[invalid-argument-type]
387 },
388 "end": {
389 "row": lineno - 1,
390 "column": resolved[
391 "end_column"
392 ], # dynamic keys # ty: ignore[invalid-argument-type]
393 },
394 }
395 del resolved["start_column"]
396 del resolved["end_column"]
397 oneline_needs.append(
398 OneLineNeed(
399 filepath,
400 remote_url,
401 source_map,
402 src_comment,
403 tagged_scope,
404 resolved, # int arguments were deleted # ty: ignore[invalid-argument-type]
405 )
406 )
407 return oneline_needs
408
[docs]409 # @Extract marked reStructuredText blocks from comments, IMPL_MRST_1, impl, [FE_RST_EXTRACTION]
410 def extract_marked_rst(
411 self,
412 text: str,
413 filepath: Path,
414 tagged_scope: TreeSitterNode | None,
415 src_comment: SourceComment,
416 ) -> MarkedRst | None:
417 """Extract marked rst from a comment.
418
419 Presumably, only one marked rst text in a comment.
420 """
421 extracted_rst = utils.extract_rst(
422 text,
423 self.analyse_config.marked_rst_config.start_sequence,
424 self.analyse_config.marked_rst_config.end_sequence,
425 )
426 if not extracted_rst:
427 return None
428 if UNIX_NEWLINE in extracted_rst["rst_text"]:
429 rst_text = utils.remove_leading_sequences(extracted_rst["rst_text"], ["*"])
430 else:
431 rst_text = extracted_rst["rst_text"]
432 lineno = src_comment.node.start_point.row + extracted_rst["row_offset"] + 1
433 remote_url = self.git_remote_url
434 if self.git_remote_url and self.git_commit_rev:
435 remote_url = utils.form_https_url(
436 self.git_remote_url,
437 self.git_commit_rev,
438 self.project_path,
439 filepath,
440 lineno,
441 )
442 source_map: SourceMap = {
443 "start": {
444 "row": lineno - 1,
445 "column": extracted_rst["start_idx"],
446 },
447 "end": {
448 "row": lineno - 1,
449 "column": extracted_rst["end_idx"],
450 },
451 }
452 return MarkedRst(
453 filepath,
454 remote_url,
455 source_map,
456 src_comment,
457 tagged_scope,
458 rst_text,
459 )
460
461 def extract_marked_content(self) -> None:
462 for src_comment in self.src_comments:
463 text = (
464 src_comment.node.text.decode("utf-8") if src_comment.node.text else None
465 )
466 if not text:
467 continue
468 filepath = (
469 src_comment.source_file.filepath if src_comment.source_file else None
470 )
471 if not filepath:
472 continue
473 if getattr(src_comment.node, "is_libclang", False):
474 tagged_scope: TreeSitterNode | None = None
475 else:
476 tagged_scope = utils.find_associated_scope(
477 src_comment.node, self.analyse_config.comment_type
478 )
479 if self.analyse_config.get_need_id_refs:
480 anchors = self.extract_anchors(
481 text, filepath, tagged_scope, src_comment
482 )
483 self.need_id_refs.extend(anchors)
484
485 if self.analyse_config.get_oneline_needs:
486 oneline_needs = self.extract_oneline_needs(
487 text,
488 filepath,
489 tagged_scope,
490 src_comment,
491 self.analyse_config.oneline_comment_style,
492 )
493 self.oneline_needs.extend(oneline_needs)
494 if self.analyse_config.get_rst:
495 marked_rst = self.extract_marked_rst(
496 text, filepath, tagged_scope, src_comment
497 )
498 if marked_rst:
499 self.marked_rst.append(marked_rst)
500
501 def merge_marked_content(self) -> None:
502 self.all_marked_content.extend(self.need_id_refs)
503 self.oneline_needs.sort(key=lambda x: x.source_map["start"]["row"])
504 self.all_marked_content.extend(self.oneline_needs)
505 self.all_marked_content.extend(self.marked_rst)
506 self.all_marked_content.sort(
507 key=lambda x: (x.filepath, x.source_map["start"]["row"])
508 )
509
510 def dump_marked_content(self, outdir: Path) -> None:
511 output_path = outdir / "marked_content.json"
512 if not output_path.parent.exists():
513 output_path.parent.mkdir(parents=True)
514 to_dump = [
515 marked_content.to_dict() for marked_content in self.all_marked_content
516 ]
517 with output_path.open("w") as f:
518 json.dump(to_dump, f)
519
520 def run(self, *, log_summary: bool = True) -> None:
521 """Parse the source files and extract their markers.
522
523 :param log_summary: Whether to print the per-project summary line; a caller
524 that prints its own (the configuration pass of the Sphinx extension) passes
525 ``False``.
526 """
527 if (
528 self.analyse_config.preprocessor is not None
529 and self.analyse_config.comment_type == CommentType.cpp
530 ):
531 self.create_src_objects_libclang()
532 else:
533 self.create_src_objects()
534 self.extract_marked_content()
535 self.merge_marked_content()
536 if log_summary:
537 self.log_summary()
538
539 def log_summary(self, extra: str = "") -> None:
540 """Emit a per-project marker (default-visible) plus a -v breakdown.
541
542 :param extra: Appended to the default-visible line (the ``src-trace``
543 directive's count of the one-line needs it did not create).
544 """
545 label = f"codelinks [{self.name}]" if self.name else "codelinks"
546 logger.info(
547 f"{label}: {_count(len(self.src_files), 'file')}, "
548 f"{_count(len(self.all_marked_content), 'marker')}{extra}"
549 )
550 logger.debug(
551 f"{label}: {_count(len(self.src_comments), 'comment')}, "
552 f"{_count(len(self.oneline_needs), 'oneline need')}, "
553 f"{_count(len(self.need_id_refs), 'id-ref')}, "
554 f"{_count(len(self.marked_rst), 'marked-rst block')}"
555 )