1import configparser
2from collections.abc import ByteString, Callable
3from pathlib import Path
4from typing import TypedDict
5from urllib.request import pathname2url
6
7from giturlparse import parse
8from tree_sitter import Language, Parser, Point, Query, QueryCursor
9from tree_sitter import Node as TreeSitterNode
10
11from sphinx_codelinks.config import UNIX_NEWLINE, CommentCategory
12from sphinx_codelinks.logger import get_logger
13from sphinx_codelinks.source_discover.config import CommentType
14
15# Language-specific node types for scope detection.
16#
17# YAML and JSONC are intentionally absent. They are data formats, not code, so a
18# comment associates with the surrounding data structure (key/value pair, list
19# item, or scalar) rather than with an enclosing or following declaration. That
20# needs a different algorithm (inline same-row association first, scalar targets,
21# grammar-specific traversal), implemented in find_yaml_associated_structure and
22# find_jsonc_associated_structure and dispatched from find_associated_scope.
23# Those bespoke finders never read this table (only find_next_scope and
24# find_enclosing_scope do), so an entry here would be dead.
25SCOPE_NODE_TYPES = {
[docs] 26 # @Python Scope Node Types, IMPL_PY_2, impl, [FE_PY]
27 CommentType.python: {"function_definition", "class_definition"},
[docs] 28 # @C and C++ Scope Node Types, IMPL_C_2, impl, [FE_C_SUPPORT, FE_CPP]
29 CommentType.cpp: {"function_definition", "class_definition"},
30 CommentType.cs: {"method_declaration", "class_declaration", "property_declaration"},
[docs] 31 # @Rust Scope Node Types, IMPL_RUST_2, impl, [FE_RUST]
32 CommentType.rust: {
33 "function_item",
34 "struct_item",
35 "enum_item",
36 "impl_item",
37 "trait_item",
38 "mod_item",
39 },
[docs] 40 # @Go Scope Node Types, IMPL_GO_4, impl, [FE_GO]
41 CommentType.go: {
42 "function_declaration",
43 "method_declaration",
44 "type_declaration",
45 "type_spec",
46 },
[docs] 47 # @Bash Scope Node Types, IMPL_BASH_2, impl, [FE_BASH]
48 CommentType.bash: {"function_definition"},
49}
50
51logger = get_logger(__name__)
52
53GIT_HOST_URL_TEMPLATE = {
54 "github": "https://github.com/{owner}/{repo}/blob/{rev}/{path}#L{lineno}",
55 "gitlab": "https://gitlab.com/{owner}/{repo}/-/blob/{rev}/{path}#L{lineno}",
56}
57
58PYTHON_QUERY = """
59 ; Match comments
60 (comment) @comment
61
62 ; Match docstrings inside modules, functions, or classes
63 (module (expression_statement (string)) @comment)
64 (function_definition (block (expression_statement (string)) @comment))
65 (class_definition (block (expression_statement (string)) @comment))
66 """
67CPP_QUERY = """(comment) @comment"""
68C_SHARP_QUERY = """(comment) @comment"""
69YAML_QUERY = """(comment) @comment"""
70RUST_QUERY = """
71 (line_comment) @comment
72 (block_comment) @comment
73"""
[docs] 74# @Go comment query for tree-sitter, IMPL_GO_3, impl, [FE_GO]
75GO_QUERY = """
76 (comment) @comment
77"""
78JSONC_QUERY = """(comment) @comment"""
[docs] 79# @Bash comment query for tree-sitter, IMPL_BASH_3, impl, [FE_BASH]
80BASH_QUERY = """(comment) @comment"""
81
82# JSON value node types that can be associated with a comment.
83JSON_STRUCTURE_TYPES = {
84 "pair",
85 "object",
86 "array",
87 "string",
88 "number",
89 "true",
90 "false",
91 "null",
92}
93
94
95def is_text_file(filepath: Path, sample_size: int = 2048) -> bool:
96 """Return True if file is likely text, False if binary."""
97 try:
98 with filepath.open("rb") as f:
99 chunk = f.read(sample_size)
100 # Quick binary heuristic: null byte present
101 if b"\x00" in chunk:
102 return False
103 # Try UTF-8 decode on the sample
104 chunk.decode("utf-8")
105 return True
106 except UnicodeDecodeError:
107 return False
108
109
[docs]110# @Tree-sitter parser initialization for multiple languages, IMPL_LANG_1, impl, [FE_C_SUPPORT, FE_CPP, FE_PY, FE_YAML, FE_RUST, FE_GO, FE_JSONC, FE_BASH]
111def init_tree_sitter(comment_type: CommentType) -> tuple[Parser, Query]:
112 if comment_type == CommentType.cpp:
113 import tree_sitter_cpp
114
115 parsed_language = Language(tree_sitter_cpp.language())
116 query = Query(parsed_language, CPP_QUERY)
117 elif comment_type == CommentType.python:
118 import tree_sitter_python
119
120 parsed_language = Language(tree_sitter_python.language())
121 query = Query(parsed_language, PYTHON_QUERY)
122 elif comment_type == CommentType.cs:
123 import tree_sitter_c_sharp
124
125 parsed_language = Language(tree_sitter_c_sharp.language())
126 query = Query(parsed_language, C_SHARP_QUERY)
127 elif comment_type == CommentType.yaml:
128 import tree_sitter_yaml
129
130 parsed_language = Language(tree_sitter_yaml.language())
131 query = Query(parsed_language, YAML_QUERY)
132 elif comment_type == CommentType.rust:
133 import tree_sitter_rust
134
135 parsed_language = Language(tree_sitter_rust.language())
136 query = Query(parsed_language, RUST_QUERY)
137 elif comment_type == CommentType.go:
138 import tree_sitter_go
139
140 parsed_language = Language(tree_sitter_go.language())
141 query = Query(parsed_language, GO_QUERY)
142 elif comment_type == CommentType.jsonc:
143 import tree_sitter_json
144
145 parsed_language = Language(tree_sitter_json.language())
146 query = Query(parsed_language, JSONC_QUERY)
147 elif comment_type == CommentType.bash:
148 import tree_sitter_bash
149
150 parsed_language = Language(tree_sitter_bash.language())
151 query = Query(parsed_language, BASH_QUERY)
152 else:
153 raise ValueError(f"Unsupported comment style: {comment_type}")
154 parser = Parser(parsed_language)
155 return parser, query
156
157
158def wrap_read_callable_point(
159 src_string: ByteString,
160) -> Callable[[int, Point], ByteString]:
161 def read_callable_byte_offset(byte_offset: int, _: Point) -> ByteString:
162 return src_string[byte_offset : byte_offset + 1]
163
164 return read_callable_byte_offset
165
166
[docs]167# @Comment extraction from source code using tree-sitter, IMPL_EXTR_1, impl, [FE_DEF]
168def extract_comments(
169 src_string: ByteString, parser: Parser, query: Query
170) -> list[TreeSitterNode] | None:
171 """Get all comments from source files by tree-sitter."""
172 read_point_fn = wrap_read_callable_point(src_string)
173 tree = parser.parse(read_point_fn)
174 query_cursor = QueryCursor(query)
175 captures: dict[str, list[TreeSitterNode]] = query_cursor.captures(tree.root_node)
176
177 return captures.get("comment")
178
179
180def find_enclosing_scope(
181 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
182) -> TreeSitterNode | None:
183 """Find the enclosing scope of a comment."""
184 scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
185 current: TreeSitterNode = node
186 while current:
187 if current.type in scope_types:
188 return current
189 current: TreeSitterNode | None = current.parent # required for node traversal
190 return None
191
192
193def find_next_scope(
194 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
195) -> TreeSitterNode | None:
196 """Find the next scope of a comment."""
197 scope_types = SCOPE_NODE_TYPES.get(comment_type, SCOPE_NODE_TYPES[CommentType.cpp])
198 current: TreeSitterNode = node
199 while current:
200 if current.type in scope_types:
201 return current
202 current: TreeSitterNode | None = (
203 current.next_named_sibling
204 ) # required for node traversal
205 if current and current.type == "block":
206 for child in current.named_children:
207 if child.type in scope_types:
208 return child
209 return None
210
211
212def _find_yaml_structure_in_block_node(
213 block_node: TreeSitterNode,
214) -> TreeSitterNode | None:
215 """Find YAML structure elements within a block_node."""
216 for grandchild in block_node.named_children:
217 if grandchild.type == "block_mapping":
218 for ggchild in grandchild.named_children:
219 if ggchild.type == "block_mapping_pair":
220 return ggchild
221 elif grandchild.type == "block_sequence":
222 for ggchild in grandchild.named_children:
223 if ggchild.type == "block_sequence_item":
224 return ggchild
225 return None
226
227
228def find_yaml_next_structure(node: TreeSitterNode) -> TreeSitterNode | None:
229 """Find the next YAML structure element after the comment node."""
230 current = node.next_named_sibling
231 while current:
232 if current.type in {
233 "block_mapping_pair",
234 "block_sequence_item",
235 "flow_mapping",
236 "flow_sequence",
237 }:
238 return current
239 if current.type == "document":
240 for child in current.named_children:
241 if child.type == "block_node":
242 result = _find_yaml_structure_in_block_node(child)
243 if result:
244 return result
245 if current.type == "block_node":
246 result = _find_yaml_structure_in_block_node(current)
247 if result:
248 return result
249 current = current.next_named_sibling
250 return None
251
252
253def find_prev_sibling_on_same_row(node: TreeSitterNode) -> TreeSitterNode | None:
254 """Find a previous named sibling that is on the same row as the comment.
255
256 Grammar-agnostic: used to detect inline comments in both YAML and JSONC.
257 """
258 comment_row = node.start_point.row
259 current = node.prev_named_sibling
260
261 while current:
262 # Check if this sibling ends on the same row as the comment starts
263 # This indicates it's an inline comment
264 if current.end_point.row == comment_row:
265 return current
266 # If we find a sibling that ends before the comment row, we can stop
267 # as we won't find any siblings on the same row going backwards
268 if current.end_point.row < comment_row:
269 break
270 current = current.prev_named_sibling
271
272 return None
273
274
275def find_yaml_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
276 """Find the YAML structure (key-value pair, list item, etc.) associated with a comment."""
277 # First, check if this is an inline comment by looking for a previous sibling on the same row
278 prev_sibling_same_row = find_prev_sibling_on_same_row(node)
279 if prev_sibling_same_row:
280 return prev_sibling_same_row
281
282 # If no previous sibling on same row, try to find the next named sibling (structure after the comment)
283 structure = find_yaml_next_structure(node)
284 if structure:
285 return structure
286
287 # If no next sibling found, traverse up to find parent structure
288 parent = node.parent
289 while parent:
290 if parent.type in {"block_mapping_pair", "block_sequence_item"}:
291 return parent
292 parent = parent.parent
293
294 return None
295
296
[docs]297# @JSONC comment-to-structure association, IMPL_JSONC_2, impl, [FE_JSONC]
298def find_jsonc_associated_structure(node: TreeSitterNode) -> TreeSitterNode | None:
299 """Find the JSON structure (key/value pair, value, list item) for a comment.
300
301 JSON is data rather than code, so association follows the same intent as YAML:
302 an inline comment belongs to the value on its row, a leading comment belongs to
303 the following structure, otherwise it belongs to the enclosing structure.
304 """
305 # Inline comment: a value/pair on the same row, before the comment
306 prev_sibling_same_row = find_prev_sibling_on_same_row(node)
307 if prev_sibling_same_row:
308 return prev_sibling_same_row
309
310 # Leading comment: the next structure following the comment
311 current = node.next_named_sibling
312 while current:
313 if current.type in JSON_STRUCTURE_TYPES:
314 return current
315 current = current.next_named_sibling
316
317 # Otherwise: the enclosing structure
318 parent = node.parent
319 while parent:
320 if parent.type in {"pair", "object", "array"}:
321 return parent
322 parent = parent.parent
323
324 return None
325
326
327def find_associated_scope(
328 node: TreeSitterNode, comment_type: CommentType = CommentType.cpp
329) -> TreeSitterNode | None:
330 """Find the associated scope of a comment."""
331 if comment_type == CommentType.yaml:
332 # YAML uses different structure association logic
333 return find_yaml_associated_structure(node)
334
335 if comment_type == CommentType.jsonc:
336 # JSONC uses data-aware structure association logic
337 return find_jsonc_associated_structure(node)
338
339 if node.type == CommentCategory.docstring:
340 # Only for python's docstring
341 return find_enclosing_scope(node, comment_type)
342 # General comments regardless of comment types
343 associated_scope = find_next_scope(node, comment_type)
344 if not associated_scope:
345 associated_scope = find_enclosing_scope(node, comment_type)
346 return associated_scope
347
348
349def _git_dir(path: Path) -> Path | None:
350 """The git directory for a checkout rooted at ``path``, or None if it is not one.
351
352 In a plain checkout ``.git`` is a directory. In a LINKED WORKTREE it is a file
353 holding ``gitdir: <path>`` -- git's documented gitfile format -- and the real
354 directory is elsewhere, under the main repository's ``.git/worktrees/<name>``.
355 Treating the file as "not a repository" is what made every worktree of this
356 repository look unversioned (useblocks/sphinx-codelinks#106).
357 """
358 dot_git = path / ".git"
359 if dot_git.is_dir():
360 return dot_git
361 if dot_git.is_file():
362 content = dot_git.read_text(encoding="utf-8", errors="replace").strip()
363 if content.startswith("gitdir:"):
364 target = Path(content.split(":", 1)[1].strip())
365 if not target.is_absolute():
366 target = (path / target).resolve()
367 if target.is_dir():
368 return target
369 return None
370
371
372def _git_common_dir(git_dir: Path) -> Path:
373 """The repository-wide git directory that ``git_dir`` belongs to.
374
375 A linked worktree's git directory carries a ``commondir`` file naming the main
376 repository's, and that is where ``config`` and every shared ref live -- only
377 per-worktree state (``HEAD``, ``refs/bisect``, ``refs/worktree``) is local. A plain
378 checkout has no ``commondir`` and the two are the same directory.
379 """
380 commondir = git_dir / "commondir"
381 if not commondir.is_file():
382 return git_dir
383 target = Path(commondir.read_text(encoding="utf-8").strip())
384 if not target.is_absolute():
385 target = (git_dir / target).resolve()
386 return target
387
388
389def find_git_root(src_dir: Path) -> Path | None:
390 """The git root :func:`locate_git_root` finds, without its warning when none."""
391 current = src_dir.resolve()
392 parents = list(current.parents)
393 parents.append(current)
394 for parent in parents:
395 if _git_dir(parent) is not None:
396 return parent
397 return None
398
399
400def locate_git_root(src_dir: Path) -> Path | None:
401 """Traverse upwards to find git root."""
402 found = find_git_root(src_dir)
403 if found is not None:
404 return found
405 logger.warning(
406 f"git root is not found in the parent of {src_dir}",
407 subtype="git_root",
408 location=str(src_dir),
409 )
410 return None
411
412
413def get_remote_url(git_root: Path, remote_name: str = "origin") -> str | None:
414 """Get remote url from the git directory's config."""
415 git_dir = _git_dir(git_root)
416 # a worktree's remotes are the main repository's, so `config` is in the common dir
417 config_path = (
418 _git_common_dir(git_dir) if git_dir is not None else git_root / ".git"
419 ) / "config"
420 if not config_path.exists():
421 logger.warning(
422 f"{config_path} does not exist",
423 subtype="git_config",
424 location=str(config_path),
425 )
426 return None
427
428 config = configparser.ConfigParser(allow_no_value=True, strict=False)
429 config.read(config_path)
430 section = f'remote "{remote_name}"'
431 if section in config and "url" in config[section]:
432 url: str = config[section]["url"]
433 return url
434 logger.warning(
435 f"remote-url is not found in {config_path}",
436 subtype="git_remote",
437 location=str(config_path),
438 )
439 return None
440
441
442def _packed_ref(common_dir: Path, ref: str) -> str | None:
443 """The sha ``packed-refs`` records for ``ref``, or None.
444
445 A ref has no file of its own once ``git gc`` (or ``git pack-refs``) has run, which
446 happens unattended on any long-lived checkout -- so a loose-file lookup alone reports
447 "no rev" for a perfectly ordinary repository. The file is one ``<sha> <refname>`` per
448 line; ``^<sha>`` lines are the peeled targets of annotated tags and are skipped, as
449 is the leading ``#`` header.
450 """
451 packed = common_dir / "packed-refs"
452 if not packed.is_file():
453 return None
454 for line in packed.read_text(encoding="utf-8", errors="replace").splitlines():
455 if not line or line.startswith(("#", "^")):
456 continue
457 sha, _, name = line.partition(" ")
458 if name.strip() == ref:
459 return sha
460 return None
461
462
463def get_current_rev(git_root: Path) -> str | None:
464 """Get current commit rev from the git directory's HEAD."""
465 # HEAD is per-worktree, so it is read from the worktree's OWN git directory
466 git_dir = _git_dir(git_root) or git_root / ".git"
467 head_path = git_dir / "HEAD"
468 if not head_path.exists():
469 logger.warning(
470 f"{head_path} does not exist",
471 subtype="git_head",
472 location=str(head_path),
473 )
474 return None
475 head_content = head_path.read_text().strip()
476 if not head_content.startswith("ref: "):
477 # Detached HEAD (e.g. CI checkouts): .git/HEAD holds the commit SHA
478 # directly, which is exactly the rev we want.
479 return head_content
480
481 ref = head_content.split(":", 1)[1].strip()
482 common_dir = _git_common_dir(git_dir)
483 ref_path = git_dir / ref
484 if not ref_path.exists():
485 # every branch is a SHARED ref, so in a worktree it lives in the common
486 # directory; only refs/bisect and refs/worktree are per-worktree
487 ref_path = common_dir / ref
488 if ref_path.exists():
489 return ref_path.read_text().strip()
490 packed = _packed_ref(common_dir, ref)
491 if packed is not None:
492 return packed
493 logger.warning(
494 f"{ref_path} does not exist",
495 subtype="git_ref",
496 location=str(ref_path),
497 )
498 return None
499
500
501def form_https_url(
502 git_url: str, rev: str, project_path: Path, filepath: Path, lineno: int
503) -> str | None:
504 parsed_url = parse(git_url)
505 template = GIT_HOST_URL_TEMPLATE.get(parsed_url.platform)
506 if not template:
507 logger.warning(
508 f"Unsupported Git host: {parsed_url.platform}",
509 subtype="git_host",
510 )
511 return git_url
512 https_url = template.format(
513 owner=parsed_url.owner,
514 repo=parsed_url.repo,
515 rev=rev,
516 path=pathname2url(str(filepath.absolute().relative_to(project_path))),
517 lineno=str(lineno),
518 )
519 return https_url
520
521
522def remove_leading_sequences(text: str, leading_sequences: list[str]) -> str:
523 lines = text.splitlines(keepends=True)
524 no_comment_lines = []
525 for line in lines:
526 leading_sequence_exist = False
527 for leading_sequence in leading_sequences:
528 leading_sequence_idx = line.find(leading_sequence)
529 if leading_sequence_idx == -1:
530 continue
531 no_comment_lines.append(
532 line[leading_sequence_idx + len(leading_sequence) :]
533 )
534 leading_sequence_exist = True
535 break
536
537 if not leading_sequence_exist:
538 no_comment_lines.append(line)
539
540 return "".join(no_comment_lines)
541
542
543class ExtractedRstType(TypedDict):
544 rst_text: str
545 row_offset: int
546 start_idx: int
547 end_idx: int
548
549
[docs]550# @Extract reStructuredText blocks embedded in comments, IMPL_RST_1, impl, [FE_RST_EXTRACTION]
551def extract_rst(
552 text: str, start_marker: str, end_marker: str
553) -> ExtractedRstType | None:
554 """Extract rst from a comment.
555
556 Two use cases:
557 1. Start_marker and end_marker one the same line.
558
559 The rst text is wrapped by start and the end markers on the same line,
560 so, there is no need to remove the leading chars.ArithmeticError
561 E.g.
562 @rst .. admonition:: title here @endrst
563
564 2. Start_marker and end_marker in different lines.
565
566 The rst text is expected to start from the next line of the start_marker
567 and ends at he previous line of the end_marker.
568 E.g.
569 @rst
570 .. admonition:: title here
571 :collapsible: open
572
573 This example is collapsible, and initially open.
574 @endrst
575 """
576 start_idx = text.find(start_marker)
577 end_idx = text.rfind(end_marker)
578 if start_idx == -1 or end_idx == -1:
579 return None
580 rst_text = text[start_idx + len(start_marker) : end_idx]
581 # count newlines, not lines: text before a one-line block on the comment's first
582 # row holds no newline, yet splitlines() would count it as one line
583 row_offset = text[:start_idx].count(UNIX_NEWLINE)
584 if not rst_text.strip():
585 # empty string is out of the interest
586 return None
587 if UNIX_NEWLINE not in rst_text:
588 # single line rst text
589 oneline_rst: ExtractedRstType = {
590 "rst_text": rst_text,
591 "row_offset": row_offset,
592 "start_idx": start_idx + len(start_marker),
593 "end_idx": end_idx,
594 }
595 return oneline_rst
596
597 # multiline rst text
598
599 first_newline_idx = rst_text.find(UNIX_NEWLINE)
600 rst_text = rst_text[first_newline_idx + len(UNIX_NEWLINE) :]
601 multiline_rst: ExtractedRstType = {
602 "rst_text": rst_text,
603 "row_offset": row_offset,
604 "start_idx": start_idx
605 + len(start_marker)
606 + first_newline_idx
607 + len(UNIX_NEWLINE),
608 "end_idx": end_idx,
609 }
610
611 return multiline_rst
Back to top