1import os
2from pathlib import Path
3
4from ignore import WalkBuilder
5from ignore.overrides import OverrideBuilder
6
7from sphinx_codelinks.logger import get_logger
8from sphinx_codelinks.source_discover.config import (
9 COMMENT_FILETYPE,
10 CommentType,
11 SourceDiscoverConfig,
12)
13
14logger = get_logger(__name__)
15
16
17def lies_within(path: Path, directory: Path) -> bool:
18 """Whether ``path`` lies below ``directory``, both resolved.
19
20 Strings, not ``Path.is_relative_to``, which costs as much as the walk itself over
21 thousands of files. ``normcase`` folds case on Windows; on a case-insensitive POSIX
22 file system (APFS) a differently-spelled ancestor still counts as outside --
23 harmless, the file is traced under its own spelling.
24 """
25 prefix = os.path.join(os.path.normcase(str(directory)), "")
26 return os.path.normcase(str(path)).startswith(prefix)
27
28
29def warn_outside_src_dir(walked: Path, resolved: Path, src_dir: Path) -> None:
30 """Say that ``walked`` -- the path discovery reached, the link -- is not traced:
31 it resolves to outside ``src_dir``. POSIX on every platform.
32
33 The walked path leads the message rather than being its location: the CLI prints
34 the message alone, and Sphinx reads a location without a ``:`` as a document
35 name (``<path>.rst``) -- while a Windows path has one. It is normalised for the
36 message only (``src/../x.cpp`` reads ``x.cpp``).
37 """
38 shown = Path(os.path.normpath(walked)).as_posix()
39 logger.warning(
40 f"{shown} resolves to {resolved.as_posix()}, outside src_dir "
41 f"{src_dir.as_posix()}: not traced (widen src_dir to cover it, or exclude "
42 "the link)",
43 subtype="outside_src_dir",
44 )
45
46
47def _json_starts_with_comment(filepath: Path, sample_size: int = 256) -> bool:
48 """Return True if a ``.json`` file's first non-whitespace content is a comment.
49
50 Used to decide whether a ``.json`` file should be treated as JSONC. Per
51 https://jsonc.org/#filename-extension a ``.json`` file should only be treated as
52 JSONC when it opens with a comment (e.g. the mode line ``// -*- mode: jsonc -*-``).
53 """
54 try:
55 with filepath.open("rb") as f:
56 chunk = f.read(sample_size)
57 except OSError:
58 return False
59 # strip a leading UTF-8 BOM, then leading whitespace
60 text = chunk.removeprefix(b"\xef\xbb\xbf").lstrip()
61 return text.startswith((b"//", b"/*"))
62
63
[docs] 64# @Source code file discovery with gitignore support, IMPL_DISC_1, impl, [FE_DISCOVERY, FE_CLI_DISCOVER]
65class SourceDiscover:
66 """The source files below ``src_discover_config.src_dir``, as ``source_paths``.
67
68 A file that resolves to outside ``boundary`` -- a symbolic link out of the tree, or
69 a file below a followed directory link -- is not listed, and a warning names
70 its walked path (``codelinks.outside_src_dir``) unless ``warn`` is false: every
71 record and copy is relative to the source directory, which cannot hold it.
72
73 :param src_discover_config: What to walk, and how.
74 :param boundary: The directory every listed file lies below; the walked directory
75 when ``None``. A ``src-trace`` scope passes its project's ``src_dir``, which
76 its ``:directory:`` lies below.
77 :param warn: Whether to warn about a file outside ``boundary``.
78 """
79
80 def __init__(
81 self,
82 src_discover_config: SourceDiscoverConfig,
83 *,
84 boundary: Path | None = None,
85 warn: bool = True,
86 ):
87 self.src_discover_config = src_discover_config
88 self.boundary = boundary
89 self.warn = warn
90 # normalize the file types to lower case with leading dot
91 self.file_types = {
92 f".{ext}" for ext in COMMENT_FILETYPE[src_discover_config.comment_type]
93 }
94
95 self.source_paths = self._discover()
96
97 def _build_overrides(self) -> OverrideBuilder | None:
98 """Build an OverrideBuilder for include/exclude patterns.
99
100 Include patterns are added as whitelist globs.
101 Exclude patterns are added as negated globs (prefixed with ``!``).
102 """
103 has_include = bool(self.src_discover_config.include)
104 has_exclude = bool(self.src_discover_config.exclude)
105
106 if not has_include and not has_exclude:
107 return None
108
109 ob = OverrideBuilder(self.src_discover_config.src_dir)
110
111 if has_include:
112 for pattern in self.src_discover_config.include:
113 ob.add(pattern)
114
115 if has_exclude:
116 for pattern in self.src_discover_config.exclude:
117 ob.add(f"!{pattern}")
118
119 return ob
120
121 def _discover(self) -> list[Path]:
122 """Discover source files recursively in the given directory."""
123 src_dir = self.src_discover_config.src_dir
124 if not src_dir.is_dir():
125 return []
126
127 gitignore = self.src_discover_config.gitignore
128
129 builder = WalkBuilder(src_dir)
130 # Replicate the Rust ignore crate's standard_filters(gitignore)
131 # followed by hidden(false), matching ubc_codelinks behaviour.
132 builder.ignore(gitignore)
133 builder.parents(gitignore)
134 builder.git_ignore(gitignore)
135 builder.git_global(gitignore)
136 builder.git_exclude(gitignore)
137 builder.hidden(False)
138 builder.follow_links(self.src_discover_config.follow_links)
139
140 override_builder = self._build_overrides()
141 if override_builder is not None:
142 builder.overrides(override_builder.build())
143
144 boundary = (self.boundary or src_dir).resolve()
145 discovered_files = []
146 for entry in builder.build():
147 filepath = entry.path()
148 if not filepath.is_file():
149 continue
150 if self.file_types and filepath.suffix.lower() not in self.file_types:
151 continue
[docs]152 # @JSONC .json files require a leading comment, IMPL_JSONC_3, impl, [FE_JSONC]
153 # A plain ``.json`` file is only treated as JSONC when it opens with a
154 # comment; otherwise it is skipped under the ``jsonc`` comment type.
155 if (
156 self.src_discover_config.comment_type == CommentType.jsonc
157 and filepath.suffix.lower() == ".json"
158 and not _json_starts_with_comment(filepath)
159 ):
160 continue
161 # resolve() produces canonical absolute paths; follow_links only
162 # controls whether the walker descends into symlinked directories
163 resolved = filepath.resolve()
164 if not lies_within(resolved, boundary):
165 # a link out of the tree: no root-relative path, no copy (#2062)
166 if self.warn:
167 warn_outside_src_dir(filepath, resolved, boundary)
168 continue
169 discovered_files.append(resolved)
170
171 # a file reached through a symbolic link inside the tree resolves to a path
172 # already found: list each file once, or every caller analyses it once per
173 # path to it
174 sorted_filepaths = sorted(
175 dict.fromkeys(discovered_files),
176 key=lambda x: os.path.normcase(os.path.normpath(x)),
177 )
178 return sorted_filepaths
Back to top