1from collections import deque
2from dataclasses import MISSING, dataclass, field, fields, replace
3from enum import Enum
4from pathlib import Path
5from typing import Any, Literal, TypedDict, cast
6
7from jsonschema import ValidationError, validate
8from sphinx.application import Sphinx
9from sphinx.config import Config as _SphinxConfig
10
11from sphinx_codelinks.source_discover.config import (
12 CommentType,
13 SourceDiscoverConfig,
14 SourceDiscoverSectionConfigType,
15)
16from sphinx_codelinks.source_discover.source_discover import SourceDiscover
17from ub_project import anchor, load_toml, select_table
18
19UNIX_NEWLINE = "\n"
20
21
22COMMENT_MARKERS = {
[docs] 23 # @Support C and C++ style comments, IMPL_C_1, impl, [FE_C_SUPPORT, FE_CPP]
24 CommentType.cpp: ["//", "/*"],
[docs] 25 # @Support Python style comments, IMPL_PY_1, impl, [FE_PY]
26 CommentType.python: ["#"],
27 CommentType.cs: ["//", "/*", "///"],
[docs] 28 # @Support Go style comments, IMPL_GO_2, impl, [FE_GO]
29 CommentType.go: ["//", "/*"],
30}
31ESCAPE = "\\"
32
33# Default C/C++ standard for the standalone/defines parse path when no
34# compile_commands.json entry supplies one.
35DEFAULT_CPP_STD = "c++17"
36
37
38class CommentCategory(str, Enum): # noqa: UP042 # StrEnum changes str(member), which reaches CLI warnings and error messages
39 comment = "comment"
40 docstring = "expression_statement"
41
42
43class NeedIdRefsConfigType(TypedDict):
44 markers: list[str]
45
46
47@dataclass
48class NeedIdRefsConfig:
49 @classmethod
50 def field_names(cls) -> set[str]:
51 return {item.name for item in fields(cls)}
52
53 markers: list[str] = field(
54 default_factory=lambda: ["@need-ids:"],
55 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
56 )
57 """The markers to extract need ids from"""
58
59 @classmethod
60 def get_schema(cls, name: str) -> dict[str, Any] | None:
61 _field = next(_field for _field in fields(cls) if _field.name is name)
62 if _field.metadata and "schema" in _field.metadata:
63 return cast(dict[str, Any], _field.metadata["schema"])
64 return None
65
66 def check_schema(self) -> list[str]:
67 errors = []
68 for _field_name in self.field_names():
69 schema = self.get_schema(_field_name)
70 value = getattr(self, _field_name)
71 try:
72 # jsonschema's stubs type `schema` as `bool | Mapping`, but
73 # `get_schema` returns an optional dict
74 validate(
75 instance=value,
76 schema=schema, # ty: ignore[invalid-argument-type]
77 )
78 except ValidationError as e:
79 errors.append(
80 f"Schema validation error in field '{_field_name}': {e.message}"
81 )
82 return errors
83
84
85class MarkedRstConfigType(TypedDict):
86 start_sequence: str
87 end_sequence: str
88
89
90@dataclass
91class MarkedRstConfig:
92 @classmethod
93 def field_names(cls) -> set[str]:
94 return {item.name for item in fields(cls)}
95
96 start_sequence: str = field(default="@rst", metadata={"schema": {"type": "string"}})
97 """Chars sequence to indicate the start of the rst text."""
98 end_sequence: str = field(
99 default="@endrst", metadata={"schema": {"type": "string"}}
100 )
101 """Chars sequence to indicate the end of the rst text."""
102
103 @classmethod
104 def get_schema(cls, name: str) -> dict[str, Any] | None:
105 _field = next(_field for _field in fields(cls) if _field.name is name)
106 if _field.metadata and "schema" in _field.metadata:
107 return cast(dict[str, Any], _field.metadata["schema"])
108 return None
109
110 def check_schema(self) -> list[str]:
111 errors = []
112 for _field_name in self.field_names():
113 schema = self.get_schema(_field_name)
114 value = getattr(self, _field_name)
115 try:
116 # jsonschema's stubs type `schema` as `bool | Mapping`, but
117 # `get_schema` returns an optional dict
118 validate(
119 instance=value,
120 schema=schema, # ty: ignore[invalid-argument-type]
121 )
122 except ValidationError as e:
123 errors.append(
124 f"Schema validation error in field '{_field_name}': {e.message}"
125 )
126 return errors
127
128 def check_sequence_mutually_exclusive(self) -> list[str]:
129 errors = []
130 if self.start_sequence == self.end_sequence:
131 errors.append("start_sequence and end_sequence cannot be the same.")
132 return errors
133
134 def check_fields_configuration(self) -> list[str]:
135 return self.check_schema() + self.check_sequence_mutually_exclusive()
136
137
138@dataclass
139class PreprocessorConfig:
140 """Opt-in libclang engine config. Presence => libclang engine for C/C++."""
141
142 compile_commands: Path | None = field(
143 default=None, metadata={"schema": {"type": ["string", "null"]}}
144 )
145 """Explicit path to compile_commands.json. If None, walk-up auto-discovery."""
146
147 defines: list[str] = field(
148 default_factory=list,
149 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
150 )
151 """Fallback -D defines applied globally when no compile_commands.json applies."""
152
153 includes: list[Path] = field(
154 default_factory=list,
155 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
156 )
157 """Fallback -I include dirs for the defines path."""
158
159 std: str = field(
160 default=DEFAULT_CPP_STD,
161 metadata={"schema": {"type": "string"}},
162 )
163 """C/C++ standard for the standalone/defines parse path (e.g. ``c++17``,
164 ``c++20``, ``c11``); libclang pins ``-x`` to match it. Files resolved from a
165 ``compile_commands.json`` entry use that entry's own ``-std`` instead."""
166
167
168def anchor_preproc_paths(preproc: PreprocessorConfig, base: Path) -> PreprocessorConfig:
169 """Resolve a preprocessor config's ``compile_commands`` and ``includes``
170 against ``base`` (the config file's directory), so a relative path resolves
171 against the TOML file rather than the process CWD — matching ``src_dir`` /
172 ``git_root``. An absolute path is not anchored, but it is resolved too
173 (symlinks followed, ``..`` folded), like every other path here.
174 """
175 return replace(
176 preproc,
177 compile_commands=(
178 anchor(preproc.compile_commands, base).resolve()
179 if preproc.compile_commands is not None
180 else None
181 ),
182 includes=[anchor(inc, base).resolve() for inc in preproc.includes],
183 )
184
185
186class FieldConfig(TypedDict, total=False):
187 name: str
188 type: Literal["str", "list[str]"]
189 default: str | list[str] | None
190
191
192class OneLineCommentStyleType(TypedDict):
193 start_sequence: str
194 end_sequence: str
195 field_split_char: str
196 needs_fields: list[FieldConfig]
197
198
199@dataclass
200class OneLineCommentStyle:
201 def __setattr__(self, name: str, value: Any) -> None:
202 if name == "needs_fields":
203 # apply default to fields
204 self.apply_needs_field_default(value)
205 return super().__setattr__(name, value)
206
207 @classmethod
208 def field_names(cls) -> set[str]:
209 return {item.name for item in fields(cls)}
210
211 start_sequence: str = field(default="@", metadata={"schema": {"type": "string"}})
212 """Chars sequence to indicate the start of the one-line comment."""
213
214 end_sequence: str = field(
215 default=UNIX_NEWLINE, metadata={"schema": {"type": "string"}}
216 )
217 """Chars sequence to indicate the end of the one-line comment."""
218
219 field_split_char: str = field(default=",", metadata={"schema": {"type": "string"}})
220 """Char sequence to split the fields."""
221
222 needs_fields: list[FieldConfig] = field(
223 default_factory=lambda: [
224 {"name": "title"},
225 {"name": "id"},
226 {"name": "type", "default": "impl"},
227 {"name": "links", "type": "list[str]", "default": []},
228 ],
229 metadata={
230 "required_fields": ["title", "type"],
231 "field_default": {
232 "type": "str",
233 },
234 "schema": {
235 "type": "array",
236 "items": {
237 "type": "object",
238 "properties": {
239 "name": {"type": "string"},
240 "type": {
241 "type": "string",
242 "enum": ["str", "list[str]"],
243 "default": "str",
244 },
245 "default": {
246 "anyOf": [
247 {"type": "string"},
248 {"type": "array", "items": {"type": "string"}},
249 ]
250 },
251 },
252 "required": ["name"],
253 "additionalProperties": False,
254 "allOf": [
255 {
256 "if": {"properties": {"type": {"const": "list[str]"}}},
257 "then": {
258 "properties": {
259 "default": {
260 "type": "array",
261 "items": {"type": "string"},
262 }
263 }
264 },
265 },
266 {
267 "if": {"properties": {"type": {"const": "str"}}},
268 "then": {"properties": {"default": {"type": "string"}}},
269 },
270 ],
271 },
272 },
273 },
274 )
275
276 @classmethod
277 def apply_needs_field_default(cls, given_fields: list[FieldConfig]) -> None:
278 field_default = next(
279 _field.metadata["field_default"]
280 for _field in fields(cls)
281 if _field.name == "needs_fields"
282 )
283
284 for _field in given_fields:
285 for _default in field_default:
286 if _default not in _field:
287 _field[_default] = field_default[
288 _default
289 ] # dynamically assign keys
290
291 @classmethod
292 def get_required_fields(cls, name: str) -> list[str] | None:
293 _field = next(_field for _field in fields(cls) if _field.name is name)
294 if _field.metadata:
295 return cast(list[str], _field.metadata["required_fields"])
296 return None
297
298 @classmethod
299 def get_schema(cls, name: str) -> dict[str, Any] | None:
300 _field = next(_field for _field in fields(cls) if _field.name is name)
301 if _field.metadata and "schema" in _field.metadata:
302 return cast(dict[str, Any], _field.metadata["schema"])
303 return None
304
305 def check_schema(self) -> list[str]:
306 errors = []
307 for _field_name in self.field_names():
308 schema = self.get_schema(_field_name)
309 value = getattr(self, _field_name)
310 try:
311 validate(
312 instance=value,
313 schema=schema, # ty: ignore[invalid-argument-type]
314 ) # validate has no type specified
315 except ValidationError as e:
316 if _field_name == "needs_fields":
317 need_field_name = value[e.path[0]]["name"]
318 errors.append(
319 f"Schema validation error in need_fields '{need_field_name}': {e.message}"
320 )
321 else:
322 errors.append(
323 f"Schema validation error in field '{_field_name}': {e.message}"
324 )
325 return errors
326
327 def check_required_fields(self) -> list[str]:
328 errors = []
329 required_fields = self.get_required_fields("needs_fields")
330 if required_fields is None:
331 errors.append("No required fields specified.")
332 return errors
333 given_field_names = [_field["name"] for _field in self.needs_fields]
334 missing_fields = set(required_fields) - set(given_field_names)
335 if len(missing_fields) != 0:
336 errors.append(f"Missing required fields: {sorted(missing_fields)}")
337
338 return errors
339
340 def check_fields_mutually_exclusive(self) -> list[str]:
341 errors = []
342 needs_field_names = set()
343 for _field in self.needs_fields:
344 if _field["name"] in needs_field_names:
345 errors.append(f"Field '{_field['name']}' is defined multiple times.")
346 needs_field_names.add(_field["name"])
347 return errors
348
349 def check_fields_default_order(self) -> list[str]:
350 errors = []
351 seen_default = False
352 first_default_field = ""
353 for _field in self.needs_fields:
354 has_default = _field.get("default") is not None
355 if has_default and not seen_default:
356 seen_default = True
357 first_default_field = _field["name"]
358 elif not has_default and seen_default:
359 errors.append(
360 f"Field '{_field['name']}' without a default follows "
361 f"field '{first_default_field}' which has a default. "
362 f"Fields without defaults must be defined before fields with defaults."
363 )
364 return errors
365
366 def check_fields_configuration(self) -> list[str]:
367 return (
368 self.check_schema()
369 + self.check_required_fields()
370 + self.check_fields_mutually_exclusive()
371 + self.check_fields_default_order()
372 )
373
374 def get_cnt_required_fields(self) -> int:
375 cnt_required_fields = 0
376 for _field in self.needs_fields:
377 if _field.get("default") is None:
378 cnt_required_fields += 1
379 return cnt_required_fields
380
381 def get_pos_list_str(self) -> list[int]:
382 pos_list_str = []
383 for idx, _field in enumerate(self.needs_fields):
384 if _field["type"] == "list[str]":
385 pos_list_str.append(idx + 1)
386 return pos_list_str
387
388
389class AnalyseSectionConfigType(TypedDict, total=False):
390 """Define typing for loading `analyse` section from the file."""
391
392 get_need_id_refs: bool
393 get_oneline_needs: bool
394 get_rst: bool
395 outdir: str
396 git_root: str
397 need_id_refs: NeedIdRefsConfigType
398 marked_rst: MarkedRstConfigType
399 oneline_comment_style: OneLineCommentStyleType
400 preprocessor: dict[str, object]
401
402
403class SourceAnalyseConfigType(TypedDict, total=False):
404 """Define typing for its API configuration."""
405
406 src_files: list[Path]
407 src_dir: Path
408 comment_type: CommentType
409 get_need_id_refs: bool
410 get_oneline_needs: bool
411 get_rst: bool
412 git_root: Path | None
413 need_id_refs_config: NeedIdRefsConfig
414 marked_rst_config: MarkedRstConfig
415 oneline_comment_style: OneLineCommentStyle
416 preprocessor: PreprocessorConfig | None
417
418
419class ProjectsAnalyseConfigType(TypedDict, total=False):
420 projects_config: dict[str, SourceAnalyseConfigType]
421
422
423@dataclass
424class SourceAnalyseConfig:
425 @classmethod
426 def field_names(cls) -> set[str]:
427 return {item.name for item in fields(cls)}
428
429 src_files: list[Path] = field(
430 default_factory=list,
431 metadata={"schema": {"type": "array", "items": {"type": "string"}}},
432 )
433 """A list of source files to be processed."""
434 src_dir: Path = field(
435 default_factory=lambda: Path("./"), metadata={"schema": {"type": "string"}}
436 )
437
438 comment_type: CommentType = field(
439 default=CommentType.cpp, metadata={"schema": {"type": "string"}}
440 )
441 """The type of comment to be processed."""
442
443 get_need_id_refs: bool = field(
444 default=True, metadata={"schema": {"type": "boolean"}}
445 )
446 """Whether to extract need id references from comments"""
447
448 get_oneline_needs: bool = field(
449 default=False, metadata={"schema": {"type": "boolean"}}
450 )
451 """Whether to extract oneline needs from comments"""
452
453 get_rst: bool = field(default=False, metadata={"schema": {"type": "boolean"}})
454 """Whether to extract rst texts from comments"""
455
456 git_root: Path | None = field(
457 default=None, metadata={"schema": {"type": ["string", "null"]}}
458 )
459 """Explicit path to the Git repository root. If not set, it will be auto-detected
460 by traversing parent directories. Useful for Bazel builds or deeply nested configs."""
461
462 need_id_refs_config: NeedIdRefsConfig = field(default_factory=NeedIdRefsConfig)
463 """Configuration for extracting need id references from comments."""
464
465 marked_rst_config: MarkedRstConfig = field(default_factory=MarkedRstConfig)
466 """Configuration for extracting rst texts from comments."""
467
468 oneline_comment_style: OneLineCommentStyle = field(
469 default_factory=OneLineCommentStyle
470 )
471 """Configuration for extracting oneline needs from comments."""
472
473 preprocessor: PreprocessorConfig | None = field(default=None)
474 """Opt-in libclang preprocessor engine. None => tree-sitter (default).
475
476 No flat ``metadata["schema"]`` here: this is a nested dataclass, like the
477 sibling ``need_id_refs_config`` / ``marked_rst_config`` /
478 ``oneline_comment_style`` fields. ``check_schema`` only validates fields that
479 declare a flat schema; giving this field one made it validate the constructed
480 ``PreprocessorConfig`` instance against JSON type ``object`` and fail at
481 ``config-inited``. Its structure is enforced by ``convert_analyse_config``.
482 """
483
484 @classmethod
485 def get_schema(cls, name: str) -> dict[str, Any] | None:
486 _field = next(_field for _field in fields(cls) if _field.name is name)
487 if _field.metadata and "schema" in _field.metadata:
488 return cast(dict[str, Any], _field.metadata["schema"])
489 return None
490
491 def check_schema(self) -> list[str]:
492 errors = []
493 for _field_name in self.field_names():
494 schema = self.get_schema(_field_name)
495 if not schema:
496 continue
497 value = getattr(self, _field_name)
498 if isinstance(value, Path): # adapt to json schema restriction
499 value = str(value)
500 if _field_name == "src_files" and isinstance(
501 value, list
502 ): # adapt to json schema restriction
503 value: list[str] = [
504 str(src_file) for src_file in value
505 ] # only for value adaptation
506 try:
507 validate(instance=value, schema=schema)
508 except ValidationError as e:
509 errors.append(
510 f"Schema validation error in field '{_field_name}': {e.message}"
511 )
512 return errors
513
514 def check_markers_mutually_exclusive(self) -> list[str]:
515 errors = set()
516 markers = set()
517 markers.add(self.oneline_comment_style.start_sequence)
518 markers.add(self.oneline_comment_style.end_sequence)
519 if self.marked_rst_config.start_sequence in markers:
520 errors.add(
521 f"Marker {self.marked_rst_config.start_sequence} is defined multiple times"
522 )
523 else:
524 markers.add(self.marked_rst_config.start_sequence)
525 if self.marked_rst_config.end_sequence in markers:
526 errors.add(
527 f"Marker {self.marked_rst_config.end_sequence} is defined multiple times"
528 )
529 else:
530 markers.add(self.marked_rst_config.end_sequence)
531
532 for marker in self.need_id_refs_config.markers:
533 if marker in markers:
534 errors.add(f"Marker {marker} is defined multiple times")
535 else:
536 markers.add(marker)
537 return list(errors)
538
539 def check_fields_configuration(self) -> list[str]:
540 errors: deque[str] = deque()
541 if self.get_need_id_refs:
542 need_id_refs_errors = self.need_id_refs_config.check_schema()
543 if need_id_refs_errors:
544 errors.appendleft("NeedIdRefs configuration errors:")
545 errors.extend(need_id_refs_errors)
546 if self.get_oneline_needs:
547 oneline_needs_errors = (
548 self.oneline_comment_style.check_fields_configuration()
549 )
550 if oneline_needs_errors:
551 errors.appendleft("OneLineCommentStyle configuration errors:")
552 errors.extend(oneline_needs_errors)
553 if self.get_rst:
554 marked_rst_errors = self.marked_rst_config.check_fields_configuration()
555 if marked_rst_errors:
556 errors.appendleft("MarkedRst configuration errors:")
557 errors.extend(self.marked_rst_config.check_fields_configuration())
558 analyse_errors = self.check_markers_mutually_exclusive() + self.check_schema()
559 if analyse_errors:
560 errors.appendleft("analyse configuration errors:")
561 errors.extend(analyse_errors)
562 return list(errors)
563
564
565# Shared ubCode project file, which other useblocks tools (sphinx-needs,
566# ubCode checker, ...) read as well, so all tools see the same projects.
567DEFAULT_CONFIG_TOML: str = "ubproject.toml"
568
569DEFAULT_REF_URL_FIELD: str = "code_url"
570"""The need field ``@need-ids:`` references are attached to, unless a project's
571``ref_url_field`` names another (ubCode's key and default; ``""`` disables the attach)."""
572
573#: The table of the TOML file that both readers, the Sphinx extension and the CLI,
574#: take their configuration from.
575CODELINKS_TABLE: str = "codelinks"
576
577
578def load_codelinks_table(path: Path) -> dict[str, object] | None:
579 """Parse *path* and return its ``[codelinks]`` table, through ub-project.
580
581 The values are returned RAW: relative paths are anchored where they are used
582 (the Sphinx extension anchors at ``confdir / Path(config_from_toml).parent``,
583 unresolved), never here.
584
585 :param path: The TOML file.
586 :return: The table, or ``None`` when the file has no ``codelinks`` key.
587 :raises ub_project.ProjectConfigError: If the file cannot be read, is not UTF-8
588 or not valid TOML, or if ``codelinks`` is not a table.
589 :raises RecursionError: For a pathologically nested file, which ``load_toml``
590 does not wrap -- why both callers also catch ``Exception``.
591 """
592 return select_table(load_toml(path), CODELINKS_TABLE, source=path)
593
594
595class CodeLinksProjectConfigType(TypedDict, total=False):
596 """TypedDict defining the configuration structure for individual SrcTrace projects.
597
598 Contains both user-provided configuration:
599 - source_discover
600 - remote_url_pattern
601 - ref_url_field
602 - analyse
603 and runtime-generated configuration objects
604 - source_discover_config
605 - analyse_config
606 """
607
608 source_discover: SourceDiscoverSectionConfigType
609 remote_url_pattern: str
610 ref_url_field: str
611 analyse: AnalyseSectionConfigType
612 source_discover_config: SourceDiscoverConfig
613 analyse_config: SourceAnalyseConfig
614
615
616class CodeLinksConfigType(TypedDict):
617 config_from_toml: str | None
618 set_local_url: bool
619 local_url_field: str
620 set_remote_url: bool
621 remote_url_field: str
622 outdir: Path
623 projects: dict[str, CodeLinksProjectConfigType]
624 debug_measurement: bool
625 debug_filters: bool
626
627
628@dataclass
629class CodeLinksConfig:
630 @classmethod
631 def from_sphinx(cls, sphinx_config: _SphinxConfig) -> "CodeLinksConfig":
632 obj = cls()
633 src_trace_projects = getattr(sphinx_config, "src_trace_projects", None)
634 if isinstance(src_trace_projects, dict):
635 generate_project_configs(src_trace_projects)
636 super().__setattr__(obj, "_sphinx_config", sphinx_config)
637 return obj
638
639 def __getattribute__(self, name: str) -> Any:
640 if name.startswith("__") or name == "_sphinx_config":
641 return super().__getattribute__(name)
642 sphinx_config = (
643 object.__getattribute__(self, "_sphinx_config")
644 if "_sphinx_config" in self.__dict__
645 else None
646 )
647 if sphinx_config:
648 return getattr(
649 super().__getattribute__("_sphinx_config"), f"src_trace_{name}"
650 )
651
652 return object.__getattribute__(self, name)
653
654 def __setattr__(self, name: str, value: Any) -> None:
655 if name == "_sphinx_config" and "src_trace_projects" in value:
656 src_trace_projects: dict[str, CodeLinksProjectConfigType] = value[
657 "src_trace_projects"
658 ]
659 generate_project_configs(src_trace_projects)
660
661 if name.startswith("__") or name == "_sphinx_config":
662 return super().__setattr__(name, value)
663
664 sphinx_config = (
665 object.__getattribute__(self, "_sphinx_config")
666 if "_sphinx_config" in self.__dict__
667 else None
668 )
669
670 if sphinx_config:
671 setattr(
672 super().__getattribute__("_sphinx_config"), f"src_trace_{name}", value
673 )
674
675 if name == "outdir" and isinstance(value, str):
676 # Ensure outdir is a Path object
677 value = Path(value)
678 return object.__setattr__(self, name, value)
679
680 @classmethod
681 def add_config_values(cls, app: Sphinx) -> None:
682 """Add all config values to Sphinx application"""
683 for item in fields(cls):
684 if item.default_factory is not MISSING:
685 default = item.default_factory()
686 elif item.default is not MISSING:
687 default = item.default
688 else:
689 raise Exception(f"Field {item.name} has no default value or factory")
690
691 name = item.name
692 app.add_config_value(
693 f"src_trace_{name}",
694 default,
695 item.metadata["rebuild"],
696 types=item.metadata["types"],
697 )
698
699 @classmethod
700 def field_names(cls) -> set[str]:
701 return {item.name for item in fields(cls)}
702
703 @classmethod
704 def get_schema(cls, name: str) -> dict[str, Any] | None:
705 """Get the schema for a config item."""
706 _field = next(field for field in fields(cls) if field.name is name)
707 if _field.metadata and "schema" in _field.metadata:
708 return _field.metadata["schema"]
709 return None
710
711 config_from_toml: str | None = field(
712 default=DEFAULT_CONFIG_TOML,
713 metadata={
714 "rebuild": "env",
715 "types": (str, type(None)),
716 "schema": {
717 "type": ["string", "null"],
718 "examples": [DEFAULT_CONFIG_TOML, None],
719 },
720 },
721 )
722 """Path to a TOML file to load configuration from.
723
724 Defaults to ``ubproject.toml`` next to :file:`conf.py`. The default is the
725 value ``ubproject.toml`` exactly -- left unset, or written in conf.py as that
726 string: missing, or without a ``[codelinks]`` table, it is silently ignored.
727 Any other value, ``./ubproject.toml`` included, is an explicit file and warns
728 in both cases. A file that exists but cannot be read or parsed warns
729 (``codelinks.config``) either way, as any configured file did at 1.4.0.
730 """
731
732 set_local_url: bool = field(
733 default=False,
734 metadata={
735 "rebuild": "env",
736 "types": (bool,),
737 "schema": {
738 "type": "boolean",
739 },
740 },
741 )
742 """Set the file URL in the extracted need."""
743
744 local_url_field: str = field(
745 default="local-url",
746 metadata={
747 "rebuild": "env",
748 "types": (str,),
749 "schema": {
750 "type": "string",
751 },
752 },
753 )
754 """The field name for the file URL in the extracted need."""
755
756 set_remote_url: bool = field(
757 default=False,
758 metadata={
759 "rebuild": "env",
760 "types": (bool,),
761 "schema": {
762 "type": "boolean",
763 },
764 },
765 )
766 remote_url_field: str = field(
767 default="remote-url",
768 metadata={
769 "rebuild": "env",
770 "types": (str,),
771 "schema": {
772 "type": "string",
773 },
774 },
775 )
776 """The field name for the remote URL in the extracted need."""
777
778 outdir: Path = field(
779 default=Path("output"),
780 metadata={"rebuild": "env", "types": (str), "schema": {"type": "string"}},
781 )
782 """The directory where the generated artifacts and their caches will be stored."""
783
784 projects: dict[str, CodeLinksProjectConfigType] = field(
785 default_factory=dict,
786 metadata={
787 "rebuild": "env",
788 "types": (),
789 "schema": {
790 "type": "object",
791 "additionalProperties": {
792 "type": "object",
793 "properties": {
794 "source_discover": {},
795 "analyse": {},
796 "remote_url_pattern": {},
797 "ref_url_field": {},
798 "source_discover_config": {},
799 "analyse_config": {},
800 },
801 "additionalProperties": False,
802 },
803 },
804 },
805 )
806 """The configuration for the source tracing projects."""
807
808 debug_measurement: bool = field(
809 default=False, metadata={"rebuild": "html", "types": (bool,)}
810 )
811 """If True, log runtime information for various functions."""
812 debug_filters: bool = field(
813 default=False, metadata={"rebuild": "html", "types": (bool,)}
814 )
815 """If True, log filter processing runtime information."""
816
817
818def check_schema(config: CodeLinksConfig) -> list[str]:
819 """Check only first layer's of schema, so that the nested dict is not validated here."""
820 errors = []
821 for _field_name in CodeLinksConfig.field_names():
822 schema = CodeLinksConfig.get_schema(_field_name)
823 if not schema:
824 continue
825 value = getattr(config, _field_name)
826 if isinstance(value, Path): # adapt to json schema restriction
827 value = str(value)
828 try:
829 validate(instance=value, schema=schema)
830 except ValidationError as e:
831 errors.append(
832 f"Schema validation error in filed '{_field_name}': {e.message}"
833 )
834 return errors
835
836
837def check_project_configuration(config: CodeLinksConfig) -> list[str]:
838 """Check nested project configurations"""
839 errors = []
840
841 for project_name, project_config in config.projects.items():
842 project_errors: list[str] = []
843
844 # validate source_discover config
845 src_discover_config: SourceDiscoverConfig | None = project_config.get(
846 "source_discover_config"
847 )
848 src_discover_errors = []
849 if src_discover_config:
850 src_discover_errors.extend(src_discover_config.check_schema())
851
852 # validate analyse config
853 analyse_config: SourceAnalyseConfig | None = project_config.get(
854 "analyse_config"
855 )
856 analyse_errors = []
857 if analyse_config:
858 analyse_errors = analyse_config.check_fields_configuration()
859
860 # validate src-trace config
861 if config.set_remote_url and "remote_url_pattern" not in project_config:
862 project_errors.append(
863 "remote_url_pattern must be given, as set_remote_url is enabled"
864 )
865
866 if "remote_url_pattern" in project_config and not isinstance(
867 project_config["remote_url_pattern"], str
868 ):
869 project_errors.append("remote_url_pattern must be a string")
870
871 ref_url_field = project_config.get("ref_url_field", DEFAULT_REF_URL_FIELD)
872 if not isinstance(ref_url_field, str):
873 project_errors.append("ref_url_field must be a string")
874 elif ref_url_field and ref_url_field in _url_fields(config):
875 project_errors.append(
876 f"ref_url_field {ref_url_field!r} must differ from local_url_field "
877 "and remote_url_field"
878 )
879
880 if analyse_errors or src_discover_errors or project_errors:
881 errors.append(f"Project '{project_name}' has the following errors:")
882 errors.extend(analyse_errors)
883 errors.extend(src_discover_errors)
884 errors.extend(project_errors)
885
886 return errors
887
888
889def config_base_dir(confdir: str | Path, config: CodeLinksConfig) -> Path:
890 """The directory a project's relative paths are anchored at.
891
892 The configuration file's directory when ``src_trace_config_from_toml`` names one
893 (itself anchored at ``confdir``), else ``confdir``: ``src_dir``, ``git_root`` and
894 the preprocessor paths all resolve against it.
895 """
896 base = Path(confdir)
897 if config.config_from_toml:
898 base = anchor(Path(config.config_from_toml).parent, base)
899 return base
900
901
902def locate_src_dir(
903 confdir: str | Path,
904 config: CodeLinksConfig,
905 discover_config: SourceDiscoverConfig,
906) -> Path:
907 """A project's source directory, anchored (see :func:`config_base_dir`) and resolved."""
908 return anchor(discover_config.src_dir, config_base_dir(confdir, config)).resolve()
909
910
911def git_root_problem(git_root: Path, src_dir: Path) -> str | None:
912 """Why a configured ``git_root`` cannot be a project's git root, or ``None``.
913
914 Every source path is relative to the git root, so it must be ``src_dir`` or a
915 directory above it (#2062). Both paths anchored and resolved by the caller: the
916 Sphinx extension (``project_analysis.configured_git_root``) and ``codelinks
917 analyse`` alike, each treating a value with a problem as unset.
918 """
919 try:
920 if not git_root.exists():
921 return f"git_root {git_root.as_posix()} does not exist"
922 if not git_root.is_dir():
923 return f"git_root {git_root.as_posix()} is not a directory"
924 except OSError:
925 return f"git_root {git_root.as_posix()} cannot be read"
926 if not src_dir.is_relative_to(git_root):
927 return (
928 f"git_root {git_root.as_posix()} does not contain src_dir "
929 f"{src_dir.as_posix()}"
930 )
931 return None
932
933
934def git_root_warning(project: str, problem: str) -> str:
935 """The one warning text for a project's ignored ``git_root`` (``codelinks.git_root``)."""
936 return (
937 f"project {project!r}: {problem}; it is ignored, and the repository root is "
938 "detected from src_dir instead"
939 )
940
941
942def remote_url_pattern_warnings(config: CodeLinksConfig) -> list[str]:
943 """Why a project's remote URL pattern will not render as one link, if it will not.
944
945 The ``remote-url`` value is the pattern filled in, and Sphinx-Needs splits every
946 string-linked value on ``,`` and ``;`` before turning the parts into links. A
947 pattern containing either -- gitweb's ``?p=repo.git;a=blob;f={path}`` is the
948 classic -- therefore renders as several links, none of them right.
949 """
950 if not config.set_remote_url or not isinstance(config.projects, dict):
951 return []
952 warnings = []
953 for name, project_config in config.projects.items():
954 if not isinstance(project_config, dict):
955 continue
956 pattern = project_config.get("remote_url_pattern")
957 if isinstance(pattern, str) and ("," in pattern or ";" in pattern):
958 warnings.append(
959 f"Project {name!r}: remote_url_pattern {pattern!r} contains ',' or "
960 "';'. Sphinx-Needs splits string-linked values on ',' and ';', so this "
961 "pattern's links will not render as one link."
962 )
963 return warnings
964
965
966def _url_fields(config: CodeLinksConfig) -> set[str]:
967 """The names of the URL fields the extension registers (when switched on)."""
968 names: set[str] = set()
969 if config.set_local_url:
970 names.add(config.local_url_field)
971 if config.set_remote_url:
972 names.add(config.remote_url_field)
973 return names
974
975
976def need_id_refs_field(
977 config: CodeLinksConfig, project_config: CodeLinksProjectConfigType
978) -> str | None:
979 """The field a project's ``@need-ids:`` references are attached to, or ``None``.
980
981 ubCode's gate: URLs are on (local or remote), references are extracted, the project
982 has at least one marker, and its ``ref_url_field`` is not ``""``. A field that is not
983 a string, or that is one of the URL fields, is a configuration error reported by
984 :func:`check_project_configuration`, and attaches nothing.
985 """
986 if not (config.set_remote_url or config.set_local_url):
987 return None
988 field_name = project_config.get("ref_url_field", DEFAULT_REF_URL_FIELD)
989 if not isinstance(field_name, str) or not field_name:
990 return None
991 if field_name in _url_fields(config):
992 return None
993 analyse_config = project_config.get("analyse_config")
994 if analyse_config is None or not analyse_config.get_need_id_refs:
995 return None
996 if not analyse_config.need_id_refs_config.markers:
997 return None
998 return field_name
999
1000
1001def need_id_refs_fields(config: CodeLinksConfig) -> dict[str, str]:
1002 """Each project whose references are attached, mapped to its field."""
1003 projects = config.projects
1004 if not isinstance(projects, dict):
1005 return {}
1006 fields_by_project: dict[str, str] = {}
1007 for name, project_config in projects.items():
1008 if not isinstance(project_config, dict):
1009 continue
1010 field_name = need_id_refs_field(config, project_config)
1011 if field_name is not None:
1012 fields_by_project[name] = field_name
1013 return fields_by_project
1014
1015
1016def check_configuration(config: CodeLinksConfig) -> list[str]:
1017 errors = []
1018 errors.extend(check_schema(config))
1019 errors.extend(check_project_configuration(config))
1020 return errors
1021
1022
1023def convert_src_discovery_config(
1024 config_dict: SourceDiscoverSectionConfigType | None,
1025) -> SourceDiscoverConfig:
1026 if config_dict:
1027 src_discover_dict = {
1028 key: (Path(value) if key == "src_dir" and isinstance(value, str) else value)
1029 for key, value in config_dict.items()
1030 }
1031 src_discover_config = SourceDiscoverConfig(**src_discover_dict) # ty: ignore[invalid-argument-type]
1032 else:
1033 src_discover_config = SourceDiscoverConfig()
1034
1035 return src_discover_config
1036
1037
1038def _validate_preprocessor_dict(preproc: dict[str, object]) -> None:
1039 """Validate the schema-less ``[preprocessor]`` TOML section.
1040
1041 The section has no ``TypedDict``, so a mistyped scalar would otherwise be
1042 coerced into garbage instead of reported: e.g. ``defines = "X"`` (a bare
1043 string) becomes ``list("X") == ["X"]`` — or worse, ``defines = "cpp17"``
1044 becomes ``["c", "p", "p", "1", "7"]`` → five bogus ``-D`` flags. Fail loud.
1045
1046 :param preproc: the raw ``[preprocessor]`` mapping from TOML.
1047 :raises TypeError: if a key has the wrong type.
1048 """
1049 for key in ("defines", "includes"):
1050 value = preproc.get(key)
1051 if value is not None and (
1052 not isinstance(value, list) or not all(isinstance(x, str) for x in value)
1053 ):
1054 raise TypeError(
1055 f"[preprocessor] {key} must be a list of strings, "
1056 f"got {type(value).__name__}: {value!r}"
1057 )
1058 for key in ("compile_commands", "std"):
1059 value = preproc.get(key)
1060 if value is not None and not isinstance(value, str):
1061 raise TypeError(
1062 f"[preprocessor] {key} must be a string, "
1063 f"got {type(value).__name__}: {value!r}"
1064 )
1065
1066
1067def convert_analyse_config(
1068 config_dict: AnalyseSectionConfigType | None,
1069 src_discover: SourceDiscover | None = None,
1070) -> SourceAnalyseConfig:
1071 analyse_config_dict: SourceAnalyseConfigType = {}
1072 if config_dict:
1073 for k, v in config_dict.items():
1074 if k not in {
1075 "online_comment_style",
1076 "need_id_refs",
1077 "marked_rst",
1078 "preprocessor",
1079 }:
1080 # Convert string paths to Path objects
1081 if k in {"src_dir", "git_root"} and isinstance(v, str):
1082 analyse_config_dict[k] = Path(v)
1083 else:
1084 # dynamical assignment
1085 analyse_config_dict[k] = v # ty: ignore[invalid-key]
1086
1087 # Get oneline_comment_style configuration
1088 oneline_comment_style_dict: OneLineCommentStyleType | None = config_dict.get(
1089 "oneline_comment_style"
1090 )
1091 oneline_comment_style: OneLineCommentStyle = (
1092 convert_oneline_comment_style_config(oneline_comment_style_dict)
1093 )
1094
1095 # Get need_id_refs configuration
1096 need_id_refs_config_dict: NeedIdRefsConfigType | None = config_dict.get(
1097 "need_id_refs"
1098 )
1099 need_id_refs_config = convert_need_id_refs_config(need_id_refs_config_dict)
1100
1101 # Get marked_rst configuration
1102 marked_rst_config_dict: MarkedRstConfigType | None = config_dict.get(
1103 "marked_rst"
1104 )
1105 marked_rst_config = convert_marked_rst_config(marked_rst_config_dict)
1106
1107 analyse_config_dict["need_id_refs_config"] = need_id_refs_config
1108 analyse_config_dict["marked_rst_config"] = marked_rst_config
1109 analyse_config_dict["oneline_comment_style"] = oneline_comment_style
1110
1111 preprocessor_dict = config_dict.get("preprocessor")
1112 if preprocessor_dict is not None:
1113 # The preprocessor section has no TypedDict; its values are dynamic
1114 # TOML (typed ``object``), so validate the shapes up front -- a mistyped
1115 # scalar would otherwise coerce into garbage flags.
1116 _validate_preprocessor_dict(preprocessor_dict)
1117 analyse_config_dict["preprocessor"] = PreprocessorConfig(
1118 compile_commands=(
1119 Path(str(preprocessor_dict["compile_commands"]))
1120 if preprocessor_dict.get("compile_commands")
1121 else None
1122 ),
1123 defines=list(preprocessor_dict.get("defines", [])), # ty: ignore[invalid-argument-type]
1124 includes=[Path(str(p)) for p in preprocessor_dict.get("includes", [])], # ty: ignore[not-iterable]
1125 std=str(preprocessor_dict.get("std", DEFAULT_CPP_STD)),
1126 )
1127
1128 if src_discover:
1129 analyse_config_dict["src_files"] = src_discover.source_paths
1130 analyse_config_dict["src_dir"] = src_discover.src_discover_config.src_dir
1131 try:
1132 analyse_config_dict["comment_type"] = CommentType(
1133 src_discover.src_discover_config.comment_type
1134 )
1135 except ValueError:
1136 # If invalid comment_type, keep the string value
1137 # Validation will catch this error later
1138 comment_type_str: str = src_discover.src_discover_config.comment_type
1139 analyse_config_dict["comment_type"] = comment_type_str # ty: ignore[invalid-assignment]
1140
1141 return SourceAnalyseConfig(**analyse_config_dict)
1142
1143
1144def convert_oneline_comment_style_config(
1145 config_dict: OneLineCommentStyleType | None,
1146) -> OneLineCommentStyle:
1147 if config_dict is None:
1148 oneline_comment_style = OneLineCommentStyle()
1149 else:
1150 try:
1151 oneline_comment_style = OneLineCommentStyle(**config_dict)
1152 except TypeError as e:
1153 raise TypeError(f"Invalid oneline comment style configuration: {e}") from e
1154 return oneline_comment_style
1155
1156
1157def convert_need_id_refs_config(
1158 config_dict: NeedIdRefsConfigType | None,
1159) -> NeedIdRefsConfig:
1160 if not config_dict:
1161 need_id_refs_config = NeedIdRefsConfig()
1162 else:
1163 try:
1164 need_id_refs_config = NeedIdRefsConfig(**config_dict)
1165 except TypeError as e:
1166 raise TypeError(f"Invalid oneline comment style configuration: {e}") from e
1167 return need_id_refs_config
1168
1169
1170def convert_marked_rst_config(
1171 config_dict: MarkedRstConfigType | None,
1172) -> MarkedRstConfig:
1173 if not config_dict:
1174 marked_rst_config = MarkedRstConfig()
1175 else:
1176 try:
1177 marked_rst_config = MarkedRstConfig(**config_dict)
1178 except TypeError as e:
1179 raise TypeError(f"Invalid oneline comment style configuration: {e}") from e
1180 return marked_rst_config
1181
1182
1183def generate_project_configs(
1184 project_configs: dict[str, CodeLinksProjectConfigType],
1185) -> None:
1186 """Generate configs of source discover and analyse from their classes dynamically."""
1187 for project_config in project_configs.values():
1188 # overwrite the config into different types on purpose
1189 # covert dicts to their own classes
1190 src_discover_section: SourceDiscoverSectionConfigType | None = cast(
1191 SourceDiscoverSectionConfigType,
1192 project_config.get("source_discover"),
1193 )
1194 source_discover_config = convert_src_discovery_config(src_discover_section)
1195 project_config["source_discover_config"] = source_discover_config
1196
1197 analyse_section_config: AnalyseSectionConfigType | None = cast(
1198 AnalyseSectionConfigType, project_config.get("analyse")
1199 )
1200 analyse_config = convert_analyse_config(analyse_section_config)
1201 analyse_config.get_oneline_needs = True # force to get oneline_need
1202 # Copy comment_type from source_discover_config to analyse_config
1203 try:
1204 analyse_config.comment_type = CommentType(
1205 source_discover_config.comment_type
1206 )
1207 except ValueError:
1208 # If invalid comment_type, keep the string value
1209 # Validation will catch this error later
1210 analyse_config.comment_type = source_discover_config.comment_type # ty: ignore[invalid-assignment]
1211 project_config["analyse_config"] = analyse_config
Back to top