[docs] 1# @Test that line numbers stay exact far past line 256, TEST_ANA_2, test, [IMPL_LNK_1, IMPL_ONE_1, IMPL_MRST_1, IMPL_JSONC_2]
2"""Line numbers far down a long source file, with the analysis in a subprocess.
3
4tree-sitter 0.26.0's ``Point.row`` and ``Point.column`` return a borrowed reference
5(py-tree-sitter#472, fixed by #466). Ints up to 256 are CPython's cached small ints, so a
6short file never notices; a row above that is freed early and the heap is corrupted. The
7analysis then dies with SIGSEGV or reports a wrong line, and every fixture elsewhere in
8this suite is too short to reach it. This module is what makes the suite see it: it is the
9reason ``tree-sitter`` excludes 0.26.0 in ``pyproject.toml``.
10
11The analysis runs in a child interpreter so that a native crash is a clean failure of one
12test, naming the exit code, rather than the death of the whole pytest process.
13
14Two paths read tree-sitter points, and each case below reaches one or both of them:
15``analyse.py`` reads ``start_point.row`` for every one-line need, need-id reference and
16marked-rst block, whatever the language; ``utils.find_prev_sibling_on_same_row`` reads
17``start_point.row`` and ``end_point.row`` to tie an inline comment to its structure, and
18only YAML and JSONC reach it.
19"""
20
21from __future__ import annotations
22
23import json
24import signal
25import subprocess
26import sys
27from dataclasses import dataclass
28from pathlib import Path
29
30import pytest
31
32# Well past the small-int cache (-5..256) at both ends of every marker: the first marker
33# sits at row FIRST_ROW, and the file runs to a few thousand lines so that there are
34# hundreds of them, each of whose rows is read more than once.
35FIRST_ROW = 300
36NUM_BLOCKS = 300
37UPSTREAM = (
38 "tree-sitter's Point getters return a borrowed reference in 0.26.0 "
39 "(https://github.com/tree-sitter/py-tree-sitter/issues/472, "
40 "fixed by https://github.com/tree-sitter/py-tree-sitter/pull/466)"
41)
42
43# The child: one SourceAnalyse over one file, every extractor on, the result written as
44# JSON to a file rather than stdout so that nothing the analysis logs can corrupt it.
45CHILD = """
46import json
47import sys
48from pathlib import Path
49
50from sphinx_codelinks.analyse.analyse import SourceAnalyse
51from sphinx_codelinks.config import SourceAnalyseConfig
52from sphinx_codelinks.source_discover.config import CommentType
53
54src, comment_type, out = Path(sys.argv[1]), CommentType(sys.argv[2]), Path(sys.argv[3])
55analyse = SourceAnalyse(
56 SourceAnalyseConfig(
57 src_files=[src],
58 src_dir=src.parent,
59 get_need_id_refs=True,
60 get_oneline_needs=True,
61 get_rst=True,
62 comment_type=comment_type,
63 )
64)
65analyse.git_remote_url = None
66analyse.git_commit_rev = None
67analyse.run()
68out.write_text(json.dumps([m.to_dict() for m in analyse.all_marked_content]))
69"""
70
71
72@dataclass(frozen=True)
73class Marker:
74 """One marker the analysis must report, as ``(kind, 0-based row, key, scope)``."""
75
76 kind: str
77 row: int
78 key: str
79 scope: str | None = None
80
81
82def _pad(lines: list[str], filler: str) -> None:
83 """Pad ``lines`` with ``filler`` (formatted with the 0-based row) up to FIRST_ROW."""
84 while len(lines) < FIRST_ROW:
85 lines.append(filler.format(n=len(lines)))
86
87
88def _cpp_source() -> tuple[str, list[Marker]]:
89 """A C++ file with a one-line need, a need-id reference and a marked-rst block per
90 stanza, some on the first line of their comment and some further down it."""
91 lines: list[str] = []
92 expected: list[Marker] = []
93 _pad(lines, "int filler_{n} = {n};")
94 for i in range(NUM_BLOCKS):
95 row = len(lines)
96 lines.append(f"// @Title {i}, IMPL_CPP_{i}, impl")
97 expected.append(Marker("need", row, f"IMPL_CPP_{i}"))
98 lines.append(f"int value_{i} = {i};")
99 # a block comment: the need-id reference is on its second line, so the row is
100 # the comment's start row plus an offset
101 lines.append("/* block")
102 expected.append(Marker("need-id-refs", len(lines), f"REQ_CPP_{i}"))
103 lines.append(f" @need-ids: REQ_CPP_{i}")
104 lines.append("*/")
105 # a marked-rst block, reported at the row of its start sequence
106 lines.append("/*")
107 expected.append(Marker("rst", len(lines), f"rst_{i}"))
108 lines.append("@rst")
109 lines.append(f"rst_{i}")
110 lines.append("@endrst")
111 lines.append("*/")
112 lines.append(f"void func_{i}() {{}}")
113 return "\n".join(lines) + "\n", expected
114
115
116def _yaml_source() -> tuple[str, list[Marker]]:
117 """A YAML mapping with an inline marker on every other key, and a marker on a line of
118 its own between them, so that both the same-row match and the early exit of the
119 sibling walk run."""
120 lines: list[str] = []
121 expected: list[Marker] = []
122 _pad(lines, "filler_{n}: {n}")
123 for i in range(NUM_BLOCKS):
124 row = len(lines)
125 lines.append(f"key_{i}: value_{i} # @Title {i}, IMPL_YAML_{i}, impl")
126 expected.append(Marker("need", row, f"IMPL_YAML_{i}", f"key_{i}: value_{i}"))
127 expected.append(
128 Marker("need", row + 1, f"IMPL_YAML_OWN_{i}", f"other_{i}: {i}")
129 )
130 lines.append(f"# @Own line {i}, IMPL_YAML_OWN_{i}, impl")
131 lines.append(f"other_{i}: {i}")
132 return "\n".join(lines) + "\n", expected
133
134
135def _jsonc_source() -> tuple[str, list[Marker]]:
136 """A JSONC object with an inline marker after every other pair, and a marker on a
137 line of its own between them."""
138 lines: list[str] = ["{"]
139 expected: list[Marker] = []
140 _pad(lines, ' "filler_{n}": {n},')
141 for i in range(NUM_BLOCKS):
142 row = len(lines)
143 lines.append(f' "key_{i}": {i}, // @Title {i}, IMPL_JSONC_{i}, impl')
144 expected.append(Marker("need", row, f"IMPL_JSONC_{i}", f'"key_{i}": {i}'))
145 expected.append(
146 Marker("need", row + 1, f"IMPL_JSONC_OWN_{i}", f'"other_{i}": {i}')
147 )
148 lines.append(f" // @Own line {i}, IMPL_JSONC_OWN_{i}, impl")
149 lines.append(f' "other_{i}": {i},')
150 lines.append(' "last": 0')
151 lines.append("}")
152 return "\n".join(lines) + "\n", expected
153
154
155def _describe_exit(returncode: int) -> str:
156 """Name a child's exit: a negative code is the signal that killed it (POSIX)."""
157 if returncode < 0:
158 try:
159 return f"exit code {returncode} ({signal.Signals(-returncode).name})"
160 except ValueError:
161 pass
162 return f"exit code {returncode} ({returncode & 0xFFFFFFFF:#010x})"
163
164
165def _reported(record: dict, with_scope: bool) -> Marker:
166 """The Marker a ``to_dict()`` record from the child describes; the scope only when
167 ``with_scope``, since it is tree-sitter point arithmetic only for YAML and JSONC."""
168 start, end = record["source_map"]["start"], record["source_map"]["end"]
169 assert start["row"] == end["row"], record
170 kind = record["type"]
171 if kind == "need":
172 key = record["need"]["id"]
173 elif kind == "need-id-refs":
174 (key,) = record["need_ids"]
175 else:
176 key = record["rst"].strip()
177 scope = record["tagged_scope"] if with_scope else None
178 return Marker(kind, start["row"], key, scope)
179
180
181@pytest.mark.parametrize(
182 ("comment_type", "suffix", "make_source"),
183 [
184 pytest.param("cpp", "cpp", _cpp_source, id="cpp"),
185 pytest.param("yaml", "yaml", _yaml_source, id="yaml"),
186 pytest.param("jsonc", "jsonc", _jsonc_source, id="jsonc"),
187 ],
188)
189def test_line_numbers_past_row_256(
190 tmp_path: Path, comment_type: str, suffix: str, make_source
191) -> None:
192 text, expected = make_source()
193 assert min(marker.row for marker in expected) > 256
194 source = tmp_path / f"long.{suffix}"
195 source.write_text(text, encoding="utf-8")
196 out = tmp_path / "marked_content.json"
197
198 result = subprocess.run(
199 [sys.executable, "-c", CHILD, str(source), comment_type, str(out)],
200 capture_output=True,
201 text=True,
202 timeout=120,
203 check=False,
204 )
205
206 if result.returncode != 0:
207 pytest.fail(
208 f"the {comment_type} analysis of a {len(text.splitlines())}-line file "
209 f"crashed with {_describe_exit(result.returncode)}; the likeliest cause is "
210 f"that {UPSTREAM}.\nstderr (tail):\n{result.stderr[-2000:]}"
211 )
212 with_scope = comment_type != "cpp"
213 reported = [_reported(record, with_scope) for record in json.loads(out.read_text())]
214 wrong = sorted(set(reported) ^ set(expected), key=lambda m: (m.row, m.key))
215 assert len(reported) == len(expected) and not wrong, (
216 f"the {comment_type} analysis reported {len(reported)} markers for "
217 f"{len(expected)} expected; the ones that differ (reported XOR expected, first "
218 f"10): {wrong[:10]}. If the rows are wrong, the likeliest cause is that {UPSTREAM}."
219 )