[docs]  1# @Test that line numbers stay exact far past line 256, TEST_ANA_2, test, [IMPL_LNK_1, IMPL_ONE_1, IMPL_MRST_1, IMPL_JSONC_2]
  2"""Line numbers far down a long source file, with the analysis in a subprocess.
  3
  4tree-sitter 0.26.0's ``Point.row`` and ``Point.column`` return a borrowed reference
  5(py-tree-sitter#472, fixed by #466). Ints up to 256 are CPython's cached small ints, so a
  6short file never notices; a row above that is freed early and the heap is corrupted. The
  7analysis then dies with SIGSEGV or reports a wrong line, and every fixture elsewhere in
  8this suite is too short to reach it. This module is what makes the suite see it: it is the
  9reason ``tree-sitter`` excludes 0.26.0 in ``pyproject.toml``.
 10
 11The analysis runs in a child interpreter so that a native crash is a clean failure of one
 12test, naming the exit code, rather than the death of the whole pytest process.
 13
 14Two paths read tree-sitter points, and each case below reaches one or both of them:
 15``analyse.py`` reads ``start_point.row`` for every one-line need, need-id reference and
 16marked-rst block, whatever the language; ``utils.find_prev_sibling_on_same_row`` reads
 17``start_point.row`` and ``end_point.row`` to tie an inline comment to its structure, and
 18only YAML and JSONC reach it.
 19"""
 20
 21from __future__ import annotations
 22
 23import json
 24import signal
 25import subprocess
 26import sys
 27from dataclasses import dataclass
 28from pathlib import Path
 29
 30import pytest
 31
 32# Well past the small-int cache (-5..256) at both ends of every marker: the first marker
 33# sits at row FIRST_ROW, and the file runs to a few thousand lines so that there are
 34# hundreds of them, each of whose rows is read more than once.
 35FIRST_ROW = 300
 36NUM_BLOCKS = 300
 37UPSTREAM = (
 38    "tree-sitter's Point getters return a borrowed reference in 0.26.0 "
 39    "(https://github.com/tree-sitter/py-tree-sitter/issues/472, "
 40    "fixed by https://github.com/tree-sitter/py-tree-sitter/pull/466)"
 41)
 42
 43# The child: one SourceAnalyse over one file, every extractor on, the result written as
 44# JSON to a file rather than stdout so that nothing the analysis logs can corrupt it.
 45CHILD = """
 46import json
 47import sys
 48from pathlib import Path
 49
 50from sphinx_codelinks.analyse.analyse import SourceAnalyse
 51from sphinx_codelinks.config import SourceAnalyseConfig
 52from sphinx_codelinks.source_discover.config import CommentType
 53
 54src, comment_type, out = Path(sys.argv[1]), CommentType(sys.argv[2]), Path(sys.argv[3])
 55analyse = SourceAnalyse(
 56    SourceAnalyseConfig(
 57        src_files=[src],
 58        src_dir=src.parent,
 59        get_need_id_refs=True,
 60        get_oneline_needs=True,
 61        get_rst=True,
 62        comment_type=comment_type,
 63    )
 64)
 65analyse.git_remote_url = None
 66analyse.git_commit_rev = None
 67analyse.run()
 68out.write_text(json.dumps([m.to_dict() for m in analyse.all_marked_content]))
 69"""
 70
 71
 72@dataclass(frozen=True)
 73class Marker:
 74    """One marker the analysis must report, as ``(kind, 0-based row, key, scope)``."""
 75
 76    kind: str
 77    row: int
 78    key: str
 79    scope: str | None = None
 80
 81
 82def _pad(lines: list[str], filler: str) -> None:
 83    """Pad ``lines`` with ``filler`` (formatted with the 0-based row) up to FIRST_ROW."""
 84    while len(lines) < FIRST_ROW:
 85        lines.append(filler.format(n=len(lines)))
 86
 87
 88def _cpp_source() -> tuple[str, list[Marker]]:
 89    """A C++ file with a one-line need, a need-id reference and a marked-rst block per
 90    stanza, some on the first line of their comment and some further down it."""
 91    lines: list[str] = []
 92    expected: list[Marker] = []
 93    _pad(lines, "int filler_{n} = {n};")
 94    for i in range(NUM_BLOCKS):
 95        row = len(lines)
 96        lines.append(f"// @Title {i}, IMPL_CPP_{i}, impl")
 97        expected.append(Marker("need", row, f"IMPL_CPP_{i}"))
 98        lines.append(f"int value_{i} = {i};")
 99        # a block comment: the need-id reference is on its second line, so the row is
100        # the comment's start row plus an offset
101        lines.append("/* block")
102        expected.append(Marker("need-id-refs", len(lines), f"REQ_CPP_{i}"))
103        lines.append(f"   @need-ids: REQ_CPP_{i}")
104        lines.append("*/")
105        # a marked-rst block, reported at the row of its start sequence
106        lines.append("/*")
107        expected.append(Marker("rst", len(lines), f"rst_{i}"))
108        lines.append("@rst")
109        lines.append(f"rst_{i}")
110        lines.append("@endrst")
111        lines.append("*/")
112        lines.append(f"void func_{i}() {{}}")
113    return "\n".join(lines) + "\n", expected
114
115
116def _yaml_source() -> tuple[str, list[Marker]]:
117    """A YAML mapping with an inline marker on every other key, and a marker on a line of
118    its own between them, so that both the same-row match and the early exit of the
119    sibling walk run."""
120    lines: list[str] = []
121    expected: list[Marker] = []
122    _pad(lines, "filler_{n}: {n}")
123    for i in range(NUM_BLOCKS):
124        row = len(lines)
125        lines.append(f"key_{i}: value_{i}  # @Title {i}, IMPL_YAML_{i}, impl")
126        expected.append(Marker("need", row, f"IMPL_YAML_{i}", f"key_{i}: value_{i}"))
127        expected.append(
128            Marker("need", row + 1, f"IMPL_YAML_OWN_{i}", f"other_{i}: {i}")
129        )
130        lines.append(f"# @Own line {i}, IMPL_YAML_OWN_{i}, impl")
131        lines.append(f"other_{i}: {i}")
132    return "\n".join(lines) + "\n", expected
133
134
135def _jsonc_source() -> tuple[str, list[Marker]]:
136    """A JSONC object with an inline marker after every other pair, and a marker on a
137    line of its own between them."""
138    lines: list[str] = ["{"]
139    expected: list[Marker] = []
140    _pad(lines, '  "filler_{n}": {n},')
141    for i in range(NUM_BLOCKS):
142        row = len(lines)
143        lines.append(f'  "key_{i}": {i}, // @Title {i}, IMPL_JSONC_{i}, impl')
144        expected.append(Marker("need", row, f"IMPL_JSONC_{i}", f'"key_{i}": {i}'))
145        expected.append(
146            Marker("need", row + 1, f"IMPL_JSONC_OWN_{i}", f'"other_{i}": {i}')
147        )
148        lines.append(f"  // @Own line {i}, IMPL_JSONC_OWN_{i}, impl")
149        lines.append(f'  "other_{i}": {i},')
150    lines.append('  "last": 0')
151    lines.append("}")
152    return "\n".join(lines) + "\n", expected
153
154
155def _describe_exit(returncode: int) -> str:
156    """Name a child's exit: a negative code is the signal that killed it (POSIX)."""
157    if returncode < 0:
158        try:
159            return f"exit code {returncode} ({signal.Signals(-returncode).name})"
160        except ValueError:
161            pass
162    return f"exit code {returncode} ({returncode & 0xFFFFFFFF:#010x})"
163
164
165def _reported(record: dict, with_scope: bool) -> Marker:
166    """The Marker a ``to_dict()`` record from the child describes; the scope only when
167    ``with_scope``, since it is tree-sitter point arithmetic only for YAML and JSONC."""
168    start, end = record["source_map"]["start"], record["source_map"]["end"]
169    assert start["row"] == end["row"], record
170    kind = record["type"]
171    if kind == "need":
172        key = record["need"]["id"]
173    elif kind == "need-id-refs":
174        (key,) = record["need_ids"]
175    else:
176        key = record["rst"].strip()
177    scope = record["tagged_scope"] if with_scope else None
178    return Marker(kind, start["row"], key, scope)
179
180
181@pytest.mark.parametrize(
182    ("comment_type", "suffix", "make_source"),
183    [
184        pytest.param("cpp", "cpp", _cpp_source, id="cpp"),
185        pytest.param("yaml", "yaml", _yaml_source, id="yaml"),
186        pytest.param("jsonc", "jsonc", _jsonc_source, id="jsonc"),
187    ],
188)
189def test_line_numbers_past_row_256(
190    tmp_path: Path, comment_type: str, suffix: str, make_source
191) -> None:
192    text, expected = make_source()
193    assert min(marker.row for marker in expected) > 256
194    source = tmp_path / f"long.{suffix}"
195    source.write_text(text, encoding="utf-8")
196    out = tmp_path / "marked_content.json"
197
198    result = subprocess.run(
199        [sys.executable, "-c", CHILD, str(source), comment_type, str(out)],
200        capture_output=True,
201        text=True,
202        timeout=120,
203        check=False,
204    )
205
206    if result.returncode != 0:
207        pytest.fail(
208            f"the {comment_type} analysis of a {len(text.splitlines())}-line file "
209            f"crashed with {_describe_exit(result.returncode)}; the likeliest cause is "
210            f"that {UPSTREAM}.\nstderr (tail):\n{result.stderr[-2000:]}"
211        )
212    with_scope = comment_type != "cpp"
213    reported = [_reported(record, with_scope) for record in json.loads(out.read_text())]
214    wrong = sorted(set(reported) ^ set(expected), key=lambda m: (m.row, m.key))
215    assert len(reported) == len(expected) and not wrong, (
216        f"the {comment_type} analysis reported {len(reported)} markers for "
217        f"{len(expected)} expected; the ones that differ (reported XOR expected, first "
218        f"10): {wrong[:10]}. If the rows are wrong, the likeliest cause is that {UPSTREAM}."
219    )