perf(miner): O(N) incremental line-number tally in chunk_text (#2054)

chunk_text recomputed line_start/line_end with a full-prefix
content.count("\n", 0, pos) per chunk (O(N*K) overall), so large files
ground for days. Keep the emitted values byte-identical but tally
newlines incrementally over the newly-scanned span (O(N) total), with a
from-scratch fallback that keeps every value exact.

mempalace/miner.py

@@ -653,6 +653,28 @@ def chunk_text(
     start = 0
     chunk_index = 0
 
+    # Running newline tallies for the 1-indexed (line_start, line_end)
+    # locators emitted below. These replace the previous per-chunk
+    # ``content.count("\n", 0, pos)`` full-prefix rescans, which were O(pos)
+    # each and O(N*K) over K chunks: a 287 MB / ~433k-chunk file scanned
+    # ~1.2e14 bytes and ran for days, indistinguishable from a hang (#2054).
+    # ``start`` and ``end`` each advance monotonically for any
+    # ``chunk_overlap < chunk_size // 2`` (the default 800/100 config and
+    # every sane override), so counting only the newly-scanned span is O(N)
+    # total. Each position sequence keeps its own anchor; a backward step
+    # (a paragraph-boundary pull under a pathological near-``chunk_size``
+    # overlap, or a negative index) falls back to the exact full-prefix
+    # count. That fallback is defensive and does not fire on a terminating
+    # mine: the current windowing cannot send a filed chunk's ``start``
+    # backward without also entering a pre-existing infinite loop (overlap
+    # >= chunk_size // 2), so the else-branches read as uncovered. They keep
+    # every value byte-identical to the old form if the windowing ever gains
+    # a loop guard that admits backward steps.
+    _nl_before_start = 0
+    _start_anchor = 0
+    _nl_before_end = 0
+    _end_anchor = 0
+
     while start < len(content):
         end = min(start + chunk_size, len(content))
 
@@ -671,13 +693,25 @@ def chunk_text(
             # Tier 6a — 1-indexed line range in the stripped source.
             # Approximate locator (±1 at boundaries is fine for "jump to
             # roughly here"); exact-quote positioning is a future tier.
-            # Use the bounds form of ``str.count`` (counts on the original
-            # string with start/end limits) instead of slicing — slicing
-            # would allocate a new substring per chunk and produce O(N^2)
-            # work on a 500MB file with 50K chunks. Per PR #1579 review
-            # (gemini-code-assist, medium priority).
-            line_start = content.count("\n", 0, start) + 1
-            line_end = content.count("\n", 0, end) + 1
+            # ``str.count`` with bounds (not slicing) still avoids allocating
+            # a substring per chunk, the original PR #1579 review concern
+            # (gemini-code-assist, medium priority). The incremental anchors
+            # additionally avoid rescanning the whole prefix each time, the
+            # actual O(N*K) cost (#2054). Two anchors because ``start`` and
+            # ``end`` are distinct monotonic sequences; a backward step
+            # re-derives the value exactly from position 0.
+            if start >= _start_anchor:
+                _nl_before_start += content.count("\n", _start_anchor, start)
+                _start_anchor = start
+                line_start = _nl_before_start + 1
+            else:
+                line_start = content.count("\n", 0, start) + 1
+            if end >= _end_anchor:
+                _nl_before_end += content.count("\n", _end_anchor, end)
+                _end_anchor = end
+                line_end = _nl_before_end + 1
+            else:
+                line_end = content.count("\n", 0, end) + 1
             chunks.append(
                 {
                     "content": chunk,

tests/test_miner.py

@@ -2042,6 +2042,79 @@ class TestChunkTextLineRanges:
         assert chunks[0]["line_end"] == 5
 
 
+def _naive_chunk_line_ranges(content, *, chunk_size, chunk_overlap, min_chunk_size):
+    """Pre-#2054 reference implementation of chunk_text's line locators.
+
+    Same windowing as ``chunk_text`` but recomputes ``(line_start, line_end)``
+    with the original full-prefix ``content.count("\\n", 0, pos)`` form. The
+    incremental-anchor rewrite must stay byte-identical to this on every input,
+    so this is the golden reference the tests below compare against.
+    """
+    content = content.strip()
+    if not content:
+        return []
+    out = []
+    start = 0
+    while start < len(content):
+        end = min(start + chunk_size, len(content))
+        if end < len(content):
+            newline_pos = content.rfind("\n\n", start, end)
+            if newline_pos > start + chunk_size // 2:
+                end = newline_pos
+            else:
+                newline_pos = content.rfind("\n", start, end)
+                if newline_pos > start + chunk_size // 2:
+                    end = newline_pos
+        chunk = content[start:end].strip()
+        if len(chunk) >= min_chunk_size:
+            out.append((content.count("\n", 0, start) + 1, content.count("\n", 0, end) + 1))
+        start = end - chunk_overlap if end < len(content) else end
+    return out
+
+
+class TestChunkTextLineRangesIncremental:
+    """#2054: the O(N) incremental newline tally must match the old O(N*K)
+    full-prefix ``str.count`` form byte-for-byte across varied corpora and
+    configs. Configs stay at ``overlap < size//2`` so ``start``/``end`` advance
+    monotonically (the fast path); the code's from-scratch fallback for a
+    backward step is a defensive guard; a real backward step would also trip a
+    pre-existing infinite loop in the windowing itself, which is out of scope
+    for this locator-perf fix.
+    """
+
+    def test_line_ranges_match_full_prefix_reference(self):
+        import random
+
+        from mempalace.miner import chunk_text
+
+        rng = random.Random(2054)
+        corpora = [
+            "\n".join(f"line {i}" for i in range(1, 501)),  # many short lines
+            "\n\n".join(f"para {i} " + "x" * rng.randint(0, 300) for i in range(60)),
+            "no newlines at all " * 500,  # zero newlines
+            "\n" * 200 + "tail",  # leading blank lines
+            "".join(rng.choice("ab \n\n") for _ in range(5000)),  # random newline density
+            "αβγ\nδεζ\n" * 400,  # non-ASCII
+        ]
+        configs = [
+            (800, 100, 50),  # default config
+            (200, 20, 10),
+            (400, 50, 5),
+            (1000, 200, 30),
+            (2000, 0, 1),  # zero overlap
+        ]
+        for content in corpora:
+            for cs, co, mc in configs:
+                chunks = chunk_text(
+                    content, "/x.md", chunk_size=cs, chunk_overlap=co, min_chunk_size=mc
+                )
+                got = [(c["line_start"], c["line_end"]) for c in chunks]
+                expected = _naive_chunk_line_ranges(
+                    content, chunk_size=cs, chunk_overlap=co, min_chunk_size=mc
+                )
+                assert got == expected, f"cs={cs} co={co} mc={mc}: {got[:6]} != {expected[:6]}"
+
+
 class TestBuildDrawerMetadataLineRange:
     """Tier 6a — _build_drawer_metadata stores optional line_start / line_end.