29 lines
1.1 KiB
Python
29 lines
1.1 KiB
Python
from typing import List, Tuple
|
|
|
|
def basic_clean(text: str) -> str:
|
|
"""Minimal cleaning only: normalize broken lines and spaces, do NOT remove headers/footers."""
|
|
# Join obvious hyphenated line-breaks and normalize spaces
|
|
text = text.replace('\r', '')
|
|
# Collapse multiple spaces but keep newlines
|
|
lines = [l.strip() for l in text.split('\n')]
|
|
return "\n".join(lines)
|
|
|
|
def chunk_text(text: str, chunk_size: int, overlap: int) -> List[Tuple[int, int, str]]:
|
|
"""Split text into overlapping chunks by character count.
|
|
Returns list of tuples: (start_char, end_char, chunk_text)."""
|
|
if chunk_size <= 0:
|
|
raise ValueError("chunk_size must be > 0")
|
|
if overlap < 0 or overlap >= chunk_size:
|
|
raise ValueError("overlap must be >= 0 and < chunk_size")
|
|
|
|
chunks: List[Tuple[int, int, str]] = []
|
|
n = len(text)
|
|
start = 0
|
|
while start < n:
|
|
end = min(start + chunk_size, n)
|
|
chunks.append((start, end, text[start:end]))
|
|
if end == n:
|
|
break
|
|
start = end - overlap # overlap ensures context continuity
|
|
return chunks
|