feat: add LaTeX (.tex, .bib) to readable and prose extensions
LaTeX source files and BibTeX bibliographies are prose-rich content that benefits from both palace mining and entity detection. Adds the two extensions to the two extension lists most relevant to them, each with a matching test. - ``mempalace/miner.py:READABLE_EXTENSIONS`` — ``.tex`` / ``.bib`` join the mining allowlist (parallel to the Swift/Kotlin PR #1368 and the PHP ecosystem PR #1819). - ``mempalace/entity_detector.py:PROSE_EXTENSIONS`` — ``.tex`` / ``.bib`` also join the *preferred* entity-detection bucket alongside ``.md`` / ``.rst`` / ``.csv``, NOT the broader code-file fallback. The reason ``PROSE_EXTENSIONS`` exists separately is documented in-code: programming-language files have lots of capitalized identifiers (class names, function names) that produce false-positive person matches. LaTeX/BibTeX don't have that problem — they're typesetting languages for prose documents. ``.bib`` in particular is almost entirely author names, one of the highest real-entity densities of any file type the detector scans. Tests follow the patterns established by the prior extension PRs: ``tests/test_miner.py::test_scan_project_includes_latex_files`` mirrors the Swift/Kotlin scan tests, and ``tests/test_entity_detector.py::test_scan_for_detection_includes_latex_prose`` mirrors ``test_scan_for_detection_finds_prose``. The existing ``test_prose_extensions`` was extended to assert the two new entries. Full env-cleared suite: 3216 passed, 20 skipped. ``ruff check .`` and ``ruff format --check .`` both clean. Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01KC5Qsknh2zFRtRvVyjXiTA
This commit is contained in:
parent
afd0428823
commit
6cec591c17
|
|
@ -215,6 +215,8 @@ PROSE_EXTENSIONS = {
|
|||
".md",
|
||||
".rst",
|
||||
".csv",
|
||||
".tex",
|
||||
".bib",
|
||||
}
|
||||
|
||||
READABLE_EXTENSIONS = {
|
||||
|
|
|
|||
|
|
@ -133,6 +133,8 @@ READABLE_EXTENSIONS = {
|
|||
".csv",
|
||||
".sql",
|
||||
".toml",
|
||||
".tex",
|
||||
".bib",
|
||||
# C# / .NET
|
||||
".cs",
|
||||
".csproj",
|
||||
|
|
|
|||
|
|
@ -531,6 +531,24 @@ def test_scan_for_detection_skips_git_dir(tmp_path):
|
|||
assert not any(".git" in f for f in file_strs)
|
||||
|
||||
|
||||
def test_scan_for_detection_includes_latex_prose(tmp_path):
|
||||
# .tex and .bib are prose-heavy (author names, abstracts, citations) and
|
||||
# belong in the preferred PROSE_EXTENSIONS bucket alongside .md / .rst,
|
||||
# not the code-file fallback. .bib in particular is almost entirely
|
||||
# author names — high entity density per byte.
|
||||
(tmp_path / "paper.tex").write_text(
|
||||
"\\documentclass{article}\\author{Leslie Lamport}\\begin{document}Body.\\end{document}"
|
||||
)
|
||||
(tmp_path / "refs.bib").write_text(
|
||||
"@article{l86, author={Leslie Lamport}, title={LaTeX}, year={1986}}"
|
||||
)
|
||||
(tmp_path / "code.py").write_text("import os")
|
||||
files = scan_for_detection(str(tmp_path))
|
||||
extensions = {os.path.splitext(str(f))[1] for f in files}
|
||||
assert ".tex" in extensions
|
||||
assert ".bib" in extensions
|
||||
|
||||
|
||||
# ── module-level constants ──────────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
@ -543,6 +561,8 @@ def test_stopwords_contains_common_words():
|
|||
def test_prose_extensions():
|
||||
assert ".txt" in PROSE_EXTENSIONS
|
||||
assert ".md" in PROSE_EXTENSIONS
|
||||
assert ".tex" in PROSE_EXTENSIONS
|
||||
assert ".bib" in PROSE_EXTENSIONS
|
||||
|
||||
|
||||
# ── _print_entity_list ─────────────────────────────────────────────────
|
||||
|
|
|
|||
|
|
@ -349,6 +349,20 @@ def test_scan_project_includes_kotlin_files():
|
|||
]
|
||||
|
||||
|
||||
def test_scan_project_includes_latex_files():
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
project_root = Path(tmpdir).resolve()
|
||||
write_file(
|
||||
project_root / "main.tex",
|
||||
"\\documentclass{article}\n\\begin{document}\nHello, world.\n\\end{document}\n" * 20,
|
||||
)
|
||||
write_file(
|
||||
project_root / "refs.bib",
|
||||
"@article{lamport1986, author={Leslie Lamport}, title={LaTeX}, year={1986}}\n" * 20,
|
||||
)
|
||||
assert scanned_files(project_root) == ["main.tex", "refs.bib"]
|
||||
|
||||
|
||||
def test_scan_project_respects_gitignore():
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
|
|
|
|||
Loading…
Reference in New Issue