feat: add exclude_patterns config key to mempalace.yaml
Allow projects to specify .gitignore-style patterns that the miner should
skip, without relying on .gitignore for mining control.
A new optional exclude_patterns list in mempalace.yaml is parsed by the
existing GitignoreMatcher class via a new from_patterns() classmethod —
same syntax, same semantics as .gitignore, no new dependency.
exclude_patterns:
- '*.md'
- '*.yaml'
- 'docs/' # dir-only: prunes entire tree without descending
- 'dist/'
- 'coverage/'
Key behaviour:
- Patterns follow .gitignore rules: anchoring (/pattern), dir-only
- dirs[:] pruning via GitignoreMatcher.matches(..., is_dir=True) so
excluded subtrees are never walked
- Checked after .gitignore filtering; force_include (--include-ignored)
bypasses exclude_patterns
- Pre-scanned files lists (init double-scan optimisation) are filtered too
- Backwards compatible: omitting exclude_patterns changes nothing
Changes:
- GitignoreMatcher.from_patterns(): new classmethod, same rule parser as
from_dir(), reads from a list instead of a file on disk
- scan_project(): builds one exclude_matcher before os.walk; used for
both dirs[:] pruning and per-file filtering
- _mine_impl(): applies the exclude matcher to pre-scanned files lists
when provided by the caller
- tests/test_miner.py: three new tests
test_scan_project_exclude_patterns_skips_matching_files
test_scan_project_exclude_patterns_prunes_entire_directory
test_scan_project_exclude_patterns_include_ignored_bypasses_exclusion
This commit is contained in:
Lochness
J.R. Murray
parent
c00e8dcc16
commit
b70f06a69e
|
|
@ -244,20 +244,16 @@ class GitignoreMatcher:
|
|||
self.base_dir = base_dir
|
||||
self.rules = rules
|
||||
|
||||
@classmethod
|
||||
def from_dir(cls, dir_path: Path):
|
||||
gitignore_path = dir_path / ".gitignore"
|
||||
if not gitignore_path.is_file():
|
||||
return None
|
||||
|
||||
try:
|
||||
lines = gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except Exception:
|
||||
return None
|
||||
@staticmethod
|
||||
def _parse_rules(lines):
|
||||
"""Parse an iterable of raw pattern strings into a list of rule dicts.
|
||||
|
||||
Shared by :meth:`from_dir` and :meth:`from_patterns` so both paths
|
||||
stay in sync with the gitignore parsing semantics.
|
||||
"""
|
||||
rules = []
|
||||
for raw_line in lines:
|
||||
line = raw_line.strip()
|
||||
line = str(raw_line).strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
|
|
@ -289,12 +285,46 @@ class GitignoreMatcher:
|
|||
"negated": negated,
|
||||
}
|
||||
)
|
||||
return rules
|
||||
|
||||
if not rules:
|
||||
@classmethod
|
||||
def from_dir(cls, dir_path: Path):
|
||||
gitignore_path = dir_path / ".gitignore"
|
||||
if not gitignore_path.is_file():
|
||||
return None
|
||||
|
||||
try:
|
||||
lines = gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
rules = cls._parse_rules(lines)
|
||||
if not rules:
|
||||
return None
|
||||
return cls(dir_path, rules)
|
||||
|
||||
@classmethod
|
||||
def from_patterns(cls, base_dir: Path, patterns):
|
||||
"""Create a matcher from an explicit list of gitignore-inspired pattern strings.
|
||||
|
||||
Supports a gitignore-inspired subset of pattern syntax for
|
||||
``exclude_patterns`` in ``mempalace.yaml``. Patterns are parsed by
|
||||
the same rule parser used for ``.gitignore`` files, so familiar
|
||||
constructs (trailing ``/`` for dir-only, leading ``/`` for anchoring,
|
||||
``!`` negation, ``**`` globs) work as expected. Full gitignore
|
||||
semantics are not guaranteed for every edge case.
|
||||
|
||||
``patterns`` must be a list of strings. A single string is coerced
|
||||
to a one-element list for convenience. Non-string entries are
|
||||
converted via ``str()``.
|
||||
"""
|
||||
if isinstance(patterns, str):
|
||||
patterns = [patterns]
|
||||
rules = cls._parse_rules(patterns)
|
||||
if not rules:
|
||||
return None
|
||||
return cls(base_dir, rules)
|
||||
|
||||
def matches(self, path: Path, is_dir: bool = None):
|
||||
try:
|
||||
relative = path.relative_to(self.base_dir).as_posix().strip("/")
|
||||
|
|
@ -1531,6 +1561,7 @@ def scan_project(
|
|||
project_dir: str,
|
||||
respect_gitignore: bool = True,
|
||||
include_ignored: list = None,
|
||||
exclude_patterns: list = None,
|
||||
) -> list:
|
||||
"""Return list of all readable file paths under ``project_dir``.
|
||||
|
||||
|
|
@ -1543,6 +1574,7 @@ def scan_project(
|
|||
active_matchers = []
|
||||
matcher_cache = {}
|
||||
include_paths = normalize_include_paths(include_ignored)
|
||||
exclude_matcher = GitignoreMatcher.from_patterns(project_path, exclude_patterns or [])
|
||||
|
||||
for root, dirs, filenames in os.walk(project_path):
|
||||
root_path = Path(root)
|
||||
|
|
@ -1570,6 +1602,13 @@ def scan_project(
|
|||
if is_force_included(root_path / d, project_path, include_paths)
|
||||
or not is_gitignored(root_path / d, active_matchers, is_dir=True)
|
||||
]
|
||||
if exclude_matcher:
|
||||
dirs[:] = [
|
||||
d
|
||||
for d in dirs
|
||||
if is_force_included(root_path / d, project_path, include_paths)
|
||||
or exclude_matcher.matches(root_path / d, is_dir=True) is not True
|
||||
]
|
||||
|
||||
for filename in filenames:
|
||||
filepath = root_path / filename
|
||||
|
|
@ -1583,6 +1622,15 @@ def scan_project(
|
|||
if respect_gitignore and active_matchers and not force_include:
|
||||
if is_gitignored(filepath, active_matchers, is_dir=False):
|
||||
continue
|
||||
# Skip files matching project-level exclude_patterns (mempalace.yaml).
|
||||
# force_include paths bypass this check so callers can selectively
|
||||
# re-include files that would otherwise be excluded.
|
||||
if (
|
||||
not force_include
|
||||
and exclude_matcher
|
||||
and exclude_matcher.matches(filepath, is_dir=False)
|
||||
):
|
||||
continue
|
||||
# Skip symlinks — prevents following links to /dev/urandom, etc.
|
||||
if filepath.is_symlink():
|
||||
rel = filepath.relative_to(project_path).as_posix()
|
||||
|
|
@ -1706,13 +1754,29 @@ def _mine_impl(
|
|||
|
||||
wing = wing_override or config["wing"]
|
||||
rooms = config.get("rooms", [{"name": "general", "description": "All project files"}])
|
||||
exclude_patterns = config.get("exclude_patterns", [])
|
||||
|
||||
if files is None:
|
||||
files = scan_project(
|
||||
project_dir,
|
||||
respect_gitignore=respect_gitignore,
|
||||
include_ignored=include_ignored,
|
||||
exclude_patterns=exclude_patterns,
|
||||
)
|
||||
elif exclude_patterns:
|
||||
# The caller provided a pre-scanned file list, so scan_project never ran.
|
||||
# Apply exclude_patterns here so that callers reusing a scan still get
|
||||
# the same per-project exclusions. force_include paths are preserved.
|
||||
include_paths = normalize_include_paths(include_ignored)
|
||||
exclude_matcher = GitignoreMatcher.from_patterns(project_path, exclude_patterns)
|
||||
if exclude_matcher is not None:
|
||||
files = [
|
||||
file_path
|
||||
for file_path in files
|
||||
if is_force_included(file_path, project_path, include_paths)
|
||||
or exclude_matcher.matches(file_path, is_dir=False) is not True
|
||||
]
|
||||
|
||||
from .embedding import describe_device
|
||||
|
||||
print(f"\n{'=' * 55}")
|
||||
|
|
|
|||
|
|
@ -589,6 +589,115 @@ def test_scan_project_logs_nested_symlink_with_relative_path(tmp_path, capsys):
|
|||
assert "(symlink)" in err
|
||||
|
||||
|
||||
def test_scan_project_exclude_patterns_skips_matching_files():
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
project_root = Path(tmpdir).resolve()
|
||||
|
||||
write_file(project_root / "src" / "app.py", "print('app')\n" * 20)
|
||||
write_file(project_root / "docs" / "guide.md", "# Guide\n" * 20)
|
||||
write_file(project_root / "README.md", "# README\n" * 20)
|
||||
|
||||
# *.md excludes README.md and docs/guide.md; docs/* would also cover docs/
|
||||
# subtree. Patterns follow .gitignore syntax via GitignoreMatcher.
|
||||
assert scanned_files(
|
||||
project_root,
|
||||
exclude_patterns=["*.md", "docs/*"],
|
||||
) == ["src/app.py"]
|
||||
finally:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
def test_scan_project_exclude_patterns_prunes_entire_directory():
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
project_root = Path(tmpdir).resolve()
|
||||
|
||||
write_file(project_root / "src" / "app.py", "print('app')\n" * 20)
|
||||
write_file(project_root / "docs" / "sub" / "deep.md", "# Deep\n" * 20)
|
||||
|
||||
# A trailing-slash pattern (dir-only) prunes the whole directory so
|
||||
# os.walk never descends into it.
|
||||
assert scanned_files(project_root, exclude_patterns=["docs/"]) == ["src/app.py"]
|
||||
finally:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
def test_scan_project_exclude_patterns_include_ignored_bypasses_exclusion():
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
project_root = Path(tmpdir).resolve()
|
||||
|
||||
write_file(project_root / "src" / "app.py", "print('app')\n" * 20)
|
||||
write_file(project_root / ".gitignore", "docs/\n")
|
||||
write_file(project_root / "docs" / "guide.md", "# Guide\n" * 20)
|
||||
|
||||
# docs/ is both gitignored and matched by exclude_patterns, but
|
||||
# include_ignored should override both filters and bring it back.
|
||||
assert scanned_files(
|
||||
project_root,
|
||||
exclude_patterns=["docs/*"],
|
||||
include_ignored=["docs"],
|
||||
) == ["docs/guide.md", "src/app.py"]
|
||||
finally:
|
||||
shutil.rmtree(tmpdir)
|
||||
|
||||
|
||||
def test_mine_exclude_patterns_applied_to_prescan_files(monkeypatch):
|
||||
"""exclude_patterns from config must be applied when mine() is given a pre-scanned list.
|
||||
|
||||
The init command calls scan_project() first to show a file-count estimate,
|
||||
then passes the result to mine(..., files=...) to avoid walking the tree
|
||||
twice. The _mine_impl elif branch must apply exclude_patterns to that list
|
||||
so per-project exclusions still take effect.
|
||||
"""
|
||||
import mempalace.miner as miner_mod
|
||||
|
||||
tmpdir = tempfile.mkdtemp()
|
||||
try:
|
||||
project_root = Path(tmpdir).resolve()
|
||||
|
||||
write_file(project_root / "src" / "app.py", "print('app')\n" * 20)
|
||||
write_file(project_root / "docs" / "guide.md", "# Guide\n" * 20)
|
||||
write_file(project_root / "README.md", "# README\n" * 20)
|
||||
|
||||
with open(project_root / "mempalace.yaml", "w") as f:
|
||||
yaml.dump(
|
||||
{
|
||||
"wing": "test_project",
|
||||
"rooms": [{"name": "general", "description": "General"}],
|
||||
"exclude_patterns": ["*.md", "docs/"],
|
||||
},
|
||||
f,
|
||||
)
|
||||
|
||||
# Simulate the init double-scan: scan without exclude_patterns first
|
||||
# (representing the caller that doesn't know the config), then pass
|
||||
# the full list to mine() so the elif branch applies the config exclusions.
|
||||
prescan = scan_project(str(project_root), exclude_patterns=None)
|
||||
assert len(prescan) >= 2, "prescan should include .md files before filtering"
|
||||
|
||||
# Capture which files process_file is called with during dry_run
|
||||
processed = []
|
||||
_real_process_file = miner_mod.process_file
|
||||
|
||||
def _capture_process_file(filepath, **kwargs):
|
||||
processed.append(filepath)
|
||||
return _real_process_file(filepath, **kwargs)
|
||||
|
||||
monkeypatch.setattr(miner_mod, "process_file", _capture_process_file)
|
||||
|
||||
palace_path = project_root / "palace"
|
||||
mine(str(project_root), str(palace_path), dry_run=True, files=prescan)
|
||||
|
||||
processed_rel = [p.relative_to(project_root).as_posix() for p in processed]
|
||||
assert "src/app.py" in processed_rel
|
||||
assert "docs/guide.md" not in processed_rel
|
||||
assert "README.md" not in processed_rel
|
||||
finally:
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
|
||||
|
||||
def test_entity_metadata_finds_cyrillic_names(monkeypatch):
|
||||
"""Entity extraction must find non-Latin names when entity_languages includes the locale."""
|
||||
import mempalace.palace as palace_mod
|
||||
|
|
|
|||
Loading…
Reference in New Issue