diff --git a/mempalace/miner.py b/mempalace/miner.py index dda0c0e..6f60d93 100644 --- a/mempalace/miner.py +++ b/mempalace/miner.py @@ -244,20 +244,16 @@ class GitignoreMatcher: self.base_dir = base_dir self.rules = rules - @classmethod - def from_dir(cls, dir_path: Path): - gitignore_path = dir_path / ".gitignore" - if not gitignore_path.is_file(): - return None - - try: - lines = gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines() - except Exception: - return None + @staticmethod + def _parse_rules(lines): + """Parse an iterable of raw pattern strings into a list of rule dicts. + Shared by :meth:`from_dir` and :meth:`from_patterns` so both paths + stay in sync with the gitignore parsing semantics. + """ rules = [] for raw_line in lines: - line = raw_line.strip() + line = str(raw_line).strip() if not line: continue @@ -289,12 +285,46 @@ class GitignoreMatcher: "negated": negated, } ) + return rules - if not rules: + @classmethod + def from_dir(cls, dir_path: Path): + gitignore_path = dir_path / ".gitignore" + if not gitignore_path.is_file(): return None + try: + lines = gitignore_path.read_text(encoding="utf-8", errors="replace").splitlines() + except Exception: + return None + + rules = cls._parse_rules(lines) + if not rules: + return None return cls(dir_path, rules) + @classmethod + def from_patterns(cls, base_dir: Path, patterns): + """Create a matcher from an explicit list of gitignore-inspired pattern strings. + + Supports a gitignore-inspired subset of pattern syntax for + ``exclude_patterns`` in ``mempalace.yaml``. Patterns are parsed by + the same rule parser used for ``.gitignore`` files, so familiar + constructs (trailing ``/`` for dir-only, leading ``/`` for anchoring, + ``!`` negation, ``**`` globs) work as expected. Full gitignore + semantics are not guaranteed for every edge case. + + ``patterns`` must be a list of strings. A single string is coerced + to a one-element list for convenience. Non-string entries are + converted via ``str()``. + """ + if isinstance(patterns, str): + patterns = [patterns] + rules = cls._parse_rules(patterns) + if not rules: + return None + return cls(base_dir, rules) + def matches(self, path: Path, is_dir: bool = None): try: relative = path.relative_to(self.base_dir).as_posix().strip("/") @@ -1531,6 +1561,7 @@ def scan_project( project_dir: str, respect_gitignore: bool = True, include_ignored: list = None, + exclude_patterns: list = None, ) -> list: """Return list of all readable file paths under ``project_dir``. @@ -1543,6 +1574,7 @@ def scan_project( active_matchers = [] matcher_cache = {} include_paths = normalize_include_paths(include_ignored) + exclude_matcher = GitignoreMatcher.from_patterns(project_path, exclude_patterns or []) for root, dirs, filenames in os.walk(project_path): root_path = Path(root) @@ -1570,6 +1602,13 @@ def scan_project( if is_force_included(root_path / d, project_path, include_paths) or not is_gitignored(root_path / d, active_matchers, is_dir=True) ] + if exclude_matcher: + dirs[:] = [ + d + for d in dirs + if is_force_included(root_path / d, project_path, include_paths) + or exclude_matcher.matches(root_path / d, is_dir=True) is not True + ] for filename in filenames: filepath = root_path / filename @@ -1583,6 +1622,15 @@ def scan_project( if respect_gitignore and active_matchers and not force_include: if is_gitignored(filepath, active_matchers, is_dir=False): continue + # Skip files matching project-level exclude_patterns (mempalace.yaml). + # force_include paths bypass this check so callers can selectively + # re-include files that would otherwise be excluded. + if ( + not force_include + and exclude_matcher + and exclude_matcher.matches(filepath, is_dir=False) + ): + continue # Skip symlinks — prevents following links to /dev/urandom, etc. if filepath.is_symlink(): rel = filepath.relative_to(project_path).as_posix() @@ -1706,13 +1754,29 @@ def _mine_impl( wing = wing_override or config["wing"] rooms = config.get("rooms", [{"name": "general", "description": "All project files"}]) + exclude_patterns = config.get("exclude_patterns", []) if files is None: files = scan_project( project_dir, respect_gitignore=respect_gitignore, include_ignored=include_ignored, + exclude_patterns=exclude_patterns, ) + elif exclude_patterns: + # The caller provided a pre-scanned file list, so scan_project never ran. + # Apply exclude_patterns here so that callers reusing a scan still get + # the same per-project exclusions. force_include paths are preserved. + include_paths = normalize_include_paths(include_ignored) + exclude_matcher = GitignoreMatcher.from_patterns(project_path, exclude_patterns) + if exclude_matcher is not None: + files = [ + file_path + for file_path in files + if is_force_included(file_path, project_path, include_paths) + or exclude_matcher.matches(file_path, is_dir=False) is not True + ] + from .embedding import describe_device print(f"\n{'=' * 55}") diff --git a/tests/test_miner.py b/tests/test_miner.py index 78ef4d4..02d021e 100644 --- a/tests/test_miner.py +++ b/tests/test_miner.py @@ -589,6 +589,115 @@ def test_scan_project_logs_nested_symlink_with_relative_path(tmp_path, capsys): assert "(symlink)" in err +def test_scan_project_exclude_patterns_skips_matching_files(): + tmpdir = tempfile.mkdtemp() + try: + project_root = Path(tmpdir).resolve() + + write_file(project_root / "src" / "app.py", "print('app')\n" * 20) + write_file(project_root / "docs" / "guide.md", "# Guide\n" * 20) + write_file(project_root / "README.md", "# README\n" * 20) + + # *.md excludes README.md and docs/guide.md; docs/* would also cover docs/ + # subtree. Patterns follow .gitignore syntax via GitignoreMatcher. + assert scanned_files( + project_root, + exclude_patterns=["*.md", "docs/*"], + ) == ["src/app.py"] + finally: + shutil.rmtree(tmpdir) + + +def test_scan_project_exclude_patterns_prunes_entire_directory(): + tmpdir = tempfile.mkdtemp() + try: + project_root = Path(tmpdir).resolve() + + write_file(project_root / "src" / "app.py", "print('app')\n" * 20) + write_file(project_root / "docs" / "sub" / "deep.md", "# Deep\n" * 20) + + # A trailing-slash pattern (dir-only) prunes the whole directory so + # os.walk never descends into it. + assert scanned_files(project_root, exclude_patterns=["docs/"]) == ["src/app.py"] + finally: + shutil.rmtree(tmpdir) + + +def test_scan_project_exclude_patterns_include_ignored_bypasses_exclusion(): + tmpdir = tempfile.mkdtemp() + try: + project_root = Path(tmpdir).resolve() + + write_file(project_root / "src" / "app.py", "print('app')\n" * 20) + write_file(project_root / ".gitignore", "docs/\n") + write_file(project_root / "docs" / "guide.md", "# Guide\n" * 20) + + # docs/ is both gitignored and matched by exclude_patterns, but + # include_ignored should override both filters and bring it back. + assert scanned_files( + project_root, + exclude_patterns=["docs/*"], + include_ignored=["docs"], + ) == ["docs/guide.md", "src/app.py"] + finally: + shutil.rmtree(tmpdir) + + +def test_mine_exclude_patterns_applied_to_prescan_files(monkeypatch): + """exclude_patterns from config must be applied when mine() is given a pre-scanned list. + + The init command calls scan_project() first to show a file-count estimate, + then passes the result to mine(..., files=...) to avoid walking the tree + twice. The _mine_impl elif branch must apply exclude_patterns to that list + so per-project exclusions still take effect. + """ + import mempalace.miner as miner_mod + + tmpdir = tempfile.mkdtemp() + try: + project_root = Path(tmpdir).resolve() + + write_file(project_root / "src" / "app.py", "print('app')\n" * 20) + write_file(project_root / "docs" / "guide.md", "# Guide\n" * 20) + write_file(project_root / "README.md", "# README\n" * 20) + + with open(project_root / "mempalace.yaml", "w") as f: + yaml.dump( + { + "wing": "test_project", + "rooms": [{"name": "general", "description": "General"}], + "exclude_patterns": ["*.md", "docs/"], + }, + f, + ) + + # Simulate the init double-scan: scan without exclude_patterns first + # (representing the caller that doesn't know the config), then pass + # the full list to mine() so the elif branch applies the config exclusions. + prescan = scan_project(str(project_root), exclude_patterns=None) + assert len(prescan) >= 2, "prescan should include .md files before filtering" + + # Capture which files process_file is called with during dry_run + processed = [] + _real_process_file = miner_mod.process_file + + def _capture_process_file(filepath, **kwargs): + processed.append(filepath) + return _real_process_file(filepath, **kwargs) + + monkeypatch.setattr(miner_mod, "process_file", _capture_process_file) + + palace_path = project_root / "palace" + mine(str(project_root), str(palace_path), dry_run=True, files=prescan) + + processed_rel = [p.relative_to(project_root).as_posix() for p in processed] + assert "src/app.py" in processed_rel + assert "docs/guide.md" not in processed_rel + assert "README.md" not in processed_rel + finally: + shutil.rmtree(tmpdir, ignore_errors=True) + + def test_entity_metadata_finds_cyrillic_names(monkeypatch): """Entity extraction must find non-Latin names when entity_languages includes the locale.""" import mempalace.palace as palace_mod