diff --git a/mempalace/cli.py b/mempalace/cli.py index daeffde..8b751d0 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -736,6 +736,7 @@ def cmd_mine(args): limit=args.limit, dry_run=args.dry_run, extract_mode=args.extract, + include_subagents=getattr(args, "include_subagents", False), ) elif args.mode == "extract": from .format_miner import mine_formats @@ -2372,6 +2373,17 @@ def main(): f"Windows if you hit ONNX bad_alloc (#1455)." ), ) + p_mine.add_argument( + "--include-subagents", + action="store_true", + default=False, + help=( + "Also mine Claude Code subagent transcripts (subagents/ dirs). " + "Excluded by default: these are short ephemeral exchanges " + "(Explore/Plan/Grep agents) already summarized in the parent " + "session, and on typical workspaces they dominate file counts." + ), + ) # sweep p_sweep = sub.add_parser( diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index d8647b4..0866153 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -412,12 +412,21 @@ def detect_convo_room(content: str) -> str: # ============================================================================= -def scan_convos(convo_dir: str) -> list: +def scan_convos(convo_dir: str, include_subagents: bool = False) -> list: """Find all potential conversation files. Skips symlinks and oversized files. Each skipped symlink is logged to ``sys.stderr`` with a `` SKIP: (symlink)`` line so the caller can tell why an apparent conversation directory yielded no files. + + By default, directories named ``subagents`` are skipped: Claude Code + records Explore/Plan/Grep subagent transcripts there, and on typical + workspaces they outnumber main session files by one to two orders of + magnitude. Pass ``include_subagents=True`` to mine them anyway. + + The match is case-insensitive on the directory name only (``subagents`` + or ``Subagents``), so directories like ``mysubagents`` or + ``subagentsbackup`` are not affected. """ # A direct conversation file is a valid source. For a file, feed only # its basename through the existing directory validation loop. @@ -429,7 +438,11 @@ def scan_convos(convo_dir: str) -> list: ) files = [] for root, dirs, filenames in scan_entries: - dirs[:] = [d for d in dirs if d not in CONVO_SKIP_DIRS] + dirs[:] = [ + d + for d in dirs + if d not in CONVO_SKIP_DIRS and (include_subagents or d.lower() != "subagents") + ] for filename in filenames: if filename.endswith(".meta.json"): continue @@ -739,12 +752,16 @@ def mine_convos( limit: int = 0, dry_run: bool = False, extract_mode: str = "exchange", + include_subagents: bool = False, ): """Mine a directory of conversation files into the palace. extract_mode: "exchange" — default exchange-pair chunking (Q+A = one unit) "general" — general extractor: decisions, preferences, milestones, problems, emotions + include_subagents: + False (default) — skip Claude Code ``subagents/`` directories + True — also mine subagent transcripts The real work is in :func:`_mine_convos_impl`; this wrapper holds the per-palace flock around it so two concurrent ``mempalace mine --mode @@ -771,6 +788,7 @@ def mine_convos( limit=limit, dry_run=dry_run, extract_mode=extract_mode, + include_subagents=include_subagents, ) with mine_palace_lock(palace_path): @@ -782,6 +800,7 @@ def mine_convos( limit=limit, dry_run=dry_run, extract_mode=extract_mode, + include_subagents=include_subagents, ) @@ -867,6 +886,7 @@ def _mine_convos_impl( limit: int = 0, dry_run: bool = False, extract_mode: str = "exchange", + include_subagents: bool = False, ): from .config import MempalaceConfig @@ -886,7 +906,7 @@ def _mine_convos_impl( convo_path = Path(convo_dir).expanduser().resolve() wing = _resolve_wing(convo_path, wing) - files = scan_convos(convo_dir) + files = scan_convos(convo_dir, include_subagents=include_subagents) print(f"\n{'=' * 55}") print(" MemPalace Mine -- Conversations") diff --git a/tests/test_cli.py b/tests/test_cli.py index fd3e763..900164b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -593,6 +593,7 @@ def test_cmd_mine_convos_mode(mock_config_cls): no_gitignore=False, include_ignored=[], extract="general", + include_subagents=False, ) with patch("mempalace.convo_miner.mine_convos") as mock_mine: cmd_mine(args) @@ -604,9 +605,32 @@ def test_cmd_mine_convos_mode(mock_config_cls): limit=10, dry_run=True, extract_mode="general", + include_subagents=False, ) +@patch("mempalace.cli.MempalaceConfig") +def test_cmd_mine_convos_mode_threads_include_subagents_flag(mock_config_cls): + mock_config_cls.return_value.palace_path = "/fake/palace" + args = argparse.Namespace( + dir="/chats", + palace=None, + mode="convos", + wing="mywing", + agent="me", + limit=10, + dry_run=True, + no_gitignore=False, + include_ignored=[], + extract="exchange", + include_subagents=True, + ) + with patch("mempalace.convo_miner.mine_convos") as mock_mine: + cmd_mine(args) + kwargs = mock_mine.call_args.kwargs + assert kwargs["include_subagents"] is True + + @patch("mempalace.cli.MempalaceConfig") def test_cmd_mine_include_ignored_comma_split(mock_config_cls): mock_config_cls.return_value.palace_path = "/fake/palace" diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index ccaf6b1..d6bb4c5 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -510,6 +510,101 @@ class TestScanConvos: assert "SKIP: unreadable.txt" in err assert "stat error" in err + def test_scan_skips_subagent_dirs_by_default(self, tmp_path): + # Mimic Claude Code layout: ~/.claude/projects///subagents/agent-*.jsonl + session_dir = tmp_path / "session-abc" + session_dir.mkdir() + (session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + subagents_dir = session_dir / "subagents" + subagents_dir.mkdir() + (subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + (subagents_dir / "agent-def.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + + files = scan_convos(str(tmp_path)) + names = [f.name for f in files] + + assert "main.jsonl" in names + assert "agent-abc.jsonl" not in names + assert "agent-def.jsonl" not in names + + def test_scan_includes_subagent_dirs_when_opted_in(self, tmp_path): + session_dir = tmp_path / "session-abc" + session_dir.mkdir() + (session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + subagents_dir = session_dir / "subagents" + subagents_dir.mkdir() + (subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + + files = scan_convos(str(tmp_path), include_subagents=True) + names = [f.name for f in files] + + assert "main.jsonl" in names + assert "agent-abc.jsonl" in names + + def test_scan_skips_subagent_dirs_at_any_depth(self, tmp_path): + # The "subagents" name match is by directory name, not by depth: verify + # both shallow (top-level) and nested subagents/ get skipped. + (tmp_path / "subagents").mkdir() + (tmp_path / "subagents" / "agent-top.jsonl").write_text("{}", encoding="utf-8") + nested = tmp_path / "session" / "subagents" + nested.mkdir(parents=True) + (nested / "agent-deep.jsonl").write_text("{}", encoding="utf-8") + (tmp_path / "session" / "main.jsonl").write_text("{}", encoding="utf-8") + + files = scan_convos(str(tmp_path)) + names = [f.name for f in files] + + assert "main.jsonl" in names + assert "agent-top.jsonl" not in names + assert "agent-deep.jsonl" not in names + + def test_scan_does_not_skip_suffix_named_dirs(self, tmp_path): + # Exact name match only: 'mysubagents' or 'subagentsbackup' must still + # be mined. Guards against future regression to substring/regex match. + for dir_name in ("mysubagents", "subagentsbackup", "subagent"): + d = tmp_path / dir_name + d.mkdir() + (d / f"{dir_name}.jsonl").write_text("{}", encoding="utf-8") + + files = scan_convos(str(tmp_path)) + names = {f.name for f in files} + + assert "mysubagents.jsonl" in names + assert "subagentsbackup.jsonl" in names + assert "subagent.jsonl" in names + + def test_scan_skips_subagents_case_insensitive(self, tmp_path): + # On Windows + macOS APFS the filesystem is case-preserving; if Claude + # Code or a plugin ever emits 'Subagents' (capitalized), the filter + # must still match. Only one variant per tmp_path because case- + # insensitive filesystems collapse 'Subagents' and 'SUBAGENTS'. + d = tmp_path / "Subagents" + d.mkdir() + (d / "agent.jsonl").write_text("{}", encoding="utf-8") + (tmp_path / "main.jsonl").write_text("{}", encoding="utf-8") + + files = scan_convos(str(tmp_path)) + names = {f.name for f in files} + + assert "main.jsonl" in names + assert "agent.jsonl" not in names + + def test_scan_mines_an_explicitly_named_file_inside_subagents(self, tmp_path): + # The skip is directory pruning, so it cannot reach a caller who names + # one file: that path feeds a single synthetic entry with no directories + # to prune. The split is deliberate -- --include-subagents governs what a + # directory walk sweeps up, while naming a path is an explicit request + # and stays honored. Pinned because the two behaviours were written + # independently and nothing else exercises them together. + subagents_dir = tmp_path / "session-abc" / "subagents" + subagents_dir.mkdir(parents=True) + target = subagents_dir / "agent-abc.jsonl" + target.write_text('{"type":"user"}\n', encoding="utf-8") + + files = scan_convos(str(target)) + + assert [f.name for f in files] == ["agent-abc.jsonl"] + class TestFileChunksLocked: def test_uses_bounded_upsert_batches(self, monkeypatch):