From 5f425ba417cfa550e2c42a39d80aa3a89d7a33d3 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Sun, 3 May 2026 20:49:08 +0500 Subject: [PATCH 1/3] fix(convo-miner): skip Claude Code subagent transcripts by default (#1217) scan_convos() now prunes any directory named 'subagents' during os.walk. Claude Code records Explore/Plan/Grep subagent transcripts in /subagents/agent-*.jsonl and on a typical workspace these outweigh main session files ~80:1, dominating mining time and producing near-zero additional signal (the parent session already summarizes them). Adds a --include-subagents opt-in flag for users who want full history. The shared SKIP_DIRS set in palace.py is left untouched, so project mining (miner.scan_project) still descends into legitimate user-created subagents/ directories in code projects. --- mempalace/cli.py | 11 ++++++++ mempalace/convo_miner.py | 17 +++++++++--- tests/test_cli.py | 24 +++++++++++++++++ tests/test_convo_miner_unit.py | 48 ++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+), 3 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 58ec3f1..39c5e2e 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -596,6 +596,7 @@ def cmd_mine(args): limit=args.limit, dry_run=args.dry_run, extract_mode=args.extract, + include_subagents=getattr(args, "include_subagents", False), ) elif args.mode == "extract": from .format_miner import mine_formats @@ -1920,6 +1921,16 @@ def main(): f"Windows if you hit ONNX bad_alloc (#1455)." ), ) + p_mine.add_argument( + "--include-subagents", + action="store_true", + help=( + "Also mine Claude Code subagent transcripts (subagents/ dirs). " + "Excluded by default — these are short ephemeral exchanges " + "(Explore/Plan/Grep agents) already summarized in the parent " + "session, and on a typical workspace outnumber main sessions ~80:1." + ), + ) # sweep p_sweep = sub.add_parser( diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 44d9f6a..3f0449d 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -412,12 +412,17 @@ def detect_convo_room(content: str) -> str: # ============================================================================= -def scan_convos(convo_dir: str) -> list: +def scan_convos(convo_dir: str, include_subagents: bool = False) -> list: """Find all potential conversation files. Skips symlinks and oversized files. Each skipped symlink is logged to ``sys.stderr`` with a `` SKIP: (symlink)`` line so the caller can tell why an apparent conversation directory yielded no files. + + By default, ``subagents/`` directories are skipped: Claude Code records + Explore/Plan/Grep subagent transcripts there, and on a typical workspace + they outweigh main session files ~80:1 (#1217). Pass + ``include_subagents=True`` to mine them anyway. """ # A direct conversation file is a valid source. For a file, feed only # its basename through the existing directory validation loop. @@ -429,7 +434,9 @@ def scan_convos(convo_dir: str) -> list: ) files = [] for root, dirs, filenames in scan_entries: - dirs[:] = [d for d in dirs if d not in CONVO_SKIP_DIRS] + dirs[:] = [ + d for d in dirs if d not in CONVO_SKIP_DIRS and (include_subagents or d != "subagents") + ] for filename in filenames: if filename.endswith(".meta.json"): continue @@ -739,12 +746,16 @@ def mine_convos( limit: int = 0, dry_run: bool = False, extract_mode: str = "exchange", + include_subagents: bool = False, ): """Mine a directory of conversation files into the palace. extract_mode: "exchange" — default exchange-pair chunking (Q+A = one unit) "general" — general extractor: decisions, preferences, milestones, problems, emotions + include_subagents: + False (default) — skip Claude Code ``subagents/`` directories + True — also mine subagent transcripts The real work is in :func:`_mine_convos_impl`; this wrapper holds the per-palace flock around it so two concurrent ``mempalace mine --mode @@ -886,7 +897,7 @@ def _mine_convos_impl( convo_path = Path(convo_dir).expanduser().resolve() wing = _resolve_wing(convo_path, wing) - files = scan_convos(convo_dir) + files = scan_convos(convo_dir, include_subagents=include_subagents) print(f"\n{'=' * 55}") print(" MemPalace Mine — Conversations") diff --git a/tests/test_cli.py b/tests/test_cli.py index a5c8b69..ffa9541 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -593,6 +593,7 @@ def test_cmd_mine_convos_mode(mock_config_cls): no_gitignore=False, include_ignored=[], extract="general", + include_subagents=False, ) with patch("mempalace.convo_miner.mine_convos") as mock_mine: cmd_mine(args) @@ -604,9 +605,32 @@ def test_cmd_mine_convos_mode(mock_config_cls): limit=10, dry_run=True, extract_mode="general", + include_subagents=False, ) +@patch("mempalace.cli.MempalaceConfig") +def test_cmd_mine_convos_mode_threads_include_subagents_flag(mock_config_cls): + mock_config_cls.return_value.palace_path = "/fake/palace" + args = argparse.Namespace( + dir="/chats", + palace=None, + mode="convos", + wing="mywing", + agent="me", + limit=10, + dry_run=True, + no_gitignore=False, + include_ignored=[], + extract="exchange", + include_subagents=True, + ) + with patch("mempalace.convo_miner.mine_convos") as mock_mine: + cmd_mine(args) + kwargs = mock_mine.call_args.kwargs + assert kwargs["include_subagents"] is True + + @patch("mempalace.cli.MempalaceConfig") def test_cmd_mine_include_ignored_comma_split(mock_config_cls): mock_config_cls.return_value.palace_path = "/fake/palace" diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index ccaf6b1..7f11e3c 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -510,6 +510,54 @@ class TestScanConvos: assert "SKIP: unreadable.txt" in err assert "stat error" in err + def test_scan_skips_subagent_dirs_by_default(self, tmp_path): + # Mimic Claude Code layout: ~/.claude/projects///subagents/agent-*.jsonl + session_dir = tmp_path / "session-abc" + session_dir.mkdir() + (session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + subagents_dir = session_dir / "subagents" + subagents_dir.mkdir() + (subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + (subagents_dir / "agent-def.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + + files = scan_convos(str(tmp_path)) + names = [f.name for f in files] + + assert "main.jsonl" in names + assert "agent-abc.jsonl" not in names + assert "agent-def.jsonl" not in names + + def test_scan_includes_subagent_dirs_when_opted_in(self, tmp_path): + session_dir = tmp_path / "session-abc" + session_dir.mkdir() + (session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + subagents_dir = session_dir / "subagents" + subagents_dir.mkdir() + (subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8") + + files = scan_convos(str(tmp_path), include_subagents=True) + names = [f.name for f in files] + + assert "main.jsonl" in names + assert "agent-abc.jsonl" in names + + def test_scan_skips_subagent_dirs_at_any_depth(self, tmp_path): + # The "subagents" name match is by directory name, not by depth — verify + # both shallow (top-level) and nested subagents/ get skipped. + (tmp_path / "subagents").mkdir() + (tmp_path / "subagents" / "agent-top.jsonl").write_text("{}", encoding="utf-8") + nested = tmp_path / "session" / "subagents" + nested.mkdir(parents=True) + (nested / "agent-deep.jsonl").write_text("{}", encoding="utf-8") + (tmp_path / "session" / "main.jsonl").write_text("{}", encoding="utf-8") + + files = scan_convos(str(tmp_path)) + names = [f.name for f in files] + + assert "main.jsonl" in names + assert "agent-top.jsonl" not in names + assert "agent-deep.jsonl" not in names + class TestFileChunksLocked: def test_uses_bounded_upsert_batches(self, monkeypatch): From a3350ec5670262849400ce8d451b698bf1424b4a Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Sun, 3 May 2026 20:58:39 +0500 Subject: [PATCH 2/3] fix(convo-miner): tighten subagent filter per pre-undraft review - Case-insensitive directory match (d.lower() == 'subagents') so the filter still kicks in if Claude Code or a plugin ever emits 'Subagents/' on case-preserving filesystems (Windows, macOS APFS). - Drop defensive getattr in cmd_mine: argparse always defines the attribute since --include-subagents is unconditionally registered. Direct args.include_subagents access matches every neighbouring field and fails loudly if the registration is ever removed. - Soften CLI help and docstring: drop the 80:1 ratio (reporter- specific) and the in-code (#1217) reference. Add explicit default=False on the argparse flag for symmetry with --extract. - Add 2 negative tests: 'mysubagents'/'subagentsbackup' must still be mined (regression guard against substring-match), and 'Subagents/' must be skipped (case-insensitive coverage). --- mempalace/cli.py | 5 +++-- mempalace/convo_miner.py | 19 ++++++++++++++----- tests/test_convo_miner_unit.py | 33 ++++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 8 deletions(-) diff --git a/mempalace/cli.py b/mempalace/cli.py index 39c5e2e..4ba9e53 100644 --- a/mempalace/cli.py +++ b/mempalace/cli.py @@ -1924,11 +1924,12 @@ def main(): p_mine.add_argument( "--include-subagents", action="store_true", + default=False, help=( "Also mine Claude Code subagent transcripts (subagents/ dirs). " - "Excluded by default — these are short ephemeral exchanges " + "Excluded by default: these are short ephemeral exchanges " "(Explore/Plan/Grep agents) already summarized in the parent " - "session, and on a typical workspace outnumber main sessions ~80:1." + "session, and on typical workspaces they dominate file counts." ), ) diff --git a/mempalace/convo_miner.py b/mempalace/convo_miner.py index 3f0449d..cd8d618 100644 --- a/mempalace/convo_miner.py +++ b/mempalace/convo_miner.py @@ -419,10 +419,14 @@ def scan_convos(convo_dir: str, include_subagents: bool = False) -> list: ``sys.stderr`` with a `` SKIP: (symlink)`` line so the caller can tell why an apparent conversation directory yielded no files. - By default, ``subagents/`` directories are skipped: Claude Code records - Explore/Plan/Grep subagent transcripts there, and on a typical workspace - they outweigh main session files ~80:1 (#1217). Pass - ``include_subagents=True`` to mine them anyway. + By default, directories named ``subagents`` are skipped: Claude Code + records Explore/Plan/Grep subagent transcripts there, and on typical + workspaces they outnumber main session files by one to two orders of + magnitude. Pass ``include_subagents=True`` to mine them anyway. + + The match is case-insensitive on the directory name only (``subagents`` + or ``Subagents``), so directories like ``mysubagents`` or + ``subagentsbackup`` are not affected. """ # A direct conversation file is a valid source. For a file, feed only # its basename through the existing directory validation loop. @@ -435,7 +439,9 @@ def scan_convos(convo_dir: str, include_subagents: bool = False) -> list: files = [] for root, dirs, filenames in scan_entries: dirs[:] = [ - d for d in dirs if d not in CONVO_SKIP_DIRS and (include_subagents or d != "subagents") + d + for d in dirs + if d not in CONVO_SKIP_DIRS and (include_subagents or d.lower() != "subagents") ] for filename in filenames: if filename.endswith(".meta.json"): @@ -782,6 +788,7 @@ def mine_convos( limit=limit, dry_run=dry_run, extract_mode=extract_mode, + include_subagents=include_subagents, ) with mine_palace_lock(palace_path): @@ -793,6 +800,7 @@ def mine_convos( limit=limit, dry_run=dry_run, extract_mode=extract_mode, + include_subagents=include_subagents, ) @@ -878,6 +886,7 @@ def _mine_convos_impl( limit: int = 0, dry_run: bool = False, extract_mode: str = "exchange", + include_subagents: bool = False, ): from .config import MempalaceConfig diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index 7f11e3c..57a50e5 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -542,7 +542,7 @@ class TestScanConvos: assert "agent-abc.jsonl" in names def test_scan_skips_subagent_dirs_at_any_depth(self, tmp_path): - # The "subagents" name match is by directory name, not by depth — verify + # The "subagents" name match is by directory name, not by depth: verify # both shallow (top-level) and nested subagents/ get skipped. (tmp_path / "subagents").mkdir() (tmp_path / "subagents" / "agent-top.jsonl").write_text("{}", encoding="utf-8") @@ -558,6 +558,37 @@ class TestScanConvos: assert "agent-top.jsonl" not in names assert "agent-deep.jsonl" not in names + def test_scan_does_not_skip_suffix_named_dirs(self, tmp_path): + # Exact name match only: 'mysubagents' or 'subagentsbackup' must still + # be mined. Guards against future regression to substring/regex match. + for dir_name in ("mysubagents", "subagentsbackup", "subagent"): + d = tmp_path / dir_name + d.mkdir() + (d / f"{dir_name}.jsonl").write_text("{}", encoding="utf-8") + + files = scan_convos(str(tmp_path)) + names = {f.name for f in files} + + assert "mysubagents.jsonl" in names + assert "subagentsbackup.jsonl" in names + assert "subagent.jsonl" in names + + def test_scan_skips_subagents_case_insensitive(self, tmp_path): + # On Windows + macOS APFS the filesystem is case-preserving; if Claude + # Code or a plugin ever emits 'Subagents' (capitalized), the filter + # must still match. Only one variant per tmp_path because case- + # insensitive filesystems collapse 'Subagents' and 'SUBAGENTS'. + d = tmp_path / "Subagents" + d.mkdir() + (d / "agent.jsonl").write_text("{}", encoding="utf-8") + (tmp_path / "main.jsonl").write_text("{}", encoding="utf-8") + + files = scan_convos(str(tmp_path)) + names = {f.name for f in files} + + assert "main.jsonl" in names + assert "agent.jsonl" not in names + class TestFileChunksLocked: def test_uses_bounded_upsert_batches(self, monkeypatch): From 5bc539d4a77baacc7eb8304746d98c2079641429 Mon Sep 17 00:00:00 2001 From: mvalentsev Date: Thu, 6 Aug 2026 16:33:16 +0500 Subject: [PATCH 3/3] test(convo-miner): pin subagent skip against explicit single-file targets The default `subagents/` skip is directory pruning, so it cannot reach a caller who names one transcript directly: that path feeds a single synthetic scan entry with no directories to prune. The split is deliberate -- `--include-subagents` governs what a directory walk sweeps up, while naming a path stays an explicit request -- but the single-file scan and this filter were written independently and nothing exercised them together, so the boundary was unpinned. --- tests/test_convo_miner_unit.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tests/test_convo_miner_unit.py b/tests/test_convo_miner_unit.py index 57a50e5..d6bb4c5 100644 --- a/tests/test_convo_miner_unit.py +++ b/tests/test_convo_miner_unit.py @@ -589,6 +589,22 @@ class TestScanConvos: assert "main.jsonl" in names assert "agent.jsonl" not in names + def test_scan_mines_an_explicitly_named_file_inside_subagents(self, tmp_path): + # The skip is directory pruning, so it cannot reach a caller who names + # one file: that path feeds a single synthetic entry with no directories + # to prune. The split is deliberate -- --include-subagents governs what a + # directory walk sweeps up, while naming a path is an explicit request + # and stays honored. Pinned because the two behaviours were written + # independently and nothing else exercises them together. + subagents_dir = tmp_path / "session-abc" / "subagents" + subagents_dir.mkdir(parents=True) + target = subagents_dir / "agent-abc.jsonl" + target.write_text('{"type":"user"}\n', encoding="utf-8") + + files = scan_convos(str(target)) + + assert [f.name for f in files] == ["agent-abc.jsonl"] + class TestFileChunksLocked: def test_uses_bounded_upsert_batches(self, monkeypatch):