Merge pull request #1330 from mvalentsev/fix/convo-miner-skip-subagents
fix(convo-miner): skip Claude Code subagent transcripts by default (#1217)
This commit is contained in:
Igor Lins e Silva
GitHub
commit
3161cae8a6
|
|
@ -736,6 +736,7 @@ def cmd_mine(args):
|
|||
limit=args.limit,
|
||||
dry_run=args.dry_run,
|
||||
extract_mode=args.extract,
|
||||
include_subagents=getattr(args, "include_subagents", False),
|
||||
)
|
||||
elif args.mode == "extract":
|
||||
from .format_miner import mine_formats
|
||||
|
|
@ -2372,6 +2373,17 @@ def main():
|
|||
f"Windows if you hit ONNX bad_alloc (#1455)."
|
||||
),
|
||||
)
|
||||
p_mine.add_argument(
|
||||
"--include-subagents",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"Also mine Claude Code subagent transcripts (subagents/ dirs). "
|
||||
"Excluded by default: these are short ephemeral exchanges "
|
||||
"(Explore/Plan/Grep agents) already summarized in the parent "
|
||||
"session, and on typical workspaces they dominate file counts."
|
||||
),
|
||||
)
|
||||
|
||||
# sweep
|
||||
p_sweep = sub.add_parser(
|
||||
|
|
|
|||
|
|
@ -412,12 +412,21 @@ def detect_convo_room(content: str) -> str:
|
|||
# =============================================================================
|
||||
|
||||
|
||||
def scan_convos(convo_dir: str) -> list:
|
||||
def scan_convos(convo_dir: str, include_subagents: bool = False) -> list:
|
||||
"""Find all potential conversation files.
|
||||
|
||||
Skips symlinks and oversized files. Each skipped symlink is logged to
|
||||
``sys.stderr`` with a `` SKIP: <relative-path> (symlink)`` line so the
|
||||
caller can tell why an apparent conversation directory yielded no files.
|
||||
|
||||
By default, directories named ``subagents`` are skipped: Claude Code
|
||||
records Explore/Plan/Grep subagent transcripts there, and on typical
|
||||
workspaces they outnumber main session files by one to two orders of
|
||||
magnitude. Pass ``include_subagents=True`` to mine them anyway.
|
||||
|
||||
The match is case-insensitive on the directory name only (``subagents``
|
||||
or ``Subagents``), so directories like ``mysubagents`` or
|
||||
``subagentsbackup`` are not affected.
|
||||
"""
|
||||
# A direct conversation file is a valid source. For a file, feed only
|
||||
# its basename through the existing directory validation loop.
|
||||
|
|
@ -429,7 +438,11 @@ def scan_convos(convo_dir: str) -> list:
|
|||
)
|
||||
files = []
|
||||
for root, dirs, filenames in scan_entries:
|
||||
dirs[:] = [d for d in dirs if d not in CONVO_SKIP_DIRS]
|
||||
dirs[:] = [
|
||||
d
|
||||
for d in dirs
|
||||
if d not in CONVO_SKIP_DIRS and (include_subagents or d.lower() != "subagents")
|
||||
]
|
||||
for filename in filenames:
|
||||
if filename.endswith(".meta.json"):
|
||||
continue
|
||||
|
|
@ -739,12 +752,16 @@ def mine_convos(
|
|||
limit: int = 0,
|
||||
dry_run: bool = False,
|
||||
extract_mode: str = "exchange",
|
||||
include_subagents: bool = False,
|
||||
):
|
||||
"""Mine a directory of conversation files into the palace.
|
||||
|
||||
extract_mode:
|
||||
"exchange" — default exchange-pair chunking (Q+A = one unit)
|
||||
"general" — general extractor: decisions, preferences, milestones, problems, emotions
|
||||
include_subagents:
|
||||
False (default) — skip Claude Code ``subagents/`` directories
|
||||
True — also mine subagent transcripts
|
||||
|
||||
The real work is in :func:`_mine_convos_impl`; this wrapper holds the
|
||||
per-palace flock around it so two concurrent ``mempalace mine --mode
|
||||
|
|
@ -771,6 +788,7 @@ def mine_convos(
|
|||
limit=limit,
|
||||
dry_run=dry_run,
|
||||
extract_mode=extract_mode,
|
||||
include_subagents=include_subagents,
|
||||
)
|
||||
|
||||
with mine_palace_lock(palace_path):
|
||||
|
|
@ -782,6 +800,7 @@ def mine_convos(
|
|||
limit=limit,
|
||||
dry_run=dry_run,
|
||||
extract_mode=extract_mode,
|
||||
include_subagents=include_subagents,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -867,6 +886,7 @@ def _mine_convos_impl(
|
|||
limit: int = 0,
|
||||
dry_run: bool = False,
|
||||
extract_mode: str = "exchange",
|
||||
include_subagents: bool = False,
|
||||
):
|
||||
from .config import MempalaceConfig
|
||||
|
||||
|
|
@ -886,7 +906,7 @@ def _mine_convos_impl(
|
|||
convo_path = Path(convo_dir).expanduser().resolve()
|
||||
wing = _resolve_wing(convo_path, wing)
|
||||
|
||||
files = scan_convos(convo_dir)
|
||||
files = scan_convos(convo_dir, include_subagents=include_subagents)
|
||||
|
||||
print(f"\n{'=' * 55}")
|
||||
print(" MemPalace Mine -- Conversations")
|
||||
|
|
|
|||
|
|
@ -593,6 +593,7 @@ def test_cmd_mine_convos_mode(mock_config_cls):
|
|||
no_gitignore=False,
|
||||
include_ignored=[],
|
||||
extract="general",
|
||||
include_subagents=False,
|
||||
)
|
||||
with patch("mempalace.convo_miner.mine_convos") as mock_mine:
|
||||
cmd_mine(args)
|
||||
|
|
@ -604,9 +605,32 @@ def test_cmd_mine_convos_mode(mock_config_cls):
|
|||
limit=10,
|
||||
dry_run=True,
|
||||
extract_mode="general",
|
||||
include_subagents=False,
|
||||
)
|
||||
|
||||
|
||||
@patch("mempalace.cli.MempalaceConfig")
|
||||
def test_cmd_mine_convos_mode_threads_include_subagents_flag(mock_config_cls):
|
||||
mock_config_cls.return_value.palace_path = "/fake/palace"
|
||||
args = argparse.Namespace(
|
||||
dir="/chats",
|
||||
palace=None,
|
||||
mode="convos",
|
||||
wing="mywing",
|
||||
agent="me",
|
||||
limit=10,
|
||||
dry_run=True,
|
||||
no_gitignore=False,
|
||||
include_ignored=[],
|
||||
extract="exchange",
|
||||
include_subagents=True,
|
||||
)
|
||||
with patch("mempalace.convo_miner.mine_convos") as mock_mine:
|
||||
cmd_mine(args)
|
||||
kwargs = mock_mine.call_args.kwargs
|
||||
assert kwargs["include_subagents"] is True
|
||||
|
||||
|
||||
@patch("mempalace.cli.MempalaceConfig")
|
||||
def test_cmd_mine_include_ignored_comma_split(mock_config_cls):
|
||||
mock_config_cls.return_value.palace_path = "/fake/palace"
|
||||
|
|
|
|||
|
|
@ -510,6 +510,101 @@ class TestScanConvos:
|
|||
assert "SKIP: unreadable.txt" in err
|
||||
assert "stat error" in err
|
||||
|
||||
def test_scan_skips_subagent_dirs_by_default(self, tmp_path):
|
||||
# Mimic Claude Code layout: ~/.claude/projects/<slug>/<session>/subagents/agent-*.jsonl
|
||||
session_dir = tmp_path / "session-abc"
|
||||
session_dir.mkdir()
|
||||
(session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
|
||||
subagents_dir = session_dir / "subagents"
|
||||
subagents_dir.mkdir()
|
||||
(subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
|
||||
(subagents_dir / "agent-def.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
|
||||
|
||||
files = scan_convos(str(tmp_path))
|
||||
names = [f.name for f in files]
|
||||
|
||||
assert "main.jsonl" in names
|
||||
assert "agent-abc.jsonl" not in names
|
||||
assert "agent-def.jsonl" not in names
|
||||
|
||||
def test_scan_includes_subagent_dirs_when_opted_in(self, tmp_path):
|
||||
session_dir = tmp_path / "session-abc"
|
||||
session_dir.mkdir()
|
||||
(session_dir / "main.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
|
||||
subagents_dir = session_dir / "subagents"
|
||||
subagents_dir.mkdir()
|
||||
(subagents_dir / "agent-abc.jsonl").write_text('{"type":"user"}\n', encoding="utf-8")
|
||||
|
||||
files = scan_convos(str(tmp_path), include_subagents=True)
|
||||
names = [f.name for f in files]
|
||||
|
||||
assert "main.jsonl" in names
|
||||
assert "agent-abc.jsonl" in names
|
||||
|
||||
def test_scan_skips_subagent_dirs_at_any_depth(self, tmp_path):
|
||||
# The "subagents" name match is by directory name, not by depth: verify
|
||||
# both shallow (top-level) and nested subagents/ get skipped.
|
||||
(tmp_path / "subagents").mkdir()
|
||||
(tmp_path / "subagents" / "agent-top.jsonl").write_text("{}", encoding="utf-8")
|
||||
nested = tmp_path / "session" / "subagents"
|
||||
nested.mkdir(parents=True)
|
||||
(nested / "agent-deep.jsonl").write_text("{}", encoding="utf-8")
|
||||
(tmp_path / "session" / "main.jsonl").write_text("{}", encoding="utf-8")
|
||||
|
||||
files = scan_convos(str(tmp_path))
|
||||
names = [f.name for f in files]
|
||||
|
||||
assert "main.jsonl" in names
|
||||
assert "agent-top.jsonl" not in names
|
||||
assert "agent-deep.jsonl" not in names
|
||||
|
||||
def test_scan_does_not_skip_suffix_named_dirs(self, tmp_path):
|
||||
# Exact name match only: 'mysubagents' or 'subagentsbackup' must still
|
||||
# be mined. Guards against future regression to substring/regex match.
|
||||
for dir_name in ("mysubagents", "subagentsbackup", "subagent"):
|
||||
d = tmp_path / dir_name
|
||||
d.mkdir()
|
||||
(d / f"{dir_name}.jsonl").write_text("{}", encoding="utf-8")
|
||||
|
||||
files = scan_convos(str(tmp_path))
|
||||
names = {f.name for f in files}
|
||||
|
||||
assert "mysubagents.jsonl" in names
|
||||
assert "subagentsbackup.jsonl" in names
|
||||
assert "subagent.jsonl" in names
|
||||
|
||||
def test_scan_skips_subagents_case_insensitive(self, tmp_path):
|
||||
# On Windows + macOS APFS the filesystem is case-preserving; if Claude
|
||||
# Code or a plugin ever emits 'Subagents' (capitalized), the filter
|
||||
# must still match. Only one variant per tmp_path because case-
|
||||
# insensitive filesystems collapse 'Subagents' and 'SUBAGENTS'.
|
||||
d = tmp_path / "Subagents"
|
||||
d.mkdir()
|
||||
(d / "agent.jsonl").write_text("{}", encoding="utf-8")
|
||||
(tmp_path / "main.jsonl").write_text("{}", encoding="utf-8")
|
||||
|
||||
files = scan_convos(str(tmp_path))
|
||||
names = {f.name for f in files}
|
||||
|
||||
assert "main.jsonl" in names
|
||||
assert "agent.jsonl" not in names
|
||||
|
||||
def test_scan_mines_an_explicitly_named_file_inside_subagents(self, tmp_path):
|
||||
# The skip is directory pruning, so it cannot reach a caller who names
|
||||
# one file: that path feeds a single synthetic entry with no directories
|
||||
# to prune. The split is deliberate -- --include-subagents governs what a
|
||||
# directory walk sweeps up, while naming a path is an explicit request
|
||||
# and stays honored. Pinned because the two behaviours were written
|
||||
# independently and nothing else exercises them together.
|
||||
subagents_dir = tmp_path / "session-abc" / "subagents"
|
||||
subagents_dir.mkdir(parents=True)
|
||||
target = subagents_dir / "agent-abc.jsonl"
|
||||
target.write_text('{"type":"user"}\n', encoding="utf-8")
|
||||
|
||||
files = scan_convos(str(target))
|
||||
|
||||
assert [f.name for f in files] == ["agent-abc.jsonl"]
|
||||
|
||||
|
||||
class TestFileChunksLocked:
|
||||
def test_uses_bounded_upsert_batches(self, monkeypatch):
|
||||
|
|
|
|||
Loading…
Reference in New Issue