diff --git a/mempalace/normalize.py b/mempalace/normalize.py index 9f9fe0a..8bce677 100644 --- a/mempalace/normalize.py +++ b/mempalace/normalize.py @@ -9,6 +9,7 @@ Supported: - Claude Code JSONL (with tool_use/tool_result block capture) - OpenAI Codex CLI JSONL - Gemini CLI JSONL (~/.gemini/tmp//chats/session-*.jsonl) + - Continue.dev session JSON (~/.continue/sessions/*.json) - Slack JSON export - Plain text (pass through for paragraph chunking) @@ -167,7 +168,7 @@ def _try_normalize_json(content: str) -> Optional[str]: except json.JSONDecodeError: return None - for parser in (_try_claude_ai_json, _try_chatgpt_json, _try_slack_json): + for parser in (_try_claude_ai_json, _try_chatgpt_json, _try_continue_json, _try_slack_json): normalized = parser(data) if normalized: return normalized @@ -485,6 +486,61 @@ def _try_slack_json(data) -> Optional[str]: return None +def _try_continue_json(data) -> Optional[str]: + """Continue.dev session JSON (~/.continue/sessions/*.json). + + Sessions contain a ``history`` array of ``{role, content}`` message objects, + plus optional metadata (``title``, ``sessionId``, ``dateCreated``). + System messages are skipped. Tool-call messages (role ``tool``) are + formatted inline when they contain text content. + """ + if not isinstance(data, dict) or "history" not in data: + return None + history = data["history"] + if not isinstance(history, list): + return None + + messages = [] + for item in history: + if not isinstance(item, dict): + continue + role = item.get("role", "") + content = item.get("content", "") + + # Extract text from string or list-of-blocks content + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + parts.append(block.get("text", "")) + elif isinstance(block, str): + parts.append(block) + text = "\n".join(p for p in parts if p).strip() + elif isinstance(content, str): + text = content.strip() + else: + continue + + if not text: + continue + + if role == "user": + messages.append(("user", text)) + elif role == "assistant": + messages.append(("assistant", text)) + elif role == "tool": + # Append tool output to the previous assistant turn if possible + if messages and messages[-1][0] == "assistant": + prev_role, prev_text = messages[-1] + messages[-1] = (prev_role, prev_text + "\n" + f"[tool] {text}") + # Skip system and other roles + + if len(messages) >= 2: + return _messages_to_transcript(messages) + return None + + def _extract_content(content, tool_use_map: dict = None) -> str: """Pull text from content — handles str, list of blocks, or dict. diff --git a/tests/test_normalize.py b/tests/test_normalize.py index f56ff3a..e57deab 100644 --- a/tests/test_normalize.py +++ b/tests/test_normalize.py @@ -12,6 +12,7 @@ from mempalace.normalize import ( _try_claude_code_jsonl, _try_codex_jsonl, _try_gemini_jsonl, + _try_continue_json, _try_normalize_json, _try_slack_json, normalize, @@ -1015,6 +1016,268 @@ def test_slack_json_sanitizes_speaker_id(): assert "\n> fake" not in result +# ── _try_continue_json ───────────────────────────────────────────────── + + +def test_continue_json_valid_multi_turn(): + data = { + "history": [ + {"role": "user", "content": "What is Python?"}, + {"role": "assistant", "content": "Python is a programming language."}, + {"role": "user", "content": "How do I install it?"}, + {"role": "assistant", "content": "Use your package manager."}, + ], + "title": "Python help", + "sessionId": "abc-123", + "dateCreated": "2025-01-15T10:30:00Z", + } + result = _try_continue_json(data) + assert result is not None + assert "> What is Python?" in result + assert "Python is a programming language." in result + assert "> How do I install it?" in result + assert "Use your package manager." in result + + +def test_continue_json_with_system_messages(): + """System messages are skipped.""" + data = { + "history": [ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "> Hello" in result + assert "helpful assistant" not in result + + +def test_continue_json_with_tool_messages(): + """Tool messages are appended to the previous assistant turn.""" + data = { + "history": [ + {"role": "user", "content": "List files"}, + {"role": "assistant", "content": "Let me check."}, + {"role": "tool", "content": "file1.py\nfile2.py"}, + {"role": "assistant", "content": "I found two files."}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "> List files" in result + assert "Let me check." in result + assert "[tool] file1.py" in result + assert "I found two files." in result + + +def test_continue_json_with_code_blocks(): + """Code blocks in content are preserved.""" + data = { + "history": [ + {"role": "user", "content": "Show me a hello world"}, + { + "role": "assistant", + "content": "Here you go:\n```python\nprint('Hello, world!')\n```", + }, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "```python" in result + assert "print('Hello, world!')" in result + + +def test_continue_json_list_content_blocks(): + """Content as a list of typed blocks (text blocks).""" + data = { + "history": [ + {"role": "user", "content": [{"type": "text", "text": "Help me"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "Sure thing"}]}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "> Help me" in result + assert "Sure thing" in result + + +def test_continue_json_empty_history(): + """Empty history returns None.""" + data = {"history": []} + result = _try_continue_json(data) + assert result is None + + +def test_continue_json_single_message(): + """Too few messages returns None.""" + data = {"history": [{"role": "user", "content": "Hello"}]} + result = _try_continue_json(data) + assert result is None + + +def test_continue_json_no_history_key(): + """Missing history key returns None.""" + data = {"title": "Some session", "sessionId": "abc"} + result = _try_continue_json(data) + assert result is None + + +def test_continue_json_not_a_dict(): + """Non-dict input returns None.""" + result = _try_continue_json([1, 2, 3]) + assert result is None + result = _try_continue_json("not a dict") + assert result is None + + +def test_continue_json_history_not_a_list(): + """history key that isn't a list returns None.""" + data = {"history": "not a list"} + result = _try_continue_json(data) + assert result is None + + +def test_continue_json_malformed_entries(): + """Non-dict entries in history are skipped.""" + data = { + "history": [ + "not a dict", + 42, + {"role": "user", "content": "Q"}, + {"role": "assistant", "content": "A"}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "> Q" in result + + +def test_continue_json_missing_role(): + """Entries without a role are skipped.""" + data = { + "history": [ + {"content": "orphan text"}, + {"role": "user", "content": "Q"}, + {"role": "assistant", "content": "A"}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "orphan" not in result + + +def test_continue_json_missing_content(): + """Entries without content are skipped.""" + data = { + "history": [ + {"role": "user"}, + {"role": "user", "content": "Real question"}, + {"role": "assistant", "content": "Real answer"}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "> Real question" in result + + +def test_continue_json_empty_content(): + """Entries with empty/whitespace content are skipped.""" + data = { + "history": [ + {"role": "user", "content": ""}, + {"role": "user", "content": " "}, + {"role": "user", "content": "Actual question"}, + {"role": "assistant", "content": "Actual answer"}, + ] + } + result = _try_continue_json(data) + assert result is not None + user_turns = [line for line in result.split("\n") if line.strip().startswith(">")] + assert len(user_turns) == 1 + + +def test_continue_json_unicode_cjk(): + """Unicode and CJK content is handled correctly.""" + data = { + "history": [ + {"role": "user", "content": "Python\u306e\u4f7f\u3044\u65b9\u3092\u6559\u3048\u3066"}, + { + "role": "assistant", + "content": "\u306f\u3044\u3001Python\u306f\u7d20\u6674\u3089\u3057\u3044\u8a00\u8a9e\u3067\u3059\u3002\ud83d\ude80", + }, + {"role": "user", "content": "\u8c22\u8c22\uff01\u975e\u5e38\u6709\u5e2e\u52a9"}, + {"role": "assistant", "content": "\u4e0d\u5ba2\u6c14 \ud83d\ude0a"}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "\u306e\u4f7f\u3044\u65b9" in result + assert "\u8c22\u8c22" in result + assert "\ud83d\ude80" in result + + +def test_continue_json_very_long_message(): + """Very long messages are handled without error.""" + long_text = "x" * 50000 + data = { + "history": [ + {"role": "user", "content": "Summarize this: " + long_text}, + {"role": "assistant", "content": "That's a lot of x's."}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "Summarize this:" in result + + +def test_continue_json_non_string_content_skipped(): + """Non-string, non-list content (e.g. int, None) is skipped.""" + data = { + "history": [ + {"role": "user", "content": 42}, + {"role": "assistant", "content": None}, + {"role": "user", "content": "Real Q"}, + {"role": "assistant", "content": "Real A"}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "> Real Q" in result + + +def test_continue_json_tool_without_preceding_assistant(): + """Tool message without a preceding assistant turn is ignored.""" + data = { + "history": [ + {"role": "tool", "content": "orphan tool output"}, + {"role": "user", "content": "Q"}, + {"role": "assistant", "content": "A"}, + ] + } + result = _try_continue_json(data) + assert result is not None + assert "orphan" not in result + + +def test_continue_json_integration_via_normalize(tmp_path): + """Continue.dev JSON is detected and parsed via the top-level normalize().""" + data = { + "history": [ + {"role": "user", "content": "What is MemPalace?"}, + {"role": "assistant", "content": "A memory system for AI."}, + ], + "title": "MemPalace overview", + "sessionId": "session-001", + } + f = tmp_path / "session.json" + f.write_text(json.dumps(data)) + result = normalize(str(f)) + assert "> What is MemPalace?" in result + assert "A memory system for AI." in result + + # ── _try_normalize_json ────────────────────────────────────────────────