xiaoxue_admin
/
mempalace
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: tighten local guards and file handling

This commit is contained in:
Igor Lins e Silva 2026-06-23 23:45:03 -03:00
parent 65fa1517a1
commit 2366582fe3
11 changed files with 220 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
hooks/mempal_precompact_hook.sh
View File

@ -182,14 +182,14 @@ echo "[$(date '+%H:%M:%S')] PRE-COMPACT triggered for session $SESSION_ID" >> "$
# 1. TRANSCRIPT_PATH (from Claude Code) → parent dir, --mode convos
# 2. MEMPAL_DIR → --mode projects
if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
"$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
>> "$STATE_DIR/hook.log" 2>&1
elif [ -n "$TRANSCRIPT_PATH" ]; then
echo "[$(date '+%H:%M:%S')] Skipping missing or invalid transcript path after normalization: $TRANSCRIPT_PATH" \
>> "$STATE_DIR/hook.log"
fi
if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
mempalace mine "$MEMPAL_DIR" --mode projects \
"$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \
>> "$STATE_DIR/hook.log" 2>&1
fi

4
hooks/mempal_save_hook.sh
View File

@ -265,14 +265,14 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then
# MEMPAL_DIR is *additive*, not an override: a user with MEMPAL_DIR
# pointed at their project still gets the active conversation mined.
if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
"$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
>> "$STATE_DIR/hook.log" 2>&1 &
elif [ -n "$TRANSCRIPT_PATH" ]; then
echo "[$(date '+%H:%M:%S')] Skipping invalid transcript path: $TRANSCRIPT_PATH" \
>> "$STATE_DIR/hook.log"
fi
if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
mempalace mine "$MEMPAL_DIR" --mode projects \
"$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \
>> "$STATE_DIR/hook.log" 2>&1 &
fi

11
mempalace/cli.py
View File

@ -313,7 +313,16 @@ def cmd_init(args):
endpoint=getattr(args, "llm_endpoint", None),
api_key=getattr(args, "llm_api_key", None),
)
ok, msg = candidate.check_available()
if (
provider_name == "openai-compat"
and getattr(candidate, "api_key_source", None) == "env"
and candidate.is_external_service
):
ok = False
msg = "external openai-compat init requires explicit --llm-api-key"
print(f" LLM skipped: {msg}")
else:
ok, msg = candidate.check_available()
if ok:
llm_provider = candidate
print(f" LLM enabled: {provider_name}/{provider_model}")

41
mempalace/convo_miner.py
View File

@ -11,6 +11,7 @@ Same palace as project mining. Different ingest strategy.
import os
import sys
import logging
import stat
from pathlib import Path
from datetime import datetime
from collections import defaultdict
@ -77,6 +78,33 @@ MAX_FILE_SIZE = 500 * 1024 * 1024 # 500 MB — skip files larger than this.
# use also scales with source size.
def _path_within_root(path: Path, root: Path) -> bool:
try:
path.expanduser().resolve().relative_to(root.expanduser().resolve())
return True
except (OSError, ValueError):
return False
def _is_regular_source_file(filepath: Path, root: Path) -> bool:
if not _path_within_root(filepath, root):
return False
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
fd = -1
try:
fd = os.open(filepath, flags)
st = os.fstat(fd)
return stat.S_ISREG(st.st_mode) and st.st_size <= MAX_FILE_SIZE
except OSError:
return False
finally:
if fd != -1:
try:
os.close(fd)
except OSError:
pass
def _register_file(collection, source_file: str, wing: str, agent: str, extract_mode: str):
"""Write a sentinel so file_already_mined() returns True for 0-chunk files.
@ -366,13 +394,10 @@ def scan_convos(convo_dir: str) -> list:
rel = filepath.relative_to(convo_path).as_posix()
try:
print(f" SKIP: {rel} (symlink)", file=sys.stderr)
except OSError:
pass
continue
try:
if filepath.stat().st_size > MAX_FILE_SIZE:
continue
except OSError:
pass
continue
if not _is_regular_source_file(filepath, convo_path):
continue
files.append(filepath)
return files
@ -633,6 +658,10 @@ def _mine_convos_impl(
files_skipped += 1
continue
if not _is_regular_source_file(filepath, Path(convo_dir).expanduser().resolve()):
files_skipped += 1
continue
# Normalize format
try:
content = normalize(str(filepath))

9
mempalace/hooks_cli.py
View File

@ -851,8 +851,13 @@ def _save_diary_direct(
def _ingest_transcript(transcript_path: str):
"""Mine a Claude Code session transcript into the palace as a conversation."""
path = Path(transcript_path).expanduser()
if not path.is_file() or path.stat().st_size < 100:
path = _validate_transcript_path(transcript_path)
if path is None:
return
try:
if not path.is_file() or path.stat().st_size < 100:
return
except OSError:
return
try:

4
mempalace/llm_client.py
View File

@ -323,7 +323,9 @@ class OpenAICompatProvider(LLMProvider):
base = base.removesuffix("/chat/completions").removesuffix("/v1")
try:
req = Request(f"{base}/v1/models")
if self.api_key:
if self.api_key and (
self.api_key_source != "env" or not self.is_external_service
):
req.add_header("Authorization", f"Bearer {self.api_key}")
with urlopen(req, timeout=5):
pass

32
mempalace/mcp_server.py
View File

@ -4572,6 +4572,7 @@ _HTTP_MAX_REQUEST_BYTES = 16 * 1024 * 1024
# bind is loopback (skip the network-exposure warning) and to pin the Host
# header against DNS rebinding when serving on loopback.
_HTTP_LOOPBACK_HOSTS = ("127.0.0.1", "localhost", "::1", "[::1]")
_HTTP_ALLOW_INSECURE_NO_TOKEN_ENV = "MEMPALACE_MCP_HTTP_ALLOW_INSECURE_NO_TOKEN"
def _http_is_loopback(host: str) -> bool:
@ -4628,6 +4629,16 @@ def _build_http_server(host: str, port: int):
from urllib.parse import urlparse
auth_token = os.environ.get("MEMPALACE_MCP_HTTP_TOKEN", "").strip()
if (
not _http_is_loopback(host)
and not auth_token
and not _truthy_env(_HTTP_ALLOW_INSECURE_NO_TOKEN_ENV)
):
raise ValueError(
"MEMPALACE_MCP_HTTP_TOKEN is required when binding MCP HTTP to a "
f"non-loopback host. Set {_HTTP_ALLOW_INSECURE_NO_TOKEN_ENV}=1 only "
"when a trusted fronting layer provides access control."
)
class _MCPHTTPServer(ThreadingHTTPServer):
daemon_threads = True
@ -4764,18 +4775,25 @@ def _serve_http(host: str, port: int) -> None:
"""
try:
httpd = _build_http_server(host, port)
except OSError as exc:
except (OSError, ValueError) as exc:
logger.error("Failed to start MCP HTTP server on %s:%s: %s", host, port, exc)
sys.exit(1)
bound_port = httpd.server_address[1]
if not _http_is_loopback(host):
logger.warning(
"MemPalace MCP HTTP server bound to non-loopback host %s — the palace "
"is now reachable from the network and /mcp is unauthenticated unless "
"you set MEMPALACE_MCP_HTTP_TOKEN. Bind 127.0.0.1 to keep it local.",
host,
)
if httpd.auth_token:
logger.warning(
"MemPalace MCP HTTP server bound to non-loopback host %s; /mcp "
"requires the configured bearer token.",
host,
)
else:
logger.warning(
"MemPalace MCP HTTP server bound to non-loopback host %s without "
"a bearer token because %s is set.",
host,
_HTTP_ALLOW_INSECURE_NO_TOKEN_ENV,
)
with httpd:
logger.info("MemPalace MCP HTTP server listening on http://%s:%s/mcp", host, bound_port)
try:

2
mempalace/migrate.py
View File

@ -295,7 +295,7 @@ def migrate(palace_path: str, dry_run: bool = False, confirm: bool = False):
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
backup_path = f"{palace_path}.pre-migrate.{timestamp}"
print(f"\n Backing up to {backup_path}...")
shutil.copytree(palace_path, backup_path)
shutil.copytree(palace_path, backup_path, symlinks=True)
# Enforce backup retention so repeated migrations cannot fill the disk
# with full-palace copies. The backup we just created is the newest, so

37
mempalace/miner.py
View File

@ -14,6 +14,7 @@ import shlex
import hashlib
import fnmatch
import logging
import stat
from pathlib import Path
from datetime import datetime
from collections import defaultdict
@ -47,6 +48,37 @@ from .ids import ID_RECIPE, make_drawer_id_from_chunk
logger = logging.getLogger("mempalace_mcp")
def _path_within_root(path: Path, root: Path) -> bool:
try:
path.expanduser().resolve().relative_to(root.expanduser().resolve())
return True
except (OSError, ValueError):
return False
def _read_text_no_follow(filepath: Path, root: Path) -> Optional[str]:
if not _path_within_root(filepath, root):
return None
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
fd = -1
try:
fd = os.open(filepath, flags)
st = os.fstat(fd)
if not stat.S_ISREG(st.st_mode) or st.st_size > MAX_FILE_SIZE:
return None
with os.fdopen(fd, "r", encoding="utf-8", errors="replace") as f:
fd = -1
return f.read()
except OSError:
return None
finally:
if fd != -1:
try:
os.close(fd)
except OSError:
pass
PHP_EXTENSIONS = {
# Compound Blade templates such as ``view.blade.php`` are covered by the
# final ``.php`` suffix.
@ -1341,9 +1373,8 @@ def process_file(
if not dry_run and file_already_mined(collection, source_file, check_mtime=True):
return 0, "general", None
try:
content = filepath.read_text(encoding="utf-8", errors="replace")
except OSError:
content = _read_text_no_follow(filepath, project_path)
if content is None:
return 0, "general", None
content = content.strip()

26
mempalace/normalize.py
View File

@ -21,6 +21,7 @@ No API key. No internet. Everything local.
import json
import os
import re
import stat
from pathlib import Path
from typing import Optional
@ -118,17 +119,28 @@ def normalize(filepath: str) -> str:
Load a file and normalize to transcript format if it's a chat export.
Plain text files pass through unchanged.
"""
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
if os.path.islink(filepath):
raise IOError(f"Could not read {filepath}: symlinked files are skipped")
fd = -1
try:
file_size = os.path.getsize(filepath)
except OSError as e:
raise IOError(f"Could not read {filepath}: {e}") from e
if file_size > 500 * 1024 * 1024: # 500 MB safety limit
raise IOError(f"File too large ({file_size // (1024 * 1024)} MB): {filepath}")
try:
with open(filepath, "r", encoding="utf-8-sig", errors="replace") as f:
fd = os.open(filepath, flags)
file_stat = os.fstat(fd)
if not stat.S_ISREG(file_stat.st_mode):
raise IOError(f"Could not read {filepath}: not a regular file")
if file_stat.st_size > 500 * 1024 * 1024: # 500 MB safety limit
raise IOError(f"File too large ({file_stat.st_size // (1024 * 1024)} MB): {filepath}")
with os.fdopen(fd, "r", encoding="utf-8-sig", errors="replace") as f:
fd = -1
content = f.read()
except OSError as e:
raise IOError(f"Could not read {filepath}: {e}") from e
finally:
if fd != -1:
try:
os.close(fd)
except OSError:
pass
if not content.strip():
return content

88
mempalace/repair.py
View File

@ -33,6 +33,7 @@ import argparse
import os
import shutil
import sqlite3
import stat
import time
from collections import defaultdict
from contextlib import closing
@ -57,6 +58,77 @@ REPAIR_TEMP_COLLECTION = f"{COLLECTION_NAME}__repair_tmp"
CLOSETS_COLLECTION_NAME = "mempalace_closets"
def _no_follow_flag() -> int:
return getattr(os, "O_NOFOLLOW", 0)
def _open_regular_file_no_follow(path: str) -> int:
if os.path.islink(path):
raise RuntimeError(f"Refusing symlinked file: {path}")
fd = os.open(path, os.O_RDONLY | _no_follow_flag())
try:
st = os.fstat(fd)
if not stat.S_ISREG(st.st_mode):
raise RuntimeError(f"Refusing non-regular file: {path}")
return fd
except Exception:
os.close(fd)
raise
def _write_text_replace_no_follow(path: str, text: str) -> None:
directory = os.path.dirname(path) or "."
basename = os.path.basename(path)
tmp_path = os.path.join(
directory,
f".{basename}.{os.getpid()}.{int(time.time() * 1_000_000)}.tmp",
)
fd = os.open(
tmp_path,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | _no_follow_flag(),
0o600,
)
try:
with os.fdopen(fd, "w", encoding="utf-8") as f:
f.write(text)
os.replace(tmp_path, path)
except Exception:
try:
os.unlink(tmp_path)
except OSError:
pass
raise
def _copy_file_no_follow(src: str, dst: str, *, replace: bool = False) -> None:
src_fd = _open_regular_file_no_follow(src)
flags = os.O_WRONLY | os.O_CREAT | _no_follow_flag()
flags |= os.O_TRUNC if replace else os.O_EXCL
dst_fd = os.open(dst, flags, 0o600)
try:
with os.fdopen(src_fd, "rb") as src_f, os.fdopen(dst_fd, "wb") as dst_f:
shutil.copyfileobj(src_f, dst_f)
try:
shutil.copystat(src, dst, follow_symlinks=False)
except OSError:
pass
except Exception:
try:
os.close(src_fd)
except OSError:
pass
try:
os.close(dst_fd)
except OSError:
pass
raise
def _unique_backup_path(path: str, label: str) -> str:
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
return f"{path}.{label}.{stamp}.{os.getpid()}"
def _drawers_collection_name() -> str:
"""Resolve the drawers collection name from user config, falling back
to the module default ``COLLECTION_NAME`` if config is unreadable.
@ -316,9 +388,10 @@ def scan_palace(palace_path=None, only_wing=None, collection_name: Optional[str]
print(f" BAD: {len(bad_set):,} ({len(bad_set) / max(len(all_ids), 1) * 100:.1f}%)")
bad_file = os.path.join(palace_path, "corrupt_ids.txt")
with open(bad_file, "w") as f:
for bid in sorted(bad_set):
f.write(bid + "\n")
_write_text_replace_no_follow(
bad_file,
"".join(f"{bid}\n" for bid in sorted(bad_set)),
)
print(f"\n Bad IDs written to: {bad_file}")
return good_set, bad_set
@ -858,10 +931,10 @@ def rebuild_index(
# Back up ONLY the SQLite database, not the bloated HNSW files
sqlite_path = os.path.join(palace_path, "chroma.sqlite3")
backup_path = sqlite_path + ".backup"
backup_path = _unique_backup_path(sqlite_path, "backup")
if os.path.exists(sqlite_path):
progress(f" Backing up chroma.sqlite3 ({os.path.getsize(sqlite_path) / 1e6:.0f} MB)...")
shutil.copy2(sqlite_path, backup_path)
_copy_file_no_follow(sqlite_path, backup_path)
progress(f" Backup: {backup_path}")
# Rebuild with correct HNSW settings
@ -1116,7 +1189,10 @@ def _preserve_knowledge_graph_sqlite(source_palace: str, dest_palace: str) -> li
continue
os.makedirs(dest_palace, exist_ok=True)
shutil.copy2(src, dst)
try:
_copy_file_no_follow(src, dst, replace=True)
except RuntimeError:
continue
copied.append(filename)
if copied: