Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8ab251c452 | ||
|
|
5cd961eef9 | ||
|
|
67c90c39d5 | ||
|
|
de74cad911 | ||
|
|
c76208507e | ||
|
|
fa97c93dd7 | ||
|
|
ec8788c2bf | ||
|
|
f62ce2696b | ||
|
|
a613c921c8 | ||
|
|
4bee65c11b | ||
|
|
6340d611cc | ||
|
|
9ecdf83c08 | ||
|
|
dbfeb82e68 | ||
|
|
68edb110a1 | ||
|
|
a824be43c8 | ||
|
|
88e454caed | ||
|
|
4c8ef8ed43 | ||
|
|
287d8f1b4d | ||
|
|
0955100a1f | ||
|
|
8a87fc66e2 | ||
|
|
f68cb8d682 | ||
|
|
acd62442be | ||
|
|
2c6cecf1fe | ||
|
|
7267de2b32 | ||
|
|
32e545eaaa | ||
|
|
28aa2c49b0 | ||
|
|
88e839bce5 | ||
|
|
0b04b79869 | ||
|
|
3da735ec6b | ||
|
|
5fdce3923e | ||
|
|
3c0ecb5b3e | ||
|
|
7f07b0d4fa | ||
|
|
4b98dd796d | ||
|
|
99464572f4 | ||
|
|
5b1c32eee1 | ||
|
|
e360a3a040 | ||
|
|
d7819d4b85 | ||
|
|
c7955d3899 | ||
|
|
18a9788961 | ||
|
|
63188686e5 | ||
|
|
09aa4034a7 | ||
|
|
a20d769b8c | ||
|
|
b0b028170a | ||
|
|
c0ae2abaff | ||
|
|
9e871b45b5 | ||
|
|
9b432699ce | ||
|
|
3f49e15545 | ||
|
|
b70f06a69e | ||
|
|
da32e562f7 | ||
|
|
1d38ae8784 | ||
|
|
07182d5adf | ||
|
|
c00e8dcc16 | ||
|
|
b459293e6f | ||
|
|
9815f0ae23 | ||
|
|
4e20ad3e8c | ||
|
|
65bedc2b33 | ||
|
|
f820b73160 | ||
|
|
aff1ee8255 | ||
|
|
39bec19956 | ||
|
|
f9db9b726f | ||
|
|
578e5775b4 | ||
|
|
adc0ef876f | ||
|
|
c54531a2a2 | ||
|
|
0104d86c53 | ||
|
|
db6c3d6552 | ||
|
|
bddbb5f7a5 | ||
|
|
2c81e22e28 | ||
|
|
9b395fe9d5 | ||
|
|
da5a48caf5 | ||
|
|
e096701333 | ||
|
|
e84221c6b7 | ||
|
|
0549a90df3 | ||
|
|
281aaf489d | ||
|
|
176c8b820b | ||
|
|
3b8d2652f4 | ||
|
|
e4924e8e94 | ||
|
|
1577346158 | ||
|
|
6cec591c17 | ||
|
|
afd0428823 | ||
|
|
cea780eb79 | ||
|
|
2211c974ff | ||
|
|
8ec284db5d | ||
|
|
5dcc46ba7e | ||
|
|
178105cbd5 | ||
|
|
833b6ab4d3 | ||
|
|
faa8643d9f | ||
|
|
cff43adb63 | ||
|
|
fd3b4e8ebb | ||
|
|
cd7a8658a2 | ||
|
|
d8b63a9521 | ||
|
|
d6cb868f75 | ||
|
|
b864766b12 | ||
|
|
1bc5c508d5 | ||
|
|
5cbc37ee9b | ||
|
|
bacb935bb3 | ||
|
|
c17d1aaf13 | ||
|
|
292cf872a5 | ||
|
|
b0a94bdef2 | ||
|
|
73e74bfb80 | ||
|
|
dc433824fd | ||
|
|
0f3f8ec652 | ||
|
|
5294d3118e | ||
|
|
aac947a5c5 | ||
|
|
86546348e4 | ||
|
|
d20ac0adf1 | ||
|
|
d479d6f245 | ||
|
|
c4ce4c89d2 | ||
|
|
ca11a455f5 | ||
|
|
4700c44163 | ||
|
|
a426587d35 | ||
|
|
c7428c3b71 | ||
|
|
c80f6537ec | ||
|
|
6ea60c6043 | ||
|
|
d59ca24647 | ||
|
|
e9746df6fd | ||
|
|
2366582fe3 | ||
|
|
2c08bb251d | ||
|
|
65fa1517a1 | ||
|
|
e8f96dd8b2 | ||
|
|
270f1aede9 | ||
|
|
37194cf543 | ||
|
|
58ea91b44d | ||
|
|
1095d684ff | ||
|
|
32fad4844d | ||
|
|
eff887d12f | ||
|
|
ce00cf0718 | ||
|
|
3e3fcf29fe | ||
|
|
569beff342 | ||
|
|
3fbab55cab | ||
|
|
1a47b78dcb | ||
|
|
960eaac851 | ||
|
|
54e9f46027 | ||
|
|
2e8ff64e3f | ||
|
|
94519b91dd | ||
|
|
2f92774eec | ||
|
|
9902e3a9a5 | ||
|
|
a91f7e06c7 | ||
|
|
971e5a75c8 | ||
|
|
7392ab8a25 | ||
|
|
e38af8c2e9 | ||
|
|
70ec8e450f | ||
|
|
a11cf4b667 | ||
|
|
a9344937df | ||
|
|
abd780ee36 | ||
|
|
db90cdf132 | ||
|
|
91e2158213 | ||
|
|
b4a6ffdf9f | ||
|
|
92ac104e31 | ||
|
|
e576c98ed2 | ||
|
|
fff8546672 | ||
|
|
b8c6bfe504 | ||
|
|
7d311af78e | ||
|
|
5ae2315f80 | ||
|
|
352eb67a90 | ||
|
|
f3ed6f796e | ||
|
|
4291fecfdf | ||
|
|
d7e182a272 | ||
|
|
65d07045bd | ||
|
|
fb8dc1f677 | ||
|
|
44016ad118 | ||
|
|
c203aacf8a | ||
|
|
4c4c4aba8d | ||
|
|
7fb7bd3b8d | ||
|
|
8723d30083 | ||
|
|
2ec48dde56 | ||
|
|
6cc5832176 | ||
|
|
386f3c9796 | ||
|
|
e0da99b2d5 | ||
|
|
3eacbe0c43 | ||
|
|
a27129a32a | ||
|
|
4791999516 | ||
|
|
fa27e41436 | ||
|
|
31fff1c354 | ||
|
|
fe460c4b8b | ||
|
|
73772cb707 | ||
|
|
38253b1f5f | ||
|
|
9d5a375017 | ||
|
|
73f455c7d0 | ||
|
|
477aa362cd | ||
|
|
2eda1f9130 | ||
|
|
8bdebe1da1 | ||
|
|
abaf09be0a | ||
|
|
0601026026 | ||
|
|
157022ab1b | ||
|
|
03f638a279 | ||
|
|
5fda4e5231 | ||
|
|
b0eddf68dd | ||
|
|
94d1ced11e | ||
|
|
49e427ba35 | ||
|
|
7fb981c538 | ||
|
|
32ec879c8b | ||
|
|
4e03ce90bc | ||
|
|
393107e4ac | ||
|
|
d09392c11a | ||
|
|
fe0391ab56 | ||
|
|
f868ee78d4 | ||
|
|
95211e84cb | ||
|
|
d859d5a565 | ||
|
|
5f58cdbfa8 | ||
|
|
3c7808e1bc | ||
|
|
4c46290b80 | ||
|
|
6bccf82a8c | ||
|
|
63a382fc6d | ||
|
|
aa96bb5623 | ||
|
|
afa749c141 | ||
|
|
f0e76c4363 | ||
|
|
5935dacc98 | ||
|
|
349485b6b6 | ||
|
|
78a32c69c1 | ||
|
|
22cf8afd1e | ||
|
|
cbf6cbe851 | ||
|
|
301aa335e1 | ||
|
|
d4391abb59 | ||
|
|
617aa0c3f3 | ||
|
|
f0b5cbf374 | ||
|
|
e594bad9d6 | ||
|
|
ea71c5234e | ||
|
|
d704433701 | ||
|
|
d8d88f4d21 | ||
|
|
ad5e7dfe2c | ||
|
|
c2f42cd57e | ||
|
|
f56e3067eb | ||
|
|
d486aef083 | ||
|
|
9f434e0bfd | ||
|
|
bc9c052b51 | ||
|
|
db6e4f0654 | ||
|
|
3f20305eda | ||
|
|
868b4c9b39 | ||
|
|
327da58c09 | ||
|
|
b5c79a1eea | ||
|
|
68a971b8a1 | ||
|
|
160a852bdb | ||
|
|
7f5bbd8be8 | ||
|
|
0cb45084bc | ||
|
|
a40e0b7a47 | ||
|
|
58c45a9905 | ||
|
|
9f16295f77 | ||
|
|
8e53959bb9 | ||
|
|
d860a008a1 | ||
|
|
5d5397bf52 | ||
|
|
f89bc08876 | ||
|
|
97ba05cfe3 | ||
|
|
422edc2dda | ||
|
|
0a2937d766 | ||
|
|
7e4bd816d1 | ||
|
|
75eb86a93a | ||
|
|
80c37602c0 | ||
|
|
b286701535 | ||
|
|
c50434594c | ||
|
|
528d711f95 | ||
|
|
4fbdaf2c32 | ||
|
|
c45d069f59 | ||
|
|
950713eef6 | ||
|
|
41e5c20191 | ||
|
|
f6b6a69197 | ||
|
|
0b00d78c98 | ||
|
|
2b457a2366 | ||
|
|
38de37adbf | ||
|
|
13d6c3e7a1 | ||
|
|
5a8f0258d6 | ||
|
|
761ebf78bb | ||
|
|
7e45720dc8 | ||
|
|
0fc3a238d7 | ||
|
|
5024e90d68 | ||
|
|
f5ee33b216 | ||
|
|
4fd12318df | ||
|
|
bf71ae7168 | ||
|
|
4187521ba4 | ||
|
|
07f8789514 | ||
|
|
8e19e815cc | ||
|
|
f31db69e03 | ||
|
|
2ec4baec3f | ||
|
|
60a4610c6b | ||
|
|
fc89a1b67e | ||
|
|
47bddb066e | ||
|
|
374400a530 | ||
|
|
f124bd2c84 | ||
|
|
df0e29ac8b | ||
|
|
692599aa94 | ||
|
|
ad67c1a121 | ||
|
|
4d60db05ba | ||
|
|
77eaa312e0 | ||
|
|
8c120bee34 | ||
|
|
140c380584 | ||
|
|
7d7dc78107 | ||
|
|
d057dee9c3 | ||
|
|
24ad98df8c | ||
|
|
d05bb11724 | ||
|
|
ef2261feb8 | ||
|
|
eab11969da | ||
|
|
d514787f95 | ||
|
|
676ee682ec | ||
|
|
ed77125ae3 | ||
|
|
4ceb880d19 | ||
|
|
fdbafe5420 | ||
|
|
13de7a6743 | ||
|
|
212051da35 | ||
|
|
52144704e7 | ||
|
|
ffa8ac142c | ||
|
|
9a8c87acfe | ||
|
|
6c5e1be3db | ||
|
|
74453ee3ac | ||
|
|
ae415b8620 | ||
|
|
ca38ae899d | ||
|
|
29d7e1edcf | ||
|
|
b1537be5b5 | ||
|
|
c4da6d500b | ||
|
|
c9dc4c466d | ||
|
|
2a66bad497 | ||
|
|
be954671ca | ||
|
|
939a076baf | ||
|
|
19d2f9216d | ||
|
|
72a69e2476 | ||
|
|
051f0cc1ec | ||
|
|
d464c28e0a | ||
|
|
d51e9434f9 | ||
|
|
15833ab01a | ||
|
|
ce929a8241 | ||
|
|
9694e2cb57 | ||
|
|
73d4e43b22 | ||
|
|
e66cfff0c0 | ||
|
|
ab716d02a5 | ||
|
|
ed772a01f9 | ||
|
|
708ef4a9fd | ||
|
|
8ec438ef57 | ||
|
|
e46374d24b | ||
|
|
6ac2de8e04 | ||
|
|
ab669fd3b1 | ||
|
|
b130301bd5 | ||
|
|
ad9874fcd5 | ||
|
|
334fc6d0be | ||
|
|
1ed7f49f13 | ||
|
|
097687cf63 | ||
|
|
46e3b22316 | ||
|
|
bbe6ff9477 | ||
|
|
b1df41b943 | ||
|
|
e1fce46006 | ||
|
|
cace2230d8 | ||
|
|
04af6a436d | ||
|
|
e00bc666e7 | ||
|
|
593cfebd79 | ||
|
|
6434370034 | ||
|
|
82af1fff27 | ||
|
|
5289bdfedb | ||
|
|
694289c423 | ||
|
|
723e45bf2b | ||
|
|
37e34f6cb5 | ||
|
|
f88951e8d4 | ||
|
|
4bc1ea8078 | ||
|
|
dc7d0be10c | ||
|
|
15852edbaf | ||
|
|
eb3ff2e61a | ||
|
|
0225eb4a08 | ||
|
|
1dadd33f35 | ||
|
|
ec5d1eb1c7 | ||
|
|
9be2b97b6c | ||
|
|
8757ba2650 | ||
|
|
717a57ff43 | ||
|
|
4e0fce010e | ||
|
|
5ba8399bb2 | ||
|
|
f5d2212577 | ||
|
|
ce46747282 | ||
|
|
02b8753d97 | ||
|
|
300e824e94 | ||
|
|
af2bd0e10a | ||
|
|
526d58ae93 | ||
|
|
4d3daf96d3 | ||
|
|
30b82ba95f | ||
|
|
6aa8e93bc9 | ||
|
|
0dd0ade3b9 | ||
|
|
b5f3915d5b | ||
|
|
e54896699b | ||
|
|
6dd99ec4b0 | ||
|
|
a329acf23c | ||
|
|
206cfbb6ca | ||
|
|
22804b8e08 | ||
|
|
c19d894f68 | ||
|
|
7b1f6247d0 | ||
|
|
b11f41484c | ||
|
|
10d1ce6a94 | ||
|
|
038a7e0e15 | ||
|
|
29db125b9b | ||
|
|
818e4f3b31 | ||
|
|
48f4eb6dfe | ||
|
|
9b7cfc9940 | ||
|
|
c420a9f66c | ||
|
|
df295bd743 | ||
|
|
387d50df27 | ||
|
|
55d4e379a7 | ||
|
|
13c38ac588 | ||
|
|
a0508cb065 | ||
|
|
2d4fae1ac3 | ||
|
|
b06cf2452e | ||
|
|
bf156fb010 | ||
|
|
df5db57e79 | ||
|
|
071fa015e0 | ||
|
|
15ac4303f5 | ||
|
|
6957c7e599 | ||
|
|
234343ef68 | ||
|
|
c4d8804b44 | ||
|
|
d88597cd72 | ||
|
|
753df201ac | ||
|
|
d0d011eb33 | ||
|
|
dbbf4c5af5 | ||
|
|
c9d8a40a37 | ||
|
|
be819da146 | ||
|
|
cbd1567b6a | ||
|
|
653e9774e8 | ||
|
|
5fe54df4c8 | ||
|
|
f51ecc1621 | ||
|
|
25b918ee7e | ||
|
|
a9a2c35e65 | ||
|
|
66b0c5066b | ||
|
|
e039a675af | ||
|
|
d12c96e43e | ||
|
|
6165aa2d17 |
@@ -0,0 +1,77 @@
|
||||
# MemPalace — Antigravity plugin
|
||||
|
||||
In-repo packaging for the MemPalace integration with Google's [Antigravity IDE](https://antigravity.google/).
|
||||
|
||||
This directory is the source of truth for what gets installed at
|
||||
`~/.gemini/config/plugins/mempalace/` when the user runs the installer.
|
||||
|
||||
## Layout
|
||||
|
||||
```
|
||||
.antigravity-plugin/
|
||||
├── plugin.json # marker manifest (verified minimal schema)
|
||||
├── mcp_config.json # auto-registers the mempalace-mcp stdio server
|
||||
├── hooks.json.tmpl # template — installer renders to hooks.json
|
||||
├── skills/
|
||||
│ ├── mempalace/
|
||||
│ │ └── SKILL.md # ops skill: setup, mine, status, CLI delegation
|
||||
│ └── mempalace-recall/
|
||||
│ └── SKILL.md # recall-only skill: search-before-answer protocol
|
||||
├── rules/
|
||||
│ └── mempalace-recall.md # optional recall rule (complements the skill)
|
||||
└── README.md # this file
|
||||
```
|
||||
|
||||
The hook scripts themselves live at `hooks/antigravity/`. The installer
|
||||
copies them into `<install-dir>/hooks/` and renders `hooks.json.tmpl`
|
||||
into a `hooks.json` whose `command` paths point at the absolute install
|
||||
location.
|
||||
|
||||
## Three recall layers
|
||||
|
||||
MemPalace can store everything, but it only helps if the agent actually
|
||||
*reads* the palace before answering. Three layers wire that in, from
|
||||
eager to on-demand:
|
||||
|
||||
1. **Wake hook** (`hooks/antigravity/mempal_wake_hook_antigravity.sh`,
|
||||
`PreInvocation` event, gated to `invocationNum == 1`). On the first
|
||||
model call of a conversation it runs `mempalace wake-up` and injects
|
||||
the **actual palace content verbatim** via Antigravity's
|
||||
`injectSteps[].ephemeralMessage` output. This is Antigravity's native
|
||||
equivalent of Cursor's `sessionStart` `additional_context`, except it
|
||||
delivers the memory itself rather than a directive to go fetch it.
|
||||
2. **Recall skill** (`skills/mempalace-recall/SKILL.md`). The
|
||||
search-before-answer protocol the agent follows when a turn is
|
||||
recall-relevant — tool selection, unhappy paths, anti-patterns. It
|
||||
covers recall only; the `mempalace` skill covers setup / mine /
|
||||
status.
|
||||
3. **Optional recall rule** (`rules/mempalace-recall.md`). A lightweight
|
||||
markdown rule that nudges the agent to search before answering when
|
||||
Antigravity's matcher decides the turn is recall-relevant. It is
|
||||
deliberately recall-scoped (not an always-on global rule) so it never
|
||||
adds latency to greenfield work, honouring MemPalace's "memory should
|
||||
feel instant" budget.
|
||||
|
||||
All three point to the single canonical protocol in
|
||||
[`integrations/shared/recall-protocol.md`](../integrations/shared/recall-protocol.md)
|
||||
so the skill and rule never drift.
|
||||
|
||||
## Install
|
||||
|
||||
```bash
|
||||
bash hooks/antigravity/install.sh
|
||||
```
|
||||
|
||||
The installer is idempotent and the uninstaller matches by basename, so
|
||||
re-runs and partial installs are safe.
|
||||
|
||||
See [website/guide/antigravity.md](../website/guide/antigravity.md) for
|
||||
the full user-facing guide and [hooks/antigravity/README.md](../hooks/antigravity/README.md)
|
||||
for the hooks-specific documentation.
|
||||
|
||||
## Verified surfaces
|
||||
|
||||
Every file in this directory maps to a surface verified against
|
||||
[Google's Antigravity docs](https://antigravity.google/docs/). See
|
||||
[hooks/antigravity/INVESTIGATION.md](../hooks/antigravity/INVESTIGATION.md)
|
||||
for the full audit, including the surfaces deliberately not shipped.
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"mempalace-save": {
|
||||
"Stop": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "__PLUGIN_DIR__/hooks/mempal_save_hook_antigravity.sh",
|
||||
"timeout": 30
|
||||
}
|
||||
]
|
||||
},
|
||||
"mempalace-wake": {
|
||||
"PreInvocation": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "__PLUGIN_DIR__/hooks/mempal_wake_hook_antigravity.sh",
|
||||
"timeout": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"mempalace": {
|
||||
"command": "mempalace-mcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"name": "mempalace"
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
# MemPalace recall
|
||||
|
||||
Before answering anything that may already be in the user's memory
|
||||
palace — past work, prior decisions, a person, a project, or "what did
|
||||
we do / decide / discuss last time?" — search the palace first:
|
||||
|
||||
1. Call `mempalace_search` with a short keyword query. Use
|
||||
`mempalace_kg_query` for relational or time-bound facts.
|
||||
2. Quote the drawer's **verbatim** text. Never summarize or paraphrase
|
||||
stored content.
|
||||
3. If results are empty, say so — do not invent an answer. If the MCP
|
||||
server is unavailable, surface the error; do not fall back to guessing.
|
||||
|
||||
Skip recall for pure greenfield work with no memory relevance (renaming
|
||||
a variable, fixing a typo). Recall is question-driven, not reflexive.
|
||||
|
||||
Full protocol: <https://github.com/MemPalace/mempalace/blob/main/integrations/shared/recall-protocol.md>. Deeper guidance:
|
||||
the `mempalace-recall` skill.
|
||||
@@ -0,0 +1,119 @@
|
||||
---
|
||||
name: mempalace-recall
|
||||
description: "Recall protocol for MemPalace — search the palace before answering about past work, people, projects, or prior decisions. Apply when the user asks what was decided, what happened before, who someone is, what was discussed last time, or anything that may already be filed in their memory palace; or when mempalace-recall is invoked. Complements the mempalace setup skill and requires the mempalace-mcp server."
|
||||
---
|
||||
|
||||
# MemPalace Recall
|
||||
|
||||
Search-before-answer protocol for MemPalace. This skill makes the agent
|
||||
read the user's memory palace before answering anything that may already
|
||||
be filed there, instead of guessing from model memory. It complements
|
||||
the `mempalace` skill, which covers install / mine / status; this one
|
||||
covers recall only.
|
||||
|
||||
## Step 0 — Verify MemPalace is available
|
||||
|
||||
Before relying on recall, confirm MemPalace is installed and reachable:
|
||||
|
||||
- Official release page: <https://github.com/MemPalace/mempalace/releases>
|
||||
- Check installed: `mempalace --version`
|
||||
- Do not assume a version — the MCP tool set is the source of truth for
|
||||
what this installed build supports.
|
||||
|
||||
If the `mempalace_*` MCP tools are not available, tell the user the
|
||||
server is not connected and point them at the `mempalace` skill to set
|
||||
it up. Do not silently fall back to answering from model memory.
|
||||
|
||||
## Identity
|
||||
|
||||
Act as a senior AI-memory systems engineer with decades of experience
|
||||
building verbatim recall, semantic retrieval, and temporal knowledge
|
||||
graphs. Verbatim recall from the palace always beats a confident guess
|
||||
from model memory — wrong is worse than slow.
|
||||
|
||||
## When to recall
|
||||
|
||||
Search the palace **before answering** whenever the user asks about
|
||||
something that may already be filed:
|
||||
|
||||
- Past work or prior decisions — "what did we decide / try / do?"
|
||||
- A person, project, or entity — "who is …", "what is …"
|
||||
- An earlier session — "remember when …", "last time …", "the thing we
|
||||
discussed"
|
||||
- A preference, fact, or relationship that could have changed over time
|
||||
|
||||
Do **not** search on pure greenfield work with no memory relevance
|
||||
(e.g. "rename this variable", "fix this typo"). Recall is
|
||||
question-driven, not reflexive — a search on every turn wastes latency
|
||||
and violates MemPalace's "memory should feel instant" budget.
|
||||
|
||||
## Protocol
|
||||
|
||||
1. On wake-up, the MemPalace PreInvocation hook injects verbatim palace
|
||||
content via `injectSteps[].ephemeralMessage` on the first model call
|
||||
of a conversation. If memory was injected, start from it before
|
||||
searching further.
|
||||
2. **Before responding** about people, projects, past events, or prior
|
||||
decisions: call `mempalace_search` first. For relational or temporal
|
||||
facts ("who reported to whom in March", "what was true then"), call
|
||||
`mempalace_kg_query` instead or as well.
|
||||
3. **If unsure** about a fact (name, age, relationship, preference): say
|
||||
"let me check the palace" and query. Wrong is worse than slow.
|
||||
4. **Return verbatim.** Quote the drawer's exact stored words. Never
|
||||
summarize, paraphrase, or lossy-compress what the palace returns —
|
||||
that is the whole point of the system.
|
||||
5. **After a substantive session**, record continuity with
|
||||
`mempalace_diary_write` (background hooks may already do this — do not
|
||||
double-file).
|
||||
6. **When a fact changes**, call `mempalace_kg_invalidate` on the old
|
||||
fact, then `mempalace_kg_add` for the new one.
|
||||
|
||||
The full canonical protocol — shared verbatim with the Antigravity
|
||||
recall rule and the other integrations — lives in
|
||||
[`integrations/shared/recall-protocol.md`](https://github.com/MemPalace/mempalace/blob/main/integrations/shared/recall-protocol.md).
|
||||
|
||||
## Tool selection
|
||||
|
||||
| You need | Tool |
|
||||
|---|---|
|
||||
| Find any memory by meaning | `mempalace_search` (start here) |
|
||||
| Relational / time-bound facts about an entity | `mempalace_kg_query` |
|
||||
| The chronological story of an entity | `mempalace_kg_timeline` |
|
||||
| Recent session continuity | `mempalace_diary_read` |
|
||||
| Which wings / rooms exist (scope unknown) | `mempalace_list_wings`, `mempalace_list_rooms` |
|
||||
| Record this session | `mempalace_diary_write` |
|
||||
|
||||
`mempalace_search` takes a short natural-language `query` (keywords or a
|
||||
question — not a system prompt or pasted conversation) plus optional
|
||||
`wing` / `room` filters and `limit` (default 5).
|
||||
|
||||
## Unhappy paths
|
||||
|
||||
- **Empty results.** Say the palace has nothing on this; do not invent an
|
||||
answer to fill the gap. Offer to widen the search (drop the wing
|
||||
filter) or to file the new information.
|
||||
- **MCP unavailable / tool error.** Surface the error plainly and suggest
|
||||
the user verify the server (`mempalace status`, or re-run the
|
||||
installer `hooks/antigravity/install.sh`). Do not silently fall back
|
||||
to guessing from model memory.
|
||||
- **Stale or conflicting facts.** Prefer the knowledge graph's
|
||||
time-valid answer; if a fact has changed, invalidate the old one and
|
||||
add the new one rather than overwriting context silently.
|
||||
|
||||
## Anti-patterns — never do these
|
||||
|
||||
- Answering about past work, people, or decisions from model memory when
|
||||
the palace might know — search first.
|
||||
- Paraphrasing or summarizing stored content instead of quoting it
|
||||
verbatim.
|
||||
- Searching reflexively on every turn, including pure greenfield coding
|
||||
with no memory relevance.
|
||||
- Pasting the full conversation or a system prompt into the `query`
|
||||
argument — keep queries short and keyword-driven.
|
||||
|
||||
## Official References
|
||||
|
||||
- MemPalace: <https://github.com/MemPalace/mempalace>
|
||||
- MemPalace releases: <https://github.com/MemPalace/mempalace/releases>
|
||||
- Antigravity documentation: <https://antigravity.google/docs>
|
||||
- Agent Skills specification: <https://agentskills.io/specification>
|
||||
@@ -0,0 +1,92 @@
|
||||
---
|
||||
name: mempalace
|
||||
description: MemPalace — mine projects and conversations into a searchable memory palace. Use when the user asks about MemPalace, memory palace, mining memories, searching memories, palace setup, wings, rooms, or drawers; or when they want to recall past work that may already be filed in their palace.
|
||||
---
|
||||
|
||||
# MemPalace
|
||||
|
||||
A searchable memory palace for AI — mine projects and conversations, then search them semantically. Verbatim storage, local-first, zero external API by default.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Ensure `mempalace` is installed:
|
||||
|
||||
```bash
|
||||
mempalace --version
|
||||
```
|
||||
|
||||
If not installed (uv recommended):
|
||||
|
||||
```bash
|
||||
uv tool install mempalace # or: pip install mempalace
|
||||
```
|
||||
|
||||
## Dynamic, version-correct instructions
|
||||
|
||||
MemPalace exposes operation-specific instructions through the CLI so this skill stays accurate as MemPalace evolves. To get instructions for any operation:
|
||||
|
||||
```bash
|
||||
mempalace instructions <command>
|
||||
```
|
||||
|
||||
Always prefer the CLI output over what is written here when the two disagree — the CLI is the single source of truth for the installed version.
|
||||
|
||||
## Common operations
|
||||
|
||||
These are the five operations users ask for most often. Each one wraps a single MemPalace CLI subcommand. The `mempalace instructions <name>` form returns the full, version-correct guidance.
|
||||
|
||||
### `help` — discover what MemPalace can do
|
||||
|
||||
```bash
|
||||
mempalace instructions help
|
||||
```
|
||||
|
||||
Use when the user is new, unsure what's possible, or asks "what can you do".
|
||||
|
||||
### `init` — first-run setup of the palace
|
||||
|
||||
```bash
|
||||
mempalace instructions init
|
||||
```
|
||||
|
||||
Use when the user has just installed MemPalace, no palace exists yet, or the user explicitly asks to set up / configure / re-initialize their palace.
|
||||
|
||||
### `mine` — ingest a project or conversation directory
|
||||
|
||||
```bash
|
||||
mempalace instructions mine
|
||||
```
|
||||
|
||||
Use when the user wants to fold a project's files into their palace, or to ingest exported conversation transcripts into the palace as searchable memory.
|
||||
|
||||
### `search` — find verbatim memories by semantic query
|
||||
|
||||
```bash
|
||||
mempalace instructions search
|
||||
```
|
||||
|
||||
Use when the user wants to recall something from the past, find a previous decision, or rediscover code/notes/conversations they already wrote.
|
||||
|
||||
### `status` — what's in the palace right now
|
||||
|
||||
```bash
|
||||
mempalace instructions status
|
||||
```
|
||||
|
||||
Use when the user asks "what's in my palace", "how big is my palace", or wants a summary of wings, rooms, and drawer counts.
|
||||
|
||||
## MCP tools (preferred over CLI)
|
||||
|
||||
Inside Antigravity, the MemPalace MCP server registers a rich set of tools. Use these instead of shelling out to the CLI for live operations (search, diary writes, drawer adds, knowledge graph queries, palace status). The MCP tools always reflect the current palace state without spawning a subprocess.
|
||||
|
||||
The MCP server is auto-registered when this plugin is installed at `~/.gemini/config/plugins/mempalace/`. If the server does not appear in Antigravity's MCP store, run `mempalace-mcp --version` to verify the binary is on PATH, then restart Antigravity.
|
||||
|
||||
## Design principles (verbatim from the project)
|
||||
|
||||
- **Verbatim always** — never summarize, paraphrase, or lossy-compress user data.
|
||||
- **Local-first, zero external API by default** — extraction, embedding, and LLM-assisted refinement happen on the user's machine.
|
||||
- **Privacy by architecture** — the system never calls out to external services for core operations.
|
||||
- **Performance budgets** — hooks under 500ms; startup injection under 100ms.
|
||||
- **Background everything** — filing, indexing, and timestamps happen via hooks in the background; zero tokens spent on bookkeeping in the chat window.
|
||||
|
||||
If a request would violate any of these principles, refuse and explain — even if it would be technically convenient.
|
||||
@@ -1,6 +1,6 @@
|
||||
# MemPalace Claude Code Plugin
|
||||
|
||||
A Claude Code plugin that gives your AI a persistent memory system. Mine projects and conversations into a searchable palace backed by ChromaDB, with 19 MCP tools, auto-save hooks, and 5 guided skills.
|
||||
A Claude Code plugin that gives your AI a persistent memory system. Mine projects and conversations into a searchable palace backed by ChromaDB, with 36 MCP tools, auto-save hooks, and 5 guided skills.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -41,16 +41,17 @@ After installing the plugin, run the init command to complete setup (installs th
|
||||
|
||||
## Hooks
|
||||
|
||||
MemPalace registers two hooks that run automatically:
|
||||
MemPalace registers three hooks that run automatically:
|
||||
|
||||
- **Stop** -- Saves conversation context every 15 messages.
|
||||
- **SessionEnd** -- Runs one final save in the background on a clean exit, so short sessions that never hit the Stop interval or a compaction are still captured.
|
||||
- **PreCompact** -- Preserves important memories before context compaction.
|
||||
|
||||
Set the `MEMPAL_DIR` environment variable to a directory path to automatically run `mempalace mine` on that directory during each save trigger.
|
||||
|
||||
## MCP Server
|
||||
|
||||
The plugin automatically configures a local MCP server with 19 tools for storing, searching, and managing memories. No manual MCP setup is required -- `/mempalace:init` handles everything.
|
||||
The plugin automatically configures a local MCP server with 36 tools for storing, searching, and managing memories. No manual MCP setup is required -- `/mempalace:init` handles everything.
|
||||
|
||||
## Full Documentation
|
||||
|
||||
|
||||
@@ -12,6 +12,17 @@
|
||||
]
|
||||
}
|
||||
],
|
||||
"SessionEnd": [
|
||||
{
|
||||
"hooks": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "bash \"${CLAUDE_PLUGIN_ROOT}/hooks/mempal-session-end-hook.sh\"",
|
||||
"timeout": 10
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"PreCompact": [
|
||||
{
|
||||
"hooks": [
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
#!/bin/bash
|
||||
# MemPalace SessionEnd Hook — thin wrapper calling the Python CLI.
|
||||
#
|
||||
# Claude Code documents a default SessionEnd hook timeout of 1.5s, and
|
||||
# "timeouts set on plugin-provided hooks do not raise the budget"
|
||||
# (https://code.claude.com/docs/en/hooks). A cold `mempalace` start alone
|
||||
# exceeds 1.5s, so the final mine must NOT run in the foreground — it would be
|
||||
# killed before it saved anything. Unlike the foreground Stop/PreCompact plugin
|
||||
# wrappers, this one backgrounds the hook and returns immediately; the detached
|
||||
# child finishes the save after the session has exited. All logic lives in
|
||||
# mempalace.hooks_cli for cross-harness extensibility.
|
||||
run_mempalace_hook() {
|
||||
if command -v mempalace >/dev/null 2>&1; then
|
||||
exec mempalace hook run "$@"
|
||||
fi
|
||||
|
||||
MEMPAL_PYTHON_BIN="${MEMPAL_PYTHON:-}"
|
||||
if [ -z "$MEMPAL_PYTHON_BIN" ] || [ ! -x "$MEMPAL_PYTHON_BIN" ]; then
|
||||
MEMPAL_PYTHON_BIN="$(command -v python3 2>/dev/null || echo python3)"
|
||||
fi
|
||||
if "$MEMPAL_PYTHON_BIN" -c "import mempalace" >/dev/null 2>&1; then
|
||||
exec "$MEMPAL_PYTHON_BIN" -m mempalace hook run "$@"
|
||||
fi
|
||||
|
||||
if command -v python >/dev/null 2>&1 && python -c "import mempalace" >/dev/null 2>&1; then
|
||||
exec python -m mempalace hook run "$@"
|
||||
fi
|
||||
|
||||
echo "MemPalace hook error: could not find a runnable mempalace command or module" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Capture stdin (the SessionEnd JSON) before backgrounding — the parent's
|
||||
# stdin is gone once we return. Forward it to the detached worker, which runs
|
||||
# the final mine on its own time and outlives this process.
|
||||
payload="$(cat)"
|
||||
(
|
||||
printf '%s' "$payload" | run_mempalace_hook --hook session-end --harness "${MEMPALACE_HOOK_HARNESS:-claude-code}"
|
||||
) >/dev/null 2>&1 </dev/null &
|
||||
disown 2>/dev/null || true
|
||||
|
||||
# Return immediately so the harness never blocks on session exit.
|
||||
printf '{}'
|
||||
@@ -8,8 +8,8 @@
|
||||
{
|
||||
"name": "mempalace",
|
||||
"source": "./.claude-plugin",
|
||||
"description": "AI memory system — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, guided setup.",
|
||||
"version": "3.3.6",
|
||||
"description": "AI memory system — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, guided setup.",
|
||||
"version": "3.6.0",
|
||||
"author": {
|
||||
"name": "milla-jovovich"
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "mempalace",
|
||||
"version": "3.3.6",
|
||||
"description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.",
|
||||
"version": "3.6.0",
|
||||
"description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.",
|
||||
"author": {
|
||||
"name": "milla-jovovich"
|
||||
},
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
---
|
||||
name: mempalace-recall
|
||||
description: Recall protocol for MemPalace — search the palace before answering about past work, prior decisions, people, or projects. Use when the user asks what was decided, what happened before, who someone is, what was discussed last time, or anything that may already be filed in their memory palace.
|
||||
allowed-tools: Bash
|
||||
---
|
||||
|
||||
# MemPalace Recall
|
||||
|
||||
Search-before-answer protocol for MemPalace. Read the user's memory
|
||||
palace before answering anything that may already be filed there,
|
||||
instead of guessing from model memory. This complements the `mempalace`
|
||||
skill (install / mine / status); this one covers recall only.
|
||||
|
||||
## Step 0 — Verify MemPalace is available
|
||||
|
||||
```bash
|
||||
mempalace --version
|
||||
```
|
||||
|
||||
If the `mempalace_*` MCP tools are not available, tell the user the
|
||||
server is not connected and point them at the `mempalace` skill or
|
||||
`/init`. Do not silently fall back to answering from model memory.
|
||||
|
||||
## When to recall
|
||||
|
||||
Search the palace **before answering** whenever the user asks about
|
||||
something that may be filed:
|
||||
|
||||
- Past work or prior decisions — "what did we decide / try / do?"
|
||||
- A person, project, or entity — "who is …", "what is …"
|
||||
- An earlier session — "remember when …", "last time …"
|
||||
- A preference, fact, or relationship that could have changed over time
|
||||
|
||||
Skip recall for pure greenfield work with no memory relevance (renaming
|
||||
a variable, fixing a typo). Recall is question-driven, not reflexive.
|
||||
|
||||
## Protocol
|
||||
|
||||
1. Before responding about people / projects / past events / prior
|
||||
decisions: call `mempalace_search` first. Use `mempalace_kg_query`
|
||||
for relational or time-bound facts.
|
||||
2. If unsure about a fact: say "let me check the palace" and query.
|
||||
3. Return the drawer's **verbatim** text — never summarize or paraphrase
|
||||
stored content.
|
||||
4. After a substantive session, record continuity with
|
||||
`mempalace_diary_write` (skip if a background hook already saved).
|
||||
5. When a fact changes: `mempalace_kg_invalidate` the old fact, then
|
||||
`mempalace_kg_add` the new one.
|
||||
|
||||
## Unhappy paths
|
||||
|
||||
- **Empty results** — say the palace has nothing on this; do not invent
|
||||
an answer. Offer to widen the search or file the new information.
|
||||
- **MCP error / server down** — surface the error, suggest `mempalace
|
||||
status` or re-running `/init`; never fall back to guessing.
|
||||
- **Palace index corrupt / compactor error** — if the server reports an
|
||||
HNSW segment-writer error, a ChromaDB compaction failure, or stays
|
||||
"Not connected" after a write, the index is out of sync with
|
||||
`chroma.sqlite3` but the rows are intact. Tell the user to stop the
|
||||
server and rebuild from SQLite (`mempalace repair --mode from-sqlite
|
||||
--archive-existing --yes`), not re-mine, which drops MCP-added drawers
|
||||
and diary entries (#1843). Do not repair in-process.
|
||||
- **Conflicting facts** — trust the knowledge graph's time-valid answer;
|
||||
invalidate-then-add rather than overwriting silently.
|
||||
|
||||
The canonical protocol, shared across all MemPalace integrations, lives
|
||||
in `integrations/shared/recall-protocol.md`.
|
||||
@@ -1,6 +1,6 @@
|
||||
# MemPalace - Codex CLI Plugin
|
||||
|
||||
Give your AI a persistent memory -- mine projects and conversations into a searchable palace backed by ChromaDB, with 19 MCP tools, auto-save hooks, and guided skills.
|
||||
Give your AI a persistent memory -- mine projects and conversations into a searchable palace backed by ChromaDB, with 36 MCP tools, auto-save hooks, and guided skills.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"name": "mempalace",
|
||||
"version": "3.3.6",
|
||||
"description": "Give your AI a memory — mine projects and conversations into a searchable palace. 19 MCP tools, auto-save hooks, and guided setup.",
|
||||
"version": "3.6.0",
|
||||
"description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, auto-save hooks, and guided setup.",
|
||||
"author": {
|
||||
"name": "milla-jovovich"
|
||||
},
|
||||
@@ -27,7 +27,7 @@
|
||||
"interface": {
|
||||
"displayName": "MemPalace",
|
||||
"shortDescription": "AI memory system for Codex",
|
||||
"longDescription": "Give your AI a persistent memory — mine projects and conversations into a searchable palace backed by ChromaDB, with 19 MCP tools, auto-save hooks, and guided skills.",
|
||||
"longDescription": "Give your AI a persistent memory — mine projects and conversations into a searchable palace backed by ChromaDB, with 36 MCP tools, auto-save hooks, and guided skills.",
|
||||
"developerName": "milla-jovovich",
|
||||
"category": "Coding",
|
||||
"capabilities": [
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
# MemPalace Cursor Plugin
|
||||
|
||||
A Cursor IDE plugin that gives your agent a persistent memory system. Auto-registers the `mempalace-mcp` server (36 MCP tools), ships 5 slash commands, two model-invocable skills (setup/mining/search and a recall protocol), and an optional recall rule.
|
||||
|
||||
> Hooks (auto-save + session-start memory recall) are shipped separately under `hooks/cursor/` so the plugin is safe to install in any Cursor workspace without touching the agent loop. See [Hooks](#hooks-optional) below.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Python 3.9+
|
||||
- Cursor 1.7+ (plugin manifest schema requires it)
|
||||
|
||||
## Installation
|
||||
|
||||
### Local clone (recommended while not in the marketplace yet)
|
||||
|
||||
Symlink (or copy) this repository into Cursor's local plugins folder:
|
||||
|
||||
```bash
|
||||
ln -s /path/to/mempalace ~/.cursor/plugins/local/mempalace
|
||||
```
|
||||
|
||||
Then in Cursor: <kbd>Cmd</kbd>-<kbd>Shift</kbd>-<kbd>P</kbd> → **Developer: Reload Window**.
|
||||
|
||||
### Marketplace
|
||||
|
||||
Once published, install via the Cursor marketplace panel and select `mempalace`. Required-plugin distribution from a team marketplace is also supported.
|
||||
|
||||
## Post-Install Setup
|
||||
|
||||
After installing the plugin, run the `init` command in a Cursor chat:
|
||||
|
||||
```
|
||||
/mempalace-init
|
||||
```
|
||||
|
||||
(Or just say "use the mempalace skill" — Cursor will model-invoke the bundled skill.)
|
||||
|
||||
This installs the `mempalace` package via `uv tool` or `pip`, initializes a palace under `~/.mempalace/`, and verifies the MCP server is reachable.
|
||||
|
||||
## Available Slash Commands
|
||||
|
||||
| Command | Description |
|
||||
|---------------------|-----------------------------------------------------------------------------------|
|
||||
| `/mempalace-help` | Show available tools, skills, CLI commands, hooks, and architecture |
|
||||
| `/mempalace-init` | Set up MemPalace — install, configure, onboard |
|
||||
| `/mempalace-search` | Search your memories across the palace using semantic search |
|
||||
| `/mempalace-mine` | Mine projects and conversations into the palace |
|
||||
| `/mempalace-status` | Show palace overview — wings, rooms, drawer counts |
|
||||
|
||||
> Cursor commands are global, not plugin-namespaced — that's why each slug is prefixed with `mempalace-` rather than appearing as `/help`, `/init`, etc. This keeps them collision-free with built-in or other-plugin commands.
|
||||
|
||||
## Skills
|
||||
|
||||
Two model-invocable skills ship at the plugin root under `skills/`:
|
||||
|
||||
| Skill | What it does |
|
||||
|-------|--------------|
|
||||
| `mempalace` | Setup, mining, status, and the dynamic `mempalace instructions` CLI. |
|
||||
| `mempalace-recall` | Search-before-answer protocol — makes the agent read the palace before answering about past work, people, projects, or prior decisions instead of guessing. |
|
||||
|
||||
Cursor surfaces these automatically when a request matches their description, or you can attach them explicitly.
|
||||
|
||||
## Recall rule (optional)
|
||||
|
||||
The plugin also ships a Cursor rule at the plugin root under `rules/mempalace-recall.mdc`:
|
||||
|
||||
```yaml
|
||||
description: When the user asks about past work, prior decisions, people, ... call mempalace_search before answering ...
|
||||
alwaysApply: false
|
||||
```
|
||||
|
||||
It is `alwaysApply: false` on purpose — Cursor loads it only when its matcher judges the turn recall-relevant, so it never fires on unrelated coding work and never adds MCP latency to greenfield tasks. The rule, the `mempalace-recall` skill, and the `sessionStart` hook all reference the same canonical protocol in [`integrations/shared/recall-protocol.md`](../integrations/shared/recall-protocol.md).
|
||||
|
||||
Want recall forced into **every** conversation regardless of context? Copy the aggressive `alwaysApply: true` variant from [`examples/cursor/rules/`](../examples/cursor/rules/README.md) into `~/.cursor/rules/`. That is a deliberate, heavier opt-in, not a default.
|
||||
|
||||
## MCP Server
|
||||
|
||||
This plugin ships `mcp.json` at the plugin root, so Cursor auto-loads the `mempalace-mcp` server on plugin install:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"mempalace": {
|
||||
"command": "mempalace-mcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
All 36 MemPalace MCP tools (`mempalace_search`, `mempalace_add_drawer`, `mempalace_diary_write`, `mempalace_check_duplicate`, `mempalace_diary_read`, …) become available to the agent immediately. No manual `~/.cursor/mcp.json` edit required.
|
||||
|
||||
If the server doesn't appear, confirm `mempalace-mcp` is on the user `$PATH`:
|
||||
|
||||
```bash
|
||||
command -v mempalace-mcp
|
||||
```
|
||||
|
||||
If it isn't, run `/init` (or `mempalace install` from a terminal) — `mempalace-mcp` is installed alongside the `mempalace` package.
|
||||
|
||||
## Hooks (optional)
|
||||
|
||||
Cursor's hooks system is configured separately from plugins (in `~/.cursor/hooks.json` or `.cursor/hooks.json`), so this plugin does **not** wire hooks itself. The MemPalace repository ships three Cursor-native hooks under [`hooks/cursor/`](../hooks/cursor/) that you install with one command.
|
||||
|
||||
User scope — writes `~/.cursor/hooks.json`, applies to every Cursor workspace (recommended):
|
||||
|
||||
```bash
|
||||
hooks/cursor/install.sh --scope user --variant full
|
||||
```
|
||||
|
||||
Project scope — writes `.cursor/hooks.json` under the current project only:
|
||||
|
||||
```bash
|
||||
hooks/cursor/install.sh --scope project --variant full
|
||||
```
|
||||
|
||||
What you get:
|
||||
|
||||
| Hook event | What it does |
|
||||
|----------------|-------------------------------------------------------------------------------------------------------|
|
||||
| `sessionStart` | Injects an `additional_context` recap of relevant memories scoped to the workspace wing |
|
||||
| `stop` | Counts agent turns; every N turns, emits a `followup_message` instructing a memory checkpoint |
|
||||
| `preCompact` | Synchronously mines the transcript before compaction, drops a marker so the next `stop` saves a diary |
|
||||
|
||||
Full details: [`website/guide/cursor-hooks.md`](../website/guide/cursor-hooks.md) and [`hooks/cursor/README.md`](../hooks/cursor/README.md).
|
||||
|
||||
## Uninstall
|
||||
|
||||
Remove the local plugin symlink:
|
||||
|
||||
```bash
|
||||
rm ~/.cursor/plugins/local/mempalace
|
||||
```
|
||||
|
||||
Then in Cursor: <kbd>Cmd</kbd>-<kbd>Shift</kbd>-<kbd>P</kbd> → **Developer: Reload Window**.
|
||||
|
||||
If you also installed the hooks, remove them (leaves any unrelated hooks in `hooks.json` untouched):
|
||||
|
||||
```bash
|
||||
hooks/cursor/install.sh --scope user --uninstall
|
||||
```
|
||||
|
||||
## Full Documentation
|
||||
|
||||
See the main [README](../README.md) for complete documentation, architecture details, and advanced usage.
|
||||
@@ -0,0 +1,17 @@
|
||||
{
|
||||
"name": "mempalace",
|
||||
"owner": {
|
||||
"name": "milla-jovovich",
|
||||
"url": "https://github.com/MemPalace"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "mempalace",
|
||||
"source": ".",
|
||||
"description": "AI memory system — mine projects and conversations into a searchable palace. 36 MCP tools, slash commands, and a guided skill for Cursor.",
|
||||
"author": {
|
||||
"name": "milla-jovovich"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"mempalace": {
|
||||
"command": "mempalace-mcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"name": "mempalace",
|
||||
"description": "Give your AI a memory — mine projects and conversations into a searchable palace. 36 MCP tools, slash commands, and a guided skill for Cursor.",
|
||||
"author": {
|
||||
"name": "milla-jovovich"
|
||||
},
|
||||
"homepage": "https://github.com/MemPalace/mempalace",
|
||||
"repository": "https://github.com/MemPalace/mempalace",
|
||||
"license": "MIT",
|
||||
"keywords": [
|
||||
"memory",
|
||||
"ai",
|
||||
"rag",
|
||||
"mcp",
|
||||
"chromadb",
|
||||
"palace",
|
||||
"search"
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,55 @@
|
||||
# Keep the build context lean — only what `uv sync` + the package need.
|
||||
|
||||
# VCS
|
||||
.git/
|
||||
.gitignore
|
||||
|
||||
# Python build artifacts / caches
|
||||
*.egg-info/
|
||||
dist/
|
||||
build/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.pytest_cache/
|
||||
.mypy_cache/
|
||||
.ruff_cache/
|
||||
htmlcov/
|
||||
.coverage
|
||||
coverage.xml
|
||||
|
||||
# Virtual environments (rebuilt inside the image)
|
||||
.venv/
|
||||
venv/
|
||||
|
||||
# Local config / secrets
|
||||
.env
|
||||
.env.*
|
||||
.envrc
|
||||
mempal.yaml
|
||||
|
||||
# Editor / OS cruft
|
||||
.idea/
|
||||
.vscode/
|
||||
*.swp
|
||||
*.swo
|
||||
*~
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
||||
# Repo material not needed at runtime
|
||||
.devcontainer/
|
||||
.github/
|
||||
.agents/
|
||||
.claude/
|
||||
.codex/
|
||||
.codex-plugin/
|
||||
.claude-plugin/
|
||||
benchmarks/
|
||||
docs/
|
||||
website/
|
||||
landing/
|
||||
assets/
|
||||
examples/
|
||||
tests/
|
||||
*.md
|
||||
!README.md
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
matrix:
|
||||
python-version: ["3.9", "3.11", "3.13"]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
@@ -24,18 +24,24 @@ jobs:
|
||||
test-windows:
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
cache: 'pip'
|
||||
- run: pip install -e ".[dev]"
|
||||
- run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10
|
||||
# ChromaDB's rust HNSW core intermittently fails compaction on Windows
|
||||
# ("Failed to apply logs to the hnsw segment writer") regardless of our
|
||||
# code — a long-standing, non-reproducible-on-Linux/macOS flake. Retry
|
||||
# ONLY that specific transient error (via --only-rerun) so real,
|
||||
# deterministic failures still fail on the first run. Linux/macOS jobs
|
||||
# deliberately run with no reruns so genuine regressions surface there.
|
||||
- run: python -m pytest tests/ -v --ignore=tests/benchmarks --cov=mempalace --cov-report=term-missing --cov-fail-under=80 --durations=10 --reruns 2 --reruns-delay 5 --only-rerun "Failed to apply logs to the hnsw segment writer"
|
||||
|
||||
test-macos:
|
||||
runs-on: macos-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
@@ -45,7 +51,7 @@ jobs:
|
||||
lint:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.11"
|
||||
|
||||
@@ -23,7 +23,7 @@ jobs:
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
name: Docker
|
||||
|
||||
on:
|
||||
push:
|
||||
# `main` is the release branch: pushes here publish the released image and
|
||||
# update `latest`; `v*` tags publish versioned images. develop does not
|
||||
# publish — it is validated via the pull_request trigger below.
|
||||
branches: [main]
|
||||
tags: ["v*"]
|
||||
pull_request:
|
||||
branches: [main, develop]
|
||||
workflow_dispatch:
|
||||
|
||||
env:
|
||||
REGISTRY: ghcr.io
|
||||
IMAGE_NAME: ${{ github.repository }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
# Needed for the emulated linux/arm64 build on real pushes.
|
||||
- name: Set up QEMU
|
||||
uses: docker/setup-qemu-action@v4
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
# Only authenticate + push for in-repo events. Fork PRs lack the
|
||||
# packages:write token, so they build (to validate the Dockerfile) but
|
||||
# do not push.
|
||||
- name: Log in to GHCR
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: docker/login-action@v4
|
||||
with:
|
||||
registry: ${{ env.REGISTRY }}
|
||||
username: ${{ github.actor }}
|
||||
password: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
- name: Extract metadata
|
||||
id: meta
|
||||
uses: docker/metadata-action@v6
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
# latest -> main (the latest release); semver tags -> released versions.
|
||||
tags: |
|
||||
type=ref,event=branch
|
||||
type=ref,event=pr
|
||||
type=semver,pattern={{version}}
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
|
||||
|
||||
- name: Build and push
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile
|
||||
# Publish multi-arch (amd64 + arm64 for Apple Silicon / ARM hosts) on
|
||||
# real pushes; keep PRs amd64-only so the emulated arm64 build does not
|
||||
# slow the PR check.
|
||||
platforms: ${{ github.event_name != 'pull_request' && 'linux/amd64,linux/arm64' || 'linux/amd64' }}
|
||||
push: ${{ github.event_name != 'pull_request' }}
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
cache-from: type=gha
|
||||
# Fork PRs get a read-only Actions cache, so writing it just emits 403
|
||||
# noise — only export cache on in-repo events.
|
||||
cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max' || '' }}
|
||||
|
||||
# Build-only validation for the CUDA image so it cannot silently rot (CUDA base
|
||||
# tag drift, cross-stage interpreter/venv copy paths, the `gpu` extra). The
|
||||
# runner has no GPU, so this only proves the image *compiles*; it is never
|
||||
# published (users build it themselves, per the README).
|
||||
build-gpu:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v4
|
||||
|
||||
- name: Build GPU image (validation only — not published)
|
||||
uses: docker/build-push-action@v7
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.gpu
|
||||
platforms: linux/amd64
|
||||
push: false
|
||||
cache-from: type=gha,scope=gpu
|
||||
cache-to: ${{ github.event_name != 'pull_request' && 'type=gha,mode=max,scope=gpu' || '' }}
|
||||
@@ -0,0 +1,124 @@
|
||||
name: Publish to PyPI
|
||||
|
||||
# Publishes mempalace to PyPI via Trusted Publishing (OIDC — no stored token).
|
||||
#
|
||||
# Triggers:
|
||||
# * release: published — the normal path (drafting + publishing a GitHub
|
||||
# Release fires this).
|
||||
# * workflow_dispatch — a manual escape hatch. GitHub does not reliably emit
|
||||
# a `release` event when a *draft* tied to a pre-existing tag is published,
|
||||
# so this lets a maintainer run the publish for any existing tag from the
|
||||
# Actions tab ("Run workflow" → enter the tag, e.g. v3.4.0).
|
||||
#
|
||||
# Build + publish run in ONE job on purpose: a two-job split hands the wheel
|
||||
# between jobs via upload/download-artifact, which failed repeatedly with
|
||||
# `BlobNotFound` on the same-run download. Building and publishing in the same
|
||||
# job removes that handoff entirely. The job is gated by the `pypi` environment
|
||||
# (manual approval), and the checks below keep it self-contained — it never
|
||||
# uploads a tag that isn't on main or doesn't match the version manifest.
|
||||
#
|
||||
# One-time setup required before the first run (see docs/RELEASING.md):
|
||||
# 1. PyPI → Manage project `mempalace` → Publishing → Add a trusted publisher:
|
||||
# Owner: MemPalace Repository: mempalace
|
||||
# Workflow: publish.yml Environment: pypi
|
||||
# 2. GitHub → repo Settings → Environments → New environment `pypi`,
|
||||
# add yourself as a Required reviewer (this is the manual approval gate).
|
||||
|
||||
on:
|
||||
release:
|
||||
types: [published]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: "Existing tag to build and publish (e.g. v3.4.0). Used when the release event didn't fire."
|
||||
required: true
|
||||
type: string
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
name: Build + publish to PyPI
|
||||
runs-on: ubuntu-latest
|
||||
# The `pypi` environment gate (required reviewer) pauses the whole job until
|
||||
# approved, and scopes the OIDC trust on the PyPI side.
|
||||
environment:
|
||||
name: pypi
|
||||
url: https://pypi.org/p/mempalace
|
||||
permissions:
|
||||
contents: read # checkout
|
||||
id-token: write # mint the short-lived OIDC token for Trusted Publishing
|
||||
steps:
|
||||
- name: Resolve the release tag
|
||||
id: tag
|
||||
env:
|
||||
RELEASE_TAG: ${{ github.event.release.tag_name }}
|
||||
INPUT_TAG: ${{ github.event.inputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
tag="${RELEASE_TAG:-${INPUT_TAG:-}}"
|
||||
if [[ -z "$tag" ]]; then
|
||||
echo "::error::no tag to publish — neither a release tag nor a workflow_dispatch input was provided"
|
||||
exit 1
|
||||
fi
|
||||
# Validate the shape before it is ever used as a git ref (ref-injection
|
||||
# guard): require vMAJOR.MINOR.PATCH with an optional -/+ suffix, so
|
||||
# loose values like 'v3' or 'v3foo' are rejected.
|
||||
re='^v[0-9]+\.[0-9]+\.[0-9]+([-+][0-9A-Za-z.-]+)?$'
|
||||
if [[ ! "$tag" =~ $re ]]; then
|
||||
echo "::error::tag '$tag' is not a valid release tag (expected vMAJOR.MINOR.PATCH[-suffix])"
|
||||
exit 1
|
||||
fi
|
||||
echo "tag=$tag" >> "$GITHUB_OUTPUT"
|
||||
echo "Resolved tag: $tag"
|
||||
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
# Fully-qualified refs/tags/ so an unqualified name can't resolve to a
|
||||
# same-named *branch* instead of the tag (checkout prefers branches).
|
||||
# steps.tag.outputs.tag is format-validated above (ref-injection guard).
|
||||
ref: refs/tags/${{ steps.tag.outputs.tag }}
|
||||
fetch-depth: 0 # full history for the ancestry check
|
||||
|
||||
- name: Verify the tag is on main
|
||||
env:
|
||||
TAG: ${{ steps.tag.outputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
git fetch --no-tags origin main
|
||||
tag_sha=$(git rev-parse HEAD) # the checked-out tag commit
|
||||
main_sha=$(git rev-parse FETCH_HEAD)
|
||||
if git merge-base --is-ancestor "$tag_sha" "$main_sha"; then
|
||||
echo "Tag $TAG ($tag_sha) is reachable from main — OK."
|
||||
else
|
||||
echo "::error::tag $TAG is not an ancestor of main. Releases must be cut from main."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Verify the tag matches the version manifest
|
||||
env:
|
||||
TAG: ${{ steps.tag.outputs.tag }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
# version.py is the source of truth (per CLAUDE.md); version-guard.yml
|
||||
# separately enforces that pyproject + the three plugin manifests agree.
|
||||
py_version=$(grep -E '^__version__' mempalace/version.py | cut -d'"' -f2)
|
||||
tag_version="${TAG#v}"
|
||||
if [[ "$tag_version" != "$py_version" ]]; then
|
||||
echo "::error::tag $TAG does not match mempalace/version.py ($py_version). Bump the version files before releasing."
|
||||
exit 1
|
||||
fi
|
||||
echo "Tag $TAG matches manifest version $py_version — OK."
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.13"
|
||||
|
||||
- name: Build sdist + wheel
|
||||
run: |
|
||||
python -m pip install --upgrade build
|
||||
python -m build
|
||||
|
||||
- name: Publish to PyPI
|
||||
uses: pypa/gh-action-pypi-publish@release/v1
|
||||
@@ -16,7 +16,7 @@ jobs:
|
||||
check-versions:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@v7
|
||||
|
||||
- name: Extract versions from all sources
|
||||
id: versions
|
||||
|
||||
+148
-1
@@ -6,6 +6,142 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
||||
|
||||
---
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
---
|
||||
|
||||
## [3.6.0] — 2026-07-14
|
||||
|
||||
### Features
|
||||
|
||||
- **Turnkey secure remote / team server.** `mempalace serve` exposes the full MCP surface over HTTP with secure defaults: non-loopback binds require a bearer token, generated tokens are stored with restrictive permissions and passed through the child environment rather than argv, native TLS 1.2+ is supported, and `--read-only` both hides and refuses mutating tools. Docker Compose, systemd, and environment templates are included for deployment. (#1877, #1897, #1900)
|
||||
|
||||
- **Milvus storage backend.** The new opt-in `milvus` backend supports embedded Milvus Lite, self-hosted Milvus, and Zilliz Cloud through `pymilvus`, with COSINE vector search, native BM25 lexical search for new collections, namespace isolation, target-mismatch protection, and an optional `milvus` dependency extra. (#1899)
|
||||
|
||||
- **Atomic knowledge-graph fact replacement.** `supersede()` / `mempalace_kg_supersede` closes an open fact and opens its successor at one shared instant, so model, employer, address, and other single-valued facts can change without a hand-written invalidate/add race. Temporal queries now use half-open intervals at timestamp precision, returning only the successor at the transition boundary while preserving whole-day semantics for date-only facts. (#1913)
|
||||
|
||||
- **Conversation chronology is preserved.** Conversation drawers now retain transcript time as `authored_at` alongside ingest time, search results expose it, and exact hybrid-score ties prefer the more recently authored drawer. An idempotent, dry-run-first backfill script adds the metadata to existing palaces without re-embedding. `mempalace_list_drawers` also accepts inclusive `since` and exclusive `before` filing-time bounds. (#1890, #1891)
|
||||
|
||||
- **Mined sessions populate the associative graph.** A precision-biased, no-LLM structural extractor records code symbols, URLs, paths, and qualified identifiers as drawer entities; conversation mining then derives hallways from them. The new `mempalace hallways` CLI command exposes the graph. (#1894, #1895)
|
||||
|
||||
- **Per-project mining exclusions.** `exclude_patterns` in `mempalace.yaml` filters pre-scanned project files without changing the corrected `--limit` semantics. (#1213, #1953)
|
||||
|
||||
- **LaTeX project coverage.** `.tex` and `.bib` files are now treated as readable prose sources. (#1901)
|
||||
|
||||
### Performance
|
||||
|
||||
- **MCP initialization no longer waits for palace-wide integrity work.** stdio answers the JSON-RPC `initialize` request immediately while startup preflight runs on a background thread; a tool arriving mid-probe joins the same verdict instead of launching duplicate work. The startup SQLite probe is also skipped above a configurable size limit (512 MiB by default), while repair preflights remain strict. (#1911, #1987)
|
||||
|
||||
- **Qdrant metadata counts use server-side facets.** Taxonomy, overview, and status paths can count metadata remotely instead of fetching and tallying every record; the embedding wrapper now forwards the facet and bulk-metadata capabilities correctly. (#1868, #1898)
|
||||
|
||||
- **pgvector metadata-only fetches omit document payloads**, reducing transfer and decoding work on enumeration paths. (#1892)
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **Chroma recovery distinguishes derived-index damage from data loss.** Valid all-layer-0 HNSW segments with an empty `link_lists.bin` are no longer quarantined repeatedly; isolated FTS5 inverted-index corruption is rebuilt from intact content during repair and after mining; embedded NUL bytes are sanitized before reaching ChromaDB. (#1716, #1872, #1878, #1927, #1928)
|
||||
|
||||
- **SQLite recovery is bounded, atomic, and honest.** Integrity checks wait up to 15 seconds for transient writer contention instead of reporting a healthy palace as corrupt. In-place recovery archives use atomic `os.rename` rather than `shutil.move`'s destructive copy/delete fallback. `repair --mode from-sqlite` now rebuilds FTS5, vacuums, and requires a clean final `PRAGMA quick_check`; cleanup failures return non-zero with recovery details instead of printing a false success banner. (#1945, #2015, #2017)
|
||||
|
||||
- **CLI search checks HNSW divergence before opening ChromaDB.** A diverged palace routes directly to the existing SQLite BM25 fallback, avoiding embedder or collection initialization against damaged native index state while leaving healthy Chroma and non-Chroma backends on their normal vector path. (#2016)
|
||||
|
||||
- **Writer leases recover instead of stranding servers read-only.** A server refused while a peer owns the palace now retries on each mutating call and promotes itself after the peer exits. Status remains read-only and cannot acquire the writer lease, the in-process re-entrant holder set is released even across asynchronous interruption, and `checkpoint` / `delete_by_source` are correctly classified as mutations for both read-only serving and peer-writer protection. (#1923, #1930, #1934, #1960, #1970, #1971)
|
||||
|
||||
- **Status reports only checks that actually ran.** Non-Chroma backends now mark the Chroma SQLite integrity check as not applicable rather than claiming an unperformed success. (#1931, #1946)
|
||||
|
||||
- **Conversation transcripts are treated as mutable.** Appended or rewritten sessions are purged and re-filed based on modification time, preventing later turns from being silently skipped. Claude Code `tool-results/` sidecars are excluded from conversation scans so raw machine dumps cannot flood the embedding space. (#1957, #2010)
|
||||
|
||||
- **Explicit palace selection now scopes derived graph state.** Project, conversation, and format mining write hallways and tunnels beside the selected `--palace` instead of leaking them into the ambient default palace. (#2018)
|
||||
|
||||
- **Remote and local server hardening.** Non-loopback HTTP now requires a token unless an explicit insecure override is supplied; optional-provider probes cannot reuse an ambient external key before the user selects that provider; hook transcript paths and file-copy/repair paths receive stricter local validation and no-follow handling. Importing the MCP server no longer clobbers the host application's root logger, HTTP writes can re-enter the process-wide palace lock safely, and routine client disconnects no longer emit server tracebacks. (#1859, #1860, #1864, #1885, #2003, #2004)
|
||||
|
||||
- **Mining and configuration correctness.** Oversized files produce a visible stderr warning, `~` in configured palace paths expands consistently, wing slugs handle special characters, and Windows background daemon / synchronous hook mines use `CREATE_NO_WINDOW`. (#923, #1852, #1857, #1863, #1865)
|
||||
|
||||
- **`mempalace init` handles non-ASCII `.gitignore` files on Windows.** The project-file ignore guard now reads and appends UTF-8 explicitly instead of relying on locale defaults such as GBK. (#1648)
|
||||
|
||||
- **L1 wake-up surfaces the latest moments.** Drawers with equal importance are now ordered by `filed_at` recency rather than insertion order, so startup context prefers recent memories instead of the oldest ones. (#1630)
|
||||
|
||||
- **Backend detection requires a SQLite magic header** before classifying a target as Chroma or `sqlite_exact`, preventing unrelated files from being mistaken for a palace. (#1893, #1896)
|
||||
|
||||
### Documentation
|
||||
|
||||
- Added a storage-backend configuration reference, a remote/team-server deployment guide, `authored_at` migration guidance, and refreshed the OpenClaw integration for the full 36-tool MCP surface. (#1719, #1877, #1890, #1904, #1905)
|
||||
|
||||
### Internal
|
||||
|
||||
- Expanded WAL crash-safety, idempotence, and redaction-path coverage (#1869); updated GitHub Actions and Ruff; repaired `uv.lock` drift so it again matches the Ruff 0.15.20 pin and includes Python 3.9 dependency markers introduced by the Milvus resolution.
|
||||
|
||||
---
|
||||
|
||||
## [3.5.0] — 2026-06-22
|
||||
|
||||
### Features
|
||||
|
||||
- **Opt-in local daemon for queued writes.** A new `mempalace daemon` queues MemPalace writes through a single local process so background mines, diary saves, and hook-driven ingests serialize against one palace handle instead of racing for it. Opt-in and local-only — nothing binds to a public interface. (#1826)
|
||||
|
||||
- **Opt-in HTTP transport for the MCP server.** `mempalace-mcp --transport http` serves JSON-RPC at `POST /mcp` (with a `GET /healthz` liveness probe) for operators running MemPalace behind a long-lived HTTP MCP client/proxy, avoiding the long-lived-stdio framing failures of #1801. stdio remains the default and is unchanged. The transport reuses the exact stdio request dispatcher (no separate write/search path), binds `127.0.0.1` by default, and is hardened against the two ways a local HTTP server leaks to the network: it pins the `Host` header to loopback on a loopback bind and rejects any non-loopback browser `Origin` (DNS-rebinding/SSRF guard), and supports an optional bearer token via `MEMPALACE_MCP_HTTP_TOKEN` (required on `/mcp`, never on `/healthz`). A 16 MiB request cap and a loud warning when bound to a non-loopback host round it out. (#1801, #1806)
|
||||
|
||||
- **`mempalace_checkpoint` batch-save MCP tool.** Collapses multiple `add_drawer` calls plus an optional diary entry into a single MCP round-trip for agents that want to file a whole session at once. Stores content verbatim and reuses the existing idempotent add/dedup path. (#1851)
|
||||
|
||||
- **`mempalace_delete_by_source` bulk-cleanup MCP tool.** Exact-match, dry-run-by-default deletion of every drawer (and its matching closet/AAAK index entries) for a given `source_file` — the recourse for benchmark/eval files mined into the same wing as real data and drowning out search. The dry run reports the drawer and closet blast radius before anything is removed, and the commit writes a WAL audit entry. (#1722, #1729)
|
||||
|
||||
- **Optional `source_file` filter for `mempalace_search`.** Scope a search to an exact stored source path. The filter is threaded through every search path (vector, BM25/SQLite fallback, lexical union, and index-mismatch fallback) so it never silently drops a matching drawer, and results now expose the full `source_path` as a round-trippable key. (#1815, #1817)
|
||||
|
||||
- **New transcript parsers / importers.** Continue.dev session parser (#731), Gemini CLI / AI Studio JSON session import (#204), and a Pi agent JSONL session normalizer (#169).
|
||||
|
||||
- **Wider miner language coverage.** C# / .NET, PHP (#1819), Swift / Kotlin (#1368), and Java project detection including rootless subprojects (#1720).
|
||||
|
||||
- **Final mine on Claude plugin `SessionEnd`.** The Claude Code plugin now runs a closing mine when a session ends so the last exchanges are captured without waiting for the next save nudge. (#1814, #1820)
|
||||
|
||||
### Performance
|
||||
|
||||
- **Overview/status MCP tools answer from the SQLite aggregate.** Large palaces no longer time out building wing/room/status overviews — the counts come from a single SQLite aggregate instead of a client-side fetch-and-tally. (#1748, #1379)
|
||||
|
||||
- **`graph_stats` SQLite fast path.** Knowledge-graph stats are computed in SQLite rather than walking the collection, fixing large-palace timeouts. (#1379)
|
||||
|
||||
- **Embedder caps ONNX-runtime intra-op threads** so a background mine no longer pins every core. (#1068)
|
||||
|
||||
- **Backend pagination pushed into the query.** `sqlite_exact` (#1841, #1842) and `pgvector` (#1830, #1840) now apply `get(limit, offset)` in SQL, and Qdrant fetches bulk metadata in a single scroll with a larger page size (#1796, #1832).
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **pgvector tolerates hostile transcript bytes.** A lone Unicode surrogate (#1833) or a NUL byte (#1829) in a transcript no longer aborts the whole mine — both are sanitized before the row is written.
|
||||
|
||||
- **SQLite read-only URIs are percent-encoded** so palace paths with spaces or special characters open correctly, and `_sqlite_graph_stats` is routed through the same `sqlite_read_uri` helper.
|
||||
|
||||
- **Stale ChromaDB HNSW divergence routes to the SQLite fallback** instead of failing the read outright. (#1816, #1822)
|
||||
|
||||
- **Diverged-index recovery now points at `repair --mode from-sqlite`, not a re-mine.** A failed ChromaDB HNSW compaction leaves the index out of sync while the rows stay intact in `chroma.sqlite3`; the old "re-mine from source" advice silently dropped MCP-added drawers and diary entries (which have no source file). Both the legacy `repair`/`rebuild_index` error messages and the `repair-status` recommendation, plus the recall skill docs, now guide users to rebuild from SQLite. (#1843, #1847, #1849)
|
||||
|
||||
- **The MCP server refuses a second writer for the same palace** rather than letting two processes race the same HNSW handle. (#1818, #1823)
|
||||
|
||||
- **Windows hook miner spawns with `CREATE_NO_WINDOW`** so background mines no longer flash a console window. (#1783, #1848)
|
||||
|
||||
- **`fact_checker` `__main__` no longer emits a runpy warning** under the test runner. (#1798)
|
||||
|
||||
### Internal
|
||||
|
||||
- Live-substrate conformance test module for pgvector (#1769); dependabot bumps for `docker/login-action` (3→4), `docker/build-push-action` (6→7), and `docker/metadata-action` (5→6) (#1788, #1787, #1786); ruff dev dependency bumped to 0.15.18.
|
||||
|
||||
---
|
||||
|
||||
## [3.4.1] — 2026-06-14
|
||||
|
||||
### Features
|
||||
|
||||
- **Cursor IDE plugin (`.cursor-plugin/`).** Drops into `~/.cursor/plugins/local/mempalace` (or installs from the Cursor marketplace once published) and auto-registers the `mempalace-mcp` server, five slash commands (`/mempalace-help`, `/mempalace-init`, `/mempalace-mine`, `/mempalace-search`, `/mempalace-status`), and the model-invocable [`mempalace` skill](skills/mempalace/SKILL.md) — no manual `~/.cursor/mcp.json` edit required. The plugin manifest deliberately omits a hardcoded `version` field — `mempalace/version.py` is the single source of truth, so there is nothing to drift on the next release (a contract test enforces the field stays absent). The canonical plugin components (`commands/`, `skills/`, `mcp.json`) are real files at the plugin root; no symlinks are committed (committed symlinks materialise as broken text files on Windows clones with `core.symlinks=false`). Mirrors the surface of [`.claude-plugin/`](.claude-plugin/) and [`.codex-plugin/`](.codex-plugin/) without duplicating their hook scripts: the Cursor hook scripts under [`hooks/cursor/`](hooks/cursor/) (shipped in the same release) remain the canonical install path for `stop` / `preCompact` / `sessionStart`, wired separately by [`hooks/cursor/install.sh`](hooks/cursor/install.sh). Contract tests in [`tests/test_cursor_plugin_manifest.py`](tests/test_cursor_plugin_manifest.py) cover manifest JSON validity, kebab-case naming, `..`-free relative paths, on-disk path resolution, marketplace alignment, MCP config shape (`mcpServers` wrapper required by Cursor, unlike Claude's flat `.mcp.json`), the version-field-absent guard, the no-symlink guard, and every skill/command frontmatter — all pure file inspection so they run on any CI platform without Cursor itself.
|
||||
|
||||
- **Cursor IDE hook support (`stop` / `preCompact` / `sessionStart`).** Three new bash hooks live under [`hooks/cursor/`](hooks/cursor/) and share a `lib/common.sh` helpers module. The save hook counts `stop` invocations per Cursor `conversation_id` and emits a `followup_message` every `MEMPAL_SAVE_INTERVAL` (default 15) so the agent files the session into MemPalace and writes a diary entry. Unlike the silent-by-default Claude Code hook, the Cursor followup fires **on by default**: Cursor's transcript format is undocumented and `normalize.py` has no Cursor parser yet, so the background `mempalace mine --mode convos` is best-effort only and the `followup_message` is the load-bearing verbatim-capture path. Users who want the Claude-style "zero tokens in the chat window" behaviour can suppress it with `MEMPAL_CURSOR_SILENT=1` (or `MEMPAL_VERBOSE=false`); the default flips to silent once a Cursor transcript parser lands. The precompact hook synchronously mines the transcript before Cursor's compaction summarises it and drops a marker so the next `stop` forces a save nudge (Cursor's `preCompact` is observational-only — it cannot block or emit a `followup_message`, unlike Claude Code's `PreCompact`); the synchronous mine is bounded by Cursor's per-hook timeout, and because `mempalace mine` is incremental/append-only a killed mine resumes cleanly on the next run rather than corrupting the palace. The wake hook is Cursor-only: `sessionStart` returns `additional_context` telling the agent to recall scoped to the wing inferred from the workspace root. Honours the same `MEMPALACE_HOOKS_AUTO_SAVE=false` kill switch as the Claude Code hooks, plus a new `MEMPAL_DISABLE_HOOK=1` alias and a `MEMPAL_STATE_DIR` env override. Per-conversation state files are garbage-collected by a daily-throttled, Cursor-namespaced TTL sweep (`MEMPAL_STATE_TTL_DAYS`, default 30) so `cursor_*.count` / `cursor_*.pending` cannot grow unbounded — shared logs and other editors' state are never touched. Includes an opt-in installer at [`hooks/cursor/install.sh`](hooks/cursor/install.sh) with `--scope user|project`, `--variant full|minimal`, `--dry-run`, and `--uninstall` (idempotent, preserves unrelated hooks via `python3`-based JSON merge — no `jq` dependency). Example wirings live at [`examples/cursor/hooks.json`](examples/cursor/hooks.json) and [`examples/cursor/hooks.minimal.json`](examples/cursor/hooks.minimal.json); they are intentionally not placed at the repo root because Cursor auto-loads project hooks from any trusted workspace and we do not arm hooks on contributor checkout. Per-event stdin/stdout schema documented at [`hooks/cursor/STDIN_SHAPE.md`](hooks/cursor/STDIN_SHAPE.md). Walkthrough at [`website/guide/cursor-hooks.md`](website/guide/cursor-hooks.md). Coverage added in [`tests/test_cursor_hooks_shell.py`](tests/test_cursor_hooks_shell.py) and [`tests/test_cursor_hooks_install.py`](tests/test_cursor_hooks_install.py).
|
||||
|
||||
- **First-class Antigravity IDE support.** New `.antigravity-plugin/` package + idempotent installer at `hooks/antigravity/install.sh` that registers MemPalace as a Google Antigravity plugin (MCP server, skill, two lifecycle hooks) at `~/.gemini/config/plugins/mempalace/`. The Stop hook background-mines the active conversation transcript every Nth fire (default 15, configurable via `MEMPAL_SAVE_INTERVAL`); the PreInvocation hook injects verbatim memory on the first model call only via Antigravity's `injectSteps[].ephemeralMessage` output, gated by `invocationNum == 1`. Both hooks are bash 3.2.57 compatible (macOS default), use the same `~/.mempalace/hook_state/` directory as the Claude Code / Codex / Cursor hooks (`antigravity_*`-namespaced state files), and respect every existing kill switch (`MEMPAL_DISABLE_HOOK`, `MEMPALACE_HOOKS_AUTO_SAVE`, `~/.mempalace/config.json` `hooks.auto_save`). Installer is `cmp`-gated (re-run produces a byte-identical install), uninstall is basename-guarded (refuses to wipe a directory whose basename isn't `mempalace`), and `--dry-run` is side-effect free. Full audit of which Antigravity surfaces we ship and which we deliberately don't is in [`hooks/antigravity/INVESTIGATION.md`](hooks/antigravity/INVESTIGATION.md). User-facing guide: [`website/guide/antigravity.md`](website/guide/antigravity.md). Standalone examples in [`examples/antigravity/`](examples/antigravity/).
|
||||
- **Zero-config interpreter resolution.** `mempal_resolve_python` now derives the Python interpreter from the `mempalace-mcp` / `mempalace` console-script shebang on `$PATH` before falling back to `python3`. The common `uv tool install mempalace` / `pipx install` layout installs the console scripts into an isolated environment whose interpreter is **not** system `python3`, so the previous `command -v python3` resolution landed on a Python that couldn't import `mempalace`, the `-m mempalace` probe failed, and mining silently never fired. Resolution is pure shebang parsing + `stat` (no Python subprocess at source time, preserving the hook performance budget). `MEMPAL_PYTHON` remains the explicit override. Documented under *How the hooks find your `mempalace` install* in the guide.
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
- **`embeddinggemma` no longer OOM-kills bulk re-embeds.** `EmbeddinggemmaONNX.__call__` ran a single `session.run` over its entire input, so a repair-scale batch (5000 docs) allocated attention buffers far beyond available RAM and the kernel killed the process silently: `mempalace repair --yes` on an `embedding_model: embeddinggemma` palace died right after `Building temporary collection:` with no traceback and no crash report (#1770). Embedding now runs in sub-batches of 32 docs (constructor-tunable `batch_size`), matching the internal batching of ChromaDB's bundled MiniLM embedder. Per-document vectors are unchanged: the model's pooled output is attention-masked, so sub-batch padding does not affect values. The embedder is also hardened for shared use: the process-wide EF cache and the lazy model load are thread-safe (concurrent first calls build exactly one ONNX session), and `__call__` handles a bare string, `None`, and empty input without triggering the model download.
|
||||
- **Backup retention to prevent unbounded disk usage.** `mempalace migrate` (full-palace `<palace>.pre-migrate.<timestamp>` copies) and `mempalace repair max-seq-id` (`chroma.sqlite3.max-seq-id-backup-<timestamp>` copies) each wrote a fresh, full-size, timestamped backup every run and never deleted the old ones. On a machine that mines or repairs on a schedule, those copies could silently accumulate until they filled the disk — one palace was found with hundreds of GB of stale backups beside a few hundred MB of live data, hidden from a normal `du` of the home directory. A new `max_backups` setting (default `10`, env `MEMPALACE_MAX_BACKUPS`, or `config.json`) now prunes the oldest backups after each new one is written. Set it to `0` to keep every backup. Pruning is keyed by filesystem mtime, scoped strictly to each backup's own naming pattern (live data is never touched), and best-effort so a deletion failure can never abort a migration or repair that already succeeded.
|
||||
|
||||
---
|
||||
|
||||
## [3.3.6] — 2026-05-24
|
||||
|
||||
### Features
|
||||
@@ -494,7 +630,18 @@ Initial public release.
|
||||
|
||||
---
|
||||
|
||||
[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.2.0...HEAD
|
||||
[Unreleased]: https://github.com/MemPalace/mempalace/compare/v3.6.0...HEAD
|
||||
[3.6.0]: https://github.com/MemPalace/mempalace/compare/v3.5.0...v3.6.0
|
||||
[3.5.0]: https://github.com/MemPalace/mempalace/compare/v3.4.1...v3.5.0
|
||||
[3.4.1]: https://github.com/MemPalace/mempalace/compare/v3.4.0...v3.4.1
|
||||
[3.4.0]: https://github.com/MemPalace/mempalace/compare/v3.3.6...v3.4.0
|
||||
[3.3.6]: https://github.com/MemPalace/mempalace/compare/v3.3.5...v3.3.6
|
||||
[3.3.5]: https://github.com/MemPalace/mempalace/compare/v3.3.4...v3.3.5
|
||||
[3.3.4]: https://github.com/MemPalace/mempalace/compare/v3.3.3...v3.3.4
|
||||
[3.3.3]: https://github.com/MemPalace/mempalace/compare/v3.3.2...v3.3.3
|
||||
[3.3.2]: https://github.com/MemPalace/mempalace/compare/v3.3.1...v3.3.2
|
||||
[3.3.1]: https://github.com/MemPalace/mempalace/compare/v3.3.0...v3.3.1
|
||||
[3.3.0]: https://github.com/MemPalace/mempalace/compare/v3.2.0...v3.3.0
|
||||
[3.2.0]: https://github.com/MemPalace/mempalace/compare/v3.1.0...v3.2.0
|
||||
[3.1.0]: https://github.com/MemPalace/mempalace/compare/v3.0.0...v3.1.0
|
||||
[3.0.0]: https://github.com/MemPalace/mempalace/releases/tag/v3.0.0
|
||||
|
||||
+102
@@ -0,0 +1,102 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# MemPalace — CPU image.
|
||||
#
|
||||
# Multi-stage build using uv (the project ships a uv.lock, so we install from
|
||||
# the frozen lockfile for reproducible images). The default runtime is the MCP
|
||||
# server over stdio; the CLI is reachable through the same entrypoint.
|
||||
#
|
||||
# Build:
|
||||
# docker build -t mempalace .
|
||||
# docker build -t mempalace --build-arg EXTRAS="extract,spellcheck" .
|
||||
#
|
||||
# Run (MCP server over stdio, palace persisted on the host):
|
||||
# docker run -i --rm -v mempalace-data:/data mempalace
|
||||
#
|
||||
# Run (CLI):
|
||||
# docker run --rm -v mempalace-data:/data mempalace search "why GraphQL"
|
||||
#
|
||||
# GPU acceleration lives in Dockerfile.gpu (it needs a CUDA base image).
|
||||
|
||||
ARG PYTHON_VERSION=3.12
|
||||
|
||||
# --- builder ----------------------------------------------------------------
|
||||
FROM python:${PYTHON_VERSION}-slim AS builder
|
||||
|
||||
# uv: fast, lockfile-driven installer. Pinned by digest-less tag for clarity;
|
||||
# bump deliberately.
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/
|
||||
|
||||
# Some transitive deps (grpcio, onnxruntime, tokenizers) ship manylinux wheels
|
||||
# for cp312, but keep a compiler around so a missing wheel degrades to a source
|
||||
# build instead of failing the image. Dropped from the final stage.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PYTHON_DOWNLOADS=never
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Optional extras baked into the image. CPU-safe by default; GPU is a separate
|
||||
# Dockerfile. Pass a comma-separated list, e.g. EXTRAS="extract,spellcheck".
|
||||
ARG EXTRAS="extract,spellcheck"
|
||||
|
||||
# Layer 1: dependencies only (no project) — cached across source changes.
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
|
||||
--mount=type=bind,source=uv.lock,target=uv.lock \
|
||||
--mount=type=bind,source=README.md,target=README.md \
|
||||
set -e; \
|
||||
flags=""; \
|
||||
for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \
|
||||
uv sync --frozen --no-install-project --no-dev ${flags}
|
||||
|
||||
# Layer 2: the project itself. --no-editable installs mempalace into the venv's
|
||||
# site-packages (instead of an .pth pointing at /app), so the runtime stage can
|
||||
# copy only /app/.venv and drop the source tree.
|
||||
COPY . /app
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
set -e; \
|
||||
flags=""; \
|
||||
for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \
|
||||
uv sync --frozen --no-dev --no-editable ${flags}
|
||||
|
||||
# --- runtime ----------------------------------------------------------------
|
||||
FROM python:${PYTHON_VERSION}-slim AS runtime
|
||||
|
||||
LABEL org.opencontainers.image.title="MemPalace" \
|
||||
org.opencontainers.image.description="Local-first AI memory — verbatim storage, MCP server + CLI." \
|
||||
org.opencontainers.image.source="https://github.com/MemPalace/mempalace" \
|
||||
org.opencontainers.image.licenses="MIT"
|
||||
|
||||
# /data is the single persistence root: HOME points here, so the palace
|
||||
# (~/.mempalace/palace), config (~/.mempalace), and the embedding-model cache
|
||||
# all land under one mountable volume. The default `minilm` model caches under
|
||||
# ~/.cache/chroma (~80 MB, from ChromaDB's S3); the optional `embeddinggemma`
|
||||
# model caches under ~/.cache/huggingface (~300 MB). Both lazy-download on
|
||||
# first use.
|
||||
ENV HOME=/data \
|
||||
PATH="/app/.venv/bin:${PATH}" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1
|
||||
|
||||
# Non-root user owning the data volume.
|
||||
RUN groupadd --gid 1000 mempalace \
|
||||
&& useradd --uid 1000 --gid 1000 --home-dir /data --create-home mempalace
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# The resolved virtualenv from the builder — no build toolchain in this layer.
|
||||
COPY --from=builder --chown=mempalace:mempalace /app/.venv /app/.venv
|
||||
COPY --chown=mempalace:mempalace docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
USER mempalace
|
||||
VOLUME ["/data"]
|
||||
|
||||
# Default to the MCP server; `docker run` it with `-i` for stdio JSON-RPC.
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["mcp"]
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
|
||||
# MemPalace — GPU (NVIDIA CUDA) image.
|
||||
#
|
||||
# Multi-stage build using uv (the project ships a uv.lock, so we install from
|
||||
# the frozen lockfile for reproducible images). The `gpu` extra pulls
|
||||
# onnxruntime-gpu, which needs CUDA + cuDNN shared libraries at runtime, so
|
||||
# this variant builds on an nvidia/cuda base instead of python:slim.
|
||||
#
|
||||
# The builder stage is dropped from the final image, so build tools
|
||||
# (compilers, apt cache) never reach production — only the uv-managed
|
||||
# interpreter and the resolved virtualenv do.
|
||||
#
|
||||
# Build:
|
||||
# docker build -f Dockerfile.gpu -t mempalace:gpu .
|
||||
#
|
||||
# Run (requires the NVIDIA Container Toolkit on the host):
|
||||
# docker run -i --rm --gpus all \
|
||||
# -e MEMPALACE_EMBEDDING_DEVICE=cuda \
|
||||
# -v mempalace-data:/data mempalace:gpu
|
||||
#
|
||||
# NOTE: onnxruntime-gpu ties itself to a CUDA major version. If embeddings
|
||||
# fail to load on the GPU, align CUDA_IMAGE below with the CUDA release that
|
||||
# the resolved onnxruntime-gpu wheel targets (see its release notes), then
|
||||
# rebuild.
|
||||
|
||||
ARG CUDA_IMAGE=nvidia/cuda:12.6.3-cudnn-runtime-ubuntu22.04
|
||||
|
||||
# --- builder ----------------------------------------------------------------
|
||||
FROM ${CUDA_IMAGE} AS builder
|
||||
|
||||
COPY --from=ghcr.io/astral-sh/uv:0.5 /uv /uvx /bin/
|
||||
|
||||
# Minimal toolchain for any source-built wheels; dropped before the runtime
|
||||
# stage so compilers never ship in the production image.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ARG PYTHON_VERSION=3.12
|
||||
ARG EXTRAS="extract,spellcheck,gpu"
|
||||
|
||||
ENV UV_COMPILE_BYTECODE=1 \
|
||||
UV_LINK_MODE=copy \
|
||||
UV_PYTHON_INSTALL_DIR=/opt/uv/python \
|
||||
UV_PYTHON_PREFERENCE=only-managed
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Layer 1: dependencies only (no project source). Bind-mounted files keep the
|
||||
# project tree out of this layer, so changing source code does not bust the
|
||||
# deps cache. The uv-managed interpreter is also installed here, into a
|
||||
# stable path that the runtime stage can copy verbatim.
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
--mount=type=bind,source=pyproject.toml,target=pyproject.toml \
|
||||
--mount=type=bind,source=uv.lock,target=uv.lock \
|
||||
--mount=type=bind,source=README.md,target=README.md \
|
||||
set -e; \
|
||||
uv python install ${PYTHON_VERSION}; \
|
||||
flags=""; \
|
||||
for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \
|
||||
uv sync --frozen --no-install-project --no-dev --python ${PYTHON_VERSION} ${flags}
|
||||
|
||||
# Layer 2: the project itself. --no-editable installs mempalace into the
|
||||
# venv's site-packages (instead of an .pth pointing at /app), so the runtime
|
||||
# stage can copy only /app/.venv and drop the source tree.
|
||||
COPY . /app
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
set -e; \
|
||||
flags=""; \
|
||||
for e in $(echo "${EXTRAS}" | tr ',' ' '); do flags="${flags} --extra ${e}"; done; \
|
||||
uv sync --frozen --no-dev --no-editable ${flags}
|
||||
|
||||
# --- runtime ----------------------------------------------------------------
|
||||
FROM ${CUDA_IMAGE} AS runtime
|
||||
|
||||
LABEL org.opencontainers.image.title="MemPalace (GPU)" \
|
||||
org.opencontainers.image.description="Local-first AI memory with CUDA-accelerated embeddings." \
|
||||
org.opencontainers.image.source="https://github.com/MemPalace/mempalace" \
|
||||
org.opencontainers.image.licenses="MIT"
|
||||
|
||||
# ca-certificates only — needed for the lazy HuggingFace model download on
|
||||
# first use. No build toolchain in this stage.
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
ENV HOME=/data \
|
||||
PATH="/app/.venv/bin:${PATH}" \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
MEMPALACE_EMBEDDING_DEVICE=cuda
|
||||
|
||||
# Non-root user owning the data volume.
|
||||
RUN groupadd --gid 1000 mempalace \
|
||||
&& useradd --uid 1000 --gid 1000 --home-dir /data --create-home mempalace
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Bring the uv-managed interpreter and the resolved venv across the stage
|
||||
# boundary. /opt/uv/python must be copied alongside .venv: the venv's
|
||||
# shebangs and binary launcher reference it.
|
||||
COPY --from=builder /opt/uv/python /opt/uv/python
|
||||
COPY --from=builder --chown=mempalace:mempalace /app/.venv /app/.venv
|
||||
COPY --chown=mempalace:mempalace docker-entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
USER mempalace
|
||||
VOLUME ["/data"]
|
||||
|
||||
ENTRYPOINT ["docker-entrypoint.sh"]
|
||||
CMD ["mcp"]
|
||||
@@ -1,21 +1,3 @@
|
||||
> [!CAUTION]
|
||||
> # 🚨 CRITICAL SECURITY WARNING: BEWARE OF SCAMS
|
||||
> **MemPalace has NO other official websites.**
|
||||
>
|
||||
> The **ONLY** official sources are:
|
||||
> 1. This **[GitHub repository](https://github.com/MemPalace/mempalace)**
|
||||
> 2. The **[PyPI package](https://pypi.org/project/mempalace/)**
|
||||
> 3. The docs at **[mempalaceofficial.com](https://mempalaceofficial.com)**
|
||||
>
|
||||
> **ANY other domain** (including `.tech`, `.net`, or other `.com` variants) is an **impostor** and may distribute **malware**. Do not download executables from untrusted sites. Details and timeline: [docs/HISTORY.md](docs/HISTORY.md).
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **🚨 Claude Code sessions expire in 30 days w/out auto-save hooks wired!** **[Read this →](https://github.com/MemPalace/mempalace/discussions/1388)**
|
||||
>
|
||||
> Need the shortest recovery/setup path? Use the
|
||||
> [Claude Code retention setup checklist](https://mempalaceofficial.com/guide/claude-code-retention.html).
|
||||
|
||||
|
||||
<div align="center">
|
||||
|
||||
<img src="assets/mempalace_logo.png" alt="MemPalace" width="240">
|
||||
@@ -31,6 +13,14 @@ Local-first AI memory. Verbatim storage, pluggable backend, 96.6% R@5 raw on Lon
|
||||
|
||||
</div>
|
||||
|
||||
> [!CAUTION]
|
||||
> **Beware of impostor sites.** MemPalace has no other official websites. The **only** official sources are this **[GitHub repository](https://github.com/MemPalace/mempalace)**, the **[PyPI package](https://pypi.org/project/mempalace/)**, and the docs at **[mempalaceofficial.com](https://mempalaceofficial.com)**. Any other domain (including `.tech`, `.net`, or other `.com` variants) is an impostor and may distribute malware. Details and timeline: [docs/HISTORY.md](docs/HISTORY.md).
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Claude Code sessions expire in 30 days without auto-save hooks wired.** [Read this →](https://github.com/MemPalace/mempalace/discussions/1388)
|
||||
>
|
||||
> Need the shortest recovery/setup path? Use the [Claude Code retention setup checklist](https://mempalaceofficial.com/guide/claude-code-retention.html).
|
||||
|
||||
---
|
||||
|
||||
## What it is
|
||||
@@ -79,6 +69,63 @@ python -m venv .venv && source .venv/bin/activate
|
||||
pip install mempalace
|
||||
```
|
||||
|
||||
### Docker
|
||||
|
||||
A container image is also available for running the MCP server or the CLI
|
||||
without a local Python toolchain. Everything persists under `/data` (palace,
|
||||
config, and the cached embedding model), so mount a volume there.
|
||||
|
||||
```bash
|
||||
# Build the image (CPU; bundles the `extract` + `spellcheck` extras)
|
||||
docker build -t mempalace .
|
||||
|
||||
# MCP server over stdio — note the `-i` flag (JSON-RPC needs stdin)
|
||||
docker run -i --rm -v mempalace-data:/data mempalace
|
||||
|
||||
# Run any CLI command instead (mount the host directory you want to mine)
|
||||
docker run --rm -v mempalace-data:/data -v /path/to/project:/work mempalace mine /work
|
||||
docker run --rm -v mempalace-data:/data mempalace search "why GraphQL"
|
||||
```
|
||||
|
||||
Wire it into an MCP client (e.g. Claude Code) as a stdio server:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"mempalace": {
|
||||
"command": "docker",
|
||||
"args": ["run", "-i", "--rm", "-v", "mempalace-data:/data", "mempalace"]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
`docker compose run --rm mcp` works too (see `docker-compose.yml`). For
|
||||
CUDA-accelerated embeddings, build the GPU variant with
|
||||
`docker build -f Dockerfile.gpu -t mempalace:gpu .` and run it with
|
||||
`--gpus all`. Customise the bundled extras at build time, e.g.
|
||||
`docker build --build-arg EXTRAS="extract,spellcheck" -t mempalace .`.
|
||||
|
||||
## Storage backends
|
||||
|
||||
ChromaDB is the default and needs no configuration. MemPalace also ships a
|
||||
pluggable backend contract, exercised across deliberately different substrates
|
||||
so the contract is never accidentally shaped around one vendor. Every
|
||||
non-default backend is opt-in.
|
||||
|
||||
| Backend | Mode | Install | Namespaces | Lexical | Configure with |
|
||||
| ------- | ---- | ------- | :--------: | :-----: | -------------- |
|
||||
| `chroma` _(default)_ | Local (embedded) | bundled | – | ✓ | – |
|
||||
| `sqlite_exact` | Local (exact) | bundled | – | ✓ | – |
|
||||
| `milvus` | Local (Lite) · Server opt-in | `mempalace[milvus]` | ✓ | ✓ | `MEMPALACE_MILVUS_URI` |
|
||||
| `qdrant` | Server (REST) | bundled | ✓ | ✓ | `MEMPALACE_QDRANT_URL` |
|
||||
| `pgvector` | Server (Postgres) | `mempalace[pgvector]` | ✓ | ✓ | `MEMPALACE_PGVECTOR_DSN` |
|
||||
|
||||
Select with `--backend <name>`, `MEMPALACE_BACKEND=<name>`, or
|
||||
`"backend": "<name>"` in `config.json`. See
|
||||
[Storage backends](/guide/configuration#storage-backends) for connection
|
||||
variables, namespace behavior, and deployment notes.
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
@@ -93,7 +140,8 @@ mempalace search "why did we switch to GraphQL"
|
||||
mempalace wake-up
|
||||
```
|
||||
|
||||
For Claude Code, Gemini CLI, MCP-compatible tools, and local models, see
|
||||
For Claude Code, Gemini CLI, [Antigravity](https://mempalaceofficial.com/guide/antigravity.html),
|
||||
MCP-compatible tools, and local models, see
|
||||
[mempalaceofficial.com/guide/getting-started](https://mempalaceofficial.com/guide/getting-started.html).
|
||||
|
||||
---
|
||||
@@ -161,7 +209,7 @@ Usage and tool reference:
|
||||
|
||||
## MCP server
|
||||
|
||||
29 MCP tools cover palace reads/writes, knowledge-graph operations,
|
||||
36 MCP tools cover palace reads/writes, knowledge-graph operations,
|
||||
cross-wing navigation, drawer management, and agent diaries. Installation
|
||||
and the full tool list:
|
||||
[mempalaceofficial.com/reference/mcp-tools](https://mempalaceofficial.com/reference/mcp-tools.html).
|
||||
@@ -175,8 +223,14 @@ system prompt:
|
||||
|
||||
## Auto-save hooks
|
||||
|
||||
Two Claude Code hooks save periodically and before context compression:
|
||||
[mempalaceofficial.com/guide/hooks](https://mempalaceofficial.com/guide/hooks.html).
|
||||
Auto-save hooks for **Claude Code, Codex CLI, and Cursor IDE** save
|
||||
periodically and before context compression:
|
||||
|
||||
- Claude Code + Codex →
|
||||
[mempalaceofficial.com/guide/hooks](https://mempalaceofficial.com/guide/hooks.html)
|
||||
- Cursor IDE (adds session-start recall and a transcript snapshot before
|
||||
compaction) →
|
||||
[mempalaceofficial.com/guide/cursor-hooks](https://mempalaceofficial.com/guide/cursor-hooks.html)
|
||||
|
||||
If you are installing under time pressure, start with the
|
||||
[Claude Code retention setup checklist](https://mempalaceofficial.com/guide/claude-code-retention.html):
|
||||
@@ -215,7 +269,7 @@ PRs welcome. See [CONTRIBUTING.md](CONTRIBUTING.md).
|
||||
MIT — see [LICENSE](LICENSE).
|
||||
|
||||
<!-- Link Definitions -->
|
||||
[version-shield]: https://img.shields.io/badge/version-3.3.6-4dc9f6?style=flat-square&labelColor=0a0e14
|
||||
[version-shield]: https://img.shields.io/badge/version-3.6.0-4dc9f6?style=flat-square&labelColor=0a0e14
|
||||
[release-link]: https://github.com/MemPalace/mempalace/releases
|
||||
[python-shield]: https://img.shields.io/badge/python-3.9+-7dd8f8?style=flat-square&labelColor=0a0e14&logo=python&logoColor=7dd8f8
|
||||
[python-link]: https://www.python.org/
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: Show comprehensive MemPalace help — available skills, MCP tools, CLI commands, hooks, and architecture.
|
||||
---
|
||||
|
||||
Invoke the `mempalace` skill from this plugin and run the `help` instructions, then follow them.
|
||||
|
||||
Concretely: run `mempalace instructions help` in a terminal, then carry out the steps it prints.
|
||||
@@ -0,0 +1,12 @@
|
||||
---
|
||||
description: Set up MemPalace — install the package, initialize a palace, register the MCP server with Cursor, and verify everything works.
|
||||
---
|
||||
|
||||
Invoke the `mempalace` skill from this plugin and run the `init` instructions, then follow them.
|
||||
|
||||
Concretely: run `mempalace instructions init` in a terminal, then carry out the steps it prints.
|
||||
|
||||
Cursor-specific extras after init:
|
||||
|
||||
1. The `mempalace-mcp` server is already auto-registered by this plugin — no manual `mcp.json` edit needed.
|
||||
2. For automatic background saves and session-start memory recall, also run `hooks/cursor/install.sh --scope user` from a cloned MemPalace repo. See `website/guide/cursor-hooks.md` for the walkthrough.
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: Mine projects and conversations into the MemPalace. Supports project files, conversation exports, and auto-classification.
|
||||
---
|
||||
|
||||
Invoke the `mempalace` skill from this plugin and run the `mine` instructions, then follow them.
|
||||
|
||||
Concretely: run `mempalace instructions mine` in a terminal, then carry out the steps it prints.
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: Search your memories across the MemPalace using semantic search with wing/room filtering.
|
||||
---
|
||||
|
||||
Invoke the `mempalace` skill from this plugin and run the `search` instructions, then follow them.
|
||||
|
||||
Concretely: run `mempalace instructions search` in a terminal, then carry out the steps it prints. The MCP tool `mempalace_search` is also available directly from this Cursor session.
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
description: Show the current state of your memory palace — wings, rooms, drawer counts, and suggestions.
|
||||
---
|
||||
|
||||
Invoke the `mempalace` skill from this plugin and run the `status` instructions, then follow them.
|
||||
|
||||
Concretely: run `mempalace instructions status` in a terminal, then carry out the steps it prints.
|
||||
@@ -0,0 +1,71 @@
|
||||
# MemPalace remote team server — MCP over HTTP, backed by a central Qdrant.
|
||||
#
|
||||
# One command stands up a shared memory service a whole team's AI clients
|
||||
# connect to. Embeddings are still produced locally inside the mempalace
|
||||
# container; only your own Qdrant ever receives the vectors and text.
|
||||
#
|
||||
# 1. cp deploy/server.env.example deploy/.env && edit deploy/.env
|
||||
# (at minimum set MEMPALACE_MCP_HTTP_TOKEN to a long random secret)
|
||||
# 2. docker compose -f deploy/docker-compose.server.yml --env-file deploy/.env up -d
|
||||
# 3. connect a client (see the Remote / Team Server guide):
|
||||
# claude mcp add --transport http mempalace http://YOUR_HOST:8765/mcp \
|
||||
# --header "Authorization: Bearer $MEMPALACE_MCP_HTTP_TOKEN"
|
||||
#
|
||||
# SECURITY: this exposes plaintext HTTP on :8765. For anything beyond a trusted
|
||||
# private network, put a TLS-terminating reverse proxy in front (nginx/Caddy/
|
||||
# Traefik) and only expose the proxy. The bearer token is mandatory for the
|
||||
# network-exposed (0.0.0.0) bind.
|
||||
|
||||
services:
|
||||
qdrant:
|
||||
image: qdrant/qdrant:latest
|
||||
restart: unless-stopped
|
||||
volumes:
|
||||
- qdrant-storage:/qdrant/storage
|
||||
# Not published to the host: only the mempalace service reaches it over the
|
||||
# internal compose network. Uncomment to inspect Qdrant directly.
|
||||
# ports:
|
||||
# - "6333:6333"
|
||||
|
||||
mempalace:
|
||||
image: ghcr.io/mempalace/mempalace:latest
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
- qdrant
|
||||
command:
|
||||
- serve
|
||||
- --host
|
||||
- "0.0.0.0"
|
||||
- --port
|
||||
- "8765"
|
||||
- --backend
|
||||
- qdrant
|
||||
# Uncomment to expose recall without write access to most clients:
|
||||
# - --read-only
|
||||
environment:
|
||||
# Required for the network-exposed bind. Set in deploy/.env.
|
||||
MEMPALACE_MCP_HTTP_TOKEN: ${MEMPALACE_MCP_HTTP_TOKEN:?set MEMPALACE_MCP_HTTP_TOKEN in deploy/.env}
|
||||
MEMPALACE_QDRANT_URL: http://qdrant:6333
|
||||
MEMPALACE_QDRANT_API_KEY: ${MEMPALACE_QDRANT_API_KEY:-}
|
||||
# Set to cuda/dml/coreml on an accelerated host (see Dockerfile.gpu).
|
||||
MEMPALACE_EMBEDDING_DEVICE: ${MEMPALACE_EMBEDDING_DEVICE:-auto}
|
||||
ports:
|
||||
- "8765:8765"
|
||||
volumes:
|
||||
- mempalace-data:/data
|
||||
healthcheck:
|
||||
# The image has no curl; use Python (always present). /healthz needs no auth.
|
||||
# If you enable TLS on the server itself, switch this to https + ssl context.
|
||||
test:
|
||||
- CMD
|
||||
- python
|
||||
- -c
|
||||
- "import urllib.request,sys; sys.exit(0) if urllib.request.urlopen('http://127.0.0.1:8765/healthz').read().strip()==b'ok' else sys.exit(1)"
|
||||
interval: 30s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
start_period: 40s
|
||||
|
||||
volumes:
|
||||
qdrant-storage:
|
||||
mempalace-data:
|
||||
@@ -0,0 +1,48 @@
|
||||
# MemPalace remote MCP server — systemd unit template.
|
||||
#
|
||||
# Install:
|
||||
# sudo useradd --system --home /var/lib/mempalace --shell /usr/sbin/nologin mempalace
|
||||
# sudo install -d -o mempalace -g mempalace -m 750 /var/lib/mempalace /etc/mempalace
|
||||
# sudo cp deploy/server.env.example /etc/mempalace/server.env # then edit + chmod 600
|
||||
# sudo install -m 600 -o mempalace -g mempalace /etc/mempalace/server.env /etc/mempalace/server.env
|
||||
# # install mempalace into a venv on PATH, or adjust ExecStart to its absolute path
|
||||
# sudo cp deploy/mempalace-server.service /etc/systemd/system/
|
||||
# sudo systemctl daemon-reload && sudo systemctl enable --now mempalace-server
|
||||
#
|
||||
# This binds 0.0.0.0:8765 and requires MEMPALACE_MCP_HTTP_TOKEN (set in the
|
||||
# EnvironmentFile). Front it with a TLS-terminating reverse proxy, or set
|
||||
# MEMPALACE_MCP_TLS_CERT / _KEY in the EnvironmentFile for native TLS.
|
||||
|
||||
[Unit]
|
||||
Description=MemPalace remote MCP server
|
||||
After=network-online.target
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=exec
|
||||
User=mempalace
|
||||
Group=mempalace
|
||||
EnvironmentFile=/etc/mempalace/server.env
|
||||
ExecStart=mempalace serve --host 0.0.0.0 --port 8765
|
||||
Restart=on-failure
|
||||
RestartSec=2
|
||||
|
||||
# --- Hardening ---------------------------------------------------------------
|
||||
NoNewPrivileges=true
|
||||
ProtectSystem=strict
|
||||
ProtectHome=true
|
||||
PrivateTmp=true
|
||||
PrivateDevices=true
|
||||
ProtectKernelTunables=true
|
||||
ProtectKernelModules=true
|
||||
ProtectControlGroups=true
|
||||
RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX
|
||||
RestrictNamespaces=true
|
||||
LockPersonality=true
|
||||
MemoryDenyWriteExecute=false
|
||||
# The palace and any local state live here; everything else is read-only.
|
||||
ReadWritePaths=/var/lib/mempalace
|
||||
StateDirectory=mempalace
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,28 @@
|
||||
# MemPalace remote server environment.
|
||||
# Copy to deploy/.env (compose) or /etc/mempalace/server.env (systemd) and edit.
|
||||
# Keep this file readable only by the service account: chmod 600.
|
||||
|
||||
# --- Required for a network-exposed (0.0.0.0) bind ------------------------------
|
||||
# Clients send: Authorization: Bearer <this value>. Generate a strong secret:
|
||||
# openssl rand -hex 32
|
||||
MEMPALACE_MCP_HTTP_TOKEN=
|
||||
|
||||
# --- Storage backend ------------------------------------------------------------
|
||||
# The team server should use a networked backend so the palace is shared.
|
||||
MEMPALACE_BACKEND=qdrant
|
||||
MEMPALACE_QDRANT_URL=http://qdrant:6333
|
||||
# MEMPALACE_QDRANT_API_KEY=
|
||||
|
||||
# --- Embedding ------------------------------------------------------------------
|
||||
# auto | cpu | cuda | dml | coreml. Use cuda on a GPU host (needs the GPU image).
|
||||
MEMPALACE_EMBEDDING_DEVICE=auto
|
||||
|
||||
# --- Optional: native TLS (otherwise terminate TLS at a reverse proxy) ----------
|
||||
# Point these at a PEM cert/key the service account can read. When set, serve
|
||||
# speaks HTTPS directly and clients connect to https://...
|
||||
# MEMPALACE_MCP_TLS_CERT=/etc/mempalace/tls/cert.pem
|
||||
# MEMPALACE_MCP_TLS_KEY=/etc/mempalace/tls/key.pem
|
||||
|
||||
# --- Palace location (systemd / bare-metal) -------------------------------------
|
||||
# In Docker the palace lives on the mempalace-data volume by default.
|
||||
# MEMPALACE_PALACE_PATH=/var/lib/mempalace/palace
|
||||
@@ -0,0 +1,32 @@
|
||||
# MemPalace via Docker Compose.
|
||||
#
|
||||
# The MCP server speaks JSON-RPC over stdio, so the `mcp` service is meant to
|
||||
# be driven interactively (`docker compose run`), not left running detached:
|
||||
#
|
||||
# docker compose build
|
||||
# docker compose run --rm mcp # MCP server over stdio
|
||||
# docker compose run --rm mcp cli search "GraphQL" # one-off CLI command
|
||||
#
|
||||
# The named volume `mempalace-data` is mounted at /data and holds the palace,
|
||||
# config, and the cached embedding model across runs.
|
||||
|
||||
services:
|
||||
mcp:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
image: mempalace:local
|
||||
# stdio transport: keep STDIN open and allocate no TTY (raw JSON-RPC).
|
||||
stdin_open: true
|
||||
tty: false
|
||||
volumes:
|
||||
- mempalace-data:/data
|
||||
environment:
|
||||
# Everything defaults correctly from HOME=/data (palace ->
|
||||
# /data/.mempalace/palace, config -> /data/.mempalace, model cache ->
|
||||
# /data/.cache). Override here only for non-default locations, e.g.:
|
||||
# - MEMPALACE_EMBEDDING_MODEL=embeddinggemma # multilingual (default: minilm)
|
||||
# - MEMPALACE_PALACE_PATH=/data/custom/palace
|
||||
|
||||
volumes:
|
||||
mempalace-data:
|
||||
@@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env sh
|
||||
# Flexible entrypoint: pick the MCP server or the CLI from the first argument.
|
||||
#
|
||||
# docker run -i mempalace -> MCP server over stdio (default)
|
||||
# docker run -i mempalace mcp -> MCP server over stdio (explicit)
|
||||
# docker run mempalace cli search "query" -> CLI passthrough (explicit)
|
||||
# docker run mempalace search "query" -> CLI passthrough (implicit)
|
||||
#
|
||||
# `mcp` and `cli` are dispatch keywords; anything else is forwarded to the
|
||||
# `mempalace` CLI verbatim so subcommands like `mine`, `search`, `wake-up`
|
||||
# work without ceremony.
|
||||
set -e
|
||||
|
||||
case "${1:-mcp}" in
|
||||
mcp)
|
||||
if [ "$#" -gt 0 ]; then
|
||||
shift
|
||||
fi
|
||||
exec mempalace-mcp "$@"
|
||||
;;
|
||||
cli)
|
||||
if [ "$#" -gt 0 ]; then
|
||||
shift
|
||||
fi
|
||||
exec mempalace "$@"
|
||||
;;
|
||||
*)
|
||||
exec mempalace "$@"
|
||||
;;
|
||||
esac
|
||||
@@ -29,3 +29,59 @@ If `pyproject.toml` has no match, **stop** — the entry point is missing and
|
||||
any fresh `pip install` will ship a broken plugin config. Investigate whether
|
||||
the release branch was cut before
|
||||
[#340](https://github.com/MemPalace/mempalace/pull/340) landed on `develop`.
|
||||
|
||||
## Publishing to PyPI
|
||||
|
||||
Releases publish automatically via the
|
||||
[`publish.yml`](../.github/workflows/publish.yml) workflow, using PyPI
|
||||
[Trusted Publishing](https://docs.pypi.org/trusted-publishers/) (OIDC). There
|
||||
is **no API token** stored anywhere — GitHub mints a short-lived identity at
|
||||
upload time. The workflow fires when a **GitHub Release is published**, builds
|
||||
the sdist + wheel, and pauses for manual approval on the `pypi` environment
|
||||
before uploading.
|
||||
|
||||
### One-time setup (owners only)
|
||||
|
||||
Done once per project; both steps require PyPI owner / GitHub admin rights.
|
||||
|
||||
1. **PyPI trusted publisher** — on PyPI, go to **Manage project `mempalace`
|
||||
→ Publishing → Add a trusted publisher** and enter exactly:
|
||||
|
||||
| Field | Value |
|
||||
| --- | --- |
|
||||
| Owner | `MemPalace` |
|
||||
| Repository name | `mempalace` |
|
||||
| Workflow filename | `publish.yml` |
|
||||
| Environment name | `pypi` |
|
||||
|
||||
2. **GitHub environment** — in the repo, **Settings → Environments → New
|
||||
environment** named `pypi`. Add yourself (and any other release approvers)
|
||||
under **Required reviewers**. This is the manual gate the workflow waits on
|
||||
before the upload step runs.
|
||||
|
||||
### Cutting a release
|
||||
|
||||
1. Bump the version in **all five** sources on `develop` so `version-guard.yml`
|
||||
stays green (it is the single source of truth at `mempalace/version.py`,
|
||||
mirrored in `pyproject.toml`, `.claude-plugin/marketplace.json`,
|
||||
`.claude-plugin/plugin.json`, and `.codex-plugin/plugin.json`).
|
||||
2. Land everything for the release on `develop`, then merge `develop → main`.
|
||||
Releases publish **only from `main`** — the workflow refuses any tag whose
|
||||
commit is not an ancestor of `main`. Don't commit the bump directly to
|
||||
`main`: it bypasses branch protection and leaves `develop` behind.
|
||||
3. Run the **entry-point alignment check** above.
|
||||
4. On GitHub, **Releases → Draft a new release**:
|
||||
- **Target:** `main`
|
||||
- **Tag:** `vX.Y.Z` (must equal `mempalace/version.py`; the workflow and
|
||||
`version-guard.yml` both reject a mismatch)
|
||||
- Write the release notes, then **Publish release**.
|
||||
5. The `publish.yml` run validates the tag (on `main`, matches the manifest),
|
||||
builds, and then waits for approval on the `pypi` environment. Approve it to
|
||||
upload to PyPI. Watch the run land the new version on
|
||||
<https://pypi.org/project/mempalace/>.
|
||||
|
||||
To stage a release candidate without shipping to end users, tag a semver
|
||||
pre-release (`vX.Y.Z-rc1`) — `version-guard.yml` skips the strict manifest
|
||||
match for pre-release tags. (Note: a published GitHub Release still triggers
|
||||
`publish.yml`; use a **draft** release, or a plain pushed tag, for dry runs you
|
||||
don't want uploaded.)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# Authored date (`authored_at`)
|
||||
|
||||
Conversation transcripts carry a per-line ISO-8601 `timestamp` (both Claude Code and
|
||||
Codex JSONL). The miner records the most recent one per file as the drawer's
|
||||
**`authored_at`** — when the content was actually written.
|
||||
|
||||
This is distinct from the ingest date:
|
||||
|
||||
| Field | Meaning |
|
||||
|-------|---------|
|
||||
| `filed_at` / result `created_at` | When the drawer was **mined** (written to the palace). A bulk re-mine collapses these to a single instant. |
|
||||
| `authored_at` | When the underlying content was **written**, recovered from the transcript timestamps. Survives re-mining. |
|
||||
|
||||
`authored_at` is surfaced in search results (and shown in the CLI `search` output), and is
|
||||
used as a deterministic tie-break in hybrid ranking: candidates with identical scores order
|
||||
with the more recently authored drawer first. Drawers without per-line timestamps (e.g.
|
||||
markdown) fall back to `filed_at`.
|
||||
|
||||
## Backfilling existing memory
|
||||
|
||||
New mines populate `authored_at` automatically. Drawers mined before this feature only have
|
||||
`filed_at`. Re-mining does **not** fix them — the scanner skips files already mined at the
|
||||
current `NORMALIZE_VERSION`. Two options:
|
||||
|
||||
1. **In-place backfill (recommended — no re-embedding).** `scripts/backfill_authored_at.py`
|
||||
reads each convos drawer's source transcript and updates only the `authored_at` metadata.
|
||||
Idempotent and safe to re-run; embeddings are untouched.
|
||||
|
||||
```bash
|
||||
python scripts/backfill_authored_at.py \
|
||||
--palace ~/.mempalace/palace \
|
||||
--sessions ~/.claude --sessions ~/.codex # dry run
|
||||
python scripts/backfill_authored_at.py \
|
||||
--palace ~/.mempalace/palace \
|
||||
--sessions ~/.claude --sessions ~/.codex --apply # write
|
||||
```
|
||||
|
||||
For the Docker MCP image, mount the volume and session dirs read-only — see the header of
|
||||
`scripts/backfill_authored_at.py` for the exact `docker run` invocation.
|
||||
|
||||
> Back up first: `tar czf palace-backup.tgz -C <palace-dir> .` (or snapshot the
|
||||
> `mempalace-data` volume).
|
||||
|
||||
2. **Drop and recreate.** Delete the affected drawers and re-mine the transcripts; the fresh
|
||||
mine stamps `authored_at`. Simpler, but re-embeds everything.
|
||||
@@ -0,0 +1,63 @@
|
||||
# Recovery: legacy wing names split after the normalization change
|
||||
|
||||
**Companion to #1675.** `normalize_wing_name` now strips leading and trailing
|
||||
separators, so a path-encoded project dir like `-home-user-proj` derives the
|
||||
wing `home_user_proj` instead of `_home_user_proj`. Palaces mined before that
|
||||
change filed drawers under the old, separator-padded name. New mining and diary
|
||||
writes land on the new name, so the two no longer meet — the history is
|
||||
**split**, not lost. `mempalace migrate-wings` re-unites them.
|
||||
|
||||
## Symptom
|
||||
|
||||
After upgrading, a project that used to surface its memories returns less than
|
||||
expected, and `mempalace status` shows two wings for one project — e.g. both
|
||||
`_home_user_proj` (old drawers) and `home_user_proj` (newly mined). MCP writes
|
||||
to the padded wing may also have been rejected, since `sanitize_name` does not
|
||||
accept a leading underscore.
|
||||
|
||||
## Recovery
|
||||
|
||||
Preview first — this never modifies anything:
|
||||
|
||||
```bash
|
||||
mempalace migrate-wings --dry-run
|
||||
mempalace migrate-wings --dry-run --palace /path/to/palace
|
||||
```
|
||||
|
||||
The plan lists each rename and flags collisions that will **merge** into an
|
||||
existing wing:
|
||||
|
||||
```
|
||||
Wing-name migration plan:
|
||||
'_home_user_proj' -> 'home_user_proj': 1284 drawer(s), 96 closet(s) (MERGE into existing wing)
|
||||
```
|
||||
|
||||
Apply it:
|
||||
|
||||
```bash
|
||||
mempalace migrate-wings # prompts for confirmation
|
||||
mempalace migrate-wings --yes # no prompt
|
||||
```
|
||||
|
||||
## What it does
|
||||
|
||||
- Re-keys the `wing` **metadata field** on drawers and closets to the normalized
|
||||
form, merging collisions into the existing wing.
|
||||
- Re-keys the `topics_by_wing` registry (merging topic lists on collision).
|
||||
|
||||
## What it leaves alone
|
||||
|
||||
- **Drawer/closet IDs** are untouched. The wing in an ID (`drawer_<wing>_…`) is
|
||||
an opaque prefix that is never decoded back into a wing, so leaving it keeps
|
||||
closet `→drawer_id` pointers valid and lets future mining still skip
|
||||
already-mined files (no duplicates). The verbatim drawer content is never
|
||||
read or rewritten.
|
||||
- **Tunnels** already normalize wing names at read time, so they resolve under
|
||||
the new name without a rewrite.
|
||||
|
||||
## Notes
|
||||
|
||||
- **Idempotent.** A second run reports "nothing to migrate" and changes nothing.
|
||||
- **Backend-agnostic.** Works on any configured storage backend.
|
||||
- Run it once per palace after upgrading. New palaces are born with normalized
|
||||
wing names and never need it.
|
||||
@@ -0,0 +1,119 @@
|
||||
# Write routing policy
|
||||
|
||||
This document defines the shared policy used by the staged Tier 3 daemon
|
||||
rollout tracked in #1963.
|
||||
|
||||
This foundation PR does not change existing hook or CLI routing. It provides
|
||||
one tested policy model that later hook and CLI PRs can consume without
|
||||
inventing different fallback rules.
|
||||
|
||||
## Policies
|
||||
|
||||
`direct`
|
||||
|
||||
Always execute through the existing direct local path.
|
||||
|
||||
`prefer`
|
||||
|
||||
Use an available daemon. A caller that is allowed to start the daemon may
|
||||
do so. Otherwise, fall back to the direct path.
|
||||
|
||||
`require`
|
||||
|
||||
Use an available daemon. A caller that is allowed to start the daemon may
|
||||
do so. If neither is possible, block the operation. Never fall back to a
|
||||
direct ChromaDB writer.
|
||||
|
||||
## Concrete routing outcomes
|
||||
|
||||
The shared decision function returns one of:
|
||||
|
||||
- `direct`
|
||||
- `daemon`
|
||||
- `blocked`
|
||||
|
||||
It also reports whether the caller should auto-start the daemon and why the
|
||||
route was selected.
|
||||
|
||||
Hooks generally pass `daemon_can_start=False` because hook execution has a
|
||||
tight latency budget.
|
||||
|
||||
Interactive CLI commands can pass `daemon_can_start=True`.
|
||||
|
||||
## Configuration
|
||||
|
||||
Global environment policy:
|
||||
|
||||
MEMPALACE_WRITE_ROUTING=direct|prefer|require
|
||||
|
||||
Hook-specific environment policy:
|
||||
|
||||
MEMPALACE_HOOK_WRITE_ROUTING=direct|prefer|require
|
||||
|
||||
CLI-specific environment policy:
|
||||
|
||||
MEMPALACE_CLI_WRITE_ROUTING=direct|prefer|require
|
||||
|
||||
Configuration-file shape:
|
||||
|
||||
{
|
||||
"write_routing": {
|
||||
"default": "direct",
|
||||
"hooks": "prefer",
|
||||
"cli": "require"
|
||||
}
|
||||
}
|
||||
|
||||
## Precedence
|
||||
|
||||
For hooks:
|
||||
|
||||
1. `MEMPALACE_HOOK_WRITE_ROUTING`
|
||||
2. `MEMPALACE_WRITE_ROUTING`
|
||||
3. legacy `MEMPALACE_HOOKS_DAEMON`
|
||||
4. `write_routing.hooks`
|
||||
5. `write_routing.default`
|
||||
6. legacy `hooks.daemon`
|
||||
7. `direct`
|
||||
|
||||
For CLI writes:
|
||||
|
||||
1. `MEMPALACE_CLI_WRITE_ROUTING`
|
||||
2. `MEMPALACE_WRITE_ROUTING`
|
||||
3. `write_routing.cli`
|
||||
4. `write_routing.default`
|
||||
5. `direct`
|
||||
|
||||
## Backward compatibility
|
||||
|
||||
The existing `MEMPALACE_HOOKS_DAEMON` environment variable and
|
||||
`hooks.daemon` config value remain supported.
|
||||
|
||||
Legacy true values map to `prefer`.
|
||||
|
||||
Legacy false values map to `direct`.
|
||||
|
||||
The existing `MempalaceConfig.hook_use_daemon` property is intentionally
|
||||
unchanged in this PR. Hook and CLI behavior remains unchanged until their
|
||||
policy-aware rollout PRs land.
|
||||
|
||||
## Invalid policy values
|
||||
|
||||
New policy settings accept only:
|
||||
|
||||
- `direct`
|
||||
- `prefer`
|
||||
- `require`
|
||||
|
||||
Invalid values fail with a source-specific error rather than silently falling
|
||||
back. This is important because silently turning a misspelled `require` into a
|
||||
direct write would violate the safety purpose of the policy.
|
||||
|
||||
## Follow-up PRs
|
||||
|
||||
PR 2 will apply the policy to hook-triggered writes.
|
||||
|
||||
PR 3 will apply the policy to routine CLI writes.
|
||||
|
||||
Maintenance operations such as repair, migration, and index rebuild are not
|
||||
ordinary routed writes. They require a separate exclusive-maintenance policy.
|
||||
@@ -0,0 +1,74 @@
|
||||
# MemPalace — Antigravity examples
|
||||
|
||||
Two standalone configs for users who don't want to use the
|
||||
`hooks/antigravity/install.sh` installer.
|
||||
|
||||
## Files
|
||||
|
||||
| File | Purpose |
|
||||
|-------------------|-----------------------------------------------------------------------------------|
|
||||
| `hooks.json` | Standalone `hooks.json` registering the Stop and PreInvocation hooks. |
|
||||
| `mcp_config.json` | Standalone MCP entry registering the `mempalace-mcp` stdio server. |
|
||||
|
||||
## Wire up `hooks.json`
|
||||
|
||||
The example uses placeholder absolute paths (`/ABSOLUTE/PATH/TO/mempalace/...`).
|
||||
You must rewrite both `command` fields to the actual absolute paths to
|
||||
the hook scripts in your cloned repo, or to whichever location holds
|
||||
them. Antigravity will not resolve relative paths reliably.
|
||||
|
||||
Then drop the file at one of:
|
||||
|
||||
- `~/.gemini/config/hooks.json` (global, applies to every workspace)
|
||||
- `<workspace>/.agents/hooks.json` (workspace-scoped)
|
||||
|
||||
Restart Antigravity to pick the file up.
|
||||
|
||||
If you'd rather have the paths absolutized automatically, run the
|
||||
installer:
|
||||
|
||||
```bash
|
||||
bash hooks/antigravity/install.sh
|
||||
```
|
||||
|
||||
That writes a fully rendered `hooks.json` to
|
||||
`~/.gemini/config/plugins/mempalace/hooks.json`.
|
||||
|
||||
## Wire up `mcp_config.json`
|
||||
|
||||
The example registers the `mempalace-mcp` stdio server. Two options:
|
||||
|
||||
### Option A — merge into the user-level Antigravity MCP config
|
||||
|
||||
Antigravity's user-level MCP config lives at
|
||||
`~/.gemini/antigravity/mcp_config.json`. Merge the `mcpServers.mempalace`
|
||||
entry from this example into that file, then restart Antigravity.
|
||||
|
||||
### Option B — drop into a plugin directory
|
||||
|
||||
If you've created a custom plugin folder (per the [Antigravity plugins docs](https://antigravity.google/docs/plugins)),
|
||||
copy this `mcp_config.json` directly into the plugin root:
|
||||
|
||||
```
|
||||
<plugin-root>/mcp_config.json
|
||||
```
|
||||
|
||||
Antigravity merges plugin-level MCP entries with the user-level config
|
||||
on launch.
|
||||
|
||||
## Verify
|
||||
|
||||
After wiring up either or both:
|
||||
|
||||
```bash
|
||||
mempalace-mcp --version # confirm binary is on PATH
|
||||
ls ~/.mempalace/ # confirm palace exists (run `mempalace init` if not)
|
||||
```
|
||||
|
||||
Restart Antigravity. The `mempalace` MCP server should appear in the
|
||||
MCP store; the Stop and PreInvocation hooks fire automatically.
|
||||
|
||||
See [`hooks/antigravity/STDIN_SHAPE.md`](../../hooks/antigravity/STDIN_SHAPE.md)
|
||||
for the exact wire format Antigravity uses, and
|
||||
[`website/guide/antigravity.md`](../../website/guide/antigravity.md)
|
||||
for the full user-facing guide.
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"mempalace-save": {
|
||||
"Stop": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "/ABSOLUTE/PATH/TO/mempalace/hooks/antigravity/mempal_save_hook_antigravity.sh",
|
||||
"timeout": 30
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
"mempalace-wake": {
|
||||
"PreInvocation": [
|
||||
{
|
||||
"type": "command",
|
||||
"command": "/ABSOLUTE/PATH/TO/mempalace/hooks/antigravity/mempal_wake_hook_antigravity.sh",
|
||||
"timeout": 5
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"mempalace": {
|
||||
"command": "mempalace-mcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
# Cursor IDE Hooks — Example `hooks.json` Files
|
||||
|
||||
Sample configurations for wiring the MemPalace Cursor hooks into the
|
||||
Cursor IDE. These are **examples only** — they are intentionally not
|
||||
placed at the repo root (`/.cursor/hooks.json`) because Cursor
|
||||
auto-loads project-level hooks from any trusted workspace, and the
|
||||
repo is regularly opened by contributors. We do not auto-arm hooks on
|
||||
contributor checkout.
|
||||
|
||||
## Variants
|
||||
|
||||
### `hooks.json` — full (recommended)
|
||||
|
||||
Three hooks wired:
|
||||
|
||||
- **`sessionStart`** — calls `mempal_wake_hook_cursor.sh`, which returns
|
||||
`additional_context` telling the agent to recall scoped to the wing
|
||||
inferred from the workspace root. Cursor-only — Claude Code has no
|
||||
equivalent.
|
||||
- **`stop`** — calls `mempal_save_hook_cursor.sh`. Counts stop
|
||||
invocations per conversation and emits a `followup_message` every
|
||||
`MEMPAL_SAVE_INTERVAL` (default 15) telling the agent to file the
|
||||
session into the palace and write a diary entry. `loop_limit: 1` is
|
||||
defense-in-depth on top of our own loop-count check.
|
||||
- **`preCompact`** — calls `mempal_precompact_hook_cursor.sh`. Runs
|
||||
`mempalace mine` synchronously on the transcript before compaction
|
||||
summarises it, then drops a marker so the next `stop` forces a save
|
||||
followup.
|
||||
|
||||
### `hooks.minimal.json` — `stop` only
|
||||
|
||||
Lightest install. Wires just the save hook. Use this if you don't
|
||||
want the sessionStart recall context or the preCompact transcript
|
||||
snapshot.
|
||||
|
||||
## How to use
|
||||
|
||||
The `$HOME` placeholder is **not** expanded by Cursor — you must
|
||||
substitute the absolute path before saving the file. Pick one:
|
||||
|
||||
### Option A — let `install.sh` do it
|
||||
|
||||
Project scope — writes `<repo>/.cursor/hooks.json`:
|
||||
|
||||
```bash
|
||||
hooks/cursor/install.sh --scope project --target /path/to/your/repo
|
||||
```
|
||||
|
||||
User scope — writes `~/.cursor/hooks.json`, applies to every Cursor workspace:
|
||||
|
||||
```bash
|
||||
hooks/cursor/install.sh --scope user
|
||||
```
|
||||
|
||||
The installer copies the hook scripts to `~/.mempalace/hooks/cursor/`,
|
||||
substitutes the absolute paths, and merges the entries into your
|
||||
existing `hooks.json` without clobbering unrelated hooks. See
|
||||
`install.sh --help` for `--dry-run`, `--uninstall`, and `--variant`.
|
||||
|
||||
### Option B — copy + edit manually
|
||||
|
||||
1. Copy the chosen example to the target location:
|
||||
- User scope: `~/.cursor/hooks.json`
|
||||
- Project scope: `<your-repo>/.cursor/hooks.json`
|
||||
2. Replace every `$HOME` with the absolute path to your home
|
||||
directory (e.g., `/Users/you` or `/home/you`).
|
||||
3. Make sure each hook script is executable
|
||||
(`chmod +x ~/.mempalace/hooks/cursor/mempal_*_hook_cursor.sh`).
|
||||
4. Restart Cursor, or wait for it to auto-reload the file.
|
||||
|
||||
## Why aren't these files at the repo root?
|
||||
|
||||
Cursor automatically loads `.cursor/hooks.json` from any trusted
|
||||
workspace. Placing a real `hooks.json` at the repo root would arm
|
||||
MemPalace's hooks on every contributor's machine the moment they open
|
||||
the repo in Cursor — which would modify their conversation behaviour
|
||||
without consent and write to `~/.mempalace/hook_state/` without
|
||||
asking. Editor configuration is sacred; opt-in only.
|
||||
|
||||
If you actually want MemPalace's hooks armed when working on the
|
||||
MemPalace repo itself, run:
|
||||
|
||||
```bash
|
||||
hooks/cursor/install.sh --scope project --target .
|
||||
```
|
||||
|
||||
That will write `./.cursor/hooks.json` for the repo workspace
|
||||
specifically — but it is your decision, not ours, and the file is
|
||||
listed in `.gitignore` paths Cursor users typically already exclude.
|
||||
|
||||
## Related: the Cursor plugin
|
||||
|
||||
The hooks here are **only one half** of MemPalace's Cursor integration. The other half is the [`.cursor-plugin/`](../../.cursor-plugin/) folder at the repo root, which packages MemPalace's MCP server, five slash commands, and the model-invocable `mempalace` skill as a regular Cursor plugin you can drop into `~/.cursor/plugins/local/mempalace`.
|
||||
|
||||
The two install paths are orthogonal — install whichever you want, in any order:
|
||||
|
||||
| You want | Install |
|
||||
|---------------------------------------------------------------------------|----------------------------------------------------------|
|
||||
| MCP tools (`mempalace_search`, `mempalace_add_drawer`, …) + slash commands | The plugin — see [`.cursor-plugin/README.md`](../../.cursor-plugin/README.md) |
|
||||
| Auto-save every N turns + sessionStart memory recall | The hooks here — see Option A above |
|
||||
| Both | Install the plugin AND run `hooks/cursor/install.sh` |
|
||||
|
||||
Hooks are deliberately **not** bundled into the plugin because Cursor's hooks system is configured per-user/per-project (in `~/.cursor/hooks.json` or `.cursor/hooks.json`), not per-plugin — so the installer here owns that file with idempotent merge semantics, while the plugin owns the MCP+commands+skill side.
|
||||
|
||||
## See also
|
||||
|
||||
- [`hooks/cursor/README.md`](../../hooks/cursor/README.md) — full reference for hooks
|
||||
- [`hooks/cursor/STDIN_SHAPE.md`](../../hooks/cursor/STDIN_SHAPE.md) — per-event schema with citations
|
||||
- [`website/guide/cursor-hooks.md`](../../website/guide/cursor-hooks.md) — rendered docs
|
||||
- [`.cursor-plugin/README.md`](../../.cursor-plugin/README.md) — Cursor plugin (MCP + commands + skill)
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"version": 1,
|
||||
"hooks": {
|
||||
"sessionStart": [
|
||||
{
|
||||
"command": "$HOME/.mempalace/hooks/cursor/mempal_wake_hook_cursor.sh"
|
||||
}
|
||||
],
|
||||
"stop": [
|
||||
{
|
||||
"command": "$HOME/.mempalace/hooks/cursor/mempal_save_hook_cursor.sh",
|
||||
"loop_limit": 1
|
||||
}
|
||||
],
|
||||
"preCompact": [
|
||||
{
|
||||
"command": "$HOME/.mempalace/hooks/cursor/mempal_precompact_hook_cursor.sh"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"version": 1,
|
||||
"hooks": {
|
||||
"stop": [
|
||||
{
|
||||
"command": "$HOME/.mempalace/hooks/cursor/mempal_save_hook_cursor.sh",
|
||||
"loop_limit": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
# Cursor Rules — MemPalace recall
|
||||
|
||||
Optional [Cursor rules](https://cursor.com/docs/rules) that make the
|
||||
agent search MemPalace before answering questions about past work,
|
||||
people, projects, or prior decisions.
|
||||
|
||||
These are for users who install MemPalace **without** the Cursor plugin
|
||||
(or who want recall behaviour in a specific project). If you installed
|
||||
the [Cursor plugin](../../../.cursor-plugin/README.md), it already ships
|
||||
the `alwaysApply: false` rule at the plugin root — you do not need to
|
||||
copy anything.
|
||||
|
||||
## Which file to use
|
||||
|
||||
| File | `alwaysApply` | Fires when | Use when |
|
||||
|------|---------------|------------|----------|
|
||||
| [`mempalace-recall.mdc`](mempalace-recall.mdc) | `false` | Cursor's matcher decides the turn is recall-relevant (from the rule `description`) | **Recommended.** Recall without paying for the rule on unrelated work. |
|
||||
| [`mempalace-recall-always.mdc`](mempalace-recall-always.mdc) | `true` | Every conversation in scope, every turn | You want recall guaranteed in context and accept the cost. |
|
||||
|
||||
The always-on variant is heavier: it sits in context on every turn and
|
||||
makes the agent more eager to call `mempalace_search`, which adds MCP
|
||||
latency and works against MemPalace's "memory should feel instant"
|
||||
budget. Prefer the `false` variant unless you specifically want recall
|
||||
forced into every conversation. Pick **one** of the two — do not install
|
||||
both.
|
||||
|
||||
## Install
|
||||
|
||||
User scope (every workspace) — copy into `~/.cursor/rules/`:
|
||||
|
||||
```bash
|
||||
mkdir -p ~/.cursor/rules
|
||||
cp examples/cursor/rules/mempalace-recall.mdc ~/.cursor/rules/
|
||||
```
|
||||
|
||||
Project scope (this repo only) — copy into `.cursor/rules/`:
|
||||
|
||||
```bash
|
||||
mkdir -p .cursor/rules
|
||||
cp examples/cursor/rules/mempalace-recall.mdc .cursor/rules/
|
||||
```
|
||||
|
||||
For the aggressive variant, copy `mempalace-recall-always.mdc` instead
|
||||
(only one of the two). Then reload Cursor:
|
||||
<kbd>Cmd</kbd>-<kbd>Shift</kbd>-<kbd>P</kbd> → **Developer: Reload Window**.
|
||||
|
||||
## How recall is delivered
|
||||
|
||||
Recall ships in three orthogonal layers — install any combination:
|
||||
|
||||
| Layer | What it does | Where |
|
||||
|-------|--------------|-------|
|
||||
| `sessionStart` hook | Injects wing-scoped recall context once per new chat | [`hooks/cursor/`](../../../hooks/cursor/) |
|
||||
| `mempalace-recall` skill | Full search-before-answer protocol, model-invoked or attached | [`skills/mempalace-recall/`](../../../skills/mempalace-recall/) |
|
||||
| Recall rule (these files) | Nudges search-before-answer on recall-relevant turns | here, or the plugin root `rules/` |
|
||||
|
||||
All three reference the same canonical protocol in
|
||||
[`integrations/shared/recall-protocol.md`](../../../integrations/shared/recall-protocol.md).
|
||||
@@ -0,0 +1,25 @@
|
||||
---
|
||||
description: Always-on MemPalace recall — search the palace before answering about past work, people, projects, or prior decisions.
|
||||
alwaysApply: true
|
||||
---
|
||||
|
||||
# MemPalace recall (always on)
|
||||
|
||||
This is the aggressive variant of the recall rule: `alwaysApply: true`
|
||||
loads it into every conversation in scope, on every turn, regardless of
|
||||
whether Cursor's matcher thinks recall is relevant.
|
||||
|
||||
Before answering anything that may already be in the user's memory
|
||||
palace — past work, prior decisions, a person, a project, or "what did
|
||||
we do / decide / discuss last time?" — search the palace first:
|
||||
|
||||
1. Call `mempalace_search` with a short keyword query. Use
|
||||
`mempalace_kg_query` for relational or time-bound facts.
|
||||
2. Quote the drawer's **verbatim** text. Never summarize or paraphrase
|
||||
stored content.
|
||||
3. If results are empty, say so — do not invent an answer. If the MCP
|
||||
server is unavailable, surface the error; do not fall back to guessing.
|
||||
|
||||
Even with this rule always loaded, only actually call the tools when the
|
||||
question touches memory. Do not search on pure greenfield work
|
||||
(renaming a variable, fixing a typo).
|
||||
@@ -0,0 +1,20 @@
|
||||
---
|
||||
description: When the user asks about past work, prior decisions, people, projects, or events that may be filed in MemPalace, call mempalace_search (or mempalace_kg_query for relational or time-bound facts) before answering from model memory. Return stored content verbatim; never guess when the palace might know.
|
||||
alwaysApply: false
|
||||
---
|
||||
|
||||
# MemPalace recall
|
||||
|
||||
Before answering anything that may already be in the user's memory
|
||||
palace — past work, prior decisions, a person, a project, or "what did
|
||||
we do / decide / discuss last time?" — search the palace first:
|
||||
|
||||
1. Call `mempalace_search` with a short keyword query. Use
|
||||
`mempalace_kg_query` for relational or time-bound facts.
|
||||
2. Quote the drawer's **verbatim** text. Never summarize or paraphrase
|
||||
stored content.
|
||||
3. If results are empty, say so — do not invent an answer. If the MCP
|
||||
server is unavailable, surface the error; do not fall back to guessing.
|
||||
|
||||
Skip recall for pure greenfield work with no memory relevance (renaming
|
||||
a variable, fixing a typo). Recall is question-driven, not reflexive.
|
||||
+47
-1
@@ -2,6 +2,13 @@
|
||||
|
||||
These hook scripts make MemPalace save automatically. No manual "save" commands needed.
|
||||
|
||||
This file covers the **Claude Code** and **Codex CLI** hooks that live
|
||||
flat under `hooks/`. For the **Cursor IDE** hooks, see
|
||||
[`hooks/cursor/README.md`](cursor/README.md) or the rendered docs at
|
||||
[`website/guide/cursor-hooks.md`](../website/guide/cursor-hooks.md). The
|
||||
two are additive and share the same `~/.mempalace/hook_state/`
|
||||
directory.
|
||||
|
||||
If you are trying to protect existing Claude Code transcripts immediately,
|
||||
use the short checklist first: [`website/guide/claude-code-retention.md`](../website/guide/claude-code-retention.md).
|
||||
It covers hook wiring, JSONL backup, and one-time backfill.
|
||||
@@ -11,6 +18,7 @@ It covers hook wiring, JSONL backup, and one-time backfill.
|
||||
| Hook | When It Fires | What Happens |
|
||||
|------|--------------|-------------|
|
||||
| **Save Hook** | Every 15 human messages | Auto-mines transcript (tool output included), then blocks the AI to save topics/decisions/quotes |
|
||||
| **SessionEnd Hook** | Clean session exit | Backgrounds a final transcript mine (when a transcript exists) so short sessions aren't lost; returns immediately so teardown is never delayed. A lightweight diary checkpoint is written in the detached child. |
|
||||
| **PreCompact Hook** | Right before context compaction | Auto-mines transcript, then emergency save — forces the AI to save EVERYTHING before losing context |
|
||||
|
||||
**Two-layer capture:** Hooks auto-mine the JSONL transcript directly into the palace (capturing raw tool output — Bash results, search findings, build errors). They also block the AI with a reason message telling it to save verbatim tool output and key context. Belt and suspenders — tool output gets stored even if the AI summarizes instead of quoting.
|
||||
@@ -30,6 +38,13 @@ Add to `.claude/settings.local.json`:
|
||||
"timeout": 30
|
||||
}]
|
||||
}],
|
||||
"SessionEnd": [{
|
||||
"hooks": [{
|
||||
"type": "command",
|
||||
"command": "/absolute/path/to/hooks/mempal_session_end_hook.sh",
|
||||
"timeout": 10
|
||||
}]
|
||||
}],
|
||||
"PreCompact": [{
|
||||
"hooks": [{
|
||||
"type": "command",
|
||||
@@ -41,11 +56,35 @@ Add to `.claude/settings.local.json`:
|
||||
}
|
||||
```
|
||||
|
||||
`SessionEnd` runs once on a clean exit and backgrounds its work, so it
|
||||
returns instantly and stays within Claude Code's SessionEnd budget. Wired
|
||||
through `settings.local.json` (above) the `timeout` can raise that budget;
|
||||
the bundled plugin cannot, which is why the hook backgrounds rather than
|
||||
mining in the foreground.
|
||||
|
||||
Make them executable:
|
||||
```bash
|
||||
chmod +x hooks/mempal_save_hook.sh hooks/mempal_precompact_hook.sh
|
||||
chmod +x hooks/mempal_save_hook.sh hooks/mempal_session_end_hook.sh hooks/mempal_precompact_hook.sh
|
||||
```
|
||||
|
||||
## Install — Antigravity (Google)
|
||||
|
||||
The Antigravity integration lives in its own subdirectory because the
|
||||
wire format (camelCase JSON, `injectSteps[]` output) and event names
|
||||
(`Stop`, `PreInvocation`) are Antigravity-specific. Use the dedicated
|
||||
installer:
|
||||
|
||||
```bash
|
||||
bash hooks/antigravity/install.sh
|
||||
```
|
||||
|
||||
This installs to `~/.gemini/config/plugins/mempalace/`, registers the
|
||||
MCP server, ships the `mempalace` skill, and wires the Stop +
|
||||
PreInvocation hooks. See [`hooks/antigravity/README.md`](antigravity/README.md)
|
||||
for the full guide and [`hooks/antigravity/INVESTIGATION.md`](antigravity/INVESTIGATION.md)
|
||||
for the source-of-truth audit of which Antigravity surfaces the
|
||||
integration uses.
|
||||
|
||||
## Install — Codex CLI (OpenAI)
|
||||
|
||||
Add to `.codex/hooks.json`:
|
||||
@@ -65,6 +104,13 @@ Add to `.codex/hooks.json`:
|
||||
}
|
||||
```
|
||||
|
||||
**Other harnesses:** the clean-exit save runs through the harness-agnostic
|
||||
`mempalace hook run --hook session-end` entry point. This release wires it
|
||||
for Claude Code. Antigravity exposes no dedicated session-end event (its
|
||||
lifecycle hooks are PreToolUse/PostToolUse/PreInvocation/PostInvocation/Stop,
|
||||
and MemPalace already saves there via `Stop`); Cursor and Codex can adopt the
|
||||
same entry point as a follow-up wherever their own session-end event is available.
|
||||
|
||||
## Configuration
|
||||
|
||||
Edit `mempal_save_hook.sh` to change:
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
# Antigravity IDE — Integration Surface Investigation
|
||||
|
||||
**Investigated**: 2026-05-27
|
||||
**Author**: undeadindustries
|
||||
**Scope**: What MemPalace can integrate with in Google's Antigravity IDE,
|
||||
what we shipped, and what we deliberately did not ship.
|
||||
|
||||
This document is the source of truth for design decisions in the
|
||||
`feat/antigravity-support` branch. It exists so a future maintainer can
|
||||
re-derive every choice without re-reading the docs cold.
|
||||
|
||||
---
|
||||
|
||||
## 1. Surfaces verified against Google's official Antigravity docs
|
||||
|
||||
All quotes are pulled verbatim from Google's official Antigravity
|
||||
documentation on 2026-05-27. URLs are the authoritative source; the
|
||||
mirrored excerpts here are for reviewer convenience.
|
||||
|
||||
### 1.1. MCP — `https://antigravity.google/docs/mcp`
|
||||
|
||||
> The configuration file is located at `~/.gemini/antigravity/mcp_config.json`.
|
||||
>
|
||||
> The configuration file has a single `mcpServers` object where you
|
||||
> define each server you want to connect to.
|
||||
|
||||
Schema (verified):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"<name>": {
|
||||
"command": "...", // stdio
|
||||
"args": [...],
|
||||
"env": {...},
|
||||
"cwd": "...",
|
||||
"serverUrl": "...", // remote
|
||||
"headers": {...},
|
||||
"authProviderType": "google_credentials",
|
||||
"oauth": {"clientId": "...", "clientSecret": "..."},
|
||||
"disabled": false,
|
||||
"disabledTools": [...]
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Per-plugin form: `mcp_config.json` at the plugin root, same shape.
|
||||
Antigravity merges plugin entries with the user's
|
||||
`~/.gemini/antigravity/mcp_config.json` rather than clobbering.
|
||||
|
||||
**Cross-checked locally**: the user's existing
|
||||
`~/.gemini/antigravity/mcp_config.json` already contains a working
|
||||
`"mempalace": {"command": "/Users/robs/.local/bin/mempalace-mcp"}`
|
||||
entry, proving the shape matches and the binary is on PATH.
|
||||
|
||||
### 1.2. Plugins — `https://antigravity.google/docs/plugins`
|
||||
|
||||
> A plugin is a directory containing a `plugin.json` file and optional
|
||||
> subdirectories for different customization types:
|
||||
>
|
||||
> ```
|
||||
> plugins/<plugin-name>/
|
||||
> ├── plugin.json # Required marker file
|
||||
> ├── mcp_config.json # Optional MCP server definitions
|
||||
> ├── hooks.json # Optional hooks definition
|
||||
> ├── skills/ # Optional skills
|
||||
> │ └── <skill-name>/
|
||||
> │ └── SKILL.md
|
||||
> └── rules/ # Optional rules
|
||||
> └── <rule-name>.md
|
||||
> ```
|
||||
|
||||
Manifest schema (verified):
|
||||
|
||||
```json
|
||||
{
|
||||
"name": "my-custom-plugin"
|
||||
}
|
||||
```
|
||||
|
||||
> The name field is optional and defaults to the directory name if omitted.
|
||||
|
||||
Install locations (verified):
|
||||
|
||||
> - Workspace Level: Place your plugin folder inside a
|
||||
> `.agents/plugins/` or `_agents/plugins/` directory at the root of
|
||||
> your opened workspace.
|
||||
> - Global Level: Place your plugin folder inside
|
||||
> `~/.gemini/config/plugins/` in your user home directory.
|
||||
|
||||
We ship the global location as the canonical install path.
|
||||
|
||||
### 1.3. Skills — `https://antigravity.google/docs/skills`
|
||||
|
||||
> A skill is a folder containing a `SKILL.md` file with instructions
|
||||
> that the agent can follow when working on specific tasks.
|
||||
|
||||
Frontmatter (verified):
|
||||
|
||||
| Field | Required | Description |
|
||||
|---------------|----------|---------------------------------------------------------------------------------------------------|
|
||||
| `name` | No | A unique identifier (lowercase, hyphens). Defaults to the folder name. |
|
||||
| `description` | Yes | A clear description of what the skill does and when to use it. |
|
||||
|
||||
Standalone discovery paths (verified):
|
||||
|
||||
| Location | Scope |
|
||||
|-------------------------------------|------------------------|
|
||||
| `<workspace>/.agents/skills/` | Workspace-specific |
|
||||
| `~/.gemini/antigravity/skills/` | Global, all workspaces |
|
||||
|
||||
In-plugin discovery: `<plugin>/skills/<skill-name>/SKILL.md`.
|
||||
|
||||
We ship the in-plugin form so a single install registers MCP, skill, and
|
||||
hooks together.
|
||||
|
||||
### 1.4. Hooks — `https://antigravity.google/docs/hooks?app=antigravity`
|
||||
|
||||
> Hooks allow you to run custom scripts or shell commands at specific
|
||||
> points during Antigravity's execution loop.
|
||||
|
||||
`hooks.json` lives at one of:
|
||||
- `~/.gemini/config/hooks.json` (global)
|
||||
- `<workspace>/.agents/hooks.json` (workspace)
|
||||
- `<plugin-root>/hooks.json` (per-plugin) — what we ship
|
||||
|
||||
Top-level schema (verified):
|
||||
|
||||
```json
|
||||
{
|
||||
"<hook-name>": {
|
||||
"enabled": true,
|
||||
"PreToolUse": [{ "matcher": "...", "hooks": [{...}] }],
|
||||
"PostToolUse": [{ "matcher": "...", "hooks": [{...}] }],
|
||||
"PreInvocation": [{ ... }],
|
||||
"PostInvocation": [{ ... }],
|
||||
"Stop": [{ ... }]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For `PreInvocation` / `PostInvocation` / `Stop`, items are flat handler
|
||||
objects; the `matcher` wrapper is only used for `PreToolUse` /
|
||||
`PostToolUse`.
|
||||
|
||||
Handler object (verified):
|
||||
|
||||
| Field | Required | Description |
|
||||
|-----------|----------|---------------------------------------------------|
|
||||
| `type` | No | Currently only `"command"` is supported. Default. |
|
||||
| `command` | Yes | The shell command to execute. |
|
||||
| `timeout` | No | Timeout in seconds. Defaults to 30. |
|
||||
|
||||
#### STDIN/STDOUT contract (verified)
|
||||
|
||||
> Hooks receive input via stdin as JSON and should return output via
|
||||
> stdout as JSON. Field names use camelCase.
|
||||
|
||||
Common stdin fields (every event):
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------------------------|-----------------|-------------------------------------------------------|
|
||||
| `conversationId` | string | The unique UUID of the active agent conversation. |
|
||||
| `workspacePaths` | array<string> | Absolute directory paths of the user's workspaces. |
|
||||
| `transcriptPath` | string | Absolute path to the persistent `transcript.jsonl`. |
|
||||
| `artifactDirectoryPath` | string | Path to conversation artifacts and screenshots. |
|
||||
|
||||
`Stop` event additional fields:
|
||||
|
||||
| Field | Type | Description |
|
||||
|---------------------|---------|----------------------------------------------------------------------------|
|
||||
| `executionNum` | integer | Sequence number of the execution attempt. |
|
||||
| `terminationReason` | string | `"model_stop"`, `"max_steps_exceeded"`, `"error"`, etc. |
|
||||
| `error` | string | Optional error message. |
|
||||
| `fullyIdle` | boolean | **Required.** True iff all background commands and async tasks are done. |
|
||||
|
||||
`Stop` event stdout (verified):
|
||||
|
||||
| Field | Type | Description |
|
||||
|------------|--------|-------------------------------------------------------------------------------------------------------|
|
||||
| `decision` | string | **Required.** Set to `"continue"` to FORCE the agent to keep running. Anything else allows the stop. |
|
||||
| `reason` | string | Optional. If `decision == "continue"`, injected as a system message. |
|
||||
|
||||
**CRITICAL**: emitting `{"decision": "continue"}` from a save hook would
|
||||
turn it into an infinite agent-loop trigger. The MemPalace save hook
|
||||
MUST emit `{}` on every code path. There is an explicit refusal in
|
||||
`mempal_save_hook_antigravity.sh` to ever print the literal word
|
||||
`"continue"` from a decision field.
|
||||
|
||||
`PreInvocation` event additional fields:
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------------------|---------|----------------------------------------------------------|
|
||||
| `invocationNum` | integer | Sequence number of the current model invocation. |
|
||||
| `initialNumSteps` | integer | Number of steps currently in the trajectory. |
|
||||
|
||||
`PreInvocation` event stdout (verified):
|
||||
|
||||
| Field | Type | Description |
|
||||
|---------------|----------------|-----------------------------------------------------------------------------------------------|
|
||||
| `injectSteps` | array<object> | Optional. Steps injected before the model is called. Each step has one of: |
|
||||
| | | `{ "toolCall": {...} }` / `{ "userMessage": "..." }` / `{ "ephemeralMessage": "..." }` |
|
||||
|
||||
We use `ephemeralMessage` for the wake-up injection: the message is
|
||||
visible to the model on this turn but does not persist to the
|
||||
transcript, so we do not pollute future model calls with the same
|
||||
injection.
|
||||
|
||||
`PreInvocation` fires before EVERY model invocation, not only at session
|
||||
start. We gate to `invocationNum == 1` to mimic Cursor's `sessionStart`
|
||||
semantics — exactly one wake injection per conversation.
|
||||
|
||||
### 1.5. Permissions — `https://antigravity.google/docs/permissions`
|
||||
|
||||
Permissions are user-side, configured via Allow / Deny / Ask lists.
|
||||
Plugins do **not** declare permissions in `plugin.json`. The
|
||||
third-party "antigravity-plugins" community skill at
|
||||
`~/.gemini/skills/antigravity-plugins/SKILL.md` documents a
|
||||
`"permissions": [...]` field in `plugin.json`; that field is fabricated
|
||||
and does not appear in any real Google-shipped plugin (`firebase`,
|
||||
`google-antigravity-sdk`, `chrome-devtools-plugin`,
|
||||
`modern-web-guidance-plugin`) inspected at
|
||||
`~/.gemini/config/plugins/`.
|
||||
|
||||
We ship a minimal `plugin.json` of `{"name": "mempalace"}`.
|
||||
|
||||
---
|
||||
|
||||
## 2. Surfaces shipped
|
||||
|
||||
| Surface | What we ship |
|
||||
|---------------------------|---------------------------------------------------------------------------|
|
||||
| Plugin manifest | `.antigravity-plugin/plugin.json` — minimal, verified shape |
|
||||
| MCP auto-registration | `.antigravity-plugin/mcp_config.json` — registers `mempalace-mcp` stdio |
|
||||
| Skill | `.antigravity-plugin/skills/mempalace/SKILL.md` — real file, frontmatter |
|
||||
| `Stop` hook | `hooks/antigravity/mempal_save_hook_antigravity.sh` — counter + auto-mine |
|
||||
| `PreInvocation` hook | `hooks/antigravity/mempal_wake_hook_antigravity.sh` — wake injection |
|
||||
| Installer | `hooks/antigravity/install.sh` — idempotent, basename-match uninstall |
|
||||
| User-facing docs | `website/guide/antigravity.md` + sidebar wiring |
|
||||
| Examples | `examples/antigravity/{hooks.json,mcp_config.json,README.md}` |
|
||||
| Tests | 3 test files mirroring the cursor blueprint |
|
||||
|
||||
---
|
||||
|
||||
## 3. Surfaces deliberately not shipped
|
||||
|
||||
### 3.1. `PreCompact` equivalent — NOT SHIPPED
|
||||
|
||||
Antigravity's external `hooks.json` does **not** expose a context-compaction event. The Python SDK has an in-process `@hooks.on_compaction`
|
||||
decorator (see `~/.gemini/config/plugins/google-antigravity-sdk/examples/getting_started/hooks.md`),
|
||||
but that fires inside a Python `LocalAgentConfig`-built agent, not the
|
||||
IDE itself. There is no way to subscribe to compaction from a
|
||||
plugin's `hooks.json`.
|
||||
|
||||
UX consequence: long conversations can auto-compact without a save
|
||||
checkpoint. The `Stop` hook still catches the conversation when the
|
||||
user actually ends the turn, so the worst case is that some mid-turn
|
||||
state is lost on auto-compaction. Verbatim transcript ingestion via
|
||||
the `Stop` path covers the long-term recall use case.
|
||||
|
||||
### 3.2. Slash-commands / `commands/` — NOT SHIPPED
|
||||
|
||||
Antigravity has no `commands/` plugin component. The Cursor and Codex
|
||||
integrations both ship five quick-reference commands
|
||||
(`mempalace-help`, `-init`, `-mine`, `-search`, `-status`) that point
|
||||
at `mempalace instructions <cmd>`. Those have been folded into the
|
||||
`SKILL.md` `## Common operations` section so the agent gets the same
|
||||
quick-reference content via Antigravity's progressive-disclosure skill
|
||||
loading. No new files, no rule-noise, same discoverability.
|
||||
|
||||
### 3.3. `rules/` — NOT SHIPPED
|
||||
|
||||
`rules/<name>.md` files are evaluated as constraints on the agent's
|
||||
behavior. Shipping rules from MemPalace risks colliding with the
|
||||
user's existing project rules (e.g. `.agents/rules/*.md` files the
|
||||
user has already authored). Users who want strict MemPalace-related
|
||||
rules can drop them into their own `<workspace>/.agents/rules/`
|
||||
directory; we do not impose them.
|
||||
|
||||
### 3.4. Workspace-level `.agents/plugins/mempalace/` install — NOT SHIPPED BY DEFAULT
|
||||
|
||||
The installer writes to the global location at
|
||||
`~/.gemini/config/plugins/mempalace/`. Workspace-scoped installs are
|
||||
documented in `hooks/antigravity/README.md` for users who want to
|
||||
limit MemPalace to one workspace; they can `cp -r .antigravity-plugin
|
||||
<workspace>/.agents/plugins/mempalace`. We do not install there
|
||||
automatically because the canonical UX is global.
|
||||
|
||||
### 3.5. `permissions` field in `plugin.json` — NOT SHIPPED
|
||||
|
||||
Antigravity permissions are user-side (`Allow` / `Deny` / `Ask`
|
||||
lists). Plugin manifests do not declare permissions. The
|
||||
`"permissions": [...]` field documented in the third-party
|
||||
"antigravity-plugins" community skill is fabricated; no
|
||||
Google-shipped plugin uses it.
|
||||
|
||||
### 3.6. `PreToolUse` / `PostToolUse` hooks — NOT SHIPPED
|
||||
|
||||
These would let MemPalace observe every tool call (e.g.
|
||||
auto-extract entities after each `write_to_file`). Out of scope
|
||||
for v1; the hook surface is real and could be added in a future
|
||||
PR if there is demand. Documenting the omission here so a future
|
||||
contributor doesn't conclude the hooks weren't supported.
|
||||
|
||||
---
|
||||
|
||||
## 4. Cross-checks against the user's running Antigravity
|
||||
|
||||
The user (`robs@`) has Antigravity 2.0 installed. The following live
|
||||
artifacts on this machine corroborate the published docs:
|
||||
|
||||
| Path | Confirms |
|
||||
|------------------------------------------------------------------------|----------------------------------------------------------------|
|
||||
| `~/.gemini/antigravity/mcp_config.json` | MCP config path + standard `mcpServers` shape |
|
||||
| `~/.gemini/antigravity/skills/<skill-name>/SKILL.md` (multiple) | Global skill discovery path |
|
||||
| `~/.gemini/config/plugins/firebase/plugin.json` | Real `plugin.json` shape (no `permissions` field) |
|
||||
| `~/.gemini/config/plugins/chrome-devtools-plugin/skills/.../SKILL.md` | In-plugin skill discovery `<plugin>/skills/<name>/SKILL.md` |
|
||||
| `~/.gemini/config/plugins/google-antigravity-sdk/examples/.../hooks.md` | SDK-side compaction hook is in-process Python only |
|
||||
| Existing `mempalace` entry in `mcp_config.json` | `mempalace-mcp` already running and discoverable |
|
||||
|
||||
---
|
||||
|
||||
## 5. Reference URLs (all 2026-05-27)
|
||||
|
||||
- `https://antigravity.google/docs/plugins`
|
||||
- `https://antigravity.google/docs/hooks?app=antigravity`
|
||||
- `https://antigravity.google/docs/skills`
|
||||
- `https://antigravity.google/docs/mcp`
|
||||
- `https://antigravity.google/docs/permissions`
|
||||
- `https://antigravity.google/docs/subagents`
|
||||
- `https://antigravity.google/blog/introducing-google-antigravity-sdk`
|
||||
@@ -0,0 +1,172 @@
|
||||
# MemPalace — Antigravity hook scripts
|
||||
|
||||
Lifecycle hooks for the [Antigravity IDE](https://antigravity.google/).
|
||||
|
||||
This is the third sibling of the Claude Code and Codex integrations
|
||||
(see `hooks/mempal_save_hook.sh` and `.codex-plugin/hooks/`). The
|
||||
overall shape is the same — a Stop event triggers a background save,
|
||||
a startup-time event injects memory into the agent — but the wire
|
||||
format and STDOUT contract are Antigravity-specific (see
|
||||
[STDIN_SHAPE.md](STDIN_SHAPE.md)).
|
||||
|
||||
## Quick start
|
||||
|
||||
From the repo root:
|
||||
|
||||
```bash
|
||||
bash hooks/antigravity/install.sh
|
||||
```
|
||||
|
||||
This installs the plugin to `~/.gemini/config/plugins/mempalace/`.
|
||||
Restart Antigravity and the MCP server, skill, and hooks all register
|
||||
automatically.
|
||||
|
||||
To dry-run first:
|
||||
|
||||
```bash
|
||||
bash hooks/antigravity/install.sh --dry-run
|
||||
```
|
||||
|
||||
To uninstall:
|
||||
|
||||
```bash
|
||||
bash hooks/antigravity/install.sh --uninstall
|
||||
```
|
||||
|
||||
## What gets installed
|
||||
|
||||
```
|
||||
~/.gemini/config/plugins/mempalace/
|
||||
├── plugin.json # marker manifest
|
||||
├── mcp_config.json # registers mempalace-mcp
|
||||
├── hooks.json # rendered from hooks.json.tmpl
|
||||
├── README.md
|
||||
├── skills/
|
||||
│ └── mempalace/
|
||||
│ └── SKILL.md
|
||||
└── hooks/
|
||||
├── lib/
|
||||
│ └── common.sh
|
||||
├── mempal_save_hook_antigravity.sh # Stop event handler
|
||||
└── mempal_wake_hook_antigravity.sh # PreInvocation handler
|
||||
```
|
||||
|
||||
`hooks.json` carries absolute paths to the two hook scripts (resolved
|
||||
from `__PLUGIN_DIR__` at install time).
|
||||
|
||||
## What the hooks do
|
||||
|
||||
### `mempal_save_hook_antigravity.sh` (Stop event)
|
||||
|
||||
Fires every time the agent's execution loop terminates. Increments a
|
||||
per-conversation counter; every `MEMPAL_SAVE_INTERVAL` fires (default
|
||||
15), spawns `mempalace mine <transcript-dir> --mode convos --wing
|
||||
<inferred>` in the background. The hook itself returns `{}` to stdout
|
||||
in under a few milliseconds — the actual mining runs detached and
|
||||
does not block the user.
|
||||
|
||||
Defers when:
|
||||
|
||||
- `fullyIdle == false` (background tasks still running)
|
||||
- `terminationReason == "error"` (transcript may be corrupt)
|
||||
- A previous save for this conversation is still running
|
||||
- Any kill switch is set
|
||||
|
||||
### `mempal_wake_hook_antigravity.sh` (PreInvocation event, gated)
|
||||
|
||||
Fires before every model invocation. Gated to `invocationNum == 1`
|
||||
(first invocation of the conversation only) — beyond that we'd be
|
||||
re-injecting on every turn. Calls `mempalace wake-up --wing <inferred>`
|
||||
with a 500ms hard timeout and emits the verbatim output as an
|
||||
`ephemeralMessage` so the agent sees relevant memory on its first
|
||||
response without polluting the persistent transcript.
|
||||
|
||||
Skips when:
|
||||
|
||||
- `invocationNum != 1`
|
||||
- Already woke this conversation (atomic `mkdir` loop guard)
|
||||
- `mempalace wake-up` exits non-zero, times out, or produces empty output
|
||||
- Any kill switch is set
|
||||
|
||||
## Kill switches
|
||||
|
||||
Any one of these disables both hooks (silent passthrough, exit 0):
|
||||
|
||||
| Knob | Value |
|
||||
|------------------------------------------|--------------------------------|
|
||||
| `MEMPAL_DISABLE_HOOK` | `1`, `true`, `yes` |
|
||||
| `MEMPALACE_HOOKS_AUTO_SAVE` | `false`, `0`, `no` |
|
||||
| `~/.mempalace/config.json` | `{"hooks": {"auto_save": false}}` |
|
||||
| Removing `~/.mempalace/` entirely | (palace nuke) |
|
||||
|
||||
## Workspace-scoped install (advanced)
|
||||
|
||||
If you want MemPalace to load only inside a specific workspace,
|
||||
manually copy the rendered plugin into your workspace's `.agents/plugins/`:
|
||||
|
||||
```bash
|
||||
bash hooks/antigravity/install.sh --install-dir /tmp/render-stage
|
||||
mkdir -p <workspace>/.agents/plugins/
|
||||
cp -r /tmp/render-stage <workspace>/.agents/plugins/mempalace
|
||||
rm -rf /tmp/render-stage
|
||||
```
|
||||
|
||||
The global install at `~/.gemini/config/plugins/mempalace/` is the
|
||||
canonical UX and what we recommend.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Hooks aren't firing
|
||||
|
||||
1. Confirm Antigravity sees the plugin: open the IDE, navigate to the
|
||||
Customizations page; `mempalace` should appear in the global plugins
|
||||
list.
|
||||
2. Check `~/.mempalace/hook_state/antigravity_hook.log` — every fire
|
||||
logs a line. No log lines = the hook is not being invoked.
|
||||
3. Verify `mempalace-mcp` is on `$PATH`: `mempalace-mcp --version`.
|
||||
4. Inspect the rendered `hooks.json` paths point at executable files:
|
||||
`bash -n ~/.gemini/config/plugins/mempalace/hooks/*.sh`.
|
||||
|
||||
### Save fires but no mining happens
|
||||
|
||||
1. Look for the most recent `[event=stop]` lines in
|
||||
`antigravity_hook.log` — `count` and `interval` should both be
|
||||
visible. Mining only triggers when `count % interval == 0`.
|
||||
2. Ensure a Python that can import `mempalace` is reachable. The hook
|
||||
runs `"$MEMPAL_PYTHON_BIN" -m mempalace`, where `MEMPAL_PYTHON_BIN`
|
||||
is resolved (in order) from `$MEMPAL_PYTHON`, the
|
||||
`mempalace-mcp` / `mempalace` console-script shebang on `$PATH`,
|
||||
then `python3`. A failed probe logs:
|
||||
|
||||
```
|
||||
ERROR: mempalace is not runnable via <python> -m mempalace; install mempalace or set MEMPAL_PYTHON
|
||||
```
|
||||
|
||||
On a GUI-launched Antigravity the harness `PATH` may differ from
|
||||
your shell `PATH`; if the shebang heuristic can't find the right
|
||||
interpreter, export `MEMPAL_PYTHON=/abs/path/python` (e.g.
|
||||
`"$(uv tool dir)/mempalace/bin/python"`) and restart.
|
||||
|
||||
### Wake injection isn't appearing
|
||||
|
||||
1. The wake hook only injects on `invocationNum == 1`. Subsequent
|
||||
invocations are gated.
|
||||
2. The atomic `mkdir` marker
|
||||
`~/.mempalace/hook_state/antigravity_woke_<conversationId>` exists
|
||||
after a successful injection. Remove it to re-inject (rare).
|
||||
3. `mempalace wake-up --wing <inferred>` may be returning empty output
|
||||
if the wing doesn't exist yet. Run `mempalace status` to verify
|
||||
wing presence.
|
||||
|
||||
## See also
|
||||
|
||||
- [INVESTIGATION.md](INVESTIGATION.md) — every Antigravity surface we
|
||||
investigated, with verbatim quotes and source URLs.
|
||||
- [STDIN_SHAPE.md](STDIN_SHAPE.md) — the exact wire format
|
||||
Antigravity uses, with worked examples.
|
||||
- [../mempal_save_hook.sh](../mempal_save_hook.sh) — Claude Code
|
||||
equivalent.
|
||||
- [../../.codex-plugin/hooks/](../../.codex-plugin/hooks/) — Codex
|
||||
equivalent.
|
||||
- [../../website/guide/antigravity.md](../../website/guide/antigravity.md)
|
||||
— full user-facing guide.
|
||||
@@ -0,0 +1,206 @@
|
||||
# Antigravity hook STDIN / STDOUT contract
|
||||
|
||||
This file documents the exact wire format the Antigravity IDE uses
|
||||
when invoking the MemPalace hook scripts. All fields are verbatim
|
||||
from Google's official Antigravity hooks documentation
|
||||
(`https://antigravity.google/docs/hooks?app=antigravity`, accessed
|
||||
2026-05-27). See [INVESTIGATION.md](INVESTIGATION.md) for the
|
||||
provenance audit.
|
||||
|
||||
## Wire format
|
||||
|
||||
Hooks receive **JSON on stdin** and must emit **JSON on stdout**.
|
||||
Field names are **camelCase**.
|
||||
|
||||
Hook execution timeout defaults to 30 seconds. The MemPalace plugin
|
||||
sets the Stop hook timeout to 30s and the PreInvocation hook timeout
|
||||
to 5s in the rendered `hooks.json`.
|
||||
|
||||
## Common stdin fields (every event)
|
||||
|
||||
| Field | Type | Notes |
|
||||
|-------------------------|-----------------|--------------------------------------------------------|
|
||||
| `conversationId` | string | UUID of the active agent conversation. |
|
||||
| `workspacePaths` | array<string> | Absolute workspace dirs. **First element is canonical**. |
|
||||
| `transcriptPath` | string | Absolute path to `transcript.jsonl`. |
|
||||
| `artifactDirectoryPath` | string | Path to conversation artifacts and screenshots. |
|
||||
|
||||
## Stop event
|
||||
|
||||
### Stdin (additional fields)
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---------------------|---------|--------------------------------------------------------------------------------------|
|
||||
| `executionNum` | integer | Sequence number of the execution attempt for this conversation. |
|
||||
| `terminationReason` | string | `"model_stop"`, `"max_steps_exceeded"`, `"error"`, etc. |
|
||||
| `error` | string | Optional. Set when termination was caused by a system error. |
|
||||
| `fullyIdle` | boolean | **Required.** True iff all background commands and async tasks have completed. |
|
||||
|
||||
### Stdout
|
||||
|
||||
| Field | Type | Notes |
|
||||
|------------|--------|--------------------------------------------------------------------------------------------------------|
|
||||
| `decision` | string | If `"continue"`, **forces** the agent to keep running. Anything else allows the stop. |
|
||||
| `reason` | string | Optional. If `decision == "continue"`, injected as a system message into the conversation. |
|
||||
|
||||
**MemPalace policy**: the save hook ALWAYS emits `{}` and exits 0. It
|
||||
NEVER emits `{"decision": "continue"}` — that would force an infinite
|
||||
agent loop. There is an explicit refusal in
|
||||
`mempal_save_hook_antigravity.sh` to ever construct a stdout JSON
|
||||
object containing the literal word `"continue"` in a decision field.
|
||||
|
||||
### MemPalace gating
|
||||
|
||||
The save hook short-circuits with `{}` (no save triggered) when ANY
|
||||
of the following hold:
|
||||
|
||||
1. `MEMPAL_DISABLE_HOOK=1` (or `true`/`yes`) is set.
|
||||
2. `MEMPALACE_HOOKS_AUTO_SAVE=false` (or `0`/`no`) is set.
|
||||
3. `~/.mempalace/config.json` has `hooks.auto_save: false`.
|
||||
4. `~/.mempalace/` directory does not exist (user nuked the palace).
|
||||
5. Stdin is malformed or empty (sentinel-guarded parse failure).
|
||||
6. `fullyIdle == false` (background tasks still running; defer save).
|
||||
7. `terminationReason == "error"` (transcript may be corrupt).
|
||||
8. `transcriptPath` validation fails (not a `.json`/`.jsonl`, or `..` traversal).
|
||||
9. The transcript file does not exist on disk.
|
||||
10. The save counter has not yet hit `count % MEMPAL_SAVE_INTERVAL == 0`.
|
||||
11. A pending save is still running for this conversation (less than 1 hour old).
|
||||
12. The `mempalace` CLI is not on `$PATH`.
|
||||
|
||||
When the modulo gate is hit and validation passes, the hook spawns
|
||||
`mempalace mine <transcript-dir> --mode convos --wing <inferred>` in
|
||||
the background and returns `{}` immediately.
|
||||
|
||||
## PreInvocation event
|
||||
|
||||
### Stdin (additional fields)
|
||||
|
||||
| Field | Type | Notes |
|
||||
|-------------------|---------|------------------------------------------------------------------|
|
||||
| `invocationNum` | integer | Sequence number of the current model invocation (1-based). |
|
||||
| `initialNumSteps` | integer | Number of steps currently in the trajectory. |
|
||||
|
||||
### Stdout
|
||||
|
||||
| Field | Type | Notes |
|
||||
|---------------|----------------|--------------------------------------------------------------------------------------------------------|
|
||||
| `injectSteps` | array<object> | Optional. Steps to inject before the model is called. Each step has one of: `{"toolCall": {...}}`, `{"userMessage": "..."}`, `{"ephemeralMessage": "..."}` |
|
||||
|
||||
The `ephemeralMessage` form is what the MemPalace wake hook emits — it
|
||||
delivers the wake-up text to the model on this turn but does not
|
||||
persist into the transcript, so subsequent invocations don't see a
|
||||
duplicate.
|
||||
|
||||
### MemPalace gating
|
||||
|
||||
The wake hook short-circuits with `{}` (no injection) when ANY of:
|
||||
|
||||
1. Any kill switch trips (same five conditions as the save hook).
|
||||
2. `invocationNum != 1` — we only inject on the first model call of
|
||||
each conversation, mimicking Cursor's `sessionStart` semantics.
|
||||
3. The atomic `mkdir`-based loop guard is already taken (this
|
||||
conversation already received a wake injection).
|
||||
4. `mempalace wake-up --wing <inferred>` exits non-zero, times out
|
||||
(500ms hard cap), or produces empty output.
|
||||
|
||||
When the gates pass and `mempalace wake-up` returns text, the hook
|
||||
emits:
|
||||
|
||||
```json
|
||||
{
|
||||
"injectSteps": [
|
||||
{ "ephemeralMessage": "<verbatim wake-up output>" }
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The wake hook NEVER emits a `decision` field — that field belongs to
|
||||
the Stop event. There is a final guard against any stdout that
|
||||
contains a `decision` key.
|
||||
|
||||
## Worked example: Stop event
|
||||
|
||||
### Input
|
||||
|
||||
```json
|
||||
{
|
||||
"executionNum": 1,
|
||||
"terminationReason": "model_stop",
|
||||
"error": "",
|
||||
"fullyIdle": true,
|
||||
"conversationId": "ec33ebf9-0cba-4100-8142-c61503f6c587",
|
||||
"workspacePaths": ["/home/me/projects/mempalace"],
|
||||
"transcriptPath": "/home/me/projects/mempalace/.gemini/jetski/transcript.jsonl",
|
||||
"artifactDirectoryPath": "/home/me/projects/mempalace/.gemini/jetski/artifacts"
|
||||
}
|
||||
```
|
||||
|
||||
### Output (always)
|
||||
|
||||
```json
|
||||
{}
|
||||
```
|
||||
|
||||
(Side effects: counter `~/.mempalace/hook_state/antigravity_save_count_<id>` is
|
||||
incremented; if the modulo gate fires, a background `mempalace mine`
|
||||
subprocess is spawned with the transcript directory and the inferred
|
||||
wing `wing_mempalace`.)
|
||||
|
||||
## Worked example: PreInvocation, first invocation
|
||||
|
||||
### Input
|
||||
|
||||
```json
|
||||
{
|
||||
"invocationNum": 1,
|
||||
"initialNumSteps": 0,
|
||||
"conversationId": "ec33ebf9-0cba-4100-8142-c61503f6c587",
|
||||
"workspacePaths": ["/home/me/projects/mempalace"],
|
||||
"transcriptPath": "/home/me/projects/mempalace/.gemini/jetski/transcript.jsonl",
|
||||
"artifactDirectoryPath": "/home/me/projects/mempalace/.gemini/jetski/artifacts"
|
||||
}
|
||||
```
|
||||
|
||||
### Output (when the palace has memory for `wing_mempalace`)
|
||||
|
||||
```json
|
||||
{
|
||||
"injectSteps": [
|
||||
{
|
||||
"ephemeralMessage": "<exact verbatim text from `mempalace wake-up --wing wing_mempalace`>"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### Output (when invocationNum != 1, or any gate trips)
|
||||
|
||||
```json
|
||||
{}
|
||||
```
|
||||
|
||||
## State files
|
||||
|
||||
All hook state lives under `~/.mempalace/hook_state/` (overridable
|
||||
via `$MEMPAL_STATE_DIR`) and is namespaced with the `antigravity_`
|
||||
prefix to coexist with Claude Code, Cursor, and Codex hook state in
|
||||
the same directory.
|
||||
|
||||
| File | Purpose |
|
||||
|-----------------------------------------------|-----------------------------------------------|
|
||||
| `antigravity_hook.log` | All hook activity, ISO8601Z timestamps. |
|
||||
| `antigravity_save_count_<conversationId>` | Per-conversation Stop counter. |
|
||||
| `antigravity_pending_<conversationId>` | Marker file for in-flight save subprocess. |
|
||||
| `antigravity_woke_<conversationId>` (dir) | Atomic mkdir marker for wake injection. |
|
||||
| `antigravity_last_input.log` | 4 KB cap, mode 0600, set on parse failure. |
|
||||
| `antigravity_last_python_err.log` | Python stderr from the JSON parser, mode 0600.|
|
||||
|
||||
## Environment variables
|
||||
|
||||
| Variable | Default | Purpose |
|
||||
|--------------------------------|----------------------|-----------------------------------------------------------|
|
||||
| `MEMPAL_PYTHON` | `$(command -v python3)` | Override the Python interpreter used by the hooks. |
|
||||
| `MEMPAL_STATE_DIR` | `~/.mempalace/hook_state` | Override the hook state directory. |
|
||||
| `MEMPAL_SAVE_INTERVAL` | `15` | Save every Nth Stop fire. Floored to >= 1 (no /0). |
|
||||
| `MEMPAL_DISABLE_HOOK` | unset | Set to `1` / `true` / `yes` to disable both hooks. |
|
||||
| `MEMPALACE_HOOKS_AUTO_SAVE` | unset | Set to `false` / `0` / `no` to disable both hooks. |
|
||||
Executable
+357
@@ -0,0 +1,357 @@
|
||||
#!/bin/bash
|
||||
# MEMPALACE ANTIGRAVITY INSTALLER
|
||||
#
|
||||
# Idempotent installer for the Antigravity plugin. Copies
|
||||
# .antigravity-plugin/* and hooks/antigravity/{lib,*.sh} into the
|
||||
# install directory (default ~/.gemini/config/plugins/mempalace/),
|
||||
# renders hooks.json.tmpl into hooks.json with absolute paths, and
|
||||
# leaves the result in a state Antigravity will discover on next
|
||||
# launch.
|
||||
#
|
||||
# === Usage ===
|
||||
#
|
||||
# bash hooks/antigravity/install.sh # install with defaults
|
||||
# bash hooks/antigravity/install.sh --dry-run # show what would happen
|
||||
# bash hooks/antigravity/install.sh --uninstall # remove plugin
|
||||
# bash hooks/antigravity/install.sh --install-dir <p> # custom install dir
|
||||
# bash hooks/antigravity/install.sh --log-level debug # noisier output
|
||||
#
|
||||
# === Idempotency ===
|
||||
#
|
||||
# Re-running the installer produces a byte-identical install dir.
|
||||
# Files are only written when their content differs from what is
|
||||
# already on disk (cmp gate). The user's ~/.gemini/config/plugins/
|
||||
# directory is never touched outside the mempalace/ subdirectory.
|
||||
#
|
||||
# === Uninstall safety ===
|
||||
#
|
||||
# Uninstall removes the install dir entirely IFF it is the
|
||||
# mempalace/ plugin directory. We match by basename of the install
|
||||
# dir, never by substring search, so a user who has a sibling plugin
|
||||
# at ~/.gemini/config/plugins/mempalace-foo/ is unaffected.
|
||||
#
|
||||
# === set -e ===
|
||||
#
|
||||
# Installer can use `set -e` (constraint #2 only forbids it in the
|
||||
# hook scripts themselves). On any error we exit non-zero so a CI run
|
||||
# fails loudly.
|
||||
|
||||
set -e
|
||||
set -u
|
||||
|
||||
# ── Repo root resolution ─────────────────────────────────────────────
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd -P)"
|
||||
PLUGIN_SRC="$REPO_ROOT/.antigravity-plugin"
|
||||
HOOKS_SRC="$REPO_ROOT/hooks/antigravity"
|
||||
|
||||
# ── Defaults ─────────────────────────────────────────────────────────
|
||||
INSTALL_DIR_DEFAULT="$HOME/.gemini/config/plugins/mempalace"
|
||||
INSTALL_DIR=""
|
||||
DRY_RUN=0
|
||||
UNINSTALL=0
|
||||
LOG_LEVEL="info"
|
||||
|
||||
# ── Args ─────────────────────────────────────────────────────────────
|
||||
print_usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: install.sh [--install-dir DIR] [--dry-run] [--uninstall] [--log-level LEVEL]
|
||||
|
||||
Options:
|
||||
--install-dir DIR Plugin install directory.
|
||||
Default: ~/.gemini/config/plugins/mempalace
|
||||
--dry-run Show what would happen without writing anything.
|
||||
--uninstall Remove the installed plugin.
|
||||
--log-level LEVEL debug | info | warn | error. Default: info.
|
||||
-h, --help Show this help.
|
||||
USAGE
|
||||
}
|
||||
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--install-dir)
|
||||
INSTALL_DIR="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--install-dir=*)
|
||||
INSTALL_DIR="${1#*=}"
|
||||
shift
|
||||
;;
|
||||
--dry-run)
|
||||
DRY_RUN=1
|
||||
shift
|
||||
;;
|
||||
--uninstall)
|
||||
UNINSTALL=1
|
||||
shift
|
||||
;;
|
||||
--log-level)
|
||||
LOG_LEVEL="${2:-info}"
|
||||
shift 2
|
||||
;;
|
||||
--log-level=*)
|
||||
LOG_LEVEL="${1#*=}"
|
||||
shift
|
||||
;;
|
||||
-h|--help)
|
||||
print_usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: unknown argument: $1" >&2
|
||||
print_usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ -z "$INSTALL_DIR" ]; then
|
||||
INSTALL_DIR="$INSTALL_DIR_DEFAULT"
|
||||
fi
|
||||
|
||||
# ── Absolutize the install dir ───────────────────────────────────────
|
||||
#
|
||||
# The cursor PR review caught that a relative --install-dir would get
|
||||
# baked into hooks.json verbatim, leaving paths like
|
||||
# `./plugins/.../mempal_save_hook_antigravity.sh` that Antigravity
|
||||
# can't resolve at runtime. Absolutize before writing anything.
|
||||
mempal_absolutize() {
|
||||
local p="$1"
|
||||
case "$p" in
|
||||
/*) printf '%s' "$p" ;;
|
||||
~*) printf '%s' "${p/#\~/$HOME}" ;;
|
||||
*)
|
||||
# Resolve relative to the user's $PWD at invocation time.
|
||||
# We never `cd` in the main shell of this installer, so
|
||||
# $PWD is already the user's invocation directory — no
|
||||
# subshell cd dance needed.
|
||||
local base="${PWD}"
|
||||
printf '%s/%s' "$base" "$p"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
INSTALL_DIR="$(mempal_absolutize "$INSTALL_DIR")"
|
||||
# Squash any `//` or `./` or `name/..` artefacts using Python's
|
||||
# os.path.normpath; falls back to the raw value if Python is missing
|
||||
# (which would be very unusual on macOS / Linux).
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
INSTALL_DIR="$(python3 -c 'import os,sys; print(os.path.normpath(sys.argv[1]))' "$INSTALL_DIR")"
|
||||
fi
|
||||
|
||||
# ── Logging ──────────────────────────────────────────────────────────
|
||||
log() {
|
||||
local lvl="$1"; shift
|
||||
local msg="$*"
|
||||
case "$lvl" in
|
||||
debug)
|
||||
[ "$LOG_LEVEL" = "debug" ] && echo "[install] DEBUG: $msg"
|
||||
return 0
|
||||
;;
|
||||
info)
|
||||
case "$LOG_LEVEL" in
|
||||
debug|info) echo "[install] $msg" ;;
|
||||
esac
|
||||
;;
|
||||
warn)
|
||||
case "$LOG_LEVEL" in
|
||||
debug|info|warn) echo "[install] WARN: $msg" >&2 ;;
|
||||
esac
|
||||
;;
|
||||
error)
|
||||
echo "[install] ERROR: $msg" >&2
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ── Action helpers (dry-run aware) ──────────────────────────────────
|
||||
run() {
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
echo "[install] DRY-RUN: $*"
|
||||
return 0
|
||||
fi
|
||||
"$@"
|
||||
}
|
||||
|
||||
# ── Render template ──────────────────────────────────────────────────
|
||||
#
|
||||
# Substitutes __PLUGIN_DIR__ in $src into $dst with $INSTALL_DIR.
|
||||
# Emits the rendered file to a temp path first, then promotes it iff
|
||||
# the content differs from what's already at $dst. The cmp gate is
|
||||
# what makes the installer idempotent: a no-op re-run produces no
|
||||
# disk writes (and the test suite asserts byte-equality).
|
||||
render_template() {
|
||||
local src="$1"
|
||||
local dst="$2"
|
||||
if [ ! -f "$src" ]; then
|
||||
log error "template not found: $src"
|
||||
return 1
|
||||
fi
|
||||
local tmp
|
||||
tmp="$(mktemp "${TMPDIR:-/tmp}/mempal_agy_render.XXXXXX")"
|
||||
# Python over awk/sed — INSTALL_DIR may legitimately contain
|
||||
# characters (spaces, colons) that would require careful escaping
|
||||
# in a sed s/// replacement. Python read+replace handles all of
|
||||
# them uniformly.
|
||||
python3 -c "
|
||||
import sys
|
||||
src, dst, install_dir = sys.argv[1:4]
|
||||
with open(src, 'r') as f:
|
||||
body = f.read()
|
||||
body = body.replace('__PLUGIN_DIR__', install_dir)
|
||||
with open(dst, 'w') as f:
|
||||
f.write(body)
|
||||
" "$src" "$tmp" "$INSTALL_DIR"
|
||||
if [ -f "$dst" ] && cmp -s "$tmp" "$dst"; then
|
||||
rm -f "$tmp"
|
||||
log debug "unchanged: $dst"
|
||||
return 0
|
||||
fi
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
echo "[install] DRY-RUN: would render $src -> $dst"
|
||||
rm -f "$tmp"
|
||||
return 0
|
||||
fi
|
||||
mv "$tmp" "$dst"
|
||||
log info "wrote: $dst"
|
||||
}
|
||||
|
||||
# ── copy_file: cmp-gated copy that preserves mode ────────────────────
|
||||
copy_file() {
|
||||
local src="$1"
|
||||
local dst="$2"
|
||||
if [ ! -f "$src" ]; then
|
||||
log error "missing source file: $src"
|
||||
return 1
|
||||
fi
|
||||
if [ -f "$dst" ] && cmp -s "$src" "$dst"; then
|
||||
log debug "unchanged: $dst"
|
||||
return 0
|
||||
fi
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
echo "[install] DRY-RUN: would copy $src -> $dst"
|
||||
return 0
|
||||
fi
|
||||
mkdir -p "$(dirname "$dst")"
|
||||
cp "$src" "$dst"
|
||||
log info "wrote: $dst"
|
||||
}
|
||||
|
||||
# ── Uninstall path ───────────────────────────────────────────────────
|
||||
#
|
||||
# We DO NOT remove the install dir by string-substring match against
|
||||
# the path. We require the install dir's basename to be exactly
|
||||
# "mempalace" — that way an unrelated sibling like
|
||||
# ~/.gemini/config/plugins/mempalace-foo/ is left alone, and a
|
||||
# malformed --install-dir like ~ or / cannot wipe the user's home.
|
||||
do_uninstall() {
|
||||
local base
|
||||
base="$(basename "$INSTALL_DIR")"
|
||||
if [ "$base" != "mempalace" ]; then
|
||||
log error "refusing to uninstall: install dir basename is '$base', expected 'mempalace'"
|
||||
log error "(safety guard: prevents accidental wipe of unrelated directories)"
|
||||
return 1
|
||||
fi
|
||||
if [ ! -d "$INSTALL_DIR" ]; then
|
||||
log info "nothing to uninstall: $INSTALL_DIR does not exist"
|
||||
return 0
|
||||
fi
|
||||
# Verify the dir LOOKS like our plugin before removing — a
|
||||
# plugin.json file with our marker is the proof.
|
||||
if [ ! -f "$INSTALL_DIR/plugin.json" ]; then
|
||||
log error "refusing to uninstall: $INSTALL_DIR has no plugin.json"
|
||||
return 1
|
||||
fi
|
||||
if ! grep -q '"name"[[:space:]]*:[[:space:]]*"mempalace"' "$INSTALL_DIR/plugin.json" 2>/dev/null; then
|
||||
log error "refusing to uninstall: $INSTALL_DIR/plugin.json is not a mempalace plugin"
|
||||
return 1
|
||||
fi
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
echo "[install] DRY-RUN: would rm -rf $INSTALL_DIR"
|
||||
return 0
|
||||
fi
|
||||
rm -rf "$INSTALL_DIR"
|
||||
log info "uninstalled: $INSTALL_DIR"
|
||||
}
|
||||
|
||||
if [ "$UNINSTALL" -eq 1 ]; then
|
||||
do_uninstall
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Pre-install sanity ───────────────────────────────────────────────
|
||||
if [ ! -d "$PLUGIN_SRC" ]; then
|
||||
log error "missing source: $PLUGIN_SRC"
|
||||
exit 1
|
||||
fi
|
||||
if [ ! -d "$HOOKS_SRC" ]; then
|
||||
log error "missing source: $HOOKS_SRC"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Soft-check that mempalace-mcp is on PATH; warn but do not fail.
|
||||
if ! command -v mempalace-mcp >/dev/null 2>&1; then
|
||||
log warn "mempalace-mcp is not on PATH; the MCP server will fail to start until it is."
|
||||
log warn " fix: 'uv tool install mempalace' or 'pip install mempalace'"
|
||||
fi
|
||||
|
||||
# Soft-check ~/.gemini exists; if missing, Antigravity isn't installed.
|
||||
if [ ! -d "$HOME/.gemini" ]; then
|
||||
log warn "$HOME/.gemini not found — Antigravity is probably not installed yet."
|
||||
log warn " the install will still proceed; Antigravity will pick up the plugin on first launch."
|
||||
fi
|
||||
|
||||
log info "install dir: $INSTALL_DIR"
|
||||
|
||||
# ── Install: directories ─────────────────────────────────────────────
|
||||
run mkdir -p "$INSTALL_DIR" \
|
||||
"$INSTALL_DIR/skills/mempalace" \
|
||||
"$INSTALL_DIR/skills/mempalace-recall" \
|
||||
"$INSTALL_DIR/rules" \
|
||||
"$INSTALL_DIR/hooks" \
|
||||
"$INSTALL_DIR/hooks/lib"
|
||||
|
||||
# ── Install: plugin metadata ─────────────────────────────────────────
|
||||
copy_file "$PLUGIN_SRC/plugin.json" "$INSTALL_DIR/plugin.json"
|
||||
copy_file "$PLUGIN_SRC/mcp_config.json" "$INSTALL_DIR/mcp_config.json"
|
||||
copy_file "$PLUGIN_SRC/README.md" "$INSTALL_DIR/README.md"
|
||||
|
||||
# ── Install: skills (real files, no symlinks at the discovery path) ──
|
||||
copy_file "$PLUGIN_SRC/skills/mempalace/SKILL.md" \
|
||||
"$INSTALL_DIR/skills/mempalace/SKILL.md"
|
||||
copy_file "$PLUGIN_SRC/skills/mempalace-recall/SKILL.md" \
|
||||
"$INSTALL_DIR/skills/mempalace-recall/SKILL.md"
|
||||
|
||||
# ── Install: optional recall rule ────────────────────────────────────
|
||||
#
|
||||
# Antigravity discovers markdown rules under the plugin's rules/
|
||||
# directory. This one is recall-only and intentionally lightweight —
|
||||
# it complements the mempalace-recall skill. Shipping it as a plugin
|
||||
# rule (not an always-on global rule) keeps it scoped to recall-
|
||||
# relevant turns, honouring MemPalace's "memory should feel instant"
|
||||
# budget.
|
||||
copy_file "$PLUGIN_SRC/rules/mempalace-recall.md" \
|
||||
"$INSTALL_DIR/rules/mempalace-recall.md"
|
||||
|
||||
# ── Install: hooks ───────────────────────────────────────────────────
|
||||
copy_file "$HOOKS_SRC/lib/common.sh" "$INSTALL_DIR/hooks/lib/common.sh"
|
||||
copy_file "$HOOKS_SRC/mempal_save_hook_antigravity.sh" "$INSTALL_DIR/hooks/mempal_save_hook_antigravity.sh"
|
||||
copy_file "$HOOKS_SRC/mempal_wake_hook_antigravity.sh" "$INSTALL_DIR/hooks/mempal_wake_hook_antigravity.sh"
|
||||
|
||||
# Ensure hook scripts are executable on the install side. cp preserves
|
||||
# mode but a fresh git clone from a tarball might not — chmod is
|
||||
# defensive, idempotent, and bash 3.2 safe.
|
||||
if [ "$DRY_RUN" -ne 1 ]; then
|
||||
chmod 755 "$INSTALL_DIR/hooks/mempal_save_hook_antigravity.sh" 2>/dev/null || true
|
||||
chmod 755 "$INSTALL_DIR/hooks/mempal_wake_hook_antigravity.sh" 2>/dev/null || true
|
||||
fi
|
||||
|
||||
# ── Install: render hooks.json from template ─────────────────────────
|
||||
render_template "$PLUGIN_SRC/hooks.json.tmpl" "$INSTALL_DIR/hooks.json"
|
||||
|
||||
# ── Done ─────────────────────────────────────────────────────────────
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
log info "DRY-RUN complete; no files written."
|
||||
else
|
||||
log info "install complete: $INSTALL_DIR"
|
||||
log info "restart Antigravity to load the plugin."
|
||||
fi
|
||||
@@ -0,0 +1,498 @@
|
||||
# shellcheck shell=bash
|
||||
# MEMPALACE ANTIGRAVITY HOOK — shared helpers
|
||||
#
|
||||
# Sourced by the two Antigravity hook scripts:
|
||||
# * mempal_save_hook_antigravity.sh (Stop event)
|
||||
# * mempal_wake_hook_antigravity.sh (PreInvocation event, gated to invocationNum==1)
|
||||
#
|
||||
# Mirrors the conventions of the existing Claude Code hook scripts
|
||||
# (hooks/mempal_save_hook.sh, hooks/mempal_precompact_hook.sh):
|
||||
#
|
||||
# * STATE_DIR layout under ~/.mempalace/hook_state/
|
||||
# * MEMPAL_PYTHON resolution order (override -> $PATH -> bare python3)
|
||||
# * MEMPALACE_HOOKS_AUTO_SAVE=false kill switch (config.json fallback)
|
||||
# * sentinel-guarded Python parser via `sed -n 'Np'` (bash 3.2 safe)
|
||||
# * fail-open on internal errors: emit valid JSON and log, never crash
|
||||
# the hook host
|
||||
#
|
||||
# Antigravity-specific contract differences from Claude / Cursor:
|
||||
#
|
||||
# * Antigravity stdin uses camelCase (transcriptPath, conversationId,
|
||||
# workspacePaths, executionNum, terminationReason, fullyIdle,
|
||||
# invocationNum, initialNumSteps), not the snake_case Claude Code
|
||||
# format (session_id, transcript_path, stop_hook_active).
|
||||
# * Antigravity stdout for Stop event MUST be {} on every success path
|
||||
# because { "decision": "continue" } would force the agent into an
|
||||
# infinite re-execution loop. The save hook explicitly refuses to
|
||||
# ever emit the "continue" decision.
|
||||
# * Antigravity stdout for PreInvocation can carry an "injectSteps"
|
||||
# array of { "ephemeralMessage": "..." } objects to inject memory
|
||||
# into the agent's first turn.
|
||||
#
|
||||
# This file is sourced, not executed, so it intentionally has no
|
||||
# shebang. The `# shellcheck shell=bash` directive above tells
|
||||
# shellcheck to treat it as bash when run standalone.
|
||||
|
||||
# bash 3.2.57 (the macOS default) is the lower bound. Do not use
|
||||
# `mapfile`, `readarray`, `declare -A`, or `${var^^}` — none of those
|
||||
# exist in 3.2. Use `sed -n 'Np'` for line extraction and case-folding
|
||||
# via `tr` instead.
|
||||
|
||||
# ── State directory + log path ────────────────────────────────────────
|
||||
#
|
||||
# Honour MEMPAL_STATE_DIR while keeping the default identical to the
|
||||
# Claude Code hooks so a user running both keeps a single state directory
|
||||
# (constraint #7 in the integration brief).
|
||||
MEMPAL_STATE_DIR="${MEMPAL_STATE_DIR:-$HOME/.mempalace/hook_state}"
|
||||
mkdir -p "$MEMPAL_STATE_DIR" 2>/dev/null
|
||||
MEMPAL_AGY_LOG="$MEMPAL_STATE_DIR/antigravity_hook.log"
|
||||
|
||||
# ── Python interpreter resolution ─────────────────────────────────────
|
||||
#
|
||||
# The hooks run mempalace as `"$MEMPAL_PYTHON_BIN" -m mempalace`, so the
|
||||
# resolved interpreter MUST be one that has the mempalace package
|
||||
# importable. The single most common install path —
|
||||
# `uv tool install mempalace` (and `pipx install`) — puts the
|
||||
# `mempalace` / `mempalace-mcp` *console scripts* on PATH inside an
|
||||
# ISOLATED environment whose interpreter is NOT the system `python3`.
|
||||
# So naively resolving `command -v python3` lands on a system Python
|
||||
# that can't import mempalace, the `-m mempalace` probe fails, and
|
||||
# mining silently never fires. (This bit a real user on PR #1633.)
|
||||
#
|
||||
# Resolution order — first hit wins:
|
||||
# 1. $MEMPAL_PYTHON — explicit operator override
|
||||
# 2. shebang of the mempalace-mcp / mempalace console script on PATH
|
||||
# — pip/uv write these with an absolute-path shebang pointing at
|
||||
# the exact interpreter that owns the package. This is the SAME
|
||||
# console script mcp_config.json launches, so if the MCP server
|
||||
# can start, the hooks resolve a working interpreter too — no
|
||||
# MEMPAL_PYTHON needed for the common install paths.
|
||||
# 3. $(command -v python3) — an activated dev venv /
|
||||
# editable install where python3 itself owns the package
|
||||
# 4. bare "python3" — last-resort fallback
|
||||
#
|
||||
# Steps 2-4 are pure string parsing + stat (no Python subprocess), so
|
||||
# resolution stays cheap enough to run at source time on every hook
|
||||
# fire, including gated-out / kill-switched ones. We deliberately do
|
||||
# NOT run an `import mempalace` probe here: building that import pays
|
||||
# the chromadb/onnx cold-start cost, which a recent perf fix
|
||||
# (df295bd) moved OFF the hook foreground on purpose. The downstream
|
||||
# `-m mempalace --version` probe (backgrounded in the save hook,
|
||||
# subprocessed in the wake hook) is the safety net that catches a
|
||||
# shebang interpreter whose package is genuinely broken.
|
||||
mempal_resolve_python() {
|
||||
# 1. Explicit override always wins.
|
||||
local p="${MEMPAL_PYTHON:-}"
|
||||
if [ -n "$p" ] && [ -x "$p" ]; then
|
||||
printf '%s' "$p"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 2. Derive the interpreter from a mempalace console-script shebang.
|
||||
local script_path shebang interp
|
||||
for script_path in mempalace-mcp mempalace; do
|
||||
script_path="$(command -v "$script_path" 2>/dev/null || true)"
|
||||
[ -n "$script_path" ] || continue
|
||||
[ -r "$script_path" ] || continue
|
||||
shebang="$(sed -n '1p' "$script_path" 2>/dev/null)"
|
||||
case "$shebang" in
|
||||
'#!'*)
|
||||
interp="${shebang#\#!}" # drop the leading '#!'
|
||||
interp="${interp%$'\r'}" # strip a trailing CR (CRLF files)
|
||||
interp="${interp# }" # drop one leading space
|
||||
interp="${interp%% *}" # first whitespace-delimited token
|
||||
# Guard against `#!/usr/bin/env python` wrappers: only
|
||||
# accept a token whose basename looks like a Python
|
||||
# interpreter and is executable. An `env`-style shebang
|
||||
# yields `/usr/bin/env` here, which we skip.
|
||||
case "${interp##*/}" in
|
||||
python*)
|
||||
if [ -x "$interp" ]; then
|
||||
printf '%s' "$interp"
|
||||
return 0
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# 3. First python3 on PATH.
|
||||
p="$(command -v python3 2>/dev/null || true)"
|
||||
if [ -n "$p" ]; then
|
||||
printf '%s' "$p"
|
||||
return 0
|
||||
fi
|
||||
|
||||
# 4. Last-resort bare name.
|
||||
printf '%s' "python3"
|
||||
}
|
||||
MEMPAL_PYTHON_BIN="$(mempal_resolve_python)"
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────
|
||||
#
|
||||
# ISO8601Z timestamps are greppable across timezones.
|
||||
mempal_log() {
|
||||
local event="${1:-?}"
|
||||
local conv="${2:-unknown}"
|
||||
local msg="${3:-}"
|
||||
local ts
|
||||
ts="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
printf '[%s] [event=%s] [conv=%s] %s\n' "$ts" "$event" "$conv" "$msg" \
|
||||
>> "$MEMPAL_AGY_LOG" 2>/dev/null
|
||||
}
|
||||
|
||||
# ── Kill switch ───────────────────────────────────────────────────────
|
||||
#
|
||||
# Disabled if ANY of:
|
||||
# * MEMPAL_DISABLE_HOOK is a truthy string
|
||||
# * MEMPALACE_HOOKS_AUTO_SAVE is false/0/no
|
||||
# * ~/.mempalace/config.json sets hooks.auto_save: false
|
||||
# * ~/.mempalace/ directory does not exist (user nuked the palace)
|
||||
#
|
||||
# Returns 0 (kill switch tripped, hook should short-circuit) or non-zero
|
||||
# (proceed normally).
|
||||
mempal_kill_switch_tripped() {
|
||||
# Palace nuke is the strongest signal: respect it before touching
|
||||
# disk for state, logging, etc.
|
||||
if [ ! -d "$HOME/.mempalace" ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
case "${MEMPAL_DISABLE_HOOK:-}" in
|
||||
1|true|TRUE|yes|YES) return 0 ;;
|
||||
esac
|
||||
|
||||
case "${MEMPALACE_HOOKS_AUTO_SAVE:-}" in
|
||||
false|FALSE|0|no|NO) return 0 ;;
|
||||
esac
|
||||
|
||||
local cfg="$HOME/.mempalace/config.json"
|
||||
if [ -f "$cfg" ]; then
|
||||
local auto
|
||||
auto=$("$MEMPAL_PYTHON_BIN" -c "
|
||||
import json, sys
|
||||
try:
|
||||
with open(sys.argv[1]) as f:
|
||||
cfg = json.load(f)
|
||||
print(str(cfg.get('hooks', {}).get('auto_save', True)).lower())
|
||||
except Exception:
|
||||
print('true')
|
||||
" "$cfg" 2>/dev/null)
|
||||
if [ "$auto" = "false" ]; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── camelCase JSON parser (Antigravity stdin) ────────────────────────
|
||||
#
|
||||
# Reads JSON from stdin once and prints a sanitized, sentinel-bracketed
|
||||
# block of fields the bash side can grab via `sed -n 'Np'`. Why a
|
||||
# sentinel and per-line layout: bash 3.2 doesn't have `mapfile` or
|
||||
# `readarray`, and `eval`-on-shell-var is the wrong shape (every value
|
||||
# is user-controllable JSON). Sentinel + line offset is the same pattern
|
||||
# the existing Claude Code hook (hooks/mempal_save_hook.sh) uses.
|
||||
#
|
||||
# Output layout (one field per line; line numbers are stable and the
|
||||
# fields are documented in STDIN_SHAPE.md):
|
||||
#
|
||||
# line 1: __MEMPAL_PARSE_OK__ — sentinel (parse success marker)
|
||||
# line 2: conversationId — sanitized to [A-Za-z0-9._-]
|
||||
# line 3: transcriptPath — sanitized to a safe path charset
|
||||
# line 4: workspacePath — workspacePaths[0], sanitized
|
||||
# line 5: artifactDirectoryPath — sanitized
|
||||
# line 6: executionNum — integer, default 0
|
||||
# line 7: terminationReason — sanitized to [a-z_]
|
||||
# line 8: fullyIdle — "True" or "False" (string)
|
||||
# line 9: invocationNum — integer, default 0
|
||||
# line 10: initialNumSteps — integer, default 0
|
||||
#
|
||||
# The sanitizers are defense-in-depth: every field is also vetted by
|
||||
# the Python json.load step, but we still strip shell-meaningful chars
|
||||
# from any field a downstream bash variable might interpolate, so that
|
||||
# a hostile / malformed harness payload cannot inject command tokens.
|
||||
#
|
||||
# Stderr from Python is captured to last_python_err.log at mode 0600 so
|
||||
# operators can debug parse failures without re-firing the hook. The
|
||||
# umask 077 on the inner subshell creates the file at 0600 atomically;
|
||||
# the explicit chmod 600 below is a belt-and-suspenders guard if a
|
||||
# future edit ever drops the umask.
|
||||
mempal_parse_stdin() {
|
||||
local input="$1"
|
||||
(
|
||||
umask 077
|
||||
printf '%s' "$input" | "$MEMPAL_PYTHON_BIN" -c "
|
||||
import sys, json, re
|
||||
|
||||
# IMPORTANT: do NOT wrap json.load in a try/except. If the input is
|
||||
# not valid JSON we want Python to exit non-zero BEFORE printing the
|
||||
# __MEMPAL_PARSE_OK__ sentinel — the bash caller looks for the
|
||||
# sentinel on line 1 to decide whether to engage its defense-in-depth
|
||||
# 'failed to parse' branch. Catching the exception and falling back
|
||||
# to data={} would let the sentinel print, masking parse failures
|
||||
# from the bash side. The traceback lands in
|
||||
# antigravity_last_python_err.log so operators can debug.
|
||||
data = json.load(sys.stdin)
|
||||
|
||||
def safe(s, allowed=r'[^a-zA-Z0-9_/.\-~]'):
|
||||
return re.sub(allowed, '', str(s))
|
||||
|
||||
def safe_id(s):
|
||||
return re.sub(r'[^a-zA-Z0-9._-]', '', str(s))
|
||||
|
||||
def safe_int(v, default=0):
|
||||
try:
|
||||
n = int(v)
|
||||
return n if n >= 0 else default
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
def safe_lower_alpha_underscore(s):
|
||||
return re.sub(r'[^a-z_]', '', str(s).lower())
|
||||
|
||||
conv_id = safe_id(data.get('conversationId', ''))
|
||||
transcript = safe(data.get('transcriptPath', ''))
|
||||
wp_arr = data.get('workspacePaths', [])
|
||||
if isinstance(wp_arr, list) and wp_arr:
|
||||
workspace = safe(wp_arr[0])
|
||||
else:
|
||||
workspace = ''
|
||||
artifact = safe(data.get('artifactDirectoryPath', ''))
|
||||
execution_num = safe_int(data.get('executionNum', 0))
|
||||
termination_reason = safe_lower_alpha_underscore(data.get('terminationReason', ''))
|
||||
fully_idle_raw = data.get('fullyIdle', None)
|
||||
if fully_idle_raw is True or str(fully_idle_raw).lower() in ('true', '1', 'yes'):
|
||||
fully_idle = 'True'
|
||||
else:
|
||||
fully_idle = 'False'
|
||||
invocation_num = safe_int(data.get('invocationNum', 0))
|
||||
initial_num_steps = safe_int(data.get('initialNumSteps', 0))
|
||||
|
||||
print('__MEMPAL_PARSE_OK__')
|
||||
print(conv_id)
|
||||
print(transcript)
|
||||
print(workspace)
|
||||
print(artifact)
|
||||
print(execution_num)
|
||||
print(termination_reason)
|
||||
print(fully_idle)
|
||||
print(invocation_num)
|
||||
print(initial_num_steps)
|
||||
" 2>"$MEMPAL_STATE_DIR/antigravity_last_python_err.log"
|
||||
)
|
||||
# Tidy up the err log: keep it iff non-empty (failure happened).
|
||||
if [ -s "$MEMPAL_STATE_DIR/antigravity_last_python_err.log" ]; then
|
||||
chmod 600 "$MEMPAL_STATE_DIR/antigravity_last_python_err.log" 2>/dev/null
|
||||
else
|
||||
rm -f "$MEMPAL_STATE_DIR/antigravity_last_python_err.log" 2>/dev/null
|
||||
fi
|
||||
}
|
||||
|
||||
# ── Transcript path validator ─────────────────────────────────────────
|
||||
#
|
||||
# Mirrors mempalace.hooks_cli._validate_transcript_path: rejects empty,
|
||||
# non-jsonl/json suffixes, and any `..` traversal segment.
|
||||
mempal_is_valid_transcript_path() {
|
||||
local path="$1"
|
||||
[ -n "$path" ] || return 1
|
||||
case "$path" in
|
||||
*.json|*.jsonl) ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
case "/$path/" in
|
||||
*/../*) return 1 ;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Wing inference ────────────────────────────────────────────────────
|
||||
#
|
||||
# Takes the first workspace path from workspacePaths[] (already
|
||||
# extracted into $1) and derives a `wing_<slug>` name from its leaf
|
||||
# directory. Hyphens become underscores; spaces become underscores.
|
||||
# Empty input yields wing_sessions, matching mempalace.hooks_cli's
|
||||
# fallback.
|
||||
mempal_infer_wing() {
|
||||
local workspace="$1"
|
||||
if [ -z "$workspace" ]; then
|
||||
printf 'wing_sessions'
|
||||
return 0
|
||||
fi
|
||||
# Strip trailing slashes
|
||||
while [ "${workspace}" != "${workspace%/}" ]; do
|
||||
workspace="${workspace%/}"
|
||||
done
|
||||
if [ -z "$workspace" ]; then
|
||||
printf 'wing_sessions'
|
||||
return 0
|
||||
fi
|
||||
local leaf="${workspace##*/}"
|
||||
if [ -z "$leaf" ]; then
|
||||
printf 'wing_sessions'
|
||||
return 0
|
||||
fi
|
||||
# Lowercase + hyphens-to-underscores. tr is bash 3.2 safe; ${var^^}
|
||||
# / ${var//-/_} on a fresh expansion are bash 4+ only.
|
||||
local slug
|
||||
slug=$(printf '%s' "$leaf" | tr 'A-Z' 'a-z' | tr ' -' '__')
|
||||
printf 'wing_%s' "$slug"
|
||||
}
|
||||
|
||||
# ── Save-interval floor ───────────────────────────────────────────────
|
||||
#
|
||||
# Reads MEMPAL_SAVE_INTERVAL from the environment, floors it to >= 1
|
||||
# so that `count % interval` cannot divide by zero. We hit the
|
||||
# divide-by-zero shape on the Cursor PR review; this guards explicitly.
|
||||
mempal_save_interval() {
|
||||
local raw="${MEMPAL_SAVE_INTERVAL:-15}"
|
||||
case "$raw" in
|
||||
''|*[!0-9]*) printf '15'; return 0 ;;
|
||||
esac
|
||||
# Strip leading zeros. bash arithmetic ($((...))) parses any token
|
||||
# starting with `0` as octal, so MEMPAL_SAVE_INTERVAL=08 would
|
||||
# crash $((COUNT % INTERVAL)) with "value too great for base".
|
||||
# Loop while the value still starts with 0 AND has length > 1, so
|
||||
# the literal string "0" is preserved (then floored to 15 below).
|
||||
while [ "${raw}" != "${raw#0}" ] && [ "${#raw}" -gt 1 ]; do
|
||||
raw="${raw#0}"
|
||||
done
|
||||
if [ "$raw" -lt 1 ] 2>/dev/null; then
|
||||
printf '15'
|
||||
return 0
|
||||
fi
|
||||
printf '%s' "$raw"
|
||||
}
|
||||
|
||||
# ── Atomic counter write ──────────────────────────────────────────────
|
||||
#
|
||||
# Writes $value to $file via a same-directory temp file + `mv -f`.
|
||||
# `mv` (rename) is atomic on a single filesystem, so a concurrent
|
||||
# reader either sees the old contents or the new contents — never a
|
||||
# half-written / truncated file. A plain `printf > file` truncates
|
||||
# first and then writes, leaving a window where a concurrent Stop fire
|
||||
# could read an empty / partial value. Concurrent fires for one
|
||||
# conversation are unlikely (Antigravity serializes turns) but the
|
||||
# previous "written atomically" comment was simply false; this makes
|
||||
# it true.
|
||||
#
|
||||
# bash 3.2 safe. The temp lives in the same directory as the target so
|
||||
# the rename stays on one filesystem (a cross-device mv would fall back
|
||||
# to copy+unlink and lose atomicity). On any failure we degrade to a
|
||||
# direct write rather than leaving the counter unwritten.
|
||||
mempal_write_counter_atomic() {
|
||||
local file="$1"
|
||||
local value="$2"
|
||||
local tmp
|
||||
tmp="$(mktemp "${file}.XXXXXX" 2>/dev/null)" || {
|
||||
printf '%s' "$value" > "$file"
|
||||
return
|
||||
}
|
||||
printf '%s' "$value" > "$tmp"
|
||||
mv -f "$tmp" "$file" 2>/dev/null || {
|
||||
rm -f "$tmp" 2>/dev/null
|
||||
printf '%s' "$value" > "$file"
|
||||
}
|
||||
}
|
||||
|
||||
# ── State-file TTL ────────────────────────────────────────────────────
|
||||
#
|
||||
# Per-conversation state artifacts (antigravity_save_count_<conv>,
|
||||
# antigravity_pending_<conv>, antigravity_woke_<conv>/) accumulate one
|
||||
# set per conversation and are never otherwise removed. Reads
|
||||
# MEMPAL_STATE_TTL_DAYS (default 30), validated digits-only and
|
||||
# leading-zero-stripped like mempal_save_interval so `find -mtime`
|
||||
# never sees a bad token. A value of 0 means "sweep aggressively"
|
||||
# (everything older than today); we floor empty/garbage to 30.
|
||||
mempal_state_ttl_days() {
|
||||
local raw="${MEMPAL_STATE_TTL_DAYS:-30}"
|
||||
case "$raw" in
|
||||
''|*[!0-9]*) printf '30'; return 0 ;;
|
||||
esac
|
||||
while [ "${raw}" != "${raw#0}" ] && [ "${#raw}" -gt 1 ]; do
|
||||
raw="${raw#0}"
|
||||
done
|
||||
printf '%s' "$raw"
|
||||
}
|
||||
|
||||
# ── Stale state GC ────────────────────────────────────────────────────
|
||||
#
|
||||
# Opportunistic sweep of per-conversation state older than the TTL.
|
||||
# Gated to run at most once per 24h via the antigravity_last_sweep
|
||||
# marker so it costs nothing on the vast majority of fires (a single
|
||||
# mtime comparison). When it does run, three `find` passes remove the
|
||||
# stale counter files, pending markers, and woke marker directories.
|
||||
#
|
||||
# The name globs are specific (antigravity_save_count_*, _pending_*,
|
||||
# _woke_*), so the shared log files (antigravity_hook.log,
|
||||
# antigravity_last_input.log, antigravity_last_python_err.log) and the
|
||||
# antigravity_last_sweep marker itself are never touched. BSD find
|
||||
# (macOS default) and GNU find both accept `-maxdepth`, `-mtime +N`,
|
||||
# and `-exec ... +`.
|
||||
#
|
||||
# Fail-open: every step is best-effort; a missing state dir, a find
|
||||
# that errors, or a permission problem must never abort the caller.
|
||||
mempal_gc_stale_state() {
|
||||
[ -d "$MEMPAL_STATE_DIR" ] || return 0
|
||||
|
||||
local marker="$MEMPAL_STATE_DIR/antigravity_last_sweep"
|
||||
if [ -f "$marker" ]; then
|
||||
local mtime now
|
||||
if mtime=$("$MEMPAL_PYTHON_BIN" -c 'import os, sys; print(int(os.path.getmtime(sys.argv[1])))' "$marker" 2>/dev/null) \
|
||||
&& now=$(date '+%s' 2>/dev/null) \
|
||||
&& [ -n "$mtime" ] \
|
||||
&& [ "$((now - mtime))" -lt 86400 ]; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
# Touch the marker first so a crash mid-sweep still throttles the
|
||||
# next fire (better to skip a sweep than to hammer the disk).
|
||||
: > "$marker" 2>/dev/null
|
||||
|
||||
local ttl
|
||||
ttl=$(mempal_state_ttl_days)
|
||||
|
||||
find "$MEMPAL_STATE_DIR" -maxdepth 1 -type f \
|
||||
-name 'antigravity_save_count_*' -mtime +"$ttl" \
|
||||
-exec rm -f {} + 2>/dev/null
|
||||
find "$MEMPAL_STATE_DIR" -maxdepth 1 -type f \
|
||||
-name 'antigravity_pending_*' -mtime +"$ttl" \
|
||||
-exec rm -f {} + 2>/dev/null
|
||||
find "$MEMPAL_STATE_DIR" -maxdepth 1 -type d \
|
||||
-name 'antigravity_woke_*' -mtime +"$ttl" \
|
||||
-exec rm -rf {} + 2>/dev/null
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Fail-open emitters ────────────────────────────────────────────────
|
||||
#
|
||||
# Every code path in both hooks must terminate by calling exactly one
|
||||
# of these emitters. Stdout is JSON. Exit status is always 0 — the hook
|
||||
# never blocks the user's IDE on its own failure (constraint #2).
|
||||
#
|
||||
# CRITICAL: mempal_emit_stop_pass MUST NEVER emit
|
||||
# {"decision":"continue"} — that would force the agent to keep running
|
||||
# instead of letting the turn end. Antigravity treats any value other
|
||||
# than "continue" (including `{}`) as "allow the stop". We enforce this
|
||||
# by hard-coding the empty object output here.
|
||||
mempal_emit_stop_pass() {
|
||||
printf '{}\n'
|
||||
}
|
||||
|
||||
mempal_emit_wake_inject() {
|
||||
local message="$1"
|
||||
if [ -z "$message" ]; then
|
||||
printf '{}\n'
|
||||
return 0
|
||||
fi
|
||||
# Encode the message as JSON via Python so embedded quotes / newlines
|
||||
# / control chars don't corrupt the output.
|
||||
"$MEMPAL_PYTHON_BIN" -c "
|
||||
import json, sys
|
||||
msg = sys.argv[1]
|
||||
print(json.dumps({'injectSteps': [{'ephemeralMessage': msg}]}))
|
||||
" "$message" 2>/dev/null || printf '{}\n'
|
||||
}
|
||||
+245
@@ -0,0 +1,245 @@
|
||||
#!/bin/bash
|
||||
# MEMPALACE ANTIGRAVITY SAVE HOOK — Stop event handler
|
||||
#
|
||||
# Antigravity fires the Stop event each time the agent's execution loop
|
||||
# terminates. We use it to background-mine the active conversation
|
||||
# transcript every Nth save into the user's MemPalace, and to write a
|
||||
# diary checkpoint via `mempalace mine --mode convos`.
|
||||
#
|
||||
# Mirrors the Claude Code (hooks/mempal_save_hook.sh) and Codex
|
||||
# (.codex-plugin/hooks/mempal-hook.sh) integrations as closely as the
|
||||
# Antigravity stdin/stdout contract allows. Differences:
|
||||
#
|
||||
# * Antigravity stdin uses camelCase: conversationId, transcriptPath,
|
||||
# workspacePaths, executionNum, terminationReason, fullyIdle.
|
||||
# * Antigravity stdout MUST be `{}` on every code path. Emitting
|
||||
# `{"decision":"continue"}` would force the agent to keep running
|
||||
# and create an infinite loop. We never call mempal_emit_stop_pass
|
||||
# with anything other than the literal empty object.
|
||||
# * Counter file is namespaced antigravity_save_count_<conversationId>
|
||||
# to coexist with Claude Code / Cursor / Codex state in the same
|
||||
# ~/.mempalace/hook_state/ directory.
|
||||
#
|
||||
# === STDIN (verified, camelCase) ===
|
||||
# {
|
||||
# "executionNum": 1,
|
||||
# "terminationReason": "model_stop",
|
||||
# "error": "",
|
||||
# "fullyIdle": true,
|
||||
# "conversationId": "<uuid>",
|
||||
# "workspacePaths": ["/abs/path/..."],
|
||||
# "transcriptPath": "/abs/path/transcript.jsonl",
|
||||
# "artifactDirectoryPath": "/abs/path/artifacts/"
|
||||
# }
|
||||
#
|
||||
# === STDOUT (always) ===
|
||||
# {}
|
||||
#
|
||||
# `set -e` is intentionally NOT enabled — a broken hook must not block
|
||||
# the user's conversation (constraint #2 in the integration brief).
|
||||
|
||||
# ── Locate this script + source common helpers ───────────────────────
|
||||
MEMPAL_AGY_HOOK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
# shellcheck source=lib/common.sh
|
||||
. "$MEMPAL_AGY_HOOK_DIR/lib/common.sh"
|
||||
|
||||
# ── Read all of stdin once ───────────────────────────────────────────
|
||||
INPUT=$(cat)
|
||||
|
||||
# ── Kill switch: short-circuit cleanly if disabled ───────────────────
|
||||
if mempal_kill_switch_tripped; then
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Opportunistic GC of stale per-conversation state ─────────────────
|
||||
#
|
||||
# Self-throttled to at most once per 24h (see mempal_gc_stale_state),
|
||||
# so this is a single mtime check on the overwhelming majority of
|
||||
# fires. Runs after the kill switch so a disabled hook touches nothing.
|
||||
mempal_gc_stale_state
|
||||
|
||||
# ── Parse stdin (camelCase, sentinel-guarded) ────────────────────────
|
||||
_parsed=$(mempal_parse_stdin "$INPUT")
|
||||
_marker=$(printf '%s\n' "$_parsed" | sed -n '1p')
|
||||
CONVERSATION_ID=$(printf '%s\n' "$_parsed" | sed -n '2p')
|
||||
TRANSCRIPT_PATH=$(printf '%s\n' "$_parsed" | sed -n '3p')
|
||||
WORKSPACE_PATH=$(printf '%s\n' "$_parsed" | sed -n '4p')
|
||||
# Line 5 (artifactDirectoryPath) is parsed but unused for save. Skip.
|
||||
EXECUTION_NUM=$(printf '%s\n' "$_parsed" | sed -n '6p')
|
||||
# Line 7 (terminationReason) is parsed but used only for logging.
|
||||
TERMINATION_REASON=$(printf '%s\n' "$_parsed" | sed -n '7p')
|
||||
FULLY_IDLE=$(printf '%s\n' "$_parsed" | sed -n '8p')
|
||||
|
||||
# ── Defense-in-depth: surface raw input on parse failure ─────────────
|
||||
#
|
||||
# When the sentinel is missing, Python crashed before reaching its
|
||||
# print() calls. Persist the offending payload (capped at 4 KB, mode
|
||||
# 0600) so the next debugger doesn't lose a day to log lines that say
|
||||
# "Session unknown".
|
||||
if [ -n "$INPUT" ] && [ "$_marker" != "__MEMPAL_PARSE_OK__" ]; then
|
||||
mempal_log "stop" "unknown" "input parse failed (sentinel missing); see antigravity_last_input.log + antigravity_last_python_err.log"
|
||||
(
|
||||
umask 077
|
||||
printf '%s' "$INPUT" | head -c 4096 > "$MEMPAL_STATE_DIR/antigravity_last_input.log"
|
||||
)
|
||||
chmod 600 "$MEMPAL_STATE_DIR/antigravity_last_input.log" 2>/dev/null
|
||||
# Continue with empty fields; the validators below will reject.
|
||||
fi
|
||||
|
||||
CONVERSATION_ID="${CONVERSATION_ID:-unknown}"
|
||||
TRANSCRIPT_PATH="${TRANSCRIPT_PATH:-}"
|
||||
WORKSPACE_PATH="${WORKSPACE_PATH:-}"
|
||||
EXECUTION_NUM="${EXECUTION_NUM:-0}"
|
||||
TERMINATION_REASON="${TERMINATION_REASON:-}"
|
||||
FULLY_IDLE="${FULLY_IDLE:-False}"
|
||||
|
||||
# Expand ~ in the transcript path
|
||||
TRANSCRIPT_PATH="${TRANSCRIPT_PATH/#\~/$HOME}"
|
||||
|
||||
# ── Bail when fullyIdle is False ─────────────────────────────────────
|
||||
#
|
||||
# If background commands or async tasks are still running, the
|
||||
# transcript is still in motion. Defer the save until the next Stop
|
||||
# event when the agent is fully done — better to skip than to ingest a
|
||||
# half-finished transcript and pollute the search index.
|
||||
if [ "$FULLY_IDLE" != "True" ]; then
|
||||
mempal_log "stop" "$CONVERSATION_ID" "deferring save: fullyIdle=False (executionNum=$EXECUTION_NUM, terminationReason=$TERMINATION_REASON)"
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Skip when terminationReason is `error` ───────────────────────────
|
||||
#
|
||||
# A model error termination usually means the transcript is corrupt or
|
||||
# truncated. Don't ingest noise.
|
||||
if [ "$TERMINATION_REASON" = "error" ]; then
|
||||
mempal_log "stop" "$CONVERSATION_ID" "skipping save: terminationReason=error"
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Increment counter (per conversation) ─────────────────────────────
|
||||
#
|
||||
# The counter is a single integer, written via mempal_write_counter_atomic
|
||||
# (same-dir temp + `mv`, which is an atomic rename on one filesystem).
|
||||
# Concurrent Stop fires for the same conversation are unlikely
|
||||
# (Antigravity serializes turns), but the atomic write means a
|
||||
# concurrent reader always sees a complete value rather than a
|
||||
# half-written / truncated file. The integer-only validation on read
|
||||
# is a second guard: any garbled value resets the count to 0.
|
||||
COUNTER_FILE="$MEMPAL_STATE_DIR/antigravity_save_count_${CONVERSATION_ID}"
|
||||
COUNT=0
|
||||
if [ -f "$COUNTER_FILE" ]; then
|
||||
raw=$(cat "$COUNTER_FILE" 2>/dev/null)
|
||||
case "$raw" in
|
||||
''|*[!0-9]*) COUNT=0 ;;
|
||||
*) COUNT="$raw" ;;
|
||||
esac
|
||||
fi
|
||||
COUNT=$((COUNT + 1))
|
||||
mempal_write_counter_atomic "$COUNTER_FILE" "$COUNT"
|
||||
|
||||
INTERVAL=$(mempal_save_interval)
|
||||
mempal_log "stop" "$CONVERSATION_ID" "count=$COUNT interval=$INTERVAL executionNum=$EXECUTION_NUM workspace=$WORKSPACE_PATH"
|
||||
|
||||
# ── Modulo gate ──────────────────────────────────────────────────────
|
||||
#
|
||||
# `count % interval == 0` triggers a save. INTERVAL has been floored to
|
||||
# >= 1 by mempal_save_interval, so the modulo cannot divide by zero
|
||||
# even if the user explicitly set MEMPAL_SAVE_INTERVAL=0 or empty.
|
||||
if [ $((COUNT % INTERVAL)) -ne 0 ]; then
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Pending-marker guard ─────────────────────────────────────────────
|
||||
#
|
||||
# If a previous save is still running (the marker file exists), skip
|
||||
# this fire. The mine subprocess removes the marker on exit, but a
|
||||
# crashed mine could leave the marker forever — guard against that by
|
||||
# treating markers older than 1 hour as stale and reclaiming them.
|
||||
PENDING_FILE="$MEMPAL_STATE_DIR/antigravity_pending_${CONVERSATION_ID}"
|
||||
if [ -f "$PENDING_FILE" ]; then
|
||||
# mtime in epoch seconds (portable; BSD/macOS `date -r` takes epoch, not a path).
|
||||
if mtime=$("$MEMPAL_PYTHON_BIN" -c 'import os, sys; print(int(os.path.getmtime(sys.argv[1])))' "$PENDING_FILE" 2>/dev/null) \
|
||||
&& now=$(date '+%s') \
|
||||
&& [ -n "$mtime" ] \
|
||||
&& [ "$((now - mtime))" -lt 3600 ]; then
|
||||
mempal_log "stop" "$CONVERSATION_ID" "pending save still in flight; skipping"
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
mempal_log "stop" "$CONVERSATION_ID" "stale pending marker reclaimed"
|
||||
rm -f "$PENDING_FILE" 2>/dev/null
|
||||
fi
|
||||
|
||||
# ── Validate transcript path ─────────────────────────────────────────
|
||||
if ! mempal_is_valid_transcript_path "$TRANSCRIPT_PATH"; then
|
||||
mempal_log "stop" "$CONVERSATION_ID" "invalid transcriptPath rejected: $TRANSCRIPT_PATH"
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f "$TRANSCRIPT_PATH" ]; then
|
||||
mempal_log "stop" "$CONVERSATION_ID" "transcriptPath does not exist: $TRANSCRIPT_PATH"
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Trigger save ─────────────────────────────────────────────────────
|
||||
WING=$(mempal_infer_wing "$WORKSPACE_PATH")
|
||||
TRANSCRIPT_DIR=$(dirname "$TRANSCRIPT_PATH")
|
||||
|
||||
mempal_log "stop" "$CONVERSATION_ID" "TRIGGERING SAVE wing=$WING transcript_dir=$TRANSCRIPT_DIR"
|
||||
|
||||
# Drop the pending marker BEFORE spawning so a near-simultaneous fire
|
||||
# sees it. If the spawn fails, remove the marker so the next fire can
|
||||
# retry.
|
||||
: > "$PENDING_FILE" 2>/dev/null
|
||||
|
||||
# Detach EVERYTHING heavy into a single background subshell: the
|
||||
# runnability probe, the mine itself, and the pending-marker cleanup.
|
||||
# The foreground returns immediately after spawning, so the hook's
|
||||
# stdout (`{}`) reaches Antigravity within milliseconds.
|
||||
#
|
||||
# Why the probe must NOT run in the foreground: `mempalace --version`
|
||||
# is NOT cheap. Building the `mine` argument parser imports
|
||||
# `mempalace.miner` (-> palace -> backends -> chromadb/onnx) before
|
||||
# argparse ever processes `--version`, so the probe pays the full
|
||||
# cold-start import cost. Running it in the foreground would block the
|
||||
# hook for that entire import and blow the <500ms save budget. Moving
|
||||
# it inside the backgrounded subshell keeps the foreground instant.
|
||||
#
|
||||
# Folding the cleanup into this same subshell also removes the need for
|
||||
# a separate process-liveness polling watcher: the `rm -f
|
||||
# "$PENDING_FILE"` simply runs after the mine returns, in the same
|
||||
# shell that owns the mine — no sibling-PID `wait` hazard, no polling
|
||||
# loop.
|
||||
#
|
||||
# We invoke mempalace as `"$MEMPAL_PYTHON_BIN" -m mempalace` rather than
|
||||
# the bare `mempalace` console script so a user with the package
|
||||
# installed only inside a venv (and the venv's bin/ not on the hook's
|
||||
# PATH, e.g. `uv tool install` in some distributions, or a manually
|
||||
# managed virtualenv) still hits a working mine. MEMPAL_PYTHON honours
|
||||
# user override; sees ``mempalace/__main__.py`` which dispatches to
|
||||
# ``mempalace.cli:main`` — identical to the console script.
|
||||
mempal_log "stop" "$CONVERSATION_ID" "spawning background mine wing=$WING transcript_dir=$TRANSCRIPT_DIR"
|
||||
(
|
||||
if "$MEMPAL_PYTHON_BIN" -m mempalace --version >/dev/null 2>&1; then
|
||||
"$MEMPAL_PYTHON_BIN" -m mempalace mine "$TRANSCRIPT_DIR" \
|
||||
--mode convos \
|
||||
--wing "$WING" \
|
||||
>> "$MEMPAL_AGY_LOG" 2>&1 < /dev/null
|
||||
mempal_log "stop" "$CONVERSATION_ID" "background mine finished wing=$WING"
|
||||
else
|
||||
mempal_log "stop" "$CONVERSATION_ID" "ERROR: mempalace is not runnable via $MEMPAL_PYTHON_BIN -m mempalace; install mempalace or set MEMPAL_PYTHON"
|
||||
fi
|
||||
rm -f "$PENDING_FILE" 2>/dev/null
|
||||
) >/dev/null 2>&1 < /dev/null &
|
||||
|
||||
# ── Always emit `{}` ─────────────────────────────────────────────────
|
||||
#
|
||||
# Never `{"decision":"continue"}`. That would force the agent into an
|
||||
# infinite re-execution loop. mempal_emit_stop_pass hard-codes `{}`.
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
+166
@@ -0,0 +1,166 @@
|
||||
#!/bin/bash
|
||||
# MEMPALACE ANTIGRAVITY WAKE HOOK — PreInvocation event handler
|
||||
#
|
||||
# Antigravity fires the PreInvocation event before every model
|
||||
# invocation, with `invocationNum` carrying the sequence number of the
|
||||
# call. We use the first invocation (invocationNum == 1) as our
|
||||
# session-start equivalent and inject a verbatim memory pointer into
|
||||
# the agent's context via the `injectSteps[].ephemeralMessage` output
|
||||
# field — the message lives for one turn and does not persist into the
|
||||
# transcript, so it doesn't pollute future invocations of this same
|
||||
# conversation.
|
||||
#
|
||||
# === STDIN (verified, camelCase) ===
|
||||
# {
|
||||
# "invocationNum": 1,
|
||||
# "initialNumSteps": 0,
|
||||
# "conversationId": "<uuid>",
|
||||
# "workspacePaths": ["/abs/path/..."],
|
||||
# "transcriptPath": "/abs/path/transcript.jsonl",
|
||||
# "artifactDirectoryPath": "/abs/path/artifacts/"
|
||||
# }
|
||||
#
|
||||
# === STDOUT ===
|
||||
# Either:
|
||||
# {} — no injection
|
||||
# Or:
|
||||
# {"injectSteps":[{"ephemeralMessage":"..."}]} — verbatim memory pointer
|
||||
#
|
||||
# Verbatim guarantee: the ephemeralMessage carries the exact text
|
||||
# emitted by `mempalace wake-up`, never paraphrased or summarized.
|
||||
#
|
||||
# Performance budget: the integration brief sets a 100ms ceiling for
|
||||
# startup injection. We enforce a 500ms hard timeout on the
|
||||
# `mempalace wake-up` subprocess (more generous than 100ms because
|
||||
# cold ChromaDB connections can dominate, and missing the budget is
|
||||
# strictly better than blocking the user) — if it doesn't return in
|
||||
# time we emit `{}` and let the conversation start without injection.
|
||||
#
|
||||
# `set -e` is intentionally NOT enabled — fail-open is mandatory.
|
||||
|
||||
# ── Locate this script + source common helpers ───────────────────────
|
||||
MEMPAL_AGY_HOOK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
|
||||
# shellcheck source=lib/common.sh
|
||||
. "$MEMPAL_AGY_HOOK_DIR/lib/common.sh"
|
||||
|
||||
# ── Read all of stdin once ───────────────────────────────────────────
|
||||
INPUT=$(cat)
|
||||
|
||||
# ── Kill switch ──────────────────────────────────────────────────────
|
||||
if mempal_kill_switch_tripped; then
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Parse stdin ──────────────────────────────────────────────────────
|
||||
_parsed=$(mempal_parse_stdin "$INPUT")
|
||||
_marker=$(printf '%s\n' "$_parsed" | sed -n '1p')
|
||||
CONVERSATION_ID=$(printf '%s\n' "$_parsed" | sed -n '2p')
|
||||
# Lines 3-5 (transcriptPath, workspacePath, artifactDirectoryPath) are
|
||||
# parsed; we use workspacePath for wing inference. transcriptPath and
|
||||
# artifactDirectoryPath are unused by the wake flow.
|
||||
# Line 4: workspacePath
|
||||
WORKSPACE_PATH=$(printf '%s\n' "$_parsed" | sed -n '4p')
|
||||
INVOCATION_NUM=$(printf '%s\n' "$_parsed" | sed -n '9p')
|
||||
|
||||
# Defense-in-depth on parse failure
|
||||
if [ -n "$INPUT" ] && [ "$_marker" != "__MEMPAL_PARSE_OK__" ]; then
|
||||
mempal_log "preInvocation" "unknown" "input parse failed (sentinel missing)"
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
|
||||
CONVERSATION_ID="${CONVERSATION_ID:-unknown}"
|
||||
WORKSPACE_PATH="${WORKSPACE_PATH:-}"
|
||||
INVOCATION_NUM="${INVOCATION_NUM:-0}"
|
||||
|
||||
# ── Gate: only inject on the FIRST invocation ────────────────────────
|
||||
#
|
||||
# PreInvocation fires before every model call. Without this gate we'd
|
||||
# inject memory on every single turn — both expensive and visually
|
||||
# noisy. invocationNum == 1 means "first model call of this
|
||||
# conversation", which is the closest thing Antigravity exposes to
|
||||
# Cursor's `sessionStart`.
|
||||
if [ "$INVOCATION_NUM" != "1" ]; then
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Loop guard ───────────────────────────────────────────────────────
|
||||
#
|
||||
# Defense in depth: even within the first invocation, we only ever
|
||||
# want to inject once per conversation. mkdir is atomic and works on
|
||||
# bash 3.2 / macOS / Linux without flock or other GNU coreutils
|
||||
# extensions.
|
||||
WOKE_MARKER="$MEMPAL_STATE_DIR/antigravity_woke_${CONVERSATION_ID}"
|
||||
if ! mkdir "$WOKE_MARKER" 2>/dev/null; then
|
||||
mempal_log "preInvocation" "$CONVERSATION_ID" "already woke this conversation; skipping"
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Run wake-up with a hard timeout ──────────────────────────────────
|
||||
#
|
||||
# `timeout` is GNU coreutils — present on most Linux installs but
|
||||
# missing from stock macOS. Wrap the subprocess in a Python timeout
|
||||
# (subprocess.run(timeout=...)) which is cross-platform. The Python
|
||||
# script also constructs the final JSON envelope for stdout, so the
|
||||
# bash side just passes the result through.
|
||||
WING=$(mempal_infer_wing "$WORKSPACE_PATH")
|
||||
mempal_log "preInvocation" "$CONVERSATION_ID" "WAKE injection wing=$WING invocationNum=$INVOCATION_NUM"
|
||||
|
||||
OUTPUT=$("$MEMPAL_PYTHON_BIN" -c "
|
||||
import json, subprocess, sys
|
||||
|
||||
wing = sys.argv[1]
|
||||
timeout_s = 0.5 # 500 ms
|
||||
|
||||
# Invoke as ``[sys.executable, '-m', 'mempalace', ...]`` rather than
|
||||
# the bare ``mempalace`` console script. sys.executable is the same
|
||||
# Python that resolved MEMPAL_PYTHON in lib/common.sh, so this binds
|
||||
# the wake-up call to the correct interpreter (and its installed
|
||||
# mempalace package) even when the venv's bin/ isn't on PATH.
|
||||
try:
|
||||
completed = subprocess.run(
|
||||
[sys.executable, '-m', 'mempalace', 'wake-up', '--wing', wing],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout_s,
|
||||
)
|
||||
if completed.returncode != 0:
|
||||
print('{}')
|
||||
sys.exit(0)
|
||||
body = (completed.stdout or '').strip()
|
||||
if not body:
|
||||
print('{}')
|
||||
sys.exit(0)
|
||||
# Verbatim — pass the wake-up text exactly as emitted, wrapped in
|
||||
# the Antigravity injectSteps envelope. json.dumps escapes embedded
|
||||
# control chars and quotes correctly.
|
||||
print(json.dumps({'injectSteps': [{'ephemeralMessage': body}]}))
|
||||
except FileNotFoundError:
|
||||
print('{}')
|
||||
except subprocess.TimeoutExpired:
|
||||
print('{}')
|
||||
except Exception:
|
||||
print('{}')
|
||||
" "$WING" 2>/dev/null)
|
||||
|
||||
if [ -z "$OUTPUT" ]; then
|
||||
OUTPUT='{}'
|
||||
fi
|
||||
|
||||
# Sanity-check: never emit `decision` from a PreInvocation hook (that
|
||||
# field belongs to the Stop event). The Python helper only ever
|
||||
# constructs `{"injectSteps": [...]}` or `{}`, so this is belt-and-
|
||||
# suspenders against a future edit ever leaking a Stop-shaped object.
|
||||
case "$OUTPUT" in
|
||||
*\"decision\"*)
|
||||
mempal_log "preInvocation" "$CONVERSATION_ID" "ERROR: refused to emit decision field from wake hook"
|
||||
mempal_emit_stop_pass
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
|
||||
printf '%s\n' "$OUTPUT"
|
||||
exit 0
|
||||
@@ -0,0 +1,157 @@
|
||||
# MemPalace Cursor IDE Hooks
|
||||
|
||||
Auto-save and session-recall hooks for the [Cursor](https://cursor.com) IDE,
|
||||
matching the behaviour of the existing Claude Code + Codex hooks at the repo
|
||||
root and adding two Cursor-only capabilities (`sessionStart` recall and a
|
||||
preCompact transcript snapshot).
|
||||
|
||||
For the rendered documentation see
|
||||
[`website/guide/cursor-hooks.md`](../../website/guide/cursor-hooks.md) or
|
||||
the published version at
|
||||
[mempalaceofficial.com/guide/cursor-hooks](https://mempalaceofficial.com/guide/cursor-hooks.html).
|
||||
|
||||
## What's here
|
||||
|
||||
| File | Role |
|
||||
|-------------------------------------|---------------------------------------------------------------------|
|
||||
| `lib/common.sh` | Shared bash helpers (parse, log, counter, wing inference, kill switch). Sourced by all three hooks. |
|
||||
| `mempal_save_hook_cursor.sh` | Cursor `stop` hook. Counts stop invocations per conversation, emits a `followup_message` every `SAVE_INTERVAL` (default 15) telling the agent to file the session into MemPalace. |
|
||||
| `mempal_precompact_hook_cursor.sh` | Cursor `preCompact` hook. Runs `mempalace mine` synchronously on the transcript before compaction, then drops a `.pending` marker so the next stop forces a save nudge. |
|
||||
| `mempal_wake_hook_cursor.sh` | Cursor `sessionStart` hook. Returns `additional_context` telling the agent to recall scoped to the wing inferred from the workspace root. Cursor-only — Claude Code has no equivalent. |
|
||||
| `install.sh` | Optional installer. Copies the scripts to `~/.mempalace/hooks/cursor/` and merges entries into `~/.cursor/hooks.json` (or `.cursor/hooks.json` for project scope). Supports `--dry-run` and `--uninstall`. |
|
||||
| `STDIN_SHAPE.md` | Reference. Per-event stdin / stdout schema with citations to the official Cursor docs. |
|
||||
|
||||
## Quick install
|
||||
|
||||
Preview first (writes nothing, prints the would-be JSON to stdout):
|
||||
|
||||
```bash
|
||||
hooks/cursor/install.sh --scope user --dry-run
|
||||
```
|
||||
|
||||
Apply — writes `~/.cursor/hooks.json` and copies the scripts to `~/.mempalace/hooks/cursor/`:
|
||||
|
||||
```bash
|
||||
hooks/cursor/install.sh --scope user
|
||||
```
|
||||
|
||||
Pass `--scope project --target <repo>` to write `<repo>/.cursor/hooks.json` instead.
|
||||
The installer never auto-runs — it is a documented opt-in step. We do not
|
||||
modify your Cursor config on `pip install mempalace` because editor config
|
||||
is sacred and should never be touched without explicit consent.
|
||||
|
||||
## Manual install (no installer)
|
||||
|
||||
The minimum wiring is `stop` only. Add to `~/.cursor/hooks.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"version": 1,
|
||||
"hooks": {
|
||||
"stop": [
|
||||
{
|
||||
"command": "/absolute/path/to/hooks/cursor/mempal_save_hook_cursor.sh",
|
||||
"loop_limit": 1
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For the full triple (recommended), also wire `sessionStart` and `preCompact`
|
||||
— see [`examples/cursor/hooks.json`](../../examples/cursor/hooks.json).
|
||||
|
||||
After editing the file, Cursor watches `hooks.json` and reloads
|
||||
automatically. If hooks still do not fire, restart Cursor and check the
|
||||
Hooks panel in Settings.
|
||||
|
||||
## Configuration
|
||||
|
||||
All knobs are env vars; defaults match the Claude Code hooks where
|
||||
possible so a single hook-state directory works for both editors.
|
||||
|
||||
| Variable | Default | Purpose |
|
||||
|--------------------------------|------------------------------------|---------|
|
||||
| `MEMPAL_SAVE_INTERVAL` | `15` | Number of `stop` events between save followups. |
|
||||
| `MEMPAL_CURSOR_SILENT` | (unset) | Set to `1`/`true`/`yes` to suppress the `followup_message`. The hook still runs its best-effort background mine and keeps its counters — it just stays silent. `MEMPAL_VERBOSE=false`/`0`/`no` does the same. See note below on why the followup is on by default. |
|
||||
| `MEMPAL_DIR` | (unset) | Optional project directory to also mine on each save. Additive — never replaces the transcript mine. |
|
||||
| `MEMPAL_PYTHON` | auto-detected | Path to a Python 3 interpreter. Fallback order: `$MEMPAL_PYTHON` → `command -v python3` → bare `python3`. Useful when Cursor is launched from a GUI on macOS and the inherited PATH lacks your installed `python3`. |
|
||||
| `MEMPAL_STATE_DIR` | `$HOME/.mempalace/hook_state` | Where the hook keeps its per-conversation counter files, pending-save markers, and `cursor_hook.log`. |
|
||||
| `MEMPAL_STATE_TTL_DAYS` | `30` | Age (days) after which stale `cursor_*.count` / `cursor_*.pending` state files are garbage-collected. A daily-throttled sweep runs from the hooks; only Cursor state is touched (shared logs and other editors' state are left alone). |
|
||||
| `MEMPAL_DISABLE_HOOK` | (unset) | Set to `1`/`true`/`yes` to disable all three hooks. Emergency kill switch. |
|
||||
| `MEMPALACE_HOOKS_AUTO_SAVE` | (unset) | Set to `false`/`0`/`no` to disable. Same semantics as the Claude Code hooks. Also honoured via `~/.mempalace/config.json` → `{"hooks": {"auto_save": false}}`. |
|
||||
|
||||
## Debugging
|
||||
|
||||
Everything appends to:
|
||||
|
||||
```bash
|
||||
cat ~/.mempalace/hook_state/cursor_hook.log
|
||||
```
|
||||
|
||||
Example log lines (ISO 8601 + event + conversation id):
|
||||
|
||||
```
|
||||
[2026-05-27T02:16:01Z] [event=sessionStart] [conv=abc123] workspace=/Users/me/proj wing=proj
|
||||
[2026-05-27T02:21:33Z] [event=stop] [conv=abc123] counter 0 -> 1 (interval=15)
|
||||
[2026-05-27T02:42:09Z] [event=stop] [conv=abc123] counter 14 -> 15 (interval=15)
|
||||
[2026-05-27T02:42:09Z] [event=stop] [conv=abc123] TRIGGERING SAVE at counter=15
|
||||
[2026-05-27T02:42:11Z] [event=stop] [conv=abc123] loop_count>0; letting agent stop
|
||||
```
|
||||
|
||||
When a hook can't parse its stdin (corrupt payload, future Cursor schema
|
||||
change), the raw input (capped at 4096 bytes, mode 0600) lands at:
|
||||
|
||||
```
|
||||
~/.mempalace/hook_state/cursor_last_input.log
|
||||
~/.mempalace/hook_state/cursor_last_python_err.log
|
||||
```
|
||||
|
||||
These are overwritten on each failure, never appended, so a repeating
|
||||
misconfiguration cannot grow disk usage.
|
||||
|
||||
## What differs from the Claude Code hooks
|
||||
|
||||
| Aspect | Claude Code hooks (`hooks/mempal_*.sh`) | Cursor hooks (`hooks/cursor/*.sh`) |
|
||||
|-------------------------|------------------------------------------------|-----------------------------------------------------|
|
||||
| Counter key | `session_id` | `conversation_id` (Cursor's stable per-conv id) |
|
||||
| Loop guard | `stop_hook_active` flag in stdin | `loop_count` field in stdin |
|
||||
| Counting method | Parses JSONL transcript for user messages | Counts `stop` invocations (transcript schema undoc) |
|
||||
| Capture path | Background `mine --mode convos` (normalize.py has a Claude parser) | Background mine is best-effort (no Cursor parser); the `followup_message` carries verbatim capture |
|
||||
| Save default | Silent — diary nudge opt-IN behind `MEMPAL_VERBOSE=true` | Followup ON by default; opt-OUT via `MEMPAL_CURSOR_SILENT=1` / `MEMPAL_VERBOSE=false` |
|
||||
| PreCompact behaviour | `decision: block` forces save before compaction | Pre-mine + pending-save marker (Cursor preCompact is observational-only) |
|
||||
| sessionStart | n/a (Claude Code has no equivalent) | `additional_context` injects recall guidance |
|
||||
| State dir | `$HOME/.mempalace/hook_state` (hardcoded) | Same default, plus `MEMPAL_STATE_DIR` env override |
|
||||
| Kill switch | `MEMPALACE_HOOKS_AUTO_SAVE=false` | Same, plus `MEMPAL_DISABLE_HOOK=1` alias |
|
||||
| Log file | `hook.log` | `cursor_hook.log` (kept separate to avoid cross-tool log churn) |
|
||||
|
||||
See [`STDIN_SHAPE.md`](STDIN_SHAPE.md) for the per-event schema and
|
||||
[`website/guide/cursor-hooks.md`](../../website/guide/cursor-hooks.md) for
|
||||
the full walkthrough with diagrams.
|
||||
|
||||
## Why the followup is on by default (Cursor-specific)
|
||||
|
||||
Unlike the Claude Code hook — which is silent by default because its
|
||||
background `mempalace mine --mode convos` captures the verbatim transcript
|
||||
on its own — Cursor's transcript format is **undocumented** and
|
||||
`mempalace/normalize.py` has **no Cursor parser**. The background mine on
|
||||
the Cursor `stop`/`preCompact` hooks is therefore **best-effort only**: it
|
||||
does not yet yield clean verbatim conversation drawers.
|
||||
|
||||
That makes the `followup_message` the **load-bearing verbatim-capture
|
||||
path** for Cursor — it drives the agent to file its own in-context
|
||||
verbatim quotes via `mempalace_add_drawer` / `mempalace_diary_write`.
|
||||
Silencing it by default would leave a default Cursor install capturing
|
||||
nothing, which is why it is on by default here. Set `MEMPAL_CURSOR_SILENT=1`
|
||||
(or `MEMPAL_VERBOSE=false`) if you prefer the Claude-style silent
|
||||
behaviour and accept the reduced capture. Once `normalize.py` learns to
|
||||
read Cursor transcripts, this default will flip to silent to match Claude.
|
||||
|
||||
## Cost
|
||||
|
||||
Zero extra LLM tokens spent by the hooks themselves. The hooks are local
|
||||
bash scripts that run on your machine. The followup message the save hook
|
||||
emits is a normal user turn — it counts the same as any other user message
|
||||
and does not invoke any extra LLM call beyond the one the user would
|
||||
otherwise make. Suppress it with `MEMPAL_CURSOR_SILENT=1` if you want zero
|
||||
followups in the chat window.
|
||||
@@ -0,0 +1,184 @@
|
||||
# Cursor Hook Stdin Shape — Reference
|
||||
|
||||
This file documents the JSON payloads the Cursor IDE sends to the
|
||||
MemPalace hook scripts in `hooks/cursor/`. It exists so a future
|
||||
contributor does not have to re-discover the schema by writing a
|
||||
probe hook.
|
||||
|
||||
**Source:** [`cursor.com/docs/hooks.md`](https://cursor.com/docs/hooks.md),
|
||||
fetched 2026-05-27. Cursor's hook system is documented as a stable
|
||||
v1 schema (`{"version": 1, ...}` at the top of `hooks.json`).
|
||||
|
||||
If you suspect Cursor has changed the payload shape since that fetch
|
||||
date, re-verify against the upstream docs and update both this file
|
||||
and `hooks/cursor/lib/common.sh::mempal_parse_stdin`. The hook
|
||||
scripts deliberately ignore fields they do not consume, so adding
|
||||
new fields is non-breaking.
|
||||
|
||||
## Common fields (all events)
|
||||
|
||||
Every hook receives these on stdin in addition to its event-specific
|
||||
fields. Source: docs section "Common schema → Input (all hooks)".
|
||||
|
||||
```json
|
||||
{
|
||||
"conversation_id": "string",
|
||||
"generation_id": "string",
|
||||
"model": "string",
|
||||
"hook_event_name": "string",
|
||||
"cursor_version": "string",
|
||||
"workspace_roots": ["<absolute path>"],
|
||||
"user_email": "string | null",
|
||||
"transcript_path": "string | null"
|
||||
}
|
||||
```
|
||||
|
||||
**Field notes (verified):**
|
||||
|
||||
- `conversation_id` is the stable per-conversation ID. The Cursor
|
||||
`stop` event does **not** carry a `session_id` — only
|
||||
`conversation_id`. MemPalace keys its counter files on this. Cursor
|
||||
`sessionStart` does carry a `session_id`, and the docs note it is
|
||||
"same as `conversation_id`".
|
||||
- `generation_id` changes every user message. We do not use it.
|
||||
- `transcript_path` may be `null` if the user has disabled
|
||||
transcripts in Cursor settings. The hooks degrade gracefully when
|
||||
the value is empty.
|
||||
- `workspace_roots` is normally a single-entry array but multi-root
|
||||
workspaces are supported; MemPalace uses index `[0]`.
|
||||
|
||||
## Event-specific fields
|
||||
|
||||
### `stop` (consumed by `mempal_save_hook_cursor.sh`)
|
||||
|
||||
```json
|
||||
{
|
||||
"status": "completed" | "aborted" | "error",
|
||||
"loop_count": 0
|
||||
}
|
||||
```
|
||||
|
||||
- `loop_count` indicates how many times this stop hook has already
|
||||
triggered an automatic followup for this conversation (starts at
|
||||
0). When `loop_count > 0` we know our own previous `followup_message`
|
||||
is currently being processed — the save hook returns `{}` so the
|
||||
agent can finish. Equivalent to Claude Code's `stop_hook_active`.
|
||||
- The per-script `loop_limit` (default 5 for Cursor hooks, configurable
|
||||
via the `loop_limit` field on the hook entry in `hooks.json`) is
|
||||
defense-in-depth on top of our own check. The example `hooks.json`
|
||||
in `examples/cursor/` sets `loop_limit: 1`.
|
||||
|
||||
**Allowed output fields** (only):
|
||||
|
||||
```json
|
||||
{ "followup_message": "<text to auto-submit as next user turn>" }
|
||||
```
|
||||
|
||||
### `preCompact` (consumed by `mempal_precompact_hook_cursor.sh`)
|
||||
|
||||
```json
|
||||
{
|
||||
"trigger": "auto" | "manual",
|
||||
"context_usage_percent": 85,
|
||||
"context_tokens": 120000,
|
||||
"context_window_size": 128000,
|
||||
"message_count": 45,
|
||||
"messages_to_compact": 30,
|
||||
"is_first_compaction": true
|
||||
}
|
||||
```
|
||||
|
||||
**Critical constraint:** preCompact is documented as **observational
|
||||
only**. It cannot block compaction and its allowed output fields are
|
||||
limited to:
|
||||
|
||||
```json
|
||||
{ "user_message": "<short message shown to the user when compaction occurs>" }
|
||||
```
|
||||
|
||||
There is **no** `followup_message` and **no** `decision: block` on
|
||||
this event — unlike Claude Code's `PreCompact`. MemPalace works
|
||||
around this by:
|
||||
|
||||
1. Running `mempalace mine` synchronously inside the hook so the
|
||||
verbatim transcript lands in the palace before compaction
|
||||
summarises it.
|
||||
2. Dropping a `cursor_<conversation_id>.pending` marker that the next
|
||||
`stop` invocation reads and uses to force a save followup
|
||||
regardless of its counter.
|
||||
|
||||
### `sessionStart` (consumed by `mempal_wake_hook_cursor.sh`)
|
||||
|
||||
```json
|
||||
{
|
||||
"session_id": "<unique session identifier>",
|
||||
"is_background_agent": true,
|
||||
"composer_mode": "agent" | "ask" | "edit"
|
||||
}
|
||||
```
|
||||
|
||||
`session_id` equals `conversation_id` on this event (docs are
|
||||
explicit about this).
|
||||
|
||||
**Allowed output fields:**
|
||||
|
||||
```json
|
||||
{
|
||||
"env": { "<key>": "<value>" },
|
||||
"additional_context": "<text added to conversation's initial system context>"
|
||||
}
|
||||
```
|
||||
|
||||
`additional_context` is the field MemPalace uses. The schema also
|
||||
accepts `continue` and `user_message` but the docs explicitly note
|
||||
"current callers do not enforce them; session creation is not
|
||||
blocked even when continue is false". We do not emit either.
|
||||
|
||||
## Environment variables (all hooks)
|
||||
|
||||
Cursor sets these env vars on every hook execution; the hook scripts
|
||||
fall back to them when JSON parsing fails for any reason.
|
||||
|
||||
| Variable | Description |
|
||||
|---------------------------|---------------------------------------------------|
|
||||
| `CURSOR_PROJECT_DIR` | Workspace root (= `workspace_roots[0]`) |
|
||||
| `CURSOR_VERSION` | Cursor version string |
|
||||
| `CURSOR_USER_EMAIL` | Authenticated user email (if logged in) |
|
||||
| `CURSOR_TRANSCRIPT_PATH` | Conversation transcript path (if transcripts on) |
|
||||
| `CURSOR_CODE_REMOTE` | `"true"` if running in a remote workspace |
|
||||
| `CLAUDE_PROJECT_DIR` | Alias for `CURSOR_PROJECT_DIR` (Claude compat) |
|
||||
|
||||
## Exit code semantics
|
||||
|
||||
Cursor interprets command-hook exit codes as follows
|
||||
(docs "Hook Types → Command-Based Hooks → Exit code behavior"):
|
||||
|
||||
- `0` — success, use the JSON output.
|
||||
- `2` — block the action (equivalent to `permission: "deny"`).
|
||||
- Other — hook failed; action proceeds (fail-open by default).
|
||||
|
||||
MemPalace hooks always exit `0` and emit either `{}` (no-op) or a
|
||||
valid JSON response. We never use exit code `2`; nothing MemPalace
|
||||
does should ever block an agent action.
|
||||
|
||||
## Working directory contract
|
||||
|
||||
- **User hooks** (`~/.cursor/hooks.json`) run from `~/.cursor/`.
|
||||
- **Project hooks** (`.cursor/hooks.json`) run from the project root.
|
||||
|
||||
The MemPalace hooks always resolve their sibling `lib/common.sh` via
|
||||
`BASH_SOURCE[0]` so the working directory does not matter for the
|
||||
script's own loading — only the `command` path in `hooks.json` needs
|
||||
to point at the absolute location of the script.
|
||||
|
||||
## Transcript file format (out of scope)
|
||||
|
||||
The format of the file at `transcript_path` is **not documented by
|
||||
Cursor** as of the fetch date above. MemPalace deliberately does not
|
||||
parse it: the save hook counts `stop` invocations (each one
|
||||
corresponds to one assistant turn) and hands the transcript to
|
||||
`mempalace mine`, which has its own normaliser layer.
|
||||
|
||||
If you need to consume the transcript directly, probe its shape with
|
||||
a throw-away hook that does `cat > /tmp/cursor-transcript-sample.txt`
|
||||
and inspect the output — there is no shortcut.
|
||||
Executable
+399
@@ -0,0 +1,399 @@
|
||||
#!/bin/bash
|
||||
# MEMPALACE CURSOR HOOK INSTALLER
|
||||
#
|
||||
# Optional helper. Copies the three Cursor hook scripts to a
|
||||
# stable install location and merges entries into a Cursor
|
||||
# `hooks.json` config file — without clobbering unrelated hooks
|
||||
# already in that file.
|
||||
#
|
||||
# This is NEVER auto-invoked. Editor config is sacred; we do not
|
||||
# modify a user's hooks.json without explicit consent. The user runs
|
||||
# this script (or wires the hooks manually) as a documented opt-in.
|
||||
#
|
||||
# === USAGE ===
|
||||
#
|
||||
# hooks/cursor/install.sh [options]
|
||||
#
|
||||
# Options:
|
||||
# --scope user|project Target scope. Default: user.
|
||||
# - user: merges into ~/.cursor/hooks.json
|
||||
# - project: merges into <target>/.cursor/hooks.json
|
||||
# --target <path> Project root for --scope project (default: $PWD).
|
||||
# Ignored for --scope user.
|
||||
# --install-dir <path> Where to copy the hook scripts.
|
||||
# Default: ~/.mempalace/hooks/cursor
|
||||
# --variant full|minimal Which hook set to wire.
|
||||
# - full: stop + preCompact + sessionStart
|
||||
# - minimal: stop only
|
||||
# Default: full.
|
||||
# --dry-run Print the would-be JSON to stdout, do not write
|
||||
# and do not copy scripts.
|
||||
# --uninstall Remove MemPalace entries from the target
|
||||
# hooks.json (preserves unrelated hooks).
|
||||
# Does NOT delete the installed scripts.
|
||||
# -h, --help Show this help and exit.
|
||||
#
|
||||
# === PORTABILITY ===
|
||||
#
|
||||
# Pure bash 3.2 + POSIX tools + python3 (which the hook scripts
|
||||
# themselves already require). No `jq` dependency.
|
||||
#
|
||||
# Python helpers are materialised to temp files rather than piped via
|
||||
# `$(... <<'PYEOF' ... PYEOF)` to dodge the bash 3.2.57 parser bug
|
||||
# that trips on parens nested inside a heredoc body that lives inside
|
||||
# a `$(...)` command substitution.
|
||||
|
||||
set -e
|
||||
|
||||
usage() {
|
||||
sed -n '2,38p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'
|
||||
}
|
||||
|
||||
# ── Defaults ──────────────────────────────────────────────────────
|
||||
SCOPE="user"
|
||||
TARGET=""
|
||||
INSTALL_DIR="$HOME/.mempalace/hooks/cursor"
|
||||
VARIANT="full"
|
||||
DRY_RUN=0
|
||||
UNINSTALL=0
|
||||
|
||||
# ── Parse args ────────────────────────────────────────────────────
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--scope)
|
||||
shift
|
||||
SCOPE="${1:-}"
|
||||
;;
|
||||
--target)
|
||||
shift
|
||||
TARGET="${1:-}"
|
||||
;;
|
||||
--install-dir)
|
||||
shift
|
||||
INSTALL_DIR="${1:-}"
|
||||
;;
|
||||
--variant)
|
||||
shift
|
||||
VARIANT="${1:-}"
|
||||
;;
|
||||
--dry-run) DRY_RUN=1 ;;
|
||||
--uninstall) UNINSTALL=1 ;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
printf 'install.sh: unknown argument: %s\n' "$1" >&2
|
||||
usage >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
shift || true
|
||||
done
|
||||
|
||||
case "$SCOPE" in
|
||||
user|project) ;;
|
||||
*)
|
||||
printf 'install.sh: --scope must be "user" or "project" (got "%s")\n' \
|
||||
"$SCOPE" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
|
||||
case "$VARIANT" in
|
||||
full|minimal) ;;
|
||||
*)
|
||||
printf 'install.sh: --variant must be "full" or "minimal" (got "%s")\n' \
|
||||
"$VARIANT" >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
|
||||
# ── Resolve paths ─────────────────────────────────────────────────
|
||||
_script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" 2>/dev/null && pwd)"
|
||||
SOURCE_DIR="$_script_dir"
|
||||
|
||||
# Resolve --install-dir to an absolute path before it gets baked into
|
||||
# hooks.json. Cursor invokes hook commands from its own working
|
||||
# directory (typically the project root), so a relative command path
|
||||
# would silently fail to launch the hook. gh-PR review caught this.
|
||||
case "$INSTALL_DIR" in
|
||||
/*) ;;
|
||||
*) INSTALL_DIR="$PWD/$INSTALL_DIR" ;;
|
||||
esac
|
||||
|
||||
# Resolve the Python interpreter the same way the hooks themselves do
|
||||
# so a user with a non-default Python is consistent across install +
|
||||
# runtime.
|
||||
if [ -n "${MEMPAL_PYTHON:-}" ] && [ -x "$MEMPAL_PYTHON" ]; then
|
||||
PYTHON_BIN="$MEMPAL_PYTHON"
|
||||
else
|
||||
PYTHON_BIN="$(command -v python3 2>/dev/null || true)"
|
||||
fi
|
||||
if [ -z "$PYTHON_BIN" ]; then
|
||||
printf 'install.sh: python3 not found on PATH; cannot proceed.\n' >&2
|
||||
printf 'Set $MEMPAL_PYTHON to an interpreter path or install python3.\n' >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Determine the target hooks.json path.
|
||||
case "$SCOPE" in
|
||||
user)
|
||||
if [ -n "$TARGET" ]; then
|
||||
printf 'install.sh: --target is only meaningful with --scope project; ignoring.\n' >&2
|
||||
fi
|
||||
TARGET_DIR="$HOME/.cursor"
|
||||
;;
|
||||
project)
|
||||
TARGET_DIR="${TARGET:-$PWD}/.cursor"
|
||||
;;
|
||||
esac
|
||||
TARGET_FILE="$TARGET_DIR/hooks.json"
|
||||
|
||||
# Determine which commands the merge / uninstall logic should
|
||||
# install or remove. Paths point at the install location, NOT the
|
||||
# source repo — once the user runs install.sh they can move / delete
|
||||
# the cloned repo without breaking the wiring.
|
||||
SAVE_CMD="$INSTALL_DIR/mempal_save_hook_cursor.sh"
|
||||
PRECOMPACT_CMD="$INSTALL_DIR/mempal_precompact_hook_cursor.sh"
|
||||
WAKE_CMD="$INSTALL_DIR/mempal_wake_hook_cursor.sh"
|
||||
|
||||
# ── Step 1: copy scripts (skipped on --dry-run / --uninstall) ─────
|
||||
if [ "$UNINSTALL" -eq 0 ] && [ "$DRY_RUN" -eq 0 ]; then
|
||||
mkdir -p "$INSTALL_DIR/lib"
|
||||
cp "$SOURCE_DIR/lib/common.sh" "$INSTALL_DIR/lib/common.sh"
|
||||
cp "$SOURCE_DIR/mempal_save_hook_cursor.sh" "$INSTALL_DIR/"
|
||||
cp "$SOURCE_DIR/mempal_precompact_hook_cursor.sh" "$INSTALL_DIR/"
|
||||
cp "$SOURCE_DIR/mempal_wake_hook_cursor.sh" "$INSTALL_DIR/"
|
||||
chmod +x "$INSTALL_DIR/mempal_save_hook_cursor.sh" \
|
||||
"$INSTALL_DIR/mempal_precompact_hook_cursor.sh" \
|
||||
"$INSTALL_DIR/mempal_wake_hook_cursor.sh"
|
||||
printf 'install.sh: copied scripts to %s\n' "$INSTALL_DIR" >&2
|
||||
fi
|
||||
|
||||
# ── Short-circuit: uninstall with no existing file is a no-op ─────
|
||||
#
|
||||
# Without this, the merge step would happily write an empty
|
||||
# {"version": 1, "hooks": {}} to a brand new file that the user
|
||||
# never asked us to create — surprising behaviour that the test
|
||||
# suite explicitly guards against.
|
||||
if [ "$UNINSTALL" -eq 1 ] && [ ! -f "$TARGET_FILE" ]; then
|
||||
printf 'install.sh: nothing to uninstall (%s does not exist)\n' \
|
||||
"$TARGET_FILE" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Step 2: merge / unmerge hooks.json via python3 ────────────────
|
||||
#
|
||||
# Materialise the merge logic to a temp .py file (see bash 3.2
|
||||
# rationale at the top of this script), then invoke it. The Python
|
||||
# script is responsible for:
|
||||
# * tolerating a missing or empty hooks.json (starts from {})
|
||||
# * preserving unrelated hook entries on install
|
||||
# * preserving unrelated hook entries on uninstall
|
||||
# * recognising MemPalace entries by basename in the `command` field
|
||||
# * idempotent install (re-running does not duplicate entries)
|
||||
|
||||
# mktemp portability: pass an explicit absolute template so we sidestep
|
||||
# the BSD vs GNU difference in `-t` semantics (BSD treats it as a
|
||||
# prefix; GNU treats it as a template). Honour TMPDIR if set, fall
|
||||
# back to /tmp. gh-PR review caught the previous `-t` form as
|
||||
# non-portable.
|
||||
MERGE_PY="$(mktemp "${TMPDIR:-/tmp}/mempal-install-merge.XXXXXX")"
|
||||
trap 'rm -f "$MERGE_PY"' EXIT
|
||||
|
||||
cat > "$MERGE_PY" <<'PYEOF'
|
||||
"""hooks.json merge helper for hooks/cursor/install.sh.
|
||||
|
||||
Argv:
|
||||
sys.argv[1]: path to hooks.json (may not exist)
|
||||
sys.argv[2]: variant ("full" or "minimal")
|
||||
sys.argv[3]: uninstall flag ("1" or "0")
|
||||
sys.argv[4]: save_cmd absolute path
|
||||
sys.argv[5]: precompact_cmd absolute path
|
||||
sys.argv[6]: wake_cmd absolute path
|
||||
|
||||
Output: prints the merged JSON to stdout. Exits 2 on a malformed
|
||||
existing config (refuses to overwrite a broken file).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
|
||||
target_file = sys.argv[1]
|
||||
variant = sys.argv[2]
|
||||
uninstall = sys.argv[3] == "1"
|
||||
save_cmd = sys.argv[4]
|
||||
precompact_cmd = sys.argv[5]
|
||||
wake_cmd = sys.argv[6]
|
||||
|
||||
# Recognise our entries by basename. The three filenames below are
|
||||
# the unique product-of-our-naming convention; any entry whose command
|
||||
# ends in one of them is treated as a MemPalace entry on
|
||||
# install (so we replace rather than duplicate it) and on uninstall
|
||||
# (so we remove it without touching unrelated entries). Matching on
|
||||
# basename rather than a full-path substring lets users pick any
|
||||
# --install-dir without breaking uninstall.
|
||||
MEMPAL_BASENAMES = (
|
||||
"mempal_save_hook_cursor.sh",
|
||||
"mempal_precompact_hook_cursor.sh",
|
||||
"mempal_wake_hook_cursor.sh",
|
||||
)
|
||||
|
||||
if os.path.exists(target_file):
|
||||
with open(target_file, "r", encoding="utf-8") as fh:
|
||||
try:
|
||||
cfg = json.load(fh)
|
||||
except Exception as exc:
|
||||
sys.stderr.write(
|
||||
"install.sh: existing %s is not valid JSON: %s\n"
|
||||
"Refusing to overwrite. Fix the file and retry.\n"
|
||||
% (target_file, exc)
|
||||
)
|
||||
sys.exit(2)
|
||||
else:
|
||||
cfg = {}
|
||||
|
||||
if not isinstance(cfg, dict):
|
||||
sys.stderr.write(
|
||||
"install.sh: %s top level must be a JSON object; got %s\n"
|
||||
% (target_file, type(cfg).__name__)
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
cfg.setdefault("version", 1)
|
||||
cfg.setdefault("hooks", {})
|
||||
if not isinstance(cfg["hooks"], dict):
|
||||
sys.stderr.write(
|
||||
"install.sh: %s 'hooks' must be a JSON object\n" % target_file
|
||||
)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def is_mempal_entry(entry):
|
||||
if not isinstance(entry, dict):
|
||||
return False
|
||||
cmd = entry.get("command", "")
|
||||
if not isinstance(cmd, str):
|
||||
return False
|
||||
# Match on basename so a customised --install-dir (e.g. /opt/...,
|
||||
# ~/.local/share/..., or anything with or without a leading dot)
|
||||
# still round-trips through uninstall.
|
||||
base = os.path.basename(cmd)
|
||||
return base in MEMPAL_BASENAMES
|
||||
|
||||
|
||||
def filter_mempal(entries):
|
||||
if not isinstance(entries, list):
|
||||
return entries
|
||||
return [e for e in entries if not is_mempal_entry(e)]
|
||||
|
||||
|
||||
def upsert(event, entry):
|
||||
existing = cfg["hooks"].get(event, [])
|
||||
if not isinstance(existing, list):
|
||||
sys.stderr.write(
|
||||
"install.sh: %s hooks[%s] must be a list\n" % (target_file, event)
|
||||
)
|
||||
sys.exit(2)
|
||||
cleaned = [e for e in existing if not is_mempal_entry(e)]
|
||||
cleaned.append(entry)
|
||||
cfg["hooks"][event] = cleaned
|
||||
|
||||
|
||||
if uninstall:
|
||||
for event in list(cfg["hooks"].keys()):
|
||||
cfg["hooks"][event] = filter_mempal(cfg["hooks"][event])
|
||||
if not cfg["hooks"][event]:
|
||||
del cfg["hooks"][event]
|
||||
else:
|
||||
upsert("stop", {"command": save_cmd, "loop_limit": 1})
|
||||
if variant == "full":
|
||||
upsert("preCompact", {"command": precompact_cmd})
|
||||
upsert("sessionStart", {"command": wake_cmd})
|
||||
|
||||
# Stable key order for the events MemPalace touches, then preserve
|
||||
# any unrelated event names in their original order so future Cursor
|
||||
# events we don't know about yet still round-trip.
|
||||
known_order = [
|
||||
"sessionStart",
|
||||
"stop",
|
||||
"preCompact",
|
||||
"sessionEnd",
|
||||
"preToolUse",
|
||||
"postToolUse",
|
||||
"postToolUseFailure",
|
||||
"subagentStart",
|
||||
"subagentStop",
|
||||
"beforeShellExecution",
|
||||
"afterShellExecution",
|
||||
"beforeMCPExecution",
|
||||
"afterMCPExecution",
|
||||
"beforeReadFile",
|
||||
"afterFileEdit",
|
||||
"beforeSubmitPrompt",
|
||||
"afterAgentResponse",
|
||||
"afterAgentThought",
|
||||
"beforeTabFileRead",
|
||||
"afterTabFileEdit",
|
||||
"workspaceOpen",
|
||||
]
|
||||
|
||||
ordered_hooks = {}
|
||||
for event in known_order:
|
||||
if event in cfg["hooks"]:
|
||||
ordered_hooks[event] = cfg["hooks"][event]
|
||||
for event, entries in cfg["hooks"].items():
|
||||
if event not in ordered_hooks:
|
||||
ordered_hooks[event] = entries
|
||||
cfg["hooks"] = ordered_hooks
|
||||
|
||||
print(json.dumps(cfg, indent=2, sort_keys=False))
|
||||
PYEOF
|
||||
|
||||
NEW_JSON="$("$PYTHON_BIN" "$MERGE_PY" \
|
||||
"$TARGET_FILE" "$VARIANT" "$UNINSTALL" \
|
||||
"$SAVE_CMD" "$PRECOMPACT_CMD" "$WAKE_CMD")"
|
||||
|
||||
# ── Step 3: emit, write, or remove ────────────────────────────────
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
printf 'install.sh: --dry-run; would write to %s\n' "$TARGET_FILE" >&2
|
||||
printf '%s\n' "$NEW_JSON"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mkdir -p "$TARGET_DIR"
|
||||
|
||||
# If --uninstall left an empty hooks object AND no other top-level
|
||||
# keys beyond version, remove the file entirely so the user's
|
||||
# `.cursor/` directory does not accumulate orphan configs.
|
||||
if [ "$UNINSTALL" -eq 1 ]; then
|
||||
# Inline the emptiness check via `python -c '...'` rather than a
|
||||
# temp .py file. The body is short enough that a tmpfile is pure
|
||||
# overhead, and removing the tmpfile eliminates a small leak
|
||||
# window if the script is interrupted between mktemp and rm -f.
|
||||
# gh-PR review suggested this simplification.
|
||||
NON_EMPTY="$(printf '%s' "$NEW_JSON" | "$PYTHON_BIN" -c '
|
||||
import json, sys
|
||||
cfg = json.load(sys.stdin)
|
||||
hooks = cfg.get("hooks", {})
|
||||
extras = [k for k in cfg.keys() if k not in ("version", "hooks")]
|
||||
print("1" if (hooks or extras) else "0")
|
||||
')"
|
||||
if [ "$NON_EMPTY" = "0" ] && [ -f "$TARGET_FILE" ]; then
|
||||
rm -f "$TARGET_FILE"
|
||||
printf 'install.sh: removed empty %s\n' "$TARGET_FILE" >&2
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
TMP_FILE="${TARGET_FILE}.tmp.$$"
|
||||
printf '%s\n' "$NEW_JSON" > "$TMP_FILE"
|
||||
mv "$TMP_FILE" "$TARGET_FILE"
|
||||
|
||||
if [ "$UNINSTALL" -eq 1 ]; then
|
||||
printf 'install.sh: removed MemPalace entries from %s\n' "$TARGET_FILE" >&2
|
||||
else
|
||||
printf 'install.sh: wrote %s\n' "$TARGET_FILE" >&2
|
||||
printf 'install.sh: restart Cursor (or wait for it to reload hooks.json)\n' >&2
|
||||
fi
|
||||
@@ -0,0 +1,490 @@
|
||||
# shellcheck shell=bash
|
||||
# MEMPALACE CURSOR HOOK — shared helpers
|
||||
#
|
||||
# Sourced by the three Cursor hooks (stop / preCompact / sessionStart).
|
||||
# Mirrors the conventions of the existing Claude Code hook scripts
|
||||
# (hooks/mempal_save_hook.sh, hooks/mempal_precompact_hook.sh) so a
|
||||
# user who already debugs one knows how to debug the other:
|
||||
#
|
||||
# * STATE_DIR layout under ~/.mempalace/hook_state/
|
||||
# * MEMPAL_PYTHON resolution order (override → $PATH → bare python3)
|
||||
# * MEMPALACE_HOOKS_AUTO_SAVE=false kill switch (config.json fallback)
|
||||
# * sentinel-guarded Python parser via `sed -n 'Np'` (bash 3.2 safe)
|
||||
# * fail-open on internal errors: emit `{}` and log, never crash the
|
||||
# hook host
|
||||
#
|
||||
# Cursor-specific additions on top of that contract:
|
||||
#
|
||||
# * MEMPAL_DISABLE_HOOK=1 as an additional kill-switch alias
|
||||
# * MEMPAL_STATE_DIR env override for the state directory
|
||||
# * conversation_id (Cursor's stable per-conversation ID) replaces
|
||||
# Claude Code's session_id in the counter file names — Cursor `stop`
|
||||
# events do not carry a session_id, only conversation_id
|
||||
# * loop_count is the loop-prevention signal in place of Claude Code's
|
||||
# stop_hook_active flag (Cursor docs, "stop" event)
|
||||
#
|
||||
# This file is sourced, not executed, so it intentionally has no
|
||||
# shebang. The `# shellcheck shell=bash` directive above tells
|
||||
# shellcheck to treat it as bash when run standalone.
|
||||
|
||||
# ── State directory + log path ────────────────────────────────────────
|
||||
#
|
||||
# Honour MEMPAL_STATE_DIR (additive override introduced for Cursor)
|
||||
# while keeping the default identical to the Claude Code hooks so a
|
||||
# user running both keeps a single state directory.
|
||||
MEMPAL_STATE_DIR="${MEMPAL_STATE_DIR:-$HOME/.mempalace/hook_state}"
|
||||
mkdir -p "$MEMPAL_STATE_DIR" 2>/dev/null
|
||||
MEMPAL_CURSOR_LOG="$MEMPAL_STATE_DIR/cursor_hook.log"
|
||||
|
||||
# ── Python interpreter resolution ─────────────────────────────────────
|
||||
#
|
||||
# Same contract as the Claude Code hooks:
|
||||
# 1. $MEMPAL_PYTHON — explicit user override (absolute path)
|
||||
# 2. $(command -v python3) — first python3 on the hook's PATH
|
||||
# 3. bare "python3" — last-resort fallback
|
||||
mempal_resolve_python() {
|
||||
local p="${MEMPAL_PYTHON:-}"
|
||||
if [ -n "$p" ] && [ -x "$p" ]; then
|
||||
printf '%s' "$p"
|
||||
return 0
|
||||
fi
|
||||
p="$(command -v python3 2>/dev/null || true)"
|
||||
if [ -n "$p" ]; then
|
||||
printf '%s' "$p"
|
||||
return 0
|
||||
fi
|
||||
printf '%s' "python3"
|
||||
}
|
||||
MEMPAL_PYTHON_BIN="$(mempal_resolve_python)"
|
||||
|
||||
# ── Logging ───────────────────────────────────────────────────────────
|
||||
#
|
||||
# Lines are `[ISO8601Z] [event=...] [conv=...] message`. ISO8601 keeps
|
||||
# the format greppable across timezones (the Claude Code log uses
|
||||
# %H:%M:%S which loses the date — we improve on that here without
|
||||
# changing the existing log file).
|
||||
mempal_log() {
|
||||
local event="${1:-?}"
|
||||
local conv="${2:-unknown}"
|
||||
local msg="${3:-}"
|
||||
local ts
|
||||
ts="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
printf '[%s] [event=%s] [conv=%s] %s\n' "$ts" "$event" "$conv" "$msg" \
|
||||
>> "$MEMPAL_CURSOR_LOG" 2>/dev/null
|
||||
}
|
||||
|
||||
# ── Kill switch ───────────────────────────────────────────────────────
|
||||
#
|
||||
# Disabled if ANY of:
|
||||
# * MEMPAL_DISABLE_HOOK is a truthy string (Cursor-prompt addition)
|
||||
# * MEMPALACE_HOOKS_AUTO_SAVE is false/0/no (Claude Code convention)
|
||||
# * ~/.mempalace/config.json has hooks.auto_save == false
|
||||
#
|
||||
# Returns 0 (true in shell) when disabled, 1 when enabled.
|
||||
mempal_is_disabled() {
|
||||
case "${MEMPAL_DISABLE_HOOK:-}" in
|
||||
1|true|yes|on) return 0 ;;
|
||||
esac
|
||||
case "${MEMPALACE_HOOKS_AUTO_SAVE:-}" in
|
||||
false|0|no|off) return 0 ;;
|
||||
esac
|
||||
local cfg="$HOME/.mempalace/config.json"
|
||||
if [ -f "$cfg" ]; then
|
||||
local result
|
||||
# Use python -c '...' with the config path as argv[1] rather
|
||||
# than a heredoc. A heredoc body that contains parens inside a
|
||||
# $(...) command substitution trips the bash 3.2.57 parser
|
||||
# bug (macOS /bin/bash default) — gh-PR review caught this.
|
||||
# The -c form is also consistent with mempal_parse_stdin below.
|
||||
result="$("$MEMPAL_PYTHON_BIN" -c '
|
||||
import json, sys
|
||||
try:
|
||||
with open(sys.argv[1]) as f:
|
||||
cfg = json.load(f)
|
||||
print(str(cfg.get("hooks", {}).get("auto_save", True)).lower())
|
||||
except Exception:
|
||||
print("true")
|
||||
' "$cfg" 2>/dev/null)"
|
||||
if [ "$result" = "false" ]; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── Stdin parser ──────────────────────────────────────────────────────
|
||||
#
|
||||
# Reads Cursor's hook JSON from $1 and exports:
|
||||
# MEMPAL_CONV_ID — conversation_id, falls back to "unknown"
|
||||
# MEMPAL_LOOP_COUNT — integer (0 if absent / non-numeric)
|
||||
# MEMPAL_TRANSCRIPT — transcript_path, may be empty
|
||||
# MEMPAL_WORKSPACE — first workspace_roots entry, falls back to
|
||||
# CURSOR_PROJECT_DIR env var, then $PWD
|
||||
# MEMPAL_TRIGGER — preCompact trigger ("auto" | "manual"), empty otherwise
|
||||
# MEMPAL_STATUS — stop status ("completed" | "aborted" | "error"),
|
||||
# empty otherwise
|
||||
# MEMPAL_PARSE_OK — "1" if parser ran cleanly, "0" otherwise
|
||||
#
|
||||
# Uses the same sentinel + `sed -n 'Np'` extraction as the Claude Code
|
||||
# hooks for bash 3.2 compatibility (mapfile/readarray are unavailable
|
||||
# on macOS /bin/bash 3.2.57; #1440 regression). Each line of output is
|
||||
# pre-sanitised by the Python side to a shell-safe character set.
|
||||
mempal_parse_stdin() {
|
||||
local input="${1:-}"
|
||||
local parsed
|
||||
# We invoke Python via -c with a single-quoted multi-line string
|
||||
# rather than ``python3 - <<'PYEOF'`` because the heredoc form
|
||||
# would shadow Python's stdin with the heredoc body, leaving
|
||||
# ``json.load(sys.stdin)`` to read nothing and silently fail. The
|
||||
# parser body deliberately uses only double-quoted Python strings
|
||||
# so the surrounding bash single-quote is safe verbatim, and uses
|
||||
# only the shell-safe character set (alphanumeric, underscore,
|
||||
# dash, slash, dot, tilde) matching the Claude Code hook's
|
||||
# sanitiser so a hostile transcript_path cannot splice
|
||||
# metacharacters into the parsed output.
|
||||
parsed="$(
|
||||
umask 077
|
||||
printf '%s' "$input" | "$MEMPAL_PYTHON_BIN" -c '
|
||||
import json, re, sys
|
||||
|
||||
def safe_str(value):
|
||||
return re.sub(r"[^a-zA-Z0-9_/.\-~]", "", str(value or ""))
|
||||
|
||||
def safe_int(value):
|
||||
try:
|
||||
return str(int(value))
|
||||
except (TypeError, ValueError):
|
||||
return "0"
|
||||
|
||||
try:
|
||||
data = json.load(sys.stdin)
|
||||
except Exception:
|
||||
sys.exit(1)
|
||||
if not isinstance(data, dict):
|
||||
sys.exit(1)
|
||||
|
||||
conv = safe_str(data.get("conversation_id") or data.get("session_id"))
|
||||
loop_count = safe_int(data.get("loop_count", 0))
|
||||
transcript = safe_str(data.get("transcript_path", ""))
|
||||
trigger = safe_str(data.get("trigger", ""))
|
||||
status = safe_str(data.get("status", ""))
|
||||
|
||||
roots = data.get("workspace_roots") or []
|
||||
workspace = ""
|
||||
if isinstance(roots, list) and roots:
|
||||
workspace = safe_str(roots[0])
|
||||
|
||||
print("__MEMPAL_PARSE_OK__")
|
||||
print(conv)
|
||||
print(loop_count)
|
||||
print(transcript)
|
||||
print(workspace)
|
||||
print(trigger)
|
||||
print(status)
|
||||
' 2>"$MEMPAL_STATE_DIR/cursor_last_python_err.log"
|
||||
)"
|
||||
|
||||
# Drop empty stderr capture on success; lock it to 0600 on failure
|
||||
# (mirrors the privacy contract in the Claude Code hooks — the
|
||||
# traceback can echo back transcript_path / home layout).
|
||||
if [ -s "$MEMPAL_STATE_DIR/cursor_last_python_err.log" ]; then
|
||||
chmod 600 "$MEMPAL_STATE_DIR/cursor_last_python_err.log" 2>/dev/null
|
||||
else
|
||||
rm -f "$MEMPAL_STATE_DIR/cursor_last_python_err.log" 2>/dev/null
|
||||
fi
|
||||
|
||||
local marker
|
||||
marker="$(printf '%s\n' "$parsed" | sed -n '1p')"
|
||||
if [ "$marker" = "__MEMPAL_PARSE_OK__" ]; then
|
||||
MEMPAL_PARSE_OK="1"
|
||||
MEMPAL_CONV_ID="$(printf '%s\n' "$parsed" | sed -n '2p')"
|
||||
MEMPAL_LOOP_COUNT="$(printf '%s\n' "$parsed" | sed -n '3p')"
|
||||
MEMPAL_TRANSCRIPT="$(printf '%s\n' "$parsed" | sed -n '4p')"
|
||||
MEMPAL_WORKSPACE="$(printf '%s\n' "$parsed" | sed -n '5p')"
|
||||
MEMPAL_TRIGGER="$(printf '%s\n' "$parsed" | sed -n '6p')"
|
||||
MEMPAL_STATUS="$(printf '%s\n' "$parsed" | sed -n '7p')"
|
||||
else
|
||||
MEMPAL_PARSE_OK="0"
|
||||
MEMPAL_CONV_ID=""
|
||||
MEMPAL_LOOP_COUNT="0"
|
||||
MEMPAL_TRANSCRIPT=""
|
||||
MEMPAL_WORKSPACE=""
|
||||
MEMPAL_TRIGGER=""
|
||||
MEMPAL_STATUS=""
|
||||
fi
|
||||
|
||||
# Defaults and environment fallbacks. The Cursor docs guarantee
|
||||
# CURSOR_TRANSCRIPT_PATH and CURSOR_PROJECT_DIR env vars are set
|
||||
# for every hook execution; if JSON parsing failed for any reason
|
||||
# (sentinel missing, malformed payload, missing interpreter) we
|
||||
# still have a usable workspace.
|
||||
MEMPAL_CONV_ID="${MEMPAL_CONV_ID:-unknown}"
|
||||
case "$MEMPAL_LOOP_COUNT" in
|
||||
''|*[!0-9]*) MEMPAL_LOOP_COUNT="0" ;;
|
||||
esac
|
||||
if [ -z "$MEMPAL_TRANSCRIPT" ] && [ -n "${CURSOR_TRANSCRIPT_PATH:-}" ]; then
|
||||
MEMPAL_TRANSCRIPT="${CURSOR_TRANSCRIPT_PATH}"
|
||||
fi
|
||||
if [ -z "$MEMPAL_WORKSPACE" ]; then
|
||||
if [ -n "${CURSOR_PROJECT_DIR:-}" ]; then
|
||||
MEMPAL_WORKSPACE="${CURSOR_PROJECT_DIR}"
|
||||
elif [ -n "${CLAUDE_PROJECT_DIR:-}" ]; then
|
||||
MEMPAL_WORKSPACE="${CLAUDE_PROJECT_DIR}"
|
||||
else
|
||||
MEMPAL_WORKSPACE="${PWD:-/}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Expand a leading ~ in the transcript path so downstream
|
||||
# ``[ -f "$path" ]`` checks resolve correctly.
|
||||
case "$MEMPAL_TRANSCRIPT" in
|
||||
'~/'*) MEMPAL_TRANSCRIPT="$HOME/${MEMPAL_TRANSCRIPT#~/}" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# ── Defense-in-depth: dump unparseable stdin ──────────────────────────
|
||||
#
|
||||
# Same shape as the Claude Code hooks' last_input.log: bounded to 4096
|
||||
# bytes, overwritten (never appended) so a misconfiguration loop cannot
|
||||
# grow disk usage, 0600 perms because the dump mirrors the raw hook
|
||||
# payload (transcript_path reveals the user's home + project layout).
|
||||
mempal_dump_bad_input() {
|
||||
local input="${1:-}"
|
||||
if [ -z "$input" ]; then
|
||||
return 0
|
||||
fi
|
||||
mempal_log "${2:-?}" "${MEMPAL_CONV_ID:-unknown}" \
|
||||
"WARN: input parse failed (sentinel missing); see $MEMPAL_STATE_DIR/cursor_last_input.log + cursor_last_python_err.log"
|
||||
(
|
||||
umask 077
|
||||
printf '%s' "$input" | head -c 4096 > "$MEMPAL_STATE_DIR/cursor_last_input.log"
|
||||
)
|
||||
chmod 600 "$MEMPAL_STATE_DIR/cursor_last_input.log" 2>/dev/null
|
||||
}
|
||||
|
||||
# ── Counter helpers ───────────────────────────────────────────────────
|
||||
#
|
||||
# One counter file per conversation_id. Atomic write via temp file
|
||||
# inside the same directory (rename is atomic on POSIX) so concurrent
|
||||
# hook invocations cannot half-write the file. Read tolerates a
|
||||
# corrupted or empty file by returning 0, never crashing.
|
||||
_mempal_counter_path() {
|
||||
local conv="${1:-unknown}"
|
||||
# Sanitise the conv id one more time: it has already been through
|
||||
# the Python sanitiser, but be defensive in case a caller passes a
|
||||
# raw string. Strip any character outside [a-zA-Z0-9_.-].
|
||||
local safe_conv
|
||||
safe_conv="$(printf '%s' "$conv" | tr -cd 'a-zA-Z0-9_.-')"
|
||||
if [ -z "$safe_conv" ]; then
|
||||
safe_conv="unknown"
|
||||
fi
|
||||
printf '%s/cursor_%s.count' "$MEMPAL_STATE_DIR" "$safe_conv"
|
||||
}
|
||||
|
||||
mempal_read_counter() {
|
||||
local path="$1"
|
||||
if [ ! -f "$path" ]; then
|
||||
printf '0'
|
||||
return 0
|
||||
fi
|
||||
local raw
|
||||
raw="$(cat "$path" 2>/dev/null)"
|
||||
case "$raw" in
|
||||
''|*[!0-9]*) printf '0' ;;
|
||||
*) printf '%s' "$raw" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
mempal_write_counter_atomic() {
|
||||
local path="$1"
|
||||
local value="$2"
|
||||
case "$value" in
|
||||
''|*[!0-9]*) value="0" ;;
|
||||
esac
|
||||
local tmp="${path}.tmp.$$"
|
||||
printf '%s' "$value" > "$tmp" 2>/dev/null || return 1
|
||||
mv "$tmp" "$path" 2>/dev/null || {
|
||||
rm -f "$tmp" 2>/dev/null
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
# ── Pending-save marker ───────────────────────────────────────────────
|
||||
#
|
||||
# Dropped by the preCompact hook (which cannot itself emit a
|
||||
# followup_message — Cursor's preCompact is observational-only) and
|
||||
# consumed by the next stop invocation so the LLM still gets a diary
|
||||
# nudge after compaction.
|
||||
_mempal_pending_path() {
|
||||
local conv="${1:-unknown}"
|
||||
local safe_conv
|
||||
safe_conv="$(printf '%s' "$conv" | tr -cd 'a-zA-Z0-9_.-')"
|
||||
if [ -z "$safe_conv" ]; then
|
||||
safe_conv="unknown"
|
||||
fi
|
||||
printf '%s/cursor_%s.pending' "$MEMPAL_STATE_DIR" "$safe_conv"
|
||||
}
|
||||
|
||||
mempal_set_pending() {
|
||||
local conv="${1:-unknown}"
|
||||
local path
|
||||
path="$(_mempal_pending_path "$conv")"
|
||||
: > "$path" 2>/dev/null || return 1
|
||||
chmod 600 "$path" 2>/dev/null
|
||||
}
|
||||
|
||||
mempal_consume_pending() {
|
||||
local conv="${1:-unknown}"
|
||||
local path
|
||||
path="$(_mempal_pending_path "$conv")"
|
||||
if [ -f "$path" ]; then
|
||||
rm -f "$path" 2>/dev/null
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# ── State-file TTL ────────────────────────────────────────────────────
|
||||
#
|
||||
# Per-conversation state artifacts (cursor_<conv>.count and
|
||||
# cursor_<conv>.pending) accumulate one set per conversation and are
|
||||
# never otherwise removed (igorls review, PR #1632 — unbounded state
|
||||
# growth). Reads MEMPAL_STATE_TTL_DAYS (default 30), validated
|
||||
# digits-only and leading-zero-stripped (mirrors the SAVE_INTERVAL
|
||||
# sanitiser) so `find -mtime` never sees a bad or octal token. Empty or
|
||||
# non-numeric floors to 30; a value of 0 means "sweep everything older
|
||||
# than today".
|
||||
mempal_state_ttl_days() {
|
||||
local raw="${MEMPAL_STATE_TTL_DAYS:-30}"
|
||||
case "$raw" in
|
||||
''|*[!0-9]*) printf '30'; return 0 ;;
|
||||
esac
|
||||
while [ "${raw}" != "${raw#0}" ] && [ "${#raw}" -gt 1 ]; do
|
||||
raw="${raw#0}"
|
||||
done
|
||||
printf '%s' "$raw"
|
||||
}
|
||||
|
||||
# ── Stale state GC ────────────────────────────────────────────────────
|
||||
#
|
||||
# Opportunistic sweep of per-conversation Cursor state older than the
|
||||
# TTL. Throttled to at most once per 24h via the cursor_last_sweep
|
||||
# marker, so it costs a single mtime comparison on the vast majority of
|
||||
# fires. When it does run, two `find` passes remove the stale counter
|
||||
# files and pending markers.
|
||||
#
|
||||
# The globs are Cursor-specific and suffix-anchored (cursor_*.count,
|
||||
# cursor_*.pending), so the shared logs (cursor_hook.log,
|
||||
# cursor_last_input.log, cursor_last_python_err.log), the
|
||||
# cursor_last_sweep marker itself, and any antigravity_*/Claude state
|
||||
# sharing the same directory are never touched. BSD find (macOS default)
|
||||
# and GNU find both accept -maxdepth, -mtime +N, and -exec ... +.
|
||||
#
|
||||
# Fail-open: every step is best-effort; a missing state dir, a find that
|
||||
# errors, or a permission problem must never abort the caller.
|
||||
mempal_gc_stale_state() {
|
||||
[ -d "$MEMPAL_STATE_DIR" ] || return 0
|
||||
|
||||
local marker="$MEMPAL_STATE_DIR/cursor_last_sweep"
|
||||
if [ -f "$marker" ]; then
|
||||
local mtime now
|
||||
if mtime=$("$MEMPAL_PYTHON_BIN" -c 'import os, sys; print(int(os.path.getmtime(sys.argv[1])))' "$marker" 2>/dev/null) \
|
||||
&& now=$(date '+%s' 2>/dev/null) \
|
||||
&& [ -n "$mtime" ] \
|
||||
&& [ "$((now - mtime))" -lt 86400 ]; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
# Touch the marker first so a crash mid-sweep still throttles the
|
||||
# next fire (better to skip a sweep than to hammer the disk).
|
||||
: > "$marker" 2>/dev/null
|
||||
|
||||
local ttl
|
||||
ttl=$(mempal_state_ttl_days)
|
||||
|
||||
find "$MEMPAL_STATE_DIR" -maxdepth 1 -type f \
|
||||
-name 'cursor_*.count' -mtime +"$ttl" \
|
||||
-exec rm -f {} + 2>/dev/null
|
||||
find "$MEMPAL_STATE_DIR" -maxdepth 1 -type f \
|
||||
-name 'cursor_*.pending' -mtime +"$ttl" \
|
||||
-exec rm -f {} + 2>/dev/null
|
||||
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── Workspace → wing inference ────────────────────────────────────────
|
||||
#
|
||||
# basename(workspace_root), normalised to [a-z0-9_-]. Edge cases:
|
||||
# / → "root"
|
||||
# /path/ → trailing slash stripped, then basename
|
||||
# "/foo bar/" → "foo_bar" (spaces collapsed to underscores)
|
||||
# "" → "cursor_session"
|
||||
# "C:\\proj" → "proj" (Windows-style path; basename via tr fallback)
|
||||
#
|
||||
# We intentionally keep this in pure bash + POSIX tools so the
|
||||
# inference is identical across the hook scripts and the test suite
|
||||
# can target it as a function via `bash -c 'source ...; ...'`.
|
||||
mempal_infer_wing() {
|
||||
local raw="${1:-}"
|
||||
if [ -z "$raw" ]; then
|
||||
printf 'cursor_session'
|
||||
return 0
|
||||
fi
|
||||
# Strip trailing slashes (but preserve the lone "/" case).
|
||||
while [ "$raw" != "/" ] && [ "${raw%/}" != "$raw" ]; do
|
||||
raw="${raw%/}"
|
||||
done
|
||||
if [ "$raw" = "/" ]; then
|
||||
printf 'root'
|
||||
return 0
|
||||
fi
|
||||
local base="${raw##*/}"
|
||||
# On Windows-style paths with backslashes, fall back to splitting
|
||||
# on backslash too so we don't return the whole path verbatim.
|
||||
case "$base" in
|
||||
*\\*) base="${base##*\\}" ;;
|
||||
esac
|
||||
# Lowercase + replace anything outside [a-z0-9_-] with underscore.
|
||||
# Collapse runs of underscores so "foo bar" doesn't become
|
||||
# "foo__bar".
|
||||
base="$(printf '%s' "$base" \
|
||||
| tr '[:upper:]' '[:lower:]' \
|
||||
| tr -c 'a-z0-9_-' '_' \
|
||||
| tr -s '_' \
|
||||
| sed 's/^_//; s/_$//')"
|
||||
if [ -z "$base" ]; then
|
||||
printf 'cursor_session'
|
||||
return 0
|
||||
fi
|
||||
printf '%s' "$base"
|
||||
}
|
||||
|
||||
# ── Transcript path validation ────────────────────────────────────────
|
||||
#
|
||||
# Mirrors hooks/mempal_save_hook.sh::is_valid_transcript_path so the
|
||||
# Cursor and Claude Code hooks reject the same shapes:
|
||||
# * non-empty
|
||||
# * .json or .jsonl suffix
|
||||
# * no .. traversal segments
|
||||
mempal_is_valid_transcript() {
|
||||
local path="${1:-}"
|
||||
[ -n "$path" ] || return 1
|
||||
case "$path" in
|
||||
*.json|*.jsonl) ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
case "/$path/" in
|
||||
*/../*) return 1 ;;
|
||||
esac
|
||||
return 0
|
||||
}
|
||||
|
||||
# ── JSON emit ─────────────────────────────────────────────────────────
|
||||
#
|
||||
# Final stdout write. Uses ``printf '%s'`` instead of ``echo`` because
|
||||
# echo interprets ``-n``/``-e``/``-E`` as flags and varies in backslash
|
||||
# handling between builtin and /bin/echo (xpg_echo shopt). Matches the
|
||||
# Claude Code hook's documented rationale.
|
||||
mempal_emit() {
|
||||
printf '%s\n' "${1:-{\}}"
|
||||
}
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
#!/bin/bash
|
||||
# MEMPALACE CURSOR PRE-COMPACT HOOK — Snapshot transcript before compaction
|
||||
#
|
||||
# Cursor "preCompact" hook. Cursor's preCompact is documented as
|
||||
# OBSERVATIONAL ONLY (cursor.com/docs/hooks.md fetched 2026-05-27):
|
||||
#
|
||||
# * It cannot block compaction.
|
||||
# * Its only output field is `user_message` (no `followup_message`,
|
||||
# no `decision: block`).
|
||||
#
|
||||
# So unlike the Claude Code PreCompact hook (which can block the AI
|
||||
# and force a save before compaction proceeds), the Cursor preCompact
|
||||
# hook can only do two useful things at this moment:
|
||||
#
|
||||
# 1. Run `mempalace mine` SYNCHRONOUSLY against the transcript file
|
||||
# so whatever Cursor's transcript contains is ingested BEFORE
|
||||
# Cursor summarises the conversation — zero LLM cost, no agent
|
||||
# interaction needed. NOTE: this is BEST-EFFORT for Cursor.
|
||||
# Cursor's transcript format is undocumented and normalize.py has
|
||||
# no Cursor parser, so this does not yet produce clean verbatim
|
||||
# drawers; it is a safety net, not the primary capture path.
|
||||
#
|
||||
# 2. Drop a `.pending` marker file keyed on conversation_id. The
|
||||
# next `stop` hook reads that marker and forces a save followup
|
||||
# regardless of its counter, so the AI still gets a "write a
|
||||
# diary entry now" nudge on the very next turn. THIS followup is
|
||||
# the load-bearing verbatim-capture path for Cursor (the agent
|
||||
# files its own in-context verbatim quotes via the MCP tools).
|
||||
#
|
||||
# === INSTALL ===
|
||||
#
|
||||
# Add to ~/.cursor/hooks.json (or .cursor/hooks.json for project
|
||||
# scope) under "preCompact":
|
||||
#
|
||||
# {
|
||||
# "version": 1,
|
||||
# "hooks": {
|
||||
# "preCompact": [
|
||||
# { "command": "/absolute/path/to/mempal_precompact_hook_cursor.sh" }
|
||||
# ]
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# No loop_limit is needed; preCompact is not a looping hook.
|
||||
|
||||
_mempal_self="${BASH_SOURCE[0]:-$0}"
|
||||
_mempal_dir="$(cd "$(dirname "$_mempal_self")" 2>/dev/null && pwd)"
|
||||
# shellcheck source=lib/common.sh
|
||||
. "$_mempal_dir/lib/common.sh"
|
||||
|
||||
# Optional additional project directory to mine before compaction
|
||||
# (parity with the Claude Code hook's MEMPAL_DIR knob).
|
||||
MEMPAL_DIR="${MEMPAL_DIR:-}"
|
||||
|
||||
if mempal_is_disabled; then
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Opportunistic, daily-throttled GC of stale per-conversation state.
|
||||
# Placed after the kill switch so a disabled hook touches nothing.
|
||||
mempal_gc_stale_state
|
||||
|
||||
INPUT="$(cat)"
|
||||
mempal_parse_stdin "$INPUT"
|
||||
|
||||
if [ "$MEMPAL_PARSE_OK" != "1" ]; then
|
||||
mempal_dump_bad_input "$INPUT" "preCompact"
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mempal_log "preCompact" "$MEMPAL_CONV_ID" \
|
||||
"trigger=${MEMPAL_TRIGGER:-?} transcript=$MEMPAL_TRANSCRIPT"
|
||||
|
||||
# ── Synchronous mine ──────────────────────────────────────────────
|
||||
#
|
||||
# This intentionally blocks the hook. Compaction is irreversible —
|
||||
# once Cursor summarises the conversation we cannot get the verbatim
|
||||
# text back — so we must finish ingesting before returning. Background
|
||||
# mining would race the compaction and lose data.
|
||||
#
|
||||
# TIMEOUT TRADEOFF (igorls review, PR #1632): on a very large transcript
|
||||
# this synchronous mine can exceed Cursor's per-hook timeout, in which
|
||||
# case Cursor kills the process mid-mine. That is acceptable and safe
|
||||
# here: `mempalace mine` is incremental and append-only (a crash mid-
|
||||
# operation leaves the existing palace untouched — see CLAUDE.md
|
||||
# "Incremental only"), so a killed mine simply resumes on the next mine
|
||||
# invocation rather than corrupting the palace. We deliberately do NOT
|
||||
# wrap this in a shorter timeout, because truncating the mine would
|
||||
# trade a recoverable partial-ingest for guaranteed silent data loss
|
||||
# right before the irreversible compaction. The pending-save marker
|
||||
# below is the backstop: the next `stop` hook re-mines and nudges a
|
||||
# verbatim save regardless of whether this mine completed.
|
||||
if command -v mempalace >/dev/null 2>&1; then
|
||||
if mempal_is_valid_transcript "$MEMPAL_TRANSCRIPT" \
|
||||
&& [ -f "$MEMPAL_TRANSCRIPT" ]; then
|
||||
mempalace mine "$(dirname "$MEMPAL_TRANSCRIPT")" --mode convos \
|
||||
>> "$MEMPAL_CURSOR_LOG" 2>&1 || \
|
||||
mempal_log "preCompact" "$MEMPAL_CONV_ID" \
|
||||
"WARN: mempalace mine convos returned non-zero"
|
||||
elif [ -n "$MEMPAL_TRANSCRIPT" ]; then
|
||||
mempal_log "preCompact" "$MEMPAL_CONV_ID" \
|
||||
"skipping invalid transcript path: $MEMPAL_TRANSCRIPT"
|
||||
fi
|
||||
if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
|
||||
mempalace mine "$MEMPAL_DIR" --mode projects \
|
||||
>> "$MEMPAL_CURSOR_LOG" 2>&1 || \
|
||||
mempal_log "preCompact" "$MEMPAL_CONV_ID" \
|
||||
"WARN: mempalace mine projects returned non-zero"
|
||||
fi
|
||||
else
|
||||
mempal_log "preCompact" "$MEMPAL_CONV_ID" \
|
||||
"mempalace CLI not on PATH; skipping synchronous mine"
|
||||
fi
|
||||
|
||||
# ── Drop the pending-save marker ──────────────────────────────────
|
||||
mempal_set_pending "$MEMPAL_CONV_ID" || \
|
||||
mempal_log "preCompact" "$MEMPAL_CONV_ID" \
|
||||
"WARN: could not write pending-save marker"
|
||||
|
||||
# Surface a short user-visible note that compaction is about to
|
||||
# happen and we've already captured the verbatim text. user_message
|
||||
# is the only output field Cursor's preCompact accepts.
|
||||
"$MEMPAL_PYTHON_BIN" -c '
|
||||
import json
|
||||
print(json.dumps({
|
||||
"user_message": (
|
||||
"MemPalace: transcript snapshotted before compaction. "
|
||||
"A diary nudge is queued for the next agent turn."
|
||||
)
|
||||
}))
|
||||
'
|
||||
Executable
+267
@@ -0,0 +1,267 @@
|
||||
#!/bin/bash
|
||||
# MEMPALACE CURSOR SAVE HOOK — Auto-save every N stop events
|
||||
#
|
||||
# Cursor "stop" hook. After every agent loop ends, this hook:
|
||||
# 1. Counts stop invocations per conversation_id (each stop ≈ one
|
||||
# assistant turn ≈ roughly one user message — see plan rationale).
|
||||
# 2. Every SAVE_INTERVAL stops, returns a followup_message telling
|
||||
# the agent to file the session into MemPalace and write a diary
|
||||
# entry. Cursor auto-submits that as the next user message.
|
||||
# 3. On the next stop, loop_count > 0 so we let the agent finish
|
||||
# without re-firing — Cursor's loop_count is the equivalent of
|
||||
# Claude Code's stop_hook_active flag.
|
||||
# 4. If the preCompact hook has left a `.pending` marker, force a
|
||||
# save followup regardless of the counter and clear the marker.
|
||||
#
|
||||
# === WHY THE FOLLOWUP FIRES BY DEFAULT (differs from the Claude hook) ===
|
||||
#
|
||||
# The Claude Code hook (hooks/mempal_save_hook.sh) is SILENT by default:
|
||||
# its background `mempalace mine --mode convos` captures the verbatim
|
||||
# transcript on its own, and the LLM-driven diary nudge is opt-IN behind
|
||||
# MEMPAL_VERBOSE. That works because mempalace/normalize.py has a Claude
|
||||
# Code JSONL parser.
|
||||
#
|
||||
# Cursor is different. Cursor's transcript format is undocumented (see
|
||||
# STDIN_SHAPE.md) and normalize.py has NO Cursor parser, so the
|
||||
# background mine below is BEST-EFFORT only — it does not yet yield clean
|
||||
# verbatim drawers for Cursor. The followup_message is therefore the
|
||||
# load-bearing verbatim-capture path: it drives the agent to call
|
||||
# mempalace_add_drawer / mempalace_diary_write from its in-context memory.
|
||||
# That is why it is ON by default here — silencing it by default would
|
||||
# leave a default Cursor install capturing nothing.
|
||||
#
|
||||
# Users who want the Claude-style "zero tokens in the chat window"
|
||||
# behaviour can silence the followup (see MEMPAL_CURSOR_SILENT /
|
||||
# MEMPAL_VERBOSE below). Once normalize.py learns to read Cursor
|
||||
# transcripts, this default should flip to silent to match Claude.
|
||||
#
|
||||
# Companion files in this directory:
|
||||
# * lib/common.sh — shared helpers (sourced)
|
||||
# * mempal_precompact_hook_cursor.sh — preCompact event
|
||||
# * mempal_wake_hook_cursor.sh — sessionStart event
|
||||
#
|
||||
# === INSTALL ===
|
||||
#
|
||||
# Recommended path: run `hooks/cursor/install.sh` from a cloned repo,
|
||||
# which copies the scripts to ~/.mempalace/hooks/cursor/ and merges
|
||||
# the wiring into your ~/.cursor/hooks.json. See hooks/cursor/README.md
|
||||
# for the full walkthrough, or website/guide/cursor-hooks.md for the
|
||||
# rendered version.
|
||||
#
|
||||
# Manual wiring (user scope: ~/.cursor/hooks.json):
|
||||
#
|
||||
# {
|
||||
# "version": 1,
|
||||
# "hooks": {
|
||||
# "stop": [
|
||||
# {
|
||||
# "command": "/absolute/path/to/mempal_save_hook_cursor.sh",
|
||||
# "loop_limit": 1
|
||||
# }
|
||||
# ]
|
||||
# }
|
||||
# }
|
||||
#
|
||||
# The `loop_limit: 1` cap is defense-in-depth — even if our own
|
||||
# loop_count check below regresses, Cursor itself will stop emitting
|
||||
# our followup after one auto-iteration.
|
||||
#
|
||||
# === KILL SWITCHES ===
|
||||
#
|
||||
# MEMPAL_DISABLE_HOOK=1 — Cursor-prompt addition
|
||||
# MEMPALACE_HOOKS_AUTO_SAVE=false — matches the Claude Code hooks
|
||||
# ~/.mempalace/config.json "hooks.auto_save": false
|
||||
#
|
||||
# Any one of these short-circuits the hook to `{}` and exits 0.
|
||||
|
||||
# Resolve the directory this script lives in so we can source the
|
||||
# sibling lib/common.sh whether the user invoked us by absolute path,
|
||||
# by relative path, or via a symlink.
|
||||
_mempal_self="${BASH_SOURCE[0]:-$0}"
|
||||
_mempal_dir="$(cd "$(dirname "$_mempal_self")" 2>/dev/null && pwd)"
|
||||
# shellcheck source=lib/common.sh
|
||||
. "$_mempal_dir/lib/common.sh"
|
||||
|
||||
SAVE_INTERVAL="${MEMPAL_SAVE_INTERVAL:-15}"
|
||||
# Coerce empty, non-numeric, AND zero to the default. SAVE_INTERVAL=0
|
||||
# would otherwise crash bash on the modulo check below ($((NEXT % 0))
|
||||
# is "division by 0"). gh-PR review caught this edge case.
|
||||
case "$SAVE_INTERVAL" in
|
||||
''|*[!0-9]*|0) SAVE_INTERVAL=15 ;;
|
||||
esac
|
||||
|
||||
# Optional additional project directory to mine on save (parity with
|
||||
# the Claude Code hook's MEMPAL_DIR knob — purely additive, never an
|
||||
# override for the transcript mine).
|
||||
MEMPAL_DIR="${MEMPAL_DIR:-}"
|
||||
|
||||
# ── Followup opt-out ──────────────────────────────────────────────
|
||||
#
|
||||
# Returns 0 (true) when the user has asked to suppress the
|
||||
# followup_message. See the header comment for why the followup is ON
|
||||
# by default for Cursor. When silenced, the hook still runs the
|
||||
# best-effort background mine and still maintains its counters/markers
|
||||
# — it just emits `{}` instead of a followup. Two equivalent signals:
|
||||
# * MEMPAL_CURSOR_SILENT=1|true|yes|on — dedicated Cursor opt-out
|
||||
# * MEMPAL_VERBOSE=false|0|no|off — cross-hook silence signal
|
||||
# (mirror-image of the Claude hook, where MEMPAL_VERBOSE=true is
|
||||
# what turns its diary nudge ON)
|
||||
mempal_followup_silenced() {
|
||||
case "${MEMPAL_CURSOR_SILENT:-}" in
|
||||
1|true|yes|on) return 0 ;;
|
||||
esac
|
||||
case "${MEMPAL_VERBOSE:-}" in
|
||||
false|0|no|off) return 0 ;;
|
||||
esac
|
||||
return 1
|
||||
}
|
||||
|
||||
# Kill switch — emit `{}` so Cursor proceeds with normal stop.
|
||||
if mempal_is_disabled; then
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Opportunistic, daily-throttled GC of stale per-conversation state.
|
||||
# Placed after the kill switch so a disabled hook touches nothing.
|
||||
mempal_gc_stale_state
|
||||
|
||||
INPUT="$(cat)"
|
||||
mempal_parse_stdin "$INPUT"
|
||||
|
||||
if [ "$MEMPAL_PARSE_OK" != "1" ]; then
|
||||
mempal_dump_bad_input "$INPUT" "stop"
|
||||
# Fail-open: don't block the host on a parse error.
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" \
|
||||
"loop_count=$MEMPAL_LOOP_COUNT status=${MEMPAL_STATUS:-?} workspace=$MEMPAL_WORKSPACE"
|
||||
|
||||
# ── Loop-prevention ────────────────────────────────────────────────
|
||||
#
|
||||
# Cursor's loop_count indicates how many times THIS stop hook has
|
||||
# already triggered an automatic followup for this conversation
|
||||
# (starts at 0). If it is > 0, our own previous followup is currently
|
||||
# being consumed by the agent — let it finish without re-firing.
|
||||
if [ "$MEMPAL_LOOP_COUNT" -gt 0 ] 2>/dev/null; then
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" "loop_count>0; letting agent stop"
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
WING="$(mempal_infer_wing "$MEMPAL_WORKSPACE")"
|
||||
|
||||
# Build the followup message once; both the pending-marker branch and
|
||||
# the threshold branch use it. Constructed via Python -c (rather than
|
||||
# a heredoc) so we can pass the inferred wing as argv[1] and so the
|
||||
# JSON encoding is correct even for wings whose name would otherwise
|
||||
# need shell quoting.
|
||||
_mempal_build_followup() {
|
||||
"$MEMPAL_PYTHON_BIN" -c '
|
||||
import json, sys
|
||||
wing = sys.argv[1] if len(sys.argv) > 1 else "cursor_session"
|
||||
msg = (
|
||||
"MemPalace save checkpoint. Call mempalace_checkpoint ONCE with: "
|
||||
"items=[{wing: " + wing + ", room: <short topic>, content: <verbatim "
|
||||
"quote>}, ...] for the key topics, decisions, and verbatim quotes from "
|
||||
"this session; and diary={agent_name: cursor-ide, wing: " + wing + ", "
|
||||
"entry: <AAAK-format summary>}. It dedups, files non-duplicates, and "
|
||||
"writes the diary in one call. Then stop."
|
||||
)
|
||||
print(json.dumps({"followup_message": msg}))
|
||||
' "$WING"
|
||||
}
|
||||
|
||||
# ── Pending-save marker from preCompact ───────────────────────────
|
||||
#
|
||||
# preCompact cannot itself emit a followup_message (Cursor docs:
|
||||
# preCompact is observational-only, output supports only user_message),
|
||||
# so it drops a marker file and we consume it here. Forces a save
|
||||
# nudge regardless of the counter.
|
||||
if mempal_consume_pending "$MEMPAL_CONV_ID"; then
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" \
|
||||
"consumed pending-save marker (post-compaction)"
|
||||
if mempal_followup_silenced; then
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" \
|
||||
"followup silenced (MEMPAL_CURSOR_SILENT/MEMPAL_VERBOSE); emitting {}"
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
_mempal_build_followup
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ── Normal counter path ───────────────────────────────────────────
|
||||
COUNTER_FILE="$(_mempal_counter_path "$MEMPAL_CONV_ID")"
|
||||
CURRENT="$(mempal_read_counter "$COUNTER_FILE")"
|
||||
NEXT=$((CURRENT + 1))
|
||||
mempal_write_counter_atomic "$COUNTER_FILE" "$NEXT" || {
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" \
|
||||
"WARN: counter write failed for $COUNTER_FILE; passing through"
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
}
|
||||
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" \
|
||||
"counter $CURRENT -> $NEXT (interval=$SAVE_INTERVAL)"
|
||||
|
||||
# Trigger when we hit a multiple of SAVE_INTERVAL. Modulo arithmetic
|
||||
# keeps the counter monotonically growing (no reset) so the log file
|
||||
# is greppable for total turns across a conversation.
|
||||
if [ "$((NEXT % SAVE_INTERVAL))" -ne 0 ]; then
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" "TRIGGERING SAVE at counter=$NEXT"
|
||||
|
||||
# ── Background mine (best effort) ─────────────────────────────────
|
||||
#
|
||||
# Two independent targets — both run if both are set:
|
||||
# 1. transcript_path → its parent directory, --mode convos
|
||||
# 2. MEMPAL_DIR (user-configured project) → --mode projects
|
||||
#
|
||||
# IMPORTANT (Cursor caveat): the --mode convos mine is BEST-EFFORT for
|
||||
# Cursor. Cursor's transcript format is undocumented and
|
||||
# mempalace/normalize.py has no Cursor parser, so this call does not
|
||||
# yet produce clean verbatim conversation drawers — at best it ingests
|
||||
# raw bytes. The verbatim-capture guarantee for Cursor is carried by
|
||||
# the followup_message below, which drives the agent to file its own
|
||||
# in-context verbatim quotes. The --mode projects target (MEMPAL_DIR)
|
||||
# is unaffected — normalize.py reads ordinary project files fine.
|
||||
#
|
||||
# Both run with stdout/stderr appended to the cursor log and are
|
||||
# backgrounded so a slow mine cannot push the hook past its
|
||||
# Cursor-configured timeout. `command -v mempalace` gates so a user
|
||||
# without the CLI on PATH (e.g. a fresh GUI-launched session) does
|
||||
# not see a noisy error.
|
||||
if command -v mempalace >/dev/null 2>&1; then
|
||||
if mempal_is_valid_transcript "$MEMPAL_TRANSCRIPT" \
|
||||
&& [ -f "$MEMPAL_TRANSCRIPT" ]; then
|
||||
( mempalace mine "$(dirname "$MEMPAL_TRANSCRIPT")" --mode convos \
|
||||
>> "$MEMPAL_CURSOR_LOG" 2>&1 ) &
|
||||
elif [ -n "$MEMPAL_TRANSCRIPT" ]; then
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" \
|
||||
"skipping invalid transcript path: $MEMPAL_TRANSCRIPT"
|
||||
fi
|
||||
if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
|
||||
( mempalace mine "$MEMPAL_DIR" --mode projects \
|
||||
>> "$MEMPAL_CURSOR_LOG" 2>&1 ) &
|
||||
fi
|
||||
else
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" \
|
||||
"mempalace CLI not on PATH; skipping background mine"
|
||||
fi
|
||||
|
||||
# The followup is the load-bearing verbatim path for Cursor (see header),
|
||||
# so it fires by default. Honour the opt-out for users who want silence.
|
||||
if mempal_followup_silenced; then
|
||||
mempal_log "stop" "$MEMPAL_CONV_ID" \
|
||||
"followup silenced (MEMPAL_CURSOR_SILENT/MEMPAL_VERBOSE); background mine only"
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
_mempal_build_followup
|
||||
Executable
+82
@@ -0,0 +1,82 @@
|
||||
#!/bin/bash
|
||||
# MEMPALACE CURSOR WAKE HOOK — Session-start memory recall
|
||||
#
|
||||
# Cursor "sessionStart" hook. This is a Cursor-only capability —
|
||||
# Claude Code's third-party-hooks compatibility layer does not have
|
||||
# an equivalent event with the same "inject context into the agent's
|
||||
# initial system message" semantics.
|
||||
#
|
||||
# Behaviour:
|
||||
# 1. Parse Cursor's sessionStart payload (conversation_id /
|
||||
# session_id, is_background_agent, composer_mode, plus the
|
||||
# common workspace_roots field).
|
||||
# 2. Infer the wing from basename(workspace_roots[0]).
|
||||
# 3. Return {"additional_context": "..."} instructing the agent to
|
||||
# scope its memory recall by calling mempalace_search +
|
||||
# mempalace_diary_read with wing=<inferred>.
|
||||
#
|
||||
# sessionStart is documented as fire-and-forget — Cursor does not
|
||||
# enforce a blocking response and does not consume "continue" /
|
||||
# "user_message" — but "additional_context" is honoured and added to
|
||||
# the conversation's initial system context. Verified in
|
||||
# cursor.com/docs/hooks.md (fetched 2026-05-27).
|
||||
#
|
||||
# === INSTALL ===
|
||||
#
|
||||
# Add to ~/.cursor/hooks.json (or .cursor/hooks.json for project
|
||||
# scope) under "sessionStart":
|
||||
#
|
||||
# {
|
||||
# "version": 1,
|
||||
# "hooks": {
|
||||
# "sessionStart": [
|
||||
# { "command": "/absolute/path/to/mempal_wake_hook_cursor.sh" }
|
||||
# ]
|
||||
# }
|
||||
# }
|
||||
|
||||
_mempal_self="${BASH_SOURCE[0]:-$0}"
|
||||
_mempal_dir="$(cd "$(dirname "$_mempal_self")" 2>/dev/null && pwd)"
|
||||
# shellcheck source=lib/common.sh
|
||||
. "$_mempal_dir/lib/common.sh"
|
||||
|
||||
if mempal_is_disabled; then
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
INPUT="$(cat)"
|
||||
mempal_parse_stdin "$INPUT"
|
||||
|
||||
if [ "$MEMPAL_PARSE_OK" != "1" ]; then
|
||||
mempal_dump_bad_input "$INPUT" "sessionStart"
|
||||
mempal_emit '{}'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
WING="$(mempal_infer_wing "$MEMPAL_WORKSPACE")"
|
||||
|
||||
mempal_log "sessionStart" "$MEMPAL_CONV_ID" \
|
||||
"workspace=$MEMPAL_WORKSPACE wing=$WING"
|
||||
|
||||
# Emit the additional_context payload via Python -c (rather than a
|
||||
# heredoc) so the JSON encoding survives wings whose name contains
|
||||
# characters that would otherwise need shell escaping, and so the
|
||||
# inferred wing arrives as an argv positional. The MCP tool names
|
||||
# referenced here are verified against mempalace/mcp_server.py:
|
||||
# mempalace_search and mempalace_diary_read both exist and accept
|
||||
# the wing parameter.
|
||||
"$MEMPAL_PYTHON_BIN" -c '
|
||||
import json, sys
|
||||
wing = sys.argv[1] if len(sys.argv) > 1 else "cursor_session"
|
||||
ctx = (
|
||||
"MemPalace wake-up. The Cursor workspace maps to wing=" + wing + ". "
|
||||
"Before answering anything that touches past work in this "
|
||||
"project, call mempalace_search (wing=" + wing + ", "
|
||||
"query=<relevant keywords>) and mempalace_diary_read "
|
||||
"(agent_name=cursor-ide, wing=" + wing + ", last_n=10). "
|
||||
"Use what you find verbatim where it answers the question; "
|
||||
"never summarise the user'"'"'s own words."
|
||||
)
|
||||
print(json.dumps({"additional_context": ctx}))
|
||||
' "$WING"
|
||||
@@ -119,16 +119,8 @@ INPUT=$(cat)
|
||||
# backslashes are not mangled by echo flag parsing.
|
||||
_mempal_parsed=$(
|
||||
umask 077
|
||||
printf '%s' "$INPUT" | "$MEMPAL_PYTHON_BIN" -c "
|
||||
import sys, json, re
|
||||
data = json.load(sys.stdin)
|
||||
sid = data.get('session_id', '')
|
||||
tp = data.get('transcript_path', '')
|
||||
safe = lambda s: re.sub(r'[^a-zA-Z0-9_/.\-~]', '', str(s))
|
||||
print('__MEMPAL_PARSE_OK__')
|
||||
print(safe(sid))
|
||||
print(safe(tp))
|
||||
" 2>"$STATE_DIR/last_python_err.log"
|
||||
printf '%s' "$INPUT" | "$MEMPAL_PYTHON_BIN" -m mempalace.hook_shell parse-precompact \
|
||||
2>"$STATE_DIR/last_python_err.log"
|
||||
)
|
||||
# Drop the empty file on success; chmod 600 on failure to mirror
|
||||
# last_input.log's privacy contract.
|
||||
@@ -190,14 +182,14 @@ echo "[$(date '+%H:%M:%S')] PRE-COMPACT triggered for session $SESSION_ID" >> "$
|
||||
# 1. TRANSCRIPT_PATH (from Claude Code) → parent dir, --mode convos
|
||||
# 2. MEMPAL_DIR → --mode projects
|
||||
if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
|
||||
mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
|
||||
"$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
|
||||
>> "$STATE_DIR/hook.log" 2>&1
|
||||
elif [ -n "$TRANSCRIPT_PATH" ]; then
|
||||
echo "[$(date '+%H:%M:%S')] Skipping invalid transcript path: $TRANSCRIPT_PATH" \
|
||||
echo "[$(date '+%H:%M:%S')] Skipping missing or invalid transcript path after normalization: $TRANSCRIPT_PATH" \
|
||||
>> "$STATE_DIR/hook.log"
|
||||
fi
|
||||
if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
|
||||
mempalace mine "$MEMPAL_DIR" --mode projects \
|
||||
"$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \
|
||||
>> "$STATE_DIR/hook.log" 2>&1
|
||||
fi
|
||||
|
||||
|
||||
@@ -149,21 +149,8 @@ INPUT=$(cat)
|
||||
# ``printf '%s'`` removes the class of bug entirely.
|
||||
_mempal_parsed=$(
|
||||
umask 077
|
||||
printf '%s' "$INPUT" | "$MEMPAL_PYTHON_BIN" -c "
|
||||
import sys, json, re
|
||||
data = json.load(sys.stdin)
|
||||
sid = data.get('session_id', '')
|
||||
sha_raw = data.get('stop_hook_active', False)
|
||||
tp = data.get('transcript_path', '')
|
||||
# Shell-safe output: only allow alphanumeric, underscore, hyphen, slash, dot, tilde
|
||||
safe = lambda s: re.sub(r'[^a-zA-Z0-9_/.\-~]', '', str(s))
|
||||
# Coerce stop_hook_active to strict boolean string
|
||||
sha = 'True' if sha_raw is True or str(sha_raw).lower() in ('true', '1', 'yes') else 'False'
|
||||
print('__MEMPAL_PARSE_OK__')
|
||||
print(safe(sid))
|
||||
print(sha)
|
||||
print(safe(tp))
|
||||
" 2>"$STATE_DIR/last_python_err.log"
|
||||
printf '%s' "$INPUT" | "$MEMPAL_PYTHON_BIN" -m mempalace.hook_shell parse-stop \
|
||||
2>"$STATE_DIR/last_python_err.log"
|
||||
)
|
||||
# The 2> redirect creates the file even when stderr is empty (success).
|
||||
# Remove the empty file so the state directory stays clean on the happy
|
||||
@@ -239,24 +226,10 @@ fi
|
||||
# Count human messages in the JSONL transcript
|
||||
# SECURITY: Pass transcript path as sys.argv to avoid shell injection via crafted paths
|
||||
if [ -f "$TRANSCRIPT_PATH" ]; then
|
||||
EXCHANGE_COUNT=$("$MEMPAL_PYTHON_BIN" - "$TRANSCRIPT_PATH" <<'PYEOF'
|
||||
import json, sys
|
||||
count = 0
|
||||
with open(sys.argv[1]) as f:
|
||||
for line in f:
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
msg = entry.get('message', {})
|
||||
if isinstance(msg, dict) and msg.get('role') == 'user':
|
||||
content = msg.get('content', '')
|
||||
if isinstance(content, str) and '<command-message>' in content:
|
||||
continue
|
||||
count += 1
|
||||
except:
|
||||
pass
|
||||
print(count)
|
||||
PYEOF
|
||||
2>/dev/null)
|
||||
EXCHANGE_COUNT=$("$MEMPAL_PYTHON_BIN" -m mempalace.hook_shell count-human-messages "$TRANSCRIPT_PATH" 2>/dev/null)
|
||||
elif [ -n "$TRANSCRIPT_PATH" ]; then
|
||||
echo "[$(date '+%H:%M:%S')] WARN: transcript_path not found after normalization: $TRANSCRIPT_PATH" >> "$STATE_DIR/hook.log"
|
||||
EXCHANGE_COUNT=0
|
||||
else
|
||||
EXCHANGE_COUNT=0
|
||||
fi
|
||||
@@ -292,14 +265,14 @@ if [ "$SINCE_LAST" -ge "$SAVE_INTERVAL" ] && [ "$EXCHANGE_COUNT" -gt 0 ]; then
|
||||
# MEMPAL_DIR is *additive*, not an override: a user with MEMPAL_DIR
|
||||
# pointed at their project still gets the active conversation mined.
|
||||
if is_valid_transcript_path "$TRANSCRIPT_PATH" && [ -f "$TRANSCRIPT_PATH" ]; then
|
||||
mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
|
||||
"$MEMPAL_PYTHON_BIN" -m mempalace mine "$(dirname "$TRANSCRIPT_PATH")" --mode convos \
|
||||
>> "$STATE_DIR/hook.log" 2>&1 &
|
||||
elif [ -n "$TRANSCRIPT_PATH" ]; then
|
||||
echo "[$(date '+%H:%M:%S')] Skipping invalid transcript path: $TRANSCRIPT_PATH" \
|
||||
>> "$STATE_DIR/hook.log"
|
||||
fi
|
||||
if [ -n "$MEMPAL_DIR" ] && [ -d "$MEMPAL_DIR" ]; then
|
||||
mempalace mine "$MEMPAL_DIR" --mode projects \
|
||||
"$MEMPAL_PYTHON_BIN" -m mempalace mine "$MEMPAL_DIR" --mode projects \
|
||||
>> "$STATE_DIR/hook.log" 2>&1 &
|
||||
fi
|
||||
|
||||
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
# MemPalace SessionEnd Hook — final save on clean exit.
|
||||
#
|
||||
# Claude Code documents a default SessionEnd hook timeout of 1.5s; a per-hook
|
||||
# "timeout" in settings.local.json can raise it (up to 60s), but a
|
||||
# plugin-provided timeout cannot (https://code.claude.com/docs/en/hooks). A cold
|
||||
# `mempalace` start alone can exceed 1.5s, so we background the hook and return
|
||||
# immediately; the detached child finishes the save after the session has
|
||||
# exited. All logic lives in mempalace.hooks_cli for cross-harness extensibility.
|
||||
run_mempalace_hook() {
|
||||
if command -v mempalace >/dev/null 2>&1; then
|
||||
exec mempalace hook run "$@"
|
||||
fi
|
||||
|
||||
MEMPAL_PYTHON_BIN="${MEMPAL_PYTHON:-}"
|
||||
if [ -z "$MEMPAL_PYTHON_BIN" ] || [ ! -x "$MEMPAL_PYTHON_BIN" ]; then
|
||||
MEMPAL_PYTHON_BIN="$(command -v python3 2>/dev/null || echo python3)"
|
||||
fi
|
||||
if "$MEMPAL_PYTHON_BIN" -c "import mempalace" >/dev/null 2>&1; then
|
||||
exec "$MEMPAL_PYTHON_BIN" -m mempalace hook run "$@"
|
||||
fi
|
||||
|
||||
if command -v python >/dev/null 2>&1 && python -c "import mempalace" >/dev/null 2>&1; then
|
||||
exec python -m mempalace hook run "$@"
|
||||
fi
|
||||
|
||||
echo "MemPalace hook error: could not find a runnable mempalace command or module" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Capture stdin (the SessionEnd JSON) before backgrounding — the parent's
|
||||
# stdin is gone once we return. Forward it to the detached worker, which runs
|
||||
# the final mine on its own time and outlives this process.
|
||||
payload="$(cat)"
|
||||
(
|
||||
printf '%s' "$payload" | run_mempalace_hook --hook session-end --harness "${MEMPALACE_HOOK_HARNESS:-claude-code}"
|
||||
) >/dev/null 2>&1 </dev/null &
|
||||
disown 2>/dev/null || true
|
||||
|
||||
# Return immediately so the harness never blocks on session exit.
|
||||
printf '{}'
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: mempalace
|
||||
description: "MemPalace — Local AI memory with 96.6% recall. Semantic search, temporal knowledge graph, palace architecture (wings/rooms/drawers). Free, no cloud, no API keys."
|
||||
version: 3.3.0
|
||||
version: 3.6.0
|
||||
homepage: https://github.com/MemPalace/mempalace
|
||||
user-invocable: true
|
||||
metadata:
|
||||
@@ -46,6 +46,10 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
|
||||
|
||||
## Available Tools
|
||||
|
||||
Full MCP surface: 36 tools. Destructive or host-level tools are documented so
|
||||
you know they exist, but use them only when the user explicitly asks or when a
|
||||
tool-specific workflow below says to.
|
||||
|
||||
### Search & Browse
|
||||
- `mempalace_search` — Semantic search across all memories. Always start here.
|
||||
- `query` (required): natural language search — keep it short, keywords or a question. Do NOT include system prompts or conversation context.
|
||||
@@ -58,6 +62,14 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
|
||||
- `mempalace_status` — Palace overview: total drawers, wings, rooms, AAAK spec
|
||||
- `mempalace_list_wings` — All wings with drawer counts
|
||||
- `mempalace_list_rooms` — Rooms within a wing (optional wing filter)
|
||||
- `mempalace_list_drawers` — Paginated drawer listing
|
||||
- `wing`, `room`: optional filters
|
||||
- `since`: only drawers filed on/after this ISO date/time
|
||||
- `before`: only drawers filed before this ISO date/time
|
||||
- `limit`: max results (default 20)
|
||||
- `offset`: pagination offset (default 0)
|
||||
- `mempalace_get_drawer` — Fetch a single drawer by ID. Returns full verbatim content and metadata.
|
||||
- `drawer_id` (required)
|
||||
- `mempalace_get_taxonomy` — Full wing/room/count tree
|
||||
- `mempalace_get_aaak_spec` — Get AAAK compression dialect specification
|
||||
|
||||
@@ -81,17 +93,59 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
|
||||
- `mempalace_traverse` — Walk from a room, find connected ideas across wings
|
||||
- `start_room` (required): room to start from
|
||||
- `max_hops`: connection depth (default 2)
|
||||
- `mempalace_find_tunnels` — Find rooms that bridge two wings
|
||||
- `wing_a`, `wing_b` (required)
|
||||
- `mempalace_find_tunnels` — Find rooms that bridge two wings via *implicit* overlap (rooms whose drawers naturally share content across wings — discovered, not declared)
|
||||
- `wing_a`, `wing_b`: optional filters; omit both to scan all wing pairs
|
||||
- `mempalace_create_tunnel` — Create an *explicit* cross-wing tunnel: a user/agent-declared link between two locations. Use when you notice content in one project relates to another (e.g. API design in `project_api` connects to schema in `project_database`).
|
||||
- `source_wing`, `source_room`, `target_wing`, `target_room` (required)
|
||||
- `label`: short description of the relationship
|
||||
- `source_drawer_id`, `target_drawer_id`: anchor to specific drawers
|
||||
- `mempalace_list_tunnels` — List all explicit tunnels, optionally filtered by wing
|
||||
- `wing`: optional filter
|
||||
- `mempalace_delete_tunnel` — Remove an explicit tunnel by ID
|
||||
- `tunnel_id` (required)
|
||||
- `mempalace_list_hallways` — List within-wing entity hallways (entity-to-entity co-occurrence links built at mine time)
|
||||
- `wing`: optional filter
|
||||
- `mempalace_delete_hallway` — Remove a hallway record by ID
|
||||
- `hallway_id` (required)
|
||||
- `mempalace_follow_tunnels` — From a room, follow explicit tunnels to connected drawers in other wings
|
||||
- `wing`, `room` (required)
|
||||
- `mempalace_graph_stats` — Graph connectivity overview
|
||||
|
||||
### Write
|
||||
- `mempalace_add_drawer` — Store verbatim content into a wing/room
|
||||
- `wing`, `room`, `content` (required)
|
||||
- `source_file`: optional source reference
|
||||
- `added_by`: optional filing agent label
|
||||
- Checks for duplicates automatically
|
||||
- `mempalace_checkpoint` — Save a whole session in one call: dedup each item, file non-duplicates, then write one diary entry
|
||||
- `items` (required): array of `{wing, room, content}`; content must be verbatim
|
||||
- `diary`: optional `{agent_name, entry, topic?, wing?}`; entry should use AAAK format
|
||||
- `dedup_threshold`: similarity threshold (default 0.9)
|
||||
- `added_by`: optional filing agent label (defaults to the diary `agent_name`, else `checkpoint`)
|
||||
- `mempalace_update_drawer` — Update an existing drawer's content and/or move it to a different wing/room
|
||||
- `drawer_id` (required)
|
||||
- `content`, `wing`, `room`: at least one must be provided (no-op otherwise)
|
||||
- `mempalace_delete_drawer` — Remove a drawer by ID
|
||||
- `drawer_id` (required)
|
||||
|
||||
### Ingest & Cleanup
|
||||
- `mempalace_mine` — Mine a directory into the palace. Host-level ingest; call only when the user asks to import files.
|
||||
- `source` (required): directory to mine
|
||||
- `mode`: `projects` (default), `convos`, or `extract`
|
||||
- `wing`: target wing (default: source directory name)
|
||||
- `agent`: recorded on every drawer (default `mempalace`)
|
||||
- `limit`: max files to process (0 = all)
|
||||
- `dry_run`: preview without writing
|
||||
- `extract`: convos extraction strategy (`exchange` default, or `general`)
|
||||
- `mempalace_sync` — Prune drawers whose source files are gitignored, deleted, or moved. Use dry-run first.
|
||||
- `project_dir`: optional project root scope
|
||||
- `wing`: optional wing scope
|
||||
- `apply`: actually delete; default is dry-run preview
|
||||
- `mempalace_delete_by_source` — Bulk-delete drawers with one exact `source_file`. Destructive; use dry-run first.
|
||||
- `source_file` (required): exact metadata value to remove
|
||||
- `dry_run`: preview match count and sample (default true)
|
||||
|
||||
### Diary & Session
|
||||
- `mempalace_diary_write` — Write a session diary entry
|
||||
- `agent_name` (required): your name/identifier
|
||||
- `entry` (required): what happened, what you learned, what matters
|
||||
@@ -99,6 +153,15 @@ You have access to a local memory palace via MCP tools. The palace stores verbat
|
||||
- `mempalace_diary_read` — Read recent diary entries
|
||||
- `agent_name` (required)
|
||||
- `last_n`: number of entries (default 10)
|
||||
- `mempalace_memories_filed_away` — Acknowledge the latest silent auto-save checkpoint.
|
||||
- Returns: how many messages were tucked into drawers since the last ack
|
||||
- When to call: at the START of a session, to confirm prior-conversation persistence
|
||||
|
||||
### System
|
||||
- `mempalace_hook_settings` — Get or set auto-save hook behavior. Host-level setting; do not change silently.
|
||||
- `silent_save`: true saves directly without MCP-level clutter
|
||||
- `desktop_toast`: true shows a desktop notification when saves complete
|
||||
- `mempalace_reconnect` — Force reconnect to the palace database after external writes or stale index state
|
||||
|
||||
## Setup
|
||||
|
||||
|
||||
@@ -0,0 +1,120 @@
|
||||
# MemPalace Recall Protocol
|
||||
|
||||
The canonical "search before answering" protocol shared across every
|
||||
MemPalace integration (Cursor, Antigravity, Claude Code, Codex,
|
||||
OpenClaw). This file is the single source of truth — skills and rules
|
||||
should link here rather than restating the protocol, so the rule never
|
||||
drifts from the skill.
|
||||
|
||||
The protocol exists to honour MemPalace's foundational promise:
|
||||
**100% recall, verbatim, never guess.** When the palace might hold the
|
||||
answer, the agent must read the palace before answering from model
|
||||
memory.
|
||||
|
||||
## When to recall
|
||||
|
||||
Search the palace **before answering** whenever the user asks about
|
||||
anything that may already be filed:
|
||||
|
||||
- Past work, prior decisions, or "what did we do / decide / try?"
|
||||
- A person, project, or entity ("who is …", "what is …")
|
||||
- Something that happened in an earlier session ("remember when …",
|
||||
"last time …", "the thing we discussed")
|
||||
- A preference, fact, or relationship that could have changed over time
|
||||
|
||||
If the question is pure greenfield work with no memory relevance (e.g.
|
||||
"rename this variable", "fix this typo"), do not search — recall is
|
||||
question-driven, not reflexive.
|
||||
|
||||
## The protocol
|
||||
|
||||
1. **On wake-up** (if a session-start hook injected context, honour its wing scoping / `additional_context`): scope recall to the wing inferred from the workspace, then continue.
|
||||
2. **Before responding** about people, projects, past events, or prior
|
||||
decisions: call `mempalace_search` first. For relational or temporal
|
||||
facts ("who reported to whom in March", "what was true then"), call
|
||||
`mempalace_kg_query` instead or as well.
|
||||
3. **If unsure** about a fact (name, age, relationship, preference): say
|
||||
"let me check the palace" and query. Wrong is worse than slow.
|
||||
4. **Return verbatim.** Quote the drawer's exact stored words. Never
|
||||
summarize, paraphrase, or lossy-compress what the palace returns —
|
||||
that is the whole point of the system.
|
||||
5. **After a substantive session**, record continuity with
|
||||
`mempalace_diary_write` (background hooks may already do this — do not
|
||||
double-file).
|
||||
6. **When a fact changes**, call `mempalace_kg_invalidate` on the old
|
||||
fact, then `mempalace_kg_add` for the new one.
|
||||
|
||||
## Tool selection
|
||||
|
||||
| You need | Tool |
|
||||
|---|---|
|
||||
| Find any memory by meaning | `mempalace_search` (start here) |
|
||||
| Relational / time-bound facts about an entity | `mempalace_kg_query` |
|
||||
| The chronological story of an entity | `mempalace_kg_timeline` |
|
||||
| Recent session continuity | `mempalace_diary_read` |
|
||||
| Which wings / rooms exist (when scope unknown) | `mempalace_list_wings`, `mempalace_list_rooms` |
|
||||
| Record this session | `mempalace_diary_write` |
|
||||
|
||||
`mempalace_search` takes a short natural-language `query` (keywords or a
|
||||
question — not a system prompt or pasted conversation) plus optional
|
||||
`wing` / `room` filters and `limit` (default 5).
|
||||
|
||||
## Unhappy paths
|
||||
|
||||
- **Empty results.** Say the palace has nothing on this; do not invent an
|
||||
answer to fill the gap. Offer to widen the search (drop the wing
|
||||
filter) or to file the new information.
|
||||
- **MCP unavailable / tool error.** Surface the error plainly and suggest
|
||||
the user verify the server (`mempalace status`, or re-run install).
|
||||
Do not silently fall back to guessing from model memory.
|
||||
- **Palace index corrupt / compactor error.** When the server returns an
|
||||
error mentioning the HNSW segment writer, a ChromaDB compaction
|
||||
failure, or a stuck "Not connected" state after a write, the on-disk
|
||||
vector index is out of sync with `chroma.sqlite3` — but the drawer rows
|
||||
are intact in SQLite. Recover by rebuilding the index from SQLite, not
|
||||
by re-mining. See "Recovering a corrupt index" below. Do not attempt an
|
||||
in-process repair from the agent; guide the user to run the CLI.
|
||||
- **Stale or conflicting facts.** Prefer the knowledge graph's
|
||||
time-valid answer; if a fact has changed, invalidate the old one and
|
||||
add the new one rather than overwriting context silently.
|
||||
|
||||
## Recovering a corrupt index
|
||||
|
||||
A ChromaDB compaction failure can leave the drawers HNSW index out of
|
||||
sync with `chroma.sqlite3` and wedge the MCP server (every call returns
|
||||
"Not connected"). The data is safe in SQLite; rebuild the index from it.
|
||||
Guide the user through these CLI steps — never run an in-process rebuild
|
||||
from the agent (it can break other live clients):
|
||||
|
||||
1. Stop the MCP server (kill the `mempalace-mcp` process, or restart the
|
||||
host editor).
|
||||
2. Optional backup of the palace directory (`--archive-existing` already
|
||||
moves the old palace aside, so this is belt-and-suspenders):
|
||||
- macOS / Linux: `cp -a ~/.mempalace/palace ~/.mempalace/palace.bak.$(date +%F)`
|
||||
- Windows (PowerShell): `Copy-Item -Recurse "$env:USERPROFILE\.mempalace\palace" "$env:USERPROFILE\.mempalace\palace.bak"`
|
||||
3. Rebuild from SQLite:
|
||||
`mempalace repair --mode from-sqlite --archive-existing --yes`
|
||||
4. Verify: `mempalace repair-status` (divergence should read 0).
|
||||
5. Restart the MCP server.
|
||||
|
||||
Do **not** re-mine from source files to recover: re-mining drops drawers
|
||||
added through the MCP server and diary entries, which have no source file
|
||||
(see MemPalace issue #1843).
|
||||
|
||||
## Anti-patterns
|
||||
|
||||
- Answering about past work, people, or decisions from model memory when
|
||||
the palace might know — search first.
|
||||
- Paraphrasing or summarizing stored content instead of quoting it
|
||||
verbatim.
|
||||
- Searching reflexively on every turn, including pure greenfield coding
|
||||
with no memory relevance.
|
||||
- Pasting the full conversation or a system prompt into the `query`
|
||||
argument — keep queries short and keyword-driven.
|
||||
|
||||
## See also
|
||||
|
||||
- [`integrations/openclaw/SKILL.md`](../openclaw/SKILL.md) — the original
|
||||
full-protocol skill this is distilled from.
|
||||
- MemPalace design principles (verbatim, local-first, never summarize):
|
||||
<https://github.com/MemPalace/mempalace>
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"mcpServers": {
|
||||
"mempalace": {
|
||||
"command": "mempalace-mcp"
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
-1
@@ -16,7 +16,7 @@ The Python package that powers MemPalace. All modules, all logic.
|
||||
| `dialect.py` | AAAK compression — entity codes, emotion markers, 30x lossless ratio |
|
||||
| `knowledge_graph.py` | Temporal entity-relationship graph — SQLite, time-filtered queries, fact invalidation |
|
||||
| `palace_graph.py` | Room-based navigation graph — BFS traversal, tunnel detection across wings |
|
||||
| `mcp_server.py` | MCP server — 19 tools, AAAK auto-teach, Palace Protocol, agent diary |
|
||||
| `mcp_server.py` | MCP server — 34 tools, AAAK auto-teach, Palace Protocol, agent diary |
|
||||
| `onboarding.py` | Guided first-run setup — asks about people/projects, generates AAAK bootstrap + wing config |
|
||||
| `entity_registry.py` | Entity code registry — maps names to AAAK codes, handles ambiguous names |
|
||||
| `entity_detector.py` | Auto-detect people and projects from file content |
|
||||
|
||||
@@ -17,6 +17,7 @@ Public surface:
|
||||
from .base import (
|
||||
BackendClosedError,
|
||||
BackendError,
|
||||
BackendMismatchError,
|
||||
BaseBackend,
|
||||
BaseCollection,
|
||||
CollectionNotInitializedError,
|
||||
@@ -24,14 +25,25 @@ from .base import (
|
||||
EmbedderIdentityMismatchError,
|
||||
GetResult,
|
||||
HealthStatus,
|
||||
LexicalHit,
|
||||
LexicalResult,
|
||||
MaintenanceResult,
|
||||
PalaceNotFoundError,
|
||||
PalaceRef,
|
||||
QueryResult,
|
||||
UnsupportedCapabilityError,
|
||||
UnsupportedFilterError,
|
||||
UnsupportedMaintenanceKindError,
|
||||
)
|
||||
from .chroma import ChromaBackend, ChromaCollection
|
||||
from .milvus import MilvusBackend, MilvusCollection
|
||||
from .pgvector import PgVectorBackend, PgVectorCollection
|
||||
from .qdrant import QdrantBackend, QdrantCollection
|
||||
from .sqlite_exact import SQLiteExactBackend, SQLiteExactCollection
|
||||
from .registry import (
|
||||
available_backends,
|
||||
detect_backend_for_path,
|
||||
detect_backends_for_path,
|
||||
get_backend,
|
||||
get_backend_class,
|
||||
register,
|
||||
@@ -43,6 +55,7 @@ from .registry import (
|
||||
__all__ = [
|
||||
"BackendClosedError",
|
||||
"BackendError",
|
||||
"BackendMismatchError",
|
||||
"BaseBackend",
|
||||
"BaseCollection",
|
||||
"ChromaBackend",
|
||||
@@ -52,11 +65,26 @@ __all__ = [
|
||||
"EmbedderIdentityMismatchError",
|
||||
"GetResult",
|
||||
"HealthStatus",
|
||||
"LexicalHit",
|
||||
"LexicalResult",
|
||||
"MaintenanceResult",
|
||||
"MilvusBackend",
|
||||
"MilvusCollection",
|
||||
"PalaceNotFoundError",
|
||||
"PalaceRef",
|
||||
"PgVectorBackend",
|
||||
"PgVectorCollection",
|
||||
"QdrantBackend",
|
||||
"QdrantCollection",
|
||||
"QueryResult",
|
||||
"SQLiteExactBackend",
|
||||
"SQLiteExactCollection",
|
||||
"UnsupportedCapabilityError",
|
||||
"UnsupportedFilterError",
|
||||
"UnsupportedMaintenanceKindError",
|
||||
"available_backends",
|
||||
"detect_backend_for_path",
|
||||
"detect_backends_for_path",
|
||||
"get_backend",
|
||||
"get_backend_class",
|
||||
"register",
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Shared embedder-identity sidecar (RFC 001).
|
||||
|
||||
A small JSON file in the palace directory, keyed by collection name, recording
|
||||
the embedder identity (``model_name`` / ``dimension``). It is deliberately
|
||||
*separate* from a backend's mismatch marker: a marker's presence signals
|
||||
"palace initialized" (reads raise ``CollectionNotInitializedError`` when the
|
||||
marker exists but the store doesn't), so recording identity at first empty open
|
||||
must not create one. The sidecar is unguarded, so a brand-new palace can record
|
||||
identity immediately — the same approach the chroma backend uses.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
EMBEDDER_SIDECAR_FILENAME = "mempalace_embedder.json"
|
||||
|
||||
|
||||
def read_embedder_sidecar(path: Optional[str], collection_name: Optional[str]):
|
||||
"""Return the recorded :class:`EmbedderIdentity` for ``collection_name``, or None.
|
||||
|
||||
Robust to a missing, unreadable, or malformed (non-dict) sidecar — any of
|
||||
those degrade to ``None`` (the ``unknown`` state) rather than raising.
|
||||
"""
|
||||
from .base import EmbedderIdentity
|
||||
|
||||
if not path or not collection_name or not os.path.isfile(path):
|
||||
return None
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
return None
|
||||
if not isinstance(data, dict):
|
||||
return None
|
||||
entry = data.get(collection_name)
|
||||
if not isinstance(entry, dict) or not entry.get("model_name"):
|
||||
return None
|
||||
return EmbedderIdentity(
|
||||
model_name=str(entry["model_name"]),
|
||||
dimension=int(entry.get("dimension") or 0),
|
||||
)
|
||||
|
||||
|
||||
def write_embedder_sidecar(path: Optional[str], collection_name: Optional[str], identity) -> None:
|
||||
"""Record ``identity`` for ``collection_name`` in the sidecar, creating it if needed.
|
||||
|
||||
No-ops for a missing path, missing collection name, or a nameless identity.
|
||||
Preserves other collections' entries; never raises on I/O failure.
|
||||
"""
|
||||
if not path or not collection_name or not identity or not getattr(identity, "model_name", ""):
|
||||
return
|
||||
data: dict = {}
|
||||
if os.path.isfile(path):
|
||||
try:
|
||||
with open(path, encoding="utf-8") as f:
|
||||
loaded = json.load(f)
|
||||
if isinstance(loaded, dict):
|
||||
data = loaded
|
||||
except (OSError, json.JSONDecodeError):
|
||||
data = {}
|
||||
data[collection_name] = {
|
||||
"model_name": str(identity.model_name),
|
||||
"dimension": int(identity.dimension or 0),
|
||||
}
|
||||
try:
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
os.chmod(path, 0o600)
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
+311
-2
@@ -14,8 +14,8 @@ conformance suite land in follow-up PRs.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import ClassVar, Optional
|
||||
from dataclasses import dataclass, field
|
||||
from typing import ClassVar, Optional, Protocol, runtime_checkable
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -58,6 +58,23 @@ class UnsupportedFilterError(BackendError):
|
||||
"""
|
||||
|
||||
|
||||
class UnsupportedCapabilityError(BackendError):
|
||||
"""Raised when a backend does not implement an optional capability."""
|
||||
|
||||
|
||||
class UnsupportedMaintenanceKindError(BackendError):
|
||||
"""Raised when ``run_maintenance(kind)`` is called with an unadvertised kind.
|
||||
|
||||
A backend MUST advertise a kind in ``maintenance_kinds`` before it accepts
|
||||
it (RFC 001). Advertising a kind it does not implement is a conformance
|
||||
failure; a kind it has no analogue for MUST be omitted, not no-op'd.
|
||||
"""
|
||||
|
||||
|
||||
class BackendMismatchError(BackendError):
|
||||
"""Raised when a selected backend does not match existing palace artifacts."""
|
||||
|
||||
|
||||
class DimensionMismatchError(BackendError):
|
||||
"""Raised when the embedding dimension on write does not match the collection."""
|
||||
|
||||
@@ -66,6 +83,15 @@ class EmbedderIdentityMismatchError(BackendError):
|
||||
"""Raised when the stored embedder model name differs from the current one."""
|
||||
|
||||
|
||||
class EmbedderIdentityUnknownWarning(UserWarning):
|
||||
"""Emitted on first open of a collection with no recorded embedder identity.
|
||||
|
||||
Legacy palaces created before identity tracking carry no model name. Per
|
||||
RFC 001 the right behavior is warn-not-fail: the identity is recorded on
|
||||
the next write and subsequent opens become strict.
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Value objects
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -78,6 +104,29 @@ class PalaceRef:
|
||||
``id`` is always present and is the key backends use to cache handles.
|
||||
``local_path`` is populated for filesystem-rooted palaces.
|
||||
``namespace`` is used by server-mode backends for tenant / prefix routing.
|
||||
|
||||
Isolation contract (RFC 001 §2.1, conformance: ``tests/test_backend_conformance.py``)
|
||||
-----------------------------------------------------------------------------------
|
||||
``id`` is the *required* isolation key. Within a single backend instance:
|
||||
|
||||
A record written for one ``PalaceRef.id`` MUST NOT be returned,
|
||||
modified, or deleted by an operation issued for a different
|
||||
``PalaceRef.id``. Cross-palace access is a spec violation.
|
||||
|
||||
``namespace`` is *additional* partitioning, honored only by backends that
|
||||
advertise the ``supports_namespace_isolation`` capability. For those
|
||||
backends the same guarantee extends to namespaces:
|
||||
|
||||
A record written under one ``namespace`` MUST NOT be returned,
|
||||
modified, or deleted by an operation issued under a different
|
||||
``namespace`` within the same backend instance. Cross-namespace
|
||||
access is a spec violation.
|
||||
|
||||
Backends that do not advertise ``supports_namespace_isolation`` (e.g.
|
||||
``sqlite_exact``, whose isolation is the on-disk path alone) MAY ignore
|
||||
``namespace`` entirely; callers MUST NOT rely on it for tenant isolation
|
||||
on such backends. Any conforming backend can self-check both guarantees by
|
||||
running the shared assertions in ``tests/_backend_conformance.py``.
|
||||
"""
|
||||
|
||||
id: str
|
||||
@@ -85,6 +134,114 @@ class PalaceRef:
|
||||
namespace: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EmbedderIdentity:
|
||||
"""Identity of the embedder that produced a collection's vectors (RFC 001).
|
||||
|
||||
``model_name`` is the stable identity persisted alongside a collection and
|
||||
checked on subsequent opens. ``dimension`` is the vector width. A
|
||||
``dimension`` of ``0`` means *unknown / not probed* — comparisons treat it
|
||||
as "no dimension signal" rather than a real zero-width vector, so a cheap
|
||||
read-path check can compare model names without loading the model.
|
||||
"""
|
||||
|
||||
model_name: str
|
||||
dimension: int = 0
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MaintenanceResult:
|
||||
"""Observable outcome of ``run_maintenance(kind)`` (RFC 001).
|
||||
|
||||
Maintenance is *not* fire-and-forget: a backend MUST serialize concurrent
|
||||
same-kind runs and report the outcome so a caller can learn it must not
|
||||
re-trigger. ``status`` is one of:
|
||||
|
||||
* ``"ran"`` — this call performed the maintenance.
|
||||
* ``"already_running"`` — another caller holds the work; this call did
|
||||
nothing and the caller MUST NOT re-trigger (the production index-build
|
||||
wedge: concurrent writers each issuing the build stacked exclusive locks).
|
||||
* ``"noop"`` — nothing needed doing (e.g. the index already exists).
|
||||
|
||||
``stats`` is free-form per kind (rows analyzed, bytes reclaimed, index
|
||||
build time) for benchmark/operator reporting.
|
||||
"""
|
||||
|
||||
kind: str
|
||||
status: str
|
||||
stats: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class Embedder(Protocol):
|
||||
"""Minimal embedder contract (RFC 001, normative for identity checking).
|
||||
|
||||
The fuller embedder RFC (batching/async/pooling) is additive; identity
|
||||
enforcement depends only on these three members.
|
||||
"""
|
||||
|
||||
model_name: str
|
||||
dimension: int
|
||||
|
||||
def embed(self, texts: list[str]) -> list[list[float]]: ...
|
||||
|
||||
|
||||
def check_embedder_identity(
|
||||
stored: Optional[EmbedderIdentity],
|
||||
current: Optional[EmbedderIdentity],
|
||||
*,
|
||||
force_model_swap: bool = False,
|
||||
) -> str:
|
||||
"""Three-state embedder-identity check (RFC 001).
|
||||
|
||||
Returns the resolved state and raises on a hard, unforced conflict:
|
||||
|
||||
* ``"unknown"`` — no identity recorded yet (legacy collection), or the
|
||||
current embedder is nameless. The caller warns and records on write.
|
||||
* ``"known_match"`` — stored name (and dimension, when both known) equal
|
||||
the current embedder. Proceed normally.
|
||||
* ``"known_mismatch"`` — names or dimensions differ. Without
|
||||
``force_model_swap`` this raises (:class:`EmbedderIdentityMismatchError`
|
||||
for a model swap, :class:`DimensionMismatchError` for a width change,
|
||||
which is checked first because mismatched vectors are physically
|
||||
unusable). With ``force_model_swap`` it returns the state so the caller
|
||||
can re-record the identity and log the swap.
|
||||
|
||||
A ``dimension`` of ``0`` on either side means "unknown" and is skipped, so
|
||||
a model-name-only check (cheap read path) still works.
|
||||
"""
|
||||
if current is None or not current.model_name:
|
||||
return "unknown"
|
||||
if stored is None:
|
||||
return "unknown"
|
||||
|
||||
dim_conflict = bool(stored.dimension and current.dimension) and (
|
||||
stored.dimension != current.dimension
|
||||
)
|
||||
name_conflict = stored.model_name != current.model_name
|
||||
|
||||
if not dim_conflict and not name_conflict:
|
||||
return "known_match"
|
||||
|
||||
if force_model_swap:
|
||||
return "known_mismatch"
|
||||
|
||||
if dim_conflict:
|
||||
raise DimensionMismatchError(
|
||||
f"collection was built with a {stored.dimension}-dim embedder "
|
||||
f"({stored.model_name!r}) but the current embedder is "
|
||||
f"{current.dimension}-dim ({current.model_name!r}); the stored "
|
||||
"vectors are incompatible. Re-embed the palace to switch models."
|
||||
)
|
||||
raise EmbedderIdentityMismatchError(
|
||||
f"collection was built with embedder {stored.model_name!r} but the "
|
||||
f"current embedder is {current.model_name!r}. Searching across a model "
|
||||
"swap silently degrades recall. Re-embed the palace, or run "
|
||||
"`mempalace palace set-embedder --model <name> --force` to record the "
|
||||
"new identity if you know the vectors are compatible."
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class HealthStatus:
|
||||
ok: bool
|
||||
@@ -177,6 +334,23 @@ class GetResult(_DictCompatMixin):
|
||||
return cls(ids=[], documents=[], metadatas=[], embeddings=None)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LexicalHit:
|
||||
"""One hit from backend lexical candidate search."""
|
||||
|
||||
id: str
|
||||
document: str
|
||||
metadata: dict
|
||||
score: float
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LexicalResult:
|
||||
"""Typed return from ``BaseCollection.lexical_search``."""
|
||||
|
||||
hits: list[LexicalHit]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Collection contract
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -253,6 +427,119 @@ class BaseCollection(ABC):
|
||||
def health(self) -> HealthStatus:
|
||||
return HealthStatus.healthy()
|
||||
|
||||
@property
|
||||
def distance_metric(self) -> str:
|
||||
"""The space this collection's ``distances`` are reported in.
|
||||
|
||||
Defaults to the owning backend's declared metric (cosine for all
|
||||
in-tree backends). Collections that can vary per-collection — e.g. a
|
||||
legacy Chroma palace built without ``hnsw:space=cosine`` — override
|
||||
this to report their actual space so core ranking converts correctly.
|
||||
"""
|
||||
return "cosine"
|
||||
|
||||
def get_stored_embedder_identity(self) -> Optional[EmbedderIdentity]:
|
||||
"""Return the embedder identity recorded for this collection, if any.
|
||||
|
||||
Returns ``None`` when nothing is recorded — a legacy collection, or a
|
||||
backend that does not yet persist identity. Core treats ``None`` as the
|
||||
``unknown`` state (warn, do not fail). Backends override this and
|
||||
:meth:`set_embedder_identity` against their own metadata store.
|
||||
"""
|
||||
return None
|
||||
|
||||
def set_embedder_identity(self, identity: EmbedderIdentity) -> None:
|
||||
"""Persist this collection's embedder identity. Default: no-op.
|
||||
|
||||
A backend without an identity slot inherits the no-op default and so
|
||||
stays permanently ``unknown`` (safe — it simply never enforces). The
|
||||
enforcement choke point calls this when recording on first write or
|
||||
on an explicit, forced model swap.
|
||||
"""
|
||||
return None
|
||||
|
||||
def effective_embedder_identity(self) -> Optional[EmbedderIdentity]:
|
||||
"""The identity of the embedder this collection actually uses.
|
||||
|
||||
For ``server_embedder`` backends that ignore the injected embedder,
|
||||
this reports the server-side embedder so the same identity rules apply
|
||||
(RFC 001). Defaults to ``None`` — the collection is embedded by the
|
||||
injected/core embedder, and the caller supplies the current identity.
|
||||
"""
|
||||
return None
|
||||
|
||||
def get_all_metadata(self, where: Optional[dict] = None) -> list[dict]:
|
||||
"""Return every matching record's metadata in one logical pass (#1796).
|
||||
|
||||
Default implementation pages through :meth:`get` using
|
||||
``limit``/``offset`` -- correct for backends with a real server-side
|
||||
cursor (e.g. Chroma's SQL OFFSET), and the same shape callers already
|
||||
relied on before this method existed.
|
||||
|
||||
Backends whose ``get(limit=, offset=)`` is implemented by fully
|
||||
materializing a result set and then Python-slicing it (no true
|
||||
server-side cursor) MUST override this method to walk their native
|
||||
cursor exactly once instead. Calling the default implementation on
|
||||
such a backend is O(n^2) in collection size: each page re-walks the
|
||||
entire collection just to discard everything outside the requested
|
||||
slice. See issue #1796.
|
||||
"""
|
||||
all_meta: list[dict] = []
|
||||
offset = 0
|
||||
page_size = 1000
|
||||
while True:
|
||||
kwargs: dict = {"include": ["metadatas"], "limit": page_size, "offset": offset}
|
||||
if where:
|
||||
kwargs["where"] = where
|
||||
batch = self.get(**kwargs)
|
||||
batch_meta = batch.metadatas if hasattr(batch, "metadatas") else batch.get("metadatas")
|
||||
if not batch_meta:
|
||||
break
|
||||
all_meta.extend(batch_meta)
|
||||
if len(batch_meta) < page_size:
|
||||
break
|
||||
offset += len(batch_meta)
|
||||
return all_meta
|
||||
|
||||
def facet_counts(
|
||||
self,
|
||||
field: str,
|
||||
where: Optional[dict] = None,
|
||||
limit: int = 1000,
|
||||
) -> dict[str, int]:
|
||||
"""Return counts for each distinct value of a metadata field."""
|
||||
raise UnsupportedCapabilityError("backend does not support facet_counts")
|
||||
|
||||
def maintenance_state(self) -> dict:
|
||||
"""Return a structured snapshot of this collection's maintenance state.
|
||||
|
||||
Free-form per backend (e.g. row count, whether a vector index exists,
|
||||
last-analyze age). Used by benchmark harnesses to record state
|
||||
alongside each latency/recall measurement so an un-analyzed store is
|
||||
not compared against a settled one (RFC 001). Defaults to empty.
|
||||
"""
|
||||
return {}
|
||||
|
||||
def run_maintenance(self, kind: str) -> "MaintenanceResult":
|
||||
"""Run a maintenance ``kind`` and return an observable result (RFC 001).
|
||||
|
||||
Backends advertise supported kinds in ``BaseBackend.maintenance_kinds``
|
||||
and override this. The default supports nothing, so every kind raises
|
||||
:class:`UnsupportedMaintenanceKindError`. Implementations MUST serialize
|
||||
concurrent same-kind runs and report ``already_running`` rather than
|
||||
stacking the work.
|
||||
"""
|
||||
raise UnsupportedMaintenanceKindError(f"backend does not support maintenance kind {kind!r}")
|
||||
|
||||
def lexical_search(
|
||||
self,
|
||||
*,
|
||||
query: str,
|
||||
n_results: int = 10,
|
||||
where: Optional[dict] = None,
|
||||
) -> LexicalResult:
|
||||
raise UnsupportedCapabilityError("backend does not support lexical_search")
|
||||
|
||||
def update(
|
||||
self,
|
||||
*,
|
||||
@@ -311,11 +598,33 @@ class BaseBackend(ABC):
|
||||
Instances are lightweight on construction — no I/O, no network. All
|
||||
connection work is deferred to ``get_collection``. Instances are thread-
|
||||
safe for concurrent ``get_collection`` calls across different palaces.
|
||||
|
||||
Every backend MUST satisfy the per-``PalaceRef.id`` isolation guarantee in
|
||||
:class:`PalaceRef`. Backends that additionally isolate by
|
||||
``PalaceRef.namespace`` (multi-tenant / hosted deployments) MUST advertise
|
||||
the ``supports_namespace_isolation`` capability token; doing so is a
|
||||
promise to satisfy the cross-namespace guarantee and to pass the namespace
|
||||
arm of the conformance suite. Backends without the token MAY ignore
|
||||
``namespace``.
|
||||
"""
|
||||
|
||||
name: ClassVar[str]
|
||||
spec_version: ClassVar[str] = "1.0"
|
||||
capabilities: ClassVar[frozenset[str]] = frozenset()
|
||||
#: The space ``query()`` reports ``distances`` in (RFC 001 §2.1).
|
||||
#: One of ``"cosine"`` | ``"l2"`` | ``"ip"``. The contract for the
|
||||
#: ``distances`` field is *lower = closer* regardless of metric; core
|
||||
#: search converts distance→similarity off this declaration rather than
|
||||
#: assuming cosine. All in-tree backends are cosine today.
|
||||
distance_metric: ClassVar[str] = "cosine"
|
||||
#: Maintenance kinds this backend implements (RFC 001). Reserved names:
|
||||
#: ``"analyze"`` (refresh planner/query statistics), ``"compact"`` (reclaim
|
||||
#: space, rewrite storage), ``"reindex"`` (build/rebuild secondary indexes).
|
||||
#: A backend with no analogue for a kind MUST omit it rather than declare a
|
||||
#: no-op, so a benchmark harness can trust the set. Backends MAY add their
|
||||
#: own kinds. ``run_maintenance`` raises ``UnsupportedMaintenanceKindError``
|
||||
#: for anything not listed here.
|
||||
maintenance_kinds: ClassVar[frozenset[str]] = frozenset()
|
||||
|
||||
@abstractmethod
|
||||
def get_collection(
|
||||
|
||||
+866
-120
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,195 @@
|
||||
"""Core-side embedding adapter for explicit-vector backends."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from .base import BaseCollection
|
||||
|
||||
|
||||
def _embed_texts(texts: list[str]) -> list[list[float]]:
|
||||
"""Embed ``texts`` with the configured local embedding function."""
|
||||
if not texts:
|
||||
return []
|
||||
from ..embedding import get_embedding_function
|
||||
|
||||
ef = get_embedding_function()
|
||||
vectors = ef(input=texts)
|
||||
return [list(v) for v in vectors]
|
||||
|
||||
|
||||
def _as_list(value):
|
||||
"""Normalize ChromaDB's ``OneOrMany`` shape (``str`` | ``dict`` | sequence) to a list.
|
||||
|
||||
A bare ``str`` (a document/id) or ``dict`` (a single metadata) must be
|
||||
*wrapped*, not iterated: ``list("abc")`` yields ``['a', 'b', 'c']`` and
|
||||
``list({"k": 1})`` yields ``['k']`` — either desyncs embeddings/metadatas
|
||||
from ``ids`` on explicit-vector backends (pgvector, sqlite_exact). A list is
|
||||
returned unchanged (no copy); any other iterable is materialized once.
|
||||
See PR #1706/#1707 review.
|
||||
"""
|
||||
if isinstance(value, (str, dict)):
|
||||
return [value]
|
||||
if isinstance(value, list):
|
||||
return value
|
||||
return list(value)
|
||||
|
||||
|
||||
class EmbeddingCollection(BaseCollection):
|
||||
"""Wrap a collection that requires explicit vectors.
|
||||
|
||||
Backends opt in with the ``requires_explicit_embeddings`` capability.
|
||||
Core callers can keep using ``documents=`` and ``query_texts=``; this
|
||||
wrapper computes vectors locally before delegating to the backend.
|
||||
"""
|
||||
|
||||
def __init__(self, inner: BaseCollection):
|
||||
self._inner = inner
|
||||
|
||||
def __getattr__(self, name):
|
||||
return getattr(self._inner, name)
|
||||
|
||||
@property
|
||||
def distance_metric(self) -> str:
|
||||
# Explicit delegation: ``BaseCollection`` defines ``distance_metric``
|
||||
# as a property, so it resolves on this subclass and ``__getattr__``
|
||||
# never fires — without this override the wrapper would report the
|
||||
# base "cosine" default and mask a wrapped non-cosine backend.
|
||||
return self._inner.distance_metric
|
||||
|
||||
# Same shadowing reason as ``distance_metric``: these are concrete methods
|
||||
# on ``BaseCollection``, so ``__getattr__`` never delegates them. Forward
|
||||
# explicitly to the wrapped backend collection's identity store.
|
||||
def get_stored_embedder_identity(self):
|
||||
return self._inner.get_stored_embedder_identity()
|
||||
|
||||
def set_embedder_identity(self, identity) -> None:
|
||||
return self._inner.set_embedder_identity(identity)
|
||||
|
||||
def effective_embedder_identity(self):
|
||||
return self._inner.effective_embedder_identity()
|
||||
|
||||
def maintenance_state(self) -> dict:
|
||||
return self._inner.maintenance_state()
|
||||
|
||||
def run_maintenance(self, kind: str):
|
||||
return self._inner.run_maintenance(kind)
|
||||
|
||||
def add(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
documents = _as_list(documents)
|
||||
ids = _as_list(ids)
|
||||
if metadatas is not None:
|
||||
metadatas = _as_list(metadatas)
|
||||
if embeddings is None:
|
||||
embeddings = _embed_texts(documents)
|
||||
return self._inner.add(
|
||||
documents=documents,
|
||||
ids=ids,
|
||||
metadatas=metadatas,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
|
||||
def upsert(self, *, documents, ids, metadatas=None, embeddings=None):
|
||||
documents = _as_list(documents)
|
||||
ids = _as_list(ids)
|
||||
if metadatas is not None:
|
||||
metadatas = _as_list(metadatas)
|
||||
if embeddings is None:
|
||||
embeddings = _embed_texts(documents)
|
||||
return self._inner.upsert(
|
||||
documents=documents,
|
||||
ids=ids,
|
||||
metadatas=metadatas,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
|
||||
def query(
|
||||
self,
|
||||
*,
|
||||
query_texts: Optional[list[str] | str] = None,
|
||||
query_embeddings: Optional[list[list[float]]] = None,
|
||||
n_results: int = 10,
|
||||
where: Optional[dict] = None,
|
||||
where_document: Optional[dict] = None,
|
||||
include: Optional[list[str]] = None,
|
||||
):
|
||||
if query_texts is not None and query_embeddings is None:
|
||||
query_embeddings = _embed_texts(_as_list(query_texts))
|
||||
query_texts = None
|
||||
return self._inner.query(
|
||||
query_texts=query_texts,
|
||||
query_embeddings=query_embeddings,
|
||||
n_results=n_results,
|
||||
where=where,
|
||||
where_document=where_document,
|
||||
include=include,
|
||||
)
|
||||
|
||||
def get(
|
||||
self, *, ids=None, where=None, where_document=None, limit=None, offset=None, include=None
|
||||
):
|
||||
return self._inner.get(
|
||||
ids=ids,
|
||||
where=where,
|
||||
where_document=where_document,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
include=include,
|
||||
)
|
||||
|
||||
def delete(self, *, ids=None, where=None):
|
||||
return self._inner.delete(ids=ids, where=where)
|
||||
|
||||
def count(self) -> int:
|
||||
return self._inner.count()
|
||||
|
||||
def estimated_count(self) -> int:
|
||||
return self._inner.estimated_count()
|
||||
|
||||
def close(self) -> None:
|
||||
return self._inner.close()
|
||||
|
||||
def health(self):
|
||||
return self._inner.health()
|
||||
|
||||
def lexical_search(self, *, query: str, n_results: int = 10, where: Optional[dict] = None):
|
||||
return self._inner.lexical_search(query=query, n_results=n_results, where=where)
|
||||
|
||||
def facet_counts(
|
||||
self, field: str, where: Optional[dict] = None, limit: int = 1000
|
||||
) -> dict[str, int]:
|
||||
# ``BaseCollection.facet_counts`` is a concrete method that raises
|
||||
# ``UnsupportedCapabilityError`` as its default. MRO resolves it on
|
||||
# this subclass before ``__getattr__`` ever fires, so without an
|
||||
# explicit forwarder every facet call against a wrapped backend
|
||||
# (qdrant, pgvector, sqlite_exact) raises and silently degrades to
|
||||
# client-side counting in mcp_server's try/except.
|
||||
return self._inner.facet_counts(field, where=where, limit=limit)
|
||||
|
||||
def get_all_metadata(self, where: Optional[dict] = None) -> list[dict]:
|
||||
# ``BaseCollection.get_all_metadata`` ships a concrete default that
|
||||
# pages through ``self.get(include=["metadatas"])``. Without this
|
||||
# forwarder, MRO resolves the call here on the subclass and runs the
|
||||
# base default — which routes back through ``self.get()`` (the
|
||||
# wrapper's get, then ``__getattr__`` to the inner's get). Result:
|
||||
# the inner's overridden ``get_all_metadata`` (e.g. pgvector's
|
||||
# ``with_document=False`` fast path from #1892) is never reached,
|
||||
# and every metadata-only fetch transfers the full document column
|
||||
# over the wire. Same MRO-shadow pattern as ``facet_counts`` /
|
||||
# ``lexical_search`` above.
|
||||
return self._inner.get_all_metadata(where=where)
|
||||
|
||||
def update(self, *, ids, documents=None, metadatas=None, embeddings=None):
|
||||
ids = _as_list(ids)
|
||||
if documents is not None:
|
||||
documents = _as_list(documents)
|
||||
if embeddings is None:
|
||||
embeddings = _embed_texts(documents)
|
||||
if metadatas is not None:
|
||||
metadatas = _as_list(metadatas)
|
||||
return self._inner.update(
|
||||
ids=ids,
|
||||
documents=documents,
|
||||
metadatas=metadatas,
|
||||
embeddings=embeddings,
|
||||
)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -125,6 +125,38 @@ def get_backend(name: str) -> BaseBackend:
|
||||
return inst
|
||||
|
||||
|
||||
def detect_backends_for_path(path: str) -> list[str]:
|
||||
"""Return all registered backend names whose artifacts are present at ``path``.
|
||||
|
||||
Detection is a migration/protection aid for local palaces. Backends are
|
||||
checked in registry-name order so callers get deterministic diagnostics if
|
||||
a broken directory contains artifacts from more than one backend.
|
||||
"""
|
||||
_discover_entry_points()
|
||||
detected = []
|
||||
for name in sorted(_registry):
|
||||
cls = _registry[name]
|
||||
try:
|
||||
if cls.detect(path):
|
||||
detected.append(name)
|
||||
except Exception:
|
||||
logger.exception("detect() raised on backend %r", name)
|
||||
return detected
|
||||
|
||||
|
||||
def detect_backend_for_path(path: str) -> Optional[str]:
|
||||
"""Return the single detected backend at ``path``, or ``None``.
|
||||
|
||||
If multiple backend artifacts are present, the first name in registry order
|
||||
is returned for backward compatibility. Callers that enforce mismatch
|
||||
protection should use :func:`detect_backends_for_path`.
|
||||
"""
|
||||
detected = detect_backends_for_path(path)
|
||||
if detected:
|
||||
return detected[0]
|
||||
return None
|
||||
|
||||
|
||||
def reset_backends() -> None:
|
||||
"""Close and drop all cached backend instances (primarily for tests)."""
|
||||
with _lock:
|
||||
@@ -161,14 +193,9 @@ def resolve_backend_for_palace(
|
||||
return candidate
|
||||
|
||||
_discover_entry_points()
|
||||
if palace_path:
|
||||
for name, cls in _registry.items():
|
||||
try:
|
||||
if cls.detect(palace_path):
|
||||
return name
|
||||
except Exception:
|
||||
logger.exception("detect() raised on backend %r", name)
|
||||
continue
|
||||
detected = detect_backend_for_path(palace_path) if palace_path else None
|
||||
if detected:
|
||||
return detected
|
||||
return default
|
||||
|
||||
|
||||
@@ -180,10 +207,22 @@ def resolve_backend_for_palace(
|
||||
def _register_builtins() -> None:
|
||||
"""Register chroma as the in-tree default."""
|
||||
from .chroma import ChromaBackend
|
||||
from .milvus import MilvusBackend
|
||||
from .pgvector import PgVectorBackend
|
||||
from .qdrant import QdrantBackend
|
||||
from .sqlite_exact import SQLiteExactBackend
|
||||
|
||||
# Use setdefault semantics so a caller that pre-registered for tests wins.
|
||||
if "chroma" not in _registry:
|
||||
_registry["chroma"] = ChromaBackend
|
||||
if "milvus" not in _registry:
|
||||
_registry["milvus"] = MilvusBackend
|
||||
if "qdrant" not in _registry:
|
||||
_registry["qdrant"] = QdrantBackend
|
||||
if "sqlite_exact" not in _registry:
|
||||
_registry["sqlite_exact"] = SQLiteExactBackend
|
||||
if "pgvector" not in _registry:
|
||||
_registry["pgvector"] = PgVectorBackend
|
||||
|
||||
|
||||
_register_builtins()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,74 @@
|
||||
"""Retention pruning for timestamped palace backups.
|
||||
|
||||
``mempalace migrate`` and ``mempalace repair max-seq-id`` each write a fresh,
|
||||
timestamped backup every time they run and historically never deleted the old
|
||||
ones. On a machine that mines or repairs on a schedule those full-size copies
|
||||
accumulate silently — a real palace was found with hundreds of gigabytes of
|
||||
backups sitting beside only a few hundred megabytes of live data, nearly
|
||||
filling the disk. This module prunes the backup set down to a bounded count
|
||||
after each new backup is written.
|
||||
|
||||
The retention count comes from ``MempalaceConfig.max_backups`` (default 10).
|
||||
"""
|
||||
|
||||
import glob
|
||||
import os
|
||||
import shutil
|
||||
|
||||
|
||||
def prune_backups(pattern, max_backups, *, log=None):
|
||||
"""Delete the oldest backups matching ``pattern`` so at most ``max_backups`` remain.
|
||||
|
||||
Args:
|
||||
pattern: A glob pattern matching the backup paths (files or
|
||||
directories). The caller is responsible for ``glob.escape``-ing
|
||||
any literal, non-wildcard portion that can contain glob
|
||||
metacharacters — palace paths sometimes do (e.g. a ``[``).
|
||||
max_backups: Number of most-recent backups to keep. ``None`` or any
|
||||
value ``<= 0`` disables pruning and returns immediately, so a
|
||||
backup set is never touched when the user has opted out.
|
||||
log: Optional callable (e.g. ``print``) for human-readable progress.
|
||||
|
||||
Returns:
|
||||
The list of paths that were successfully removed.
|
||||
|
||||
Recency is determined by filesystem mtime rather than by parsing the
|
||||
timestamp out of the name, so it stays correct even when two backup
|
||||
producers use different timestamp formats. Deletion failures are logged
|
||||
and skipped: pruning is best-effort cleanup and must never abort the
|
||||
migrate/repair operation that just completed successfully.
|
||||
"""
|
||||
if max_backups is None or max_backups <= 0:
|
||||
return []
|
||||
|
||||
scored = []
|
||||
for path in glob.glob(pattern):
|
||||
try:
|
||||
scored.append((os.path.getmtime(path), path))
|
||||
except OSError:
|
||||
# Vanished between glob and stat (concurrent prune / cleanup);
|
||||
# nothing for us to remove.
|
||||
continue
|
||||
|
||||
if len(scored) <= max_backups:
|
||||
return []
|
||||
|
||||
# Newest first; the path breaks mtime ties so ordering is deterministic.
|
||||
scored.sort(key=lambda item: (item[0], item[1]), reverse=True)
|
||||
|
||||
removed = []
|
||||
for _mtime, path in scored[max_backups:]:
|
||||
try:
|
||||
if os.path.isdir(path) and not os.path.islink(path):
|
||||
shutil.rmtree(path)
|
||||
else:
|
||||
os.remove(path)
|
||||
except OSError as exc:
|
||||
if log:
|
||||
log(f" Backup prune: could not remove {path}: {exc}")
|
||||
continue
|
||||
removed.append(path)
|
||||
if log:
|
||||
log(f" Backup prune: removed old backup {path}")
|
||||
|
||||
return removed
|
||||
+692
-27
@@ -51,6 +51,51 @@ _PASS_ZERO_PER_FILE_CAP = 100_000 # 100KB per file is generous for prose
|
||||
_PASS_ZERO_TOTAL_CAP = 5_000_000 # 5MB total ceiling — bounds memory
|
||||
_PASS_ZERO_LLM_PER_SAMPLE = 2_000 # for Tier 2 LLM call only
|
||||
_PASS_ZERO_LLM_MAX_SAMPLES = 20 # caps the LLM-tier sample count
|
||||
_EXPLICIT_BACKEND_ENV = "MEMPALACE_BACKEND_EXPLICIT"
|
||||
|
||||
# Keep parser construction lightweight for --version and hook commands.
|
||||
# This mirrors miner.MAX_CHUNKS_PER_FILE without importing miner here;
|
||||
# importing miner pulls in Chroma dependencies before argparse can handle
|
||||
# lightweight exits such as --version.
|
||||
_CLI_MAX_CHUNKS_PER_FILE_DEFAULT = 50_000
|
||||
|
||||
|
||||
def _backend_arg(args):
|
||||
"""Return a CLI-selected backend from subcommand or global flags."""
|
||||
return getattr(args, "backend", None) or getattr(args, "global_backend", None)
|
||||
|
||||
|
||||
def _apply_backend_arg(args) -> None:
|
||||
backend = _backend_arg(args)
|
||||
if not backend:
|
||||
return
|
||||
backend = str(backend).strip().lower()
|
||||
from .backends import get_backend_class
|
||||
|
||||
get_backend_class(backend)
|
||||
os.environ[_EXPLICIT_BACKEND_ENV] = backend
|
||||
os.environ["MEMPALACE_BACKEND"] = backend
|
||||
|
||||
|
||||
def _selected_backend_for_palace(palace_path: str) -> str:
|
||||
from .palace import resolve_backend_name
|
||||
|
||||
return resolve_backend_name(palace_path, explicit=os.environ.get(_EXPLICIT_BACKEND_ENV))
|
||||
|
||||
|
||||
def _maintenance_requires_chroma(palace_path: str, command_name: str) -> bool:
|
||||
try:
|
||||
backend_name = _selected_backend_for_palace(palace_path)
|
||||
except Exception as exc: # noqa: BLE001 - user-facing guard before maintenance imports
|
||||
print(f"\n {command_name} cannot resolve the palace backend: {exc}", file=sys.stderr)
|
||||
return False
|
||||
if backend_name == "chroma":
|
||||
return True
|
||||
print(
|
||||
f"\n {command_name} is Chroma-only in this release (selected backend: {backend_name}).",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _gather_origin_samples(project_dir) -> list:
|
||||
@@ -214,14 +259,16 @@ def _ensure_mempalace_files_gitignored(project_dir) -> bool:
|
||||
if not (project_path / ".git").exists():
|
||||
return False
|
||||
gitignore = project_path / ".gitignore"
|
||||
existing = gitignore.read_text() if gitignore.exists() else ""
|
||||
# Force UTF-8: Windows defaults to GBK and chokes on non-ASCII .gitignore
|
||||
# comments, killing auto-init even though the file is valid UTF-8.
|
||||
existing = gitignore.read_text(encoding="utf-8", errors="replace") if gitignore.exists() else ""
|
||||
existing_lines = {line.strip() for line in existing.splitlines()}
|
||||
missing = [p for p in _MEMPALACE_PROJECT_FILES if p not in existing_lines]
|
||||
if not missing:
|
||||
return False
|
||||
prefix = "" if not existing or existing.endswith("\n") else "\n"
|
||||
block = prefix + "\n# MemPalace per-project files (issue #185)\n" + "\n".join(missing) + "\n"
|
||||
with open(gitignore, "a") as f:
|
||||
with open(gitignore, "a", encoding="utf-8") as f:
|
||||
f.write(block)
|
||||
print(f" Added {', '.join(missing)} to {gitignore.name}")
|
||||
return True
|
||||
@@ -268,7 +315,16 @@ def cmd_init(args):
|
||||
endpoint=getattr(args, "llm_endpoint", None),
|
||||
api_key=getattr(args, "llm_api_key", None),
|
||||
)
|
||||
ok, msg = candidate.check_available()
|
||||
if (
|
||||
provider_name == "openai-compat"
|
||||
and getattr(candidate, "api_key_source", None) == "env"
|
||||
and candidate.is_external_service
|
||||
):
|
||||
ok = False
|
||||
msg = "external openai-compat init requires explicit --llm-api-key"
|
||||
print(f" LLM skipped: {msg}")
|
||||
else:
|
||||
ok, msg = candidate.check_available()
|
||||
if ok:
|
||||
llm_provider = candidate
|
||||
print(f" LLM enabled: {provider_name}/{provider_model}")
|
||||
@@ -380,6 +436,9 @@ def cmd_init(args):
|
||||
# Pass 2: detect rooms from folder structure
|
||||
detect_rooms_local(project_dir=args.dir, yes=getattr(args, "yes", False))
|
||||
cfg.init()
|
||||
backend = _backend_arg(args)
|
||||
if backend:
|
||||
cfg.set_backend(backend)
|
||||
|
||||
# Pass 3: protect git repos from accidentally committing per-project files
|
||||
_ensure_mempalace_files_gitignored(args.dir)
|
||||
@@ -492,6 +551,27 @@ def cmd_mine(args):
|
||||
for raw in args.include_ignored or []:
|
||||
include_ignored.extend(part.strip() for part in raw.split(",") if part.strip())
|
||||
|
||||
if getattr(args, "background", False) and not getattr(args, "daemon", False):
|
||||
print("mempalace: --background requires --daemon", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
if getattr(args, "daemon", False):
|
||||
payload = {
|
||||
"source": args.dir,
|
||||
"mode": args.mode,
|
||||
"wing": args.wing,
|
||||
"agent": args.agent,
|
||||
"limit": args.limit,
|
||||
"dry_run": args.dry_run,
|
||||
"extract": args.extract,
|
||||
"no_gitignore": args.no_gitignore,
|
||||
"include_ignored": include_ignored,
|
||||
"max_chunks_per_file": getattr(args, "max_chunks_per_file", None),
|
||||
"redetect_origin": getattr(args, "redetect_origin", False),
|
||||
}
|
||||
_submit_daemon_cli_job("mine", payload, args, background=getattr(args, "background", False))
|
||||
return
|
||||
|
||||
# --redetect-origin re-runs corpus_origin on the current corpus state
|
||||
# and overwrites <palace>/.mempalace/origin.json before mining proceeds.
|
||||
# Heuristic-only by design — full LLM detection lives on `mempalace init`.
|
||||
@@ -613,17 +693,41 @@ def cmd_sweep(args):
|
||||
|
||||
def cmd_sync(args):
|
||||
"""Prune drawers whose source files are gitignored, deleted, or moved (#1252)."""
|
||||
from .mcp_server import _wal_log
|
||||
from .palace import MineAlreadyRunning
|
||||
from .sync import sync_palace
|
||||
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
|
||||
if getattr(args, "background", False) and not getattr(args, "daemon", False):
|
||||
print("mempalace: --background requires --daemon", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
if getattr(args, "daemon", False):
|
||||
payload = {
|
||||
"dir": args.dir,
|
||||
"root": list(args.root or []),
|
||||
"wing": args.wing,
|
||||
"dry_run": args.dry_run,
|
||||
}
|
||||
_submit_daemon_cli_job("sync", payload, args, background=getattr(args, "background", False))
|
||||
return
|
||||
|
||||
from .palace import MineAlreadyRunning
|
||||
from .wal import _wal_log
|
||||
from .backends import detect_backend_for_path
|
||||
from .palace import _backend_artifact_label, resolve_backend_name
|
||||
from .sync import sync_palace
|
||||
|
||||
if not os.path.isdir(palace_path):
|
||||
print(f"\n No palace found at {palace_path}")
|
||||
return
|
||||
if not os.path.isfile(os.path.join(palace_path, "chroma.sqlite3")):
|
||||
print(f"\n Palace dir at {palace_path} exists but has no chroma.sqlite3 yet.")
|
||||
try:
|
||||
backend_name = resolve_backend_name(palace_path)
|
||||
except Exception as exc: # noqa: BLE001 - user-facing CLI guard
|
||||
print(f"\n Could not resolve palace backend: {exc}", file=sys.stderr)
|
||||
return
|
||||
if detect_backend_for_path(palace_path) is None:
|
||||
print(
|
||||
f"\n Palace dir at {palace_path} exists but has no "
|
||||
f"{_backend_artifact_label(backend_name)} yet."
|
||||
)
|
||||
print(" Run: mempalace mine <dir>")
|
||||
return
|
||||
|
||||
@@ -693,6 +797,133 @@ def cmd_sync(args):
|
||||
print(f"\n{'=' * 55}\n")
|
||||
|
||||
|
||||
def _submit_daemon_cli_job(kind: str, payload: dict, args, *, background: bool) -> None:
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
backend = _backend_arg(args)
|
||||
from .daemon import DaemonError, submit_job
|
||||
|
||||
try:
|
||||
job = submit_job(
|
||||
kind,
|
||||
payload,
|
||||
palace_path=palace_path,
|
||||
backend=backend,
|
||||
wait=not background,
|
||||
auto_start=True,
|
||||
)
|
||||
except DaemonError as exc:
|
||||
print(f"mempalace: daemon submission failed: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
if background:
|
||||
print(f"Submitted daemon job {job['id']} ({kind})")
|
||||
return
|
||||
|
||||
result = job.get("result") or {}
|
||||
from .service import print_job_result
|
||||
|
||||
exit_code = print_job_result(result)
|
||||
if job.get("state") != "succeeded" and exit_code == 0:
|
||||
error = job.get("error") or {}
|
||||
print(
|
||||
f"mempalace: daemon job failed: {error.get('message', 'unknown error')}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
exit_code = 1
|
||||
if exit_code:
|
||||
sys.exit(exit_code)
|
||||
|
||||
|
||||
def cmd_daemon(args):
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
backend = _backend_arg(args)
|
||||
from .daemon import (
|
||||
TERMINAL_STATES,
|
||||
DaemonError,
|
||||
QueueStore,
|
||||
get_client_if_running,
|
||||
job_to_dict,
|
||||
queue_path,
|
||||
start_daemon,
|
||||
stop_daemon,
|
||||
)
|
||||
|
||||
action = getattr(args, "daemon_action", None)
|
||||
try:
|
||||
if action == "start":
|
||||
if args.foreground:
|
||||
start_daemon(palace_path, backend=backend, foreground=True)
|
||||
return
|
||||
client = start_daemon(palace_path, backend=backend, foreground=False)
|
||||
health = client.health()
|
||||
print(f"MemPalace daemon running on 127.0.0.1:{client.port}")
|
||||
print(f" Palace: {health.get('palace_path')}")
|
||||
print(f" PID: {health.get('pid')}")
|
||||
return
|
||||
|
||||
if action == "stop":
|
||||
if stop_daemon(palace_path):
|
||||
print("MemPalace daemon stopping")
|
||||
else:
|
||||
print("MemPalace daemon is not running")
|
||||
return
|
||||
|
||||
if action == "status":
|
||||
client = get_client_if_running(palace_path)
|
||||
if client is None:
|
||||
print("MemPalace daemon is not running")
|
||||
sys.exit(1)
|
||||
health = client.health()
|
||||
print("MemPalace daemon is running")
|
||||
print(f" Palace: {health.get('palace_path')}")
|
||||
print(f" PID: {health.get('pid')}")
|
||||
print(f" Active: {health.get('active_job_id') or '-'}")
|
||||
print(f" Jobs: {health.get('counts') or {}}")
|
||||
return
|
||||
|
||||
if action == "jobs":
|
||||
client = get_client_if_running(palace_path)
|
||||
if client is not None:
|
||||
jobs = client.list_jobs(limit=args.limit)
|
||||
else:
|
||||
qpath = queue_path(palace_path)
|
||||
if not qpath.exists():
|
||||
jobs = []
|
||||
else:
|
||||
jobs = [
|
||||
job_to_dict(job, include_payload=False)
|
||||
for job in QueueStore(qpath).list(args.limit)
|
||||
]
|
||||
for job in jobs:
|
||||
print(f"{job['id']} {job['state']:<9} {job['kind']:<10} {job['created_at']}")
|
||||
return
|
||||
|
||||
if action == "wait":
|
||||
client = get_client_if_running(palace_path)
|
||||
if client is not None:
|
||||
job = client.wait(args.job_id)
|
||||
else:
|
||||
qpath = queue_path(palace_path)
|
||||
if not qpath.exists():
|
||||
raise DaemonError("daemon is not running")
|
||||
job = job_to_dict(QueueStore(qpath).get(args.job_id))
|
||||
if job.get("state") not in TERMINAL_STATES:
|
||||
raise DaemonError(f"daemon is not running; job {args.job_id} is {job['state']}")
|
||||
result = job.get("result") or {}
|
||||
from .service import print_job_result
|
||||
|
||||
exit_code = print_job_result(result)
|
||||
if job.get("state") != "succeeded" and exit_code == 0:
|
||||
print(f"mempalace: daemon job failed: {job.get('error')}", file=sys.stderr)
|
||||
exit_code = 1
|
||||
if exit_code:
|
||||
sys.exit(exit_code)
|
||||
return
|
||||
except DaemonError as exc:
|
||||
print(f"mempalace: daemon error: {exc}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def cmd_search(args):
|
||||
from .searcher import search, SearchError
|
||||
|
||||
@@ -748,9 +979,11 @@ def cmd_split(args):
|
||||
|
||||
def cmd_migrate(args):
|
||||
"""Migrate palace from a different ChromaDB version."""
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
if not _maintenance_requires_chroma(palace_path, "migrate"):
|
||||
raise SystemExit(2)
|
||||
from .migrate import migrate
|
||||
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
migrate(
|
||||
palace_path=palace_path,
|
||||
dry_run=args.dry_run,
|
||||
@@ -758,6 +991,41 @@ def cmd_migrate(args):
|
||||
)
|
||||
|
||||
|
||||
def cmd_migrate_wings(args):
|
||||
"""Normalize legacy wing names (strip leading/trailing separators)."""
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
from .migrate import migrate_wing_names
|
||||
|
||||
migrate_wing_names(
|
||||
palace_path=palace_path,
|
||||
dry_run=args.dry_run,
|
||||
confirm=getattr(args, "yes", False),
|
||||
)
|
||||
|
||||
|
||||
def cmd_hallways(args):
|
||||
"""List within-wing entity hallways (the auto-built associative graph)."""
|
||||
from .hallways import list_hallways
|
||||
|
||||
palace_path = (
|
||||
os.path.expanduser(args.palace)
|
||||
if getattr(args, "palace", None)
|
||||
else MempalaceConfig().palace_path
|
||||
)
|
||||
rows = list_hallways(
|
||||
getattr(args, "wing", None),
|
||||
config=MempalaceConfig(palace_path=palace_path),
|
||||
)
|
||||
if not rows:
|
||||
print("No hallways yet — they are built from drawer entities when you mine.")
|
||||
return
|
||||
rows.sort(key=lambda h: h.get("co_occurrence_count", 0), reverse=True)
|
||||
print(f" {len(rows)} hallway(s):")
|
||||
for h in rows[: max(0, args.limit)]:
|
||||
label = h.get("label") or f"{h.get('entity_a', '?')} <-> {h.get('entity_b', '?')}"
|
||||
print(f" {label}")
|
||||
|
||||
|
||||
def cmd_status(args):
|
||||
from .miner import status
|
||||
|
||||
@@ -765,16 +1033,78 @@ def cmd_status(args):
|
||||
status(palace_path=palace_path)
|
||||
|
||||
|
||||
def cmd_palace_set_embedder(args):
|
||||
"""Record (or force-override) a palace's embedder identity (RFC 001).
|
||||
|
||||
Resolves the ``unknown`` state for a legacy palace, or records a specific
|
||||
model with ``--model``. It records identity on the palace only; it does not
|
||||
change the configured model — when the two differ it prints how to align
|
||||
``MEMPALACE_EMBEDDING_MODEL``. ``--force`` overwrites an existing,
|
||||
differently-named identity.
|
||||
"""
|
||||
from .backends.base import EmbedderIdentityMismatchError
|
||||
from .palace import set_palace_embedder_identity
|
||||
|
||||
config = MempalaceConfig()
|
||||
palace_path = os.path.abspath(
|
||||
os.path.expanduser(args.palace) if args.palace else config.palace_path
|
||||
)
|
||||
model = getattr(args, "model", None)
|
||||
try:
|
||||
old, new = set_palace_embedder_identity(
|
||||
palace_path,
|
||||
model=model,
|
||||
force=getattr(args, "force", False),
|
||||
backend=_backend_arg(args),
|
||||
)
|
||||
except EmbedderIdentityMismatchError as exc:
|
||||
print(f" ✗ {exc}")
|
||||
raise SystemExit(2) from exc
|
||||
if old is None:
|
||||
print(f" ✓ recorded embedder identity: {new.model_name} (dim={new.dimension})")
|
||||
elif old.model_name == new.model_name:
|
||||
print(f" ✓ embedder identity unchanged: {new.model_name} (dim={new.dimension})")
|
||||
else:
|
||||
print(
|
||||
f" ✓ embedder identity changed: {old.model_name} → {new.model_name} "
|
||||
f"(dim={new.dimension})"
|
||||
)
|
||||
# set-embedder records the palace's identity; it does not change the
|
||||
# configured model. If they differ, the next normal open would mismatch —
|
||||
# tell the user how to align them.
|
||||
configured = config.embedding_model
|
||||
if new.model_name and configured and new.model_name != configured:
|
||||
print(
|
||||
f" ⚠ configured model is {configured!r}; set MEMPALACE_EMBEDDING_MODEL="
|
||||
f"{new.model_name} (or run onboarding) so normal opens of this palace match."
|
||||
)
|
||||
|
||||
|
||||
def cmd_repair_status(args):
|
||||
"""Read-only HNSW capacity health check (#1222)."""
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
if not _maintenance_requires_chroma(palace_path, "repair-status"):
|
||||
raise SystemExit(2)
|
||||
from .repair import status as repair_status
|
||||
|
||||
palace_path = os.path.expanduser(args.palace) if args.palace else MempalaceConfig().palace_path
|
||||
repair_status(palace_path=palace_path)
|
||||
|
||||
|
||||
def cmd_repair(args):
|
||||
"""Rebuild palace vector index from SQLite metadata."""
|
||||
"""Rebuild palace vector index from SQLite metadata.
|
||||
|
||||
On success the palace SQLite file is VACUUMed and the FTS5 index is
|
||||
rebuilt, so the next repair's integrity preflight reads a consistent
|
||||
database (#1747).
|
||||
"""
|
||||
config = MempalaceConfig()
|
||||
collection_name = config.collection_name
|
||||
palace_path = os.path.abspath(
|
||||
os.path.expanduser(args.palace) if args.palace else config.palace_path
|
||||
)
|
||||
if not _maintenance_requires_chroma(palace_path, "repair"):
|
||||
raise SystemExit(2)
|
||||
|
||||
import shutil
|
||||
from .backends.chroma import ChromaBackend
|
||||
from .migrate import confirm_destructive_action, contains_palace_database
|
||||
@@ -783,18 +1113,19 @@ def cmd_repair(args):
|
||||
TruncationDetected,
|
||||
_close_chroma_handles,
|
||||
_extract_drawers,
|
||||
_post_rebuild_cleanup,
|
||||
_rebuild_collection_via_temp,
|
||||
check_extraction_safety,
|
||||
index_read_recovery_guidance,
|
||||
maybe_autoheal_fts5_index,
|
||||
maybe_repair_poisoned_max_seq_id_before_rebuild,
|
||||
print_sqlite_integrity_abort,
|
||||
sqlite_integrity_errors,
|
||||
)
|
||||
|
||||
config = MempalaceConfig()
|
||||
collection_name = config.collection_name
|
||||
palace_path = os.path.abspath(
|
||||
os.path.expanduser(args.palace) if args.palace else config.palace_path
|
||||
)
|
||||
if getattr(args, "repair_action", None) == "rebuild-index":
|
||||
args.mode = "from-sqlite"
|
||||
args.archive_existing = True
|
||||
|
||||
if getattr(args, "mode", "legacy") == "max-seq-id":
|
||||
from .repair import repair_max_seq_id
|
||||
@@ -811,7 +1142,7 @@ def cmd_repair(args):
|
||||
|
||||
if getattr(args, "mode", "legacy") == "from-sqlite":
|
||||
from .migrate import confirm_destructive_action
|
||||
from .repair import RebuildPartialError, rebuild_from_sqlite
|
||||
from .repair import RebuildCleanupError, RebuildPartialError, rebuild_from_sqlite
|
||||
|
||||
source_path = getattr(args, "source", None)
|
||||
source_path = (
|
||||
@@ -849,6 +1180,13 @@ def cmd_repair(args):
|
||||
f"Failed in collection: {exc.failed_collection}"
|
||||
)
|
||||
sys.exit(1)
|
||||
except RebuildCleanupError:
|
||||
# All rows may have landed, but rebuild_from_sqlite deliberately
|
||||
# withholds success until FTS5 rebuild, VACUUM, and quick_check are
|
||||
# clean. Its exception already includes the retained destination
|
||||
# and archive/source recovery paths.
|
||||
print("\n Rebuild cleanup failed — see recovery details above.")
|
||||
sys.exit(1)
|
||||
# An empty counts dict is rebuild_from_sqlite's documented signal
|
||||
# for a validation refusal (missing source, existing dest,
|
||||
# in-place without --archive-existing). The library already
|
||||
@@ -877,6 +1215,8 @@ def cmd_repair(args):
|
||||
# here so we can surface the clear recovery instructions and exit
|
||||
# cleanly before chromadb's compactor touches the disk.
|
||||
sqlite_errors = sqlite_integrity_errors(palace_path)
|
||||
if sqlite_errors:
|
||||
sqlite_errors = maybe_autoheal_fts5_index(palace_path, sqlite_errors)
|
||||
if sqlite_errors:
|
||||
print_sqlite_integrity_abort(palace_path, sqlite_errors)
|
||||
sys.exit(1)
|
||||
@@ -904,7 +1244,7 @@ def cmd_repair(args):
|
||||
print(f" Drawers found: {total}")
|
||||
except Exception as e:
|
||||
print(f" Error reading palace: {e}")
|
||||
print(" Cannot recover — palace may need to be re-mined from source files.")
|
||||
print(index_read_recovery_guidance())
|
||||
return
|
||||
|
||||
if total == 0:
|
||||
@@ -982,6 +1322,10 @@ def cmd_repair(args):
|
||||
print(f" Backup location: {backup_path}")
|
||||
sys.exit(1)
|
||||
|
||||
# The bulk delete + re-upsert cycle above leaves the FTS5 inverted index
|
||||
# inconsistent, which fails the next repair's integrity preflight (#1747).
|
||||
_post_rebuild_cleanup(palace_path, backend=backend, progress=print)
|
||||
|
||||
print(f"\n Repair complete. {filed} drawers rebuilt.")
|
||||
print(f" Backup saved at {backup_path}")
|
||||
print(f"\n{'=' * 55}\n")
|
||||
@@ -1004,12 +1348,15 @@ def cmd_instructions(args):
|
||||
def cmd_mcp(args):
|
||||
"""Show how to wire MemPalace into MCP-capable hosts."""
|
||||
base_server_cmd = "mempalace-mcp"
|
||||
cmd_parts = [base_server_cmd]
|
||||
|
||||
if args.palace:
|
||||
resolved_palace = str(Path(args.palace).expanduser())
|
||||
server_cmd = f"{base_server_cmd} --palace {shlex.quote(resolved_palace)}"
|
||||
else:
|
||||
server_cmd = base_server_cmd
|
||||
cmd_parts.extend(["--palace", shlex.quote(resolved_palace)])
|
||||
backend = _backend_arg(args)
|
||||
if backend:
|
||||
cmd_parts.extend(["--backend", shlex.quote(str(backend).strip().lower())])
|
||||
server_cmd = " ".join(cmd_parts)
|
||||
|
||||
print("MemPalace MCP quick setup:")
|
||||
print(f" claude mcp add mempalace -- {server_cmd}")
|
||||
@@ -1024,6 +1371,154 @@ def cmd_mcp(args):
|
||||
print(f" {base_server_cmd} --palace /path/to/palace")
|
||||
|
||||
|
||||
_SERVER_LOOPBACK_HOSTS = {"127.0.0.1", "localhost", "::1", "[::1]"}
|
||||
_SERVER_BIND_ALL_HOSTS = {"0.0.0.0", "::", "[::]"}
|
||||
|
||||
|
||||
def _server_is_loopback(host: str) -> bool:
|
||||
return (host or "").strip().lower() in _SERVER_LOOPBACK_HOSTS
|
||||
|
||||
|
||||
def _server_token_path(palace_path: str) -> Path:
|
||||
"""Per-palace location for the auto-generated server bearer token.
|
||||
|
||||
Distinct from the daemon's token dir; keyed by the canonical palace path so
|
||||
one server per palace reuses a stable token across restarts.
|
||||
"""
|
||||
import hashlib
|
||||
|
||||
canonical = os.path.abspath(os.path.realpath(os.path.expanduser(palace_path)))
|
||||
key = hashlib.sha256(os.path.normcase(canonical).encode("utf-8")).hexdigest()[:24]
|
||||
return Path.home() / ".mempalace" / "server" / key / "token"
|
||||
|
||||
|
||||
def _load_or_create_server_token(palace_path: str) -> tuple[str, bool]:
|
||||
"""Return (token, created). Reuse an existing 0600 token or mint a new one."""
|
||||
import secrets
|
||||
|
||||
token_path = _server_token_path(palace_path)
|
||||
if token_path.exists():
|
||||
existing = token_path.read_text(encoding="utf-8").strip()
|
||||
if existing:
|
||||
return existing, False
|
||||
token = secrets.token_urlsafe(32)
|
||||
token_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
os.chmod(str(token_path.parent), 0o700)
|
||||
except OSError:
|
||||
pass
|
||||
# O_CREAT with 0600 so the token is never briefly world-readable on disk.
|
||||
fd = os.open(str(token_path), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
||||
fh.write(token + "\n")
|
||||
return token, True
|
||||
|
||||
|
||||
def cmd_serve(args):
|
||||
"""Run a secure remote HTTP MCP server for a team to share one palace (#1877).
|
||||
|
||||
A turnkey wrapper over ``mempalace-mcp --transport http``: it resolves a
|
||||
bearer token (auto-generating a strong one for non-loopback binds), prints a
|
||||
ready-to-paste client config, then execs the real server in the foreground so
|
||||
Docker/systemd own the process lifecycle. The token is passed via the
|
||||
environment, never argv, so it can't leak through ``ps``.
|
||||
"""
|
||||
host = args.host
|
||||
port = int(args.port)
|
||||
loopback = _server_is_loopback(host)
|
||||
palace_path = (
|
||||
os.path.abspath(os.path.expanduser(args.palace))
|
||||
if args.palace
|
||||
else MempalaceConfig().palace_path
|
||||
)
|
||||
backend = _backend_arg(args)
|
||||
|
||||
tls_cert = os.path.expanduser(args.tls_cert) if args.tls_cert else None
|
||||
tls_key = os.path.expanduser(args.tls_key) if args.tls_key else None
|
||||
if bool(tls_cert) != bool(tls_key):
|
||||
print("mempalace: --tls-cert and --tls-key must be given together", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
for label, path in (("--tls-cert", tls_cert), ("--tls-key", tls_key)):
|
||||
if path and not os.path.isfile(path):
|
||||
print(f"mempalace: {label} file not found: {path}", file=sys.stderr)
|
||||
sys.exit(2)
|
||||
scheme = "https" if tls_cert else "http"
|
||||
|
||||
# Token resolution. Explicit flag > existing env > (non-loopback) auto-generated.
|
||||
token = (args.token or os.environ.get("MEMPALACE_MCP_HTTP_TOKEN", "")).strip()
|
||||
token_created = False
|
||||
if not token and not loopback and not args.allow_insecure:
|
||||
token, token_created = _load_or_create_server_token(palace_path)
|
||||
|
||||
# Build the child environment. Token rides in the env (never argv) so it
|
||||
# stays out of the process table.
|
||||
env = dict(os.environ)
|
||||
env["MEMPALACE_PALACE_PATH"] = palace_path
|
||||
if backend:
|
||||
env["MEMPALACE_BACKEND"] = str(backend).strip().lower()
|
||||
if token:
|
||||
env["MEMPALACE_MCP_HTTP_TOKEN"] = token
|
||||
if args.allow_insecure:
|
||||
env["MEMPALACE_MCP_HTTP_ALLOW_INSECURE_NO_TOKEN"] = "1"
|
||||
|
||||
child = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"mempalace.mcp_server",
|
||||
"--transport",
|
||||
"http",
|
||||
"--host",
|
||||
host,
|
||||
"--port",
|
||||
str(port),
|
||||
]
|
||||
if backend:
|
||||
child += ["--backend", str(backend).strip().lower()]
|
||||
child += ["--palace", palace_path]
|
||||
if tls_cert:
|
||||
child += ["--tls-cert", tls_cert, "--tls-key", tls_key]
|
||||
if args.read_only:
|
||||
child.append("--read-only")
|
||||
|
||||
# Client-facing address: 0.0.0.0/:: means "all interfaces" — clients dial a
|
||||
# real reachable host, so show a placeholder rather than the bind wildcard.
|
||||
client_host = "YOUR_SERVER_HOST" if host.strip().lower() in _SERVER_BIND_ALL_HOSTS else host
|
||||
url = f"{scheme}://{client_host}:{port}/mcp"
|
||||
|
||||
print("Starting MemPalace remote MCP server")
|
||||
print(f" palace : {palace_path}")
|
||||
print(f" backend : {(backend or 'default').strip().lower() if backend else 'default'}")
|
||||
print(f" bind : {host}:{port} ({'loopback' if loopback else 'network-exposed'})")
|
||||
print(f" tls : {'on' if tls_cert else 'off (plaintext — terminate TLS at a proxy)'}")
|
||||
print(f" read-only: {'yes' if args.read_only else 'no'}")
|
||||
if token_created:
|
||||
print("\n A new bearer token was generated and stored 0600 at:")
|
||||
print(f" {_server_token_path(palace_path)}")
|
||||
print(" Store it securely — clients need it to connect:")
|
||||
print(f" {token}")
|
||||
print("\nConnect a client:")
|
||||
if token:
|
||||
print(
|
||||
f" claude mcp add --transport http mempalace {url} "
|
||||
f'--header "Authorization: Bearer {token if token_created else "$MEMPALACE_MCP_HTTP_TOKEN"}"'
|
||||
)
|
||||
else:
|
||||
print(f" claude mcp add --transport http mempalace {url}")
|
||||
print(f" curl {scheme}://{client_host}:{port}/healthz # liveness (no auth)\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
# Foreground: hand the process to the real server so signals (SIGTERM from
|
||||
# Docker/systemd) reach it directly. exec on POSIX; subprocess on Windows
|
||||
# (no exec semantics) propagating the exit code.
|
||||
if os.name == "posix":
|
||||
os.execve(sys.executable, child, env)
|
||||
else:
|
||||
import subprocess
|
||||
|
||||
completed = subprocess.run(child, env=env)
|
||||
sys.exit(completed.returncode)
|
||||
|
||||
|
||||
def cmd_compress(args):
|
||||
"""Compress drawers in a wing using AAAK Dialect."""
|
||||
from .dialect import Dialect
|
||||
@@ -1204,12 +1699,23 @@ def main():
|
||||
default=None,
|
||||
help="Where the palace lives (default: from ~/.mempalace/config.json or ~/.mempalace/palace)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
dest="global_backend",
|
||||
default=None,
|
||||
help="Storage backend to use for this command (default: config/env/detected/chroma)",
|
||||
)
|
||||
|
||||
sub = parser.add_subparsers(dest="command")
|
||||
|
||||
# init
|
||||
p_init = sub.add_parser("init", help="Detect rooms from your folder structure")
|
||||
p_init.add_argument("dir", help="Project directory to set up")
|
||||
p_init.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to persist for this palace (default: chroma)",
|
||||
)
|
||||
p_init.add_argument(
|
||||
"--yes",
|
||||
action="store_true",
|
||||
@@ -1292,6 +1798,11 @@ def main():
|
||||
# mine
|
||||
p_mine = sub.add_parser("mine", help="Mine files into the palace")
|
||||
p_mine.add_argument("dir", help="Directory to mine")
|
||||
p_mine.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to use for this mine (default: config/env/detected/chroma)",
|
||||
)
|
||||
p_mine.add_argument(
|
||||
"--mode",
|
||||
choices=["projects", "convos", "extract"],
|
||||
@@ -1334,13 +1845,22 @@ def main():
|
||||
p_mine.add_argument(
|
||||
"--dry-run", action="store_true", help="Show what would be filed without filing"
|
||||
)
|
||||
p_mine.add_argument(
|
||||
"--daemon",
|
||||
action="store_true",
|
||||
help="Submit this mine to the opt-in local daemon queue",
|
||||
)
|
||||
p_mine.add_argument(
|
||||
"--background",
|
||||
action="store_true",
|
||||
help="With --daemon, return a job id immediately instead of waiting",
|
||||
)
|
||||
p_mine.add_argument(
|
||||
"--extract",
|
||||
choices=["exchange", "general"],
|
||||
default="exchange",
|
||||
help="Extraction strategy for convos mode: 'exchange' (default) or 'general' (5 memory types)",
|
||||
)
|
||||
from . import miner as _miner_for_default
|
||||
|
||||
p_mine.add_argument(
|
||||
"--max-chunks-per-file",
|
||||
@@ -1349,7 +1869,7 @@ def main():
|
||||
metavar="N",
|
||||
help=(
|
||||
f"Per-file chunk cap; files producing more chunks are skipped with a "
|
||||
f"summary counter. Default {_miner_for_default.MAX_CHUNKS_PER_FILE} "
|
||||
f"summary counter. Default {_CLI_MAX_CHUNKS_PER_FILE_DEFAULT} "
|
||||
f"(or MEMPALACE_MAX_CHUNKS_PER_FILE). Set 0 to disable. Lower this on "
|
||||
f"Windows if you hit ONNX bad_alloc (#1455)."
|
||||
),
|
||||
@@ -1397,10 +1917,25 @@ def main():
|
||||
action="store_false",
|
||||
help="Actually delete drawers (overrides --dry-run; requires --wing or a project root)",
|
||||
)
|
||||
p_sync.add_argument(
|
||||
"--daemon",
|
||||
action="store_true",
|
||||
help="Submit this sync to the opt-in local daemon queue",
|
||||
)
|
||||
p_sync.add_argument(
|
||||
"--background",
|
||||
action="store_true",
|
||||
help="With --daemon, return a job id immediately instead of waiting",
|
||||
)
|
||||
|
||||
# search
|
||||
p_search = sub.add_parser("search", help="Find anything, exact words")
|
||||
p_search.add_argument("query", help="What to search for")
|
||||
p_search.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to use for this search (default: config/env/detected/chroma)",
|
||||
)
|
||||
p_search.add_argument("--wing", default=None, help="Limit to one project")
|
||||
p_search.add_argument("--room", default=None, help="Limit to one room")
|
||||
p_search.add_argument("--results", type=int, default=5, help="Number of results")
|
||||
@@ -1454,7 +1989,7 @@ def main():
|
||||
p_hook_run.add_argument(
|
||||
"--hook",
|
||||
required=True,
|
||||
choices=["session-start", "stop", "precompact"],
|
||||
choices=["session-start", "stop", "session-end", "precompact"],
|
||||
help="Hook name to run",
|
||||
)
|
||||
p_hook_run.add_argument(
|
||||
@@ -1484,6 +2019,15 @@ def main():
|
||||
p_repair.add_argument(
|
||||
"--yes", action="store_true", help="Skip confirmation for destructive changes"
|
||||
)
|
||||
p_repair.add_argument(
|
||||
"repair_action",
|
||||
nargs="?",
|
||||
choices=["rebuild-index"],
|
||||
help=(
|
||||
"Re-embed the palace from SQLite using the current embedding model "
|
||||
"(alias for --mode from-sqlite --archive-existing)."
|
||||
),
|
||||
)
|
||||
p_repair.add_argument(
|
||||
"--confirm-truncation-ok",
|
||||
action="store_true",
|
||||
@@ -1554,11 +2098,69 @@ def main():
|
||||
help="Compare sqlite vs HNSW element counts (read-only; never opens a chromadb client)",
|
||||
)
|
||||
|
||||
# daemon
|
||||
p_daemon = sub.add_parser("daemon", help="Manage the opt-in long-lived daemon")
|
||||
daemon_sub = p_daemon.add_subparsers(dest="daemon_action")
|
||||
p_daemon_start = daemon_sub.add_parser("start", help="Start the daemon")
|
||||
p_daemon_start.add_argument(
|
||||
"--foreground",
|
||||
action="store_true",
|
||||
help="Run in the foreground for debugging or process supervisors",
|
||||
)
|
||||
p_daemon_start.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend for this daemon (default: config/env/detected/chroma)",
|
||||
)
|
||||
daemon_sub.add_parser("stop", help="Stop the daemon")
|
||||
daemon_sub.add_parser("status", help="Show daemon status")
|
||||
p_daemon_jobs = daemon_sub.add_parser("jobs", help="List recent daemon jobs")
|
||||
p_daemon_jobs.add_argument("--limit", type=int, default=20, help="Max jobs to show")
|
||||
p_daemon_wait = daemon_sub.add_parser("wait", help="Wait for a daemon job")
|
||||
p_daemon_wait.add_argument("job_id", help="Job id returned by --background")
|
||||
|
||||
# mcp
|
||||
sub.add_parser(
|
||||
p_mcp = sub.add_parser(
|
||||
"mcp",
|
||||
help="Show MCP setup command for connecting MemPalace to your AI client",
|
||||
)
|
||||
p_mcp.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to include in the MCP startup command",
|
||||
)
|
||||
|
||||
# serve — turnkey remote HTTP MCP server (#1877)
|
||||
p_serve = sub.add_parser(
|
||||
"serve",
|
||||
help="Run a secure remote HTTP MCP server for a team to share one palace",
|
||||
)
|
||||
p_serve.add_argument(
|
||||
"--host", default="127.0.0.1", help="Bind address (use 0.0.0.0 for remote clients)"
|
||||
)
|
||||
p_serve.add_argument("--port", type=int, default=8765, help="Bind port (default: 8765)")
|
||||
p_serve.add_argument(
|
||||
"--backend", default=None, help="Storage backend (default: config/env/detected)"
|
||||
)
|
||||
p_serve.add_argument("--palace", default=None, help="Palace path (overrides config/env)")
|
||||
p_serve.add_argument(
|
||||
"--token",
|
||||
default=None,
|
||||
help="Bearer token clients must present. Default: reuse/auto-generate one for "
|
||||
"non-loopback binds (stored 0600 under ~/.mempalace/server/).",
|
||||
)
|
||||
p_serve.add_argument("--tls-cert", default=None, help="PEM certificate to enable TLS")
|
||||
p_serve.add_argument("--tls-key", default=None, help="PEM private key matching --tls-cert")
|
||||
p_serve.add_argument(
|
||||
"--read-only",
|
||||
action="store_true",
|
||||
help="Expose recall only: mutating tools are hidden and refused",
|
||||
)
|
||||
p_serve.add_argument(
|
||||
"--allow-insecure",
|
||||
action="store_true",
|
||||
help="Permit a non-loopback bind with no token (only behind a trusted proxy)",
|
||||
)
|
||||
|
||||
# status
|
||||
# migrate
|
||||
@@ -1575,9 +2177,55 @@ def main():
|
||||
"--yes", action="store_true", help="Skip confirmation for destructive changes"
|
||||
)
|
||||
|
||||
sub.add_parser("status", help="Show what's been filed")
|
||||
# migrate-wings
|
||||
p_migrate_wings = sub.add_parser(
|
||||
"migrate-wings",
|
||||
help="Normalize legacy wing names (strip leading/trailing separators) so pre-#1675 palaces stay discoverable",
|
||||
)
|
||||
p_migrate_wings.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Show what would change without modifying the palace",
|
||||
)
|
||||
p_migrate_wings.add_argument("--yes", action="store_true", help="Skip the confirmation prompt")
|
||||
|
||||
p_hallways = sub.add_parser("hallways", help="List entity hallways (associative graph)")
|
||||
p_hallways.add_argument("--wing", default=None, help="Filter to one wing")
|
||||
p_hallways.add_argument("--limit", type=int, default=50, help="Max hallways to show")
|
||||
p_status = sub.add_parser("status", help="Show what's been filed")
|
||||
p_status.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend to use for status (default: config/env/detected/chroma)",
|
||||
)
|
||||
|
||||
p_palace = sub.add_parser("palace", help="Palace maintenance commands")
|
||||
palace_sub = p_palace.add_subparsers(dest="palace_action")
|
||||
p_set_embedder = palace_sub.add_parser(
|
||||
"set-embedder",
|
||||
help="Record/override the palace's embedder identity (resolve 'unknown', or switch models)",
|
||||
)
|
||||
p_set_embedder.add_argument(
|
||||
"--model",
|
||||
default=None,
|
||||
help="Embedder model to record (default: current configured model). "
|
||||
"Records identity on the palace only; does not change the configured "
|
||||
"model (prints how to align MEMPALACE_EMBEDDING_MODEL if they differ).",
|
||||
)
|
||||
p_set_embedder.add_argument(
|
||||
"--force",
|
||||
action="store_true",
|
||||
help="Overwrite an existing identity that names a different model "
|
||||
"(only if you know the stored vectors are compatible)",
|
||||
)
|
||||
p_set_embedder.add_argument(
|
||||
"--backend",
|
||||
default=None,
|
||||
help="Storage backend (default: config/env/detected/chroma)",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
_apply_backend_arg(args)
|
||||
|
||||
if not args.command:
|
||||
parser.print_help()
|
||||
@@ -1600,6 +2248,20 @@ def main():
|
||||
cmd_instructions(args)
|
||||
return
|
||||
|
||||
if args.command == "palace":
|
||||
if getattr(args, "palace_action", None) == "set-embedder":
|
||||
cmd_palace_set_embedder(args)
|
||||
else:
|
||||
p_palace.print_help()
|
||||
return
|
||||
|
||||
if args.command == "daemon":
|
||||
if not getattr(args, "daemon_action", None):
|
||||
p_daemon.print_help()
|
||||
return
|
||||
cmd_daemon(args)
|
||||
return
|
||||
|
||||
dispatch = {
|
||||
"init": cmd_init,
|
||||
"mine": cmd_mine,
|
||||
@@ -1608,11 +2270,14 @@ def main():
|
||||
"sweep": cmd_sweep,
|
||||
"sync": cmd_sync,
|
||||
"mcp": cmd_mcp,
|
||||
"serve": cmd_serve,
|
||||
"compress": cmd_compress,
|
||||
"wake-up": cmd_wakeup,
|
||||
"repair": cmd_repair,
|
||||
"repair-status": cmd_repair_status,
|
||||
"migrate": cmd_migrate,
|
||||
"migrate-wings": cmd_migrate_wings,
|
||||
"hallways": cmd_hallways,
|
||||
"status": cmd_status,
|
||||
}
|
||||
dispatch[args.command](args)
|
||||
|
||||
@@ -264,7 +264,7 @@ def regenerate_closets(
|
||||
f"Regenerating closets for {len(sources)} source files via {cfg.endpoint} ({cfg.model})..."
|
||||
)
|
||||
if dry_run:
|
||||
print("DRY RUN — no changes will be written")
|
||||
print("DRY RUN - no changes will be written")
|
||||
|
||||
processed = 0
|
||||
failed = 0
|
||||
@@ -286,7 +286,7 @@ def regenerate_closets(
|
||||
parsed, usage = _call_llm(cfg, source, w, r, content)
|
||||
if not parsed:
|
||||
failed += 1
|
||||
print(f" [{i}/{len(sources)}] ✗ {os.path.basename(source)} — LLM failed")
|
||||
print(f" [{i}/{len(sources)}] [FAIL] {os.path.basename(source)} - LLM failed")
|
||||
continue
|
||||
|
||||
if usage:
|
||||
@@ -323,7 +323,7 @@ def regenerate_closets(
|
||||
|
||||
processed += 1
|
||||
n_topics = len(parsed.get("topics", []))
|
||||
print(f" [{i}/{len(sources)}] ✓ {os.path.basename(source)} — {n_topics} topics")
|
||||
print(f" [{i}/{len(sources)}] [OK] {os.path.basename(source)} - {n_topics} topics")
|
||||
|
||||
print(f"\nDone. {processed} regenerated, {failed} failed.")
|
||||
if total_input or total_output:
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
"""Pre-mining defense against drawer_id collisions.
|
||||
|
||||
Runs immediately before a batched chromadb upsert. Computes the union of
|
||||
incoming drawer_ids and existing drawer_ids that share a key with the
|
||||
batch; raises ``CollisionError`` if any drawer_id appears more than once
|
||||
in that union with conflicting ``(source_file, chunk_index)`` metadata.
|
||||
|
||||
Under the v2 hash recipe (see :mod:`mempalace.ids`) accidental collisions
|
||||
are vanishingly rare — SHA-256 truncated to 24 hex chars makes random
|
||||
collision ~2^-96. The scan exists for two reasons:
|
||||
|
||||
1. Catch upstream bugs that emit duplicate ``(source_file, chunk_index)``
|
||||
pairs in the same batch with conflicting content. ChromaDB would
|
||||
silently let the last-write win; the scan surfaces it as an
|
||||
actionable error naming both call sites.
|
||||
2. Catch the astronomical-but-possible SHA-256 hash collision with a
|
||||
clear message instead of a silent overwrite at upsert time.
|
||||
|
||||
The scan does NOT fire on idempotent re-mines — when an incoming drawer
|
||||
matches an existing one with the SAME ``(source_file, chunk_index)``
|
||||
metadata, that is normal re-write behavior, not collision.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
class CollisionError(Exception):
|
||||
"""Raised by :func:`assert_no_collisions` when the pre-mining scan
|
||||
detects a drawer_id that would silently overwrite existing content
|
||||
or duplicate within a batch with conflicting metadata.
|
||||
|
||||
The exception message names every colliding ``drawer_id`` and the
|
||||
full set of ``(source_file, chunk_index)`` pairs producing each one,
|
||||
so a user fixing one collision does not have to rediscover the next
|
||||
by re-running the mine.
|
||||
"""
|
||||
|
||||
|
||||
def _metadata_key(meta: dict) -> tuple:
|
||||
"""Reduce a drawer metadata dict to the tuple used for collision
|
||||
discrimination. Two metadata dicts are 'the same chunk' iff their
|
||||
key tuples match. Falls back to ``(source_file,)`` when
|
||||
``chunk_index`` is absent (diary entries, sentinels)."""
|
||||
source_file = meta.get("source_file")
|
||||
chunk_index = meta.get("chunk_index")
|
||||
if chunk_index is None:
|
||||
return (source_file,)
|
||||
return (source_file, chunk_index)
|
||||
|
||||
|
||||
def assert_no_collisions(
|
||||
proposed: list[tuple[str, dict]],
|
||||
collection,
|
||||
) -> None:
|
||||
"""Abort the mine via ``CollisionError`` if any proposed drawer_id
|
||||
collides with itself or with an existing drawer in ``collection``.
|
||||
|
||||
Args:
|
||||
proposed: list of ``(drawer_id, metadata)`` tuples for the
|
||||
chunks about to be upserted. ``metadata`` must carry at
|
||||
least ``source_file``; ``chunk_index`` is used when
|
||||
present.
|
||||
collection: a ChromaDB-shaped collection with ``get(ids=...)``
|
||||
returning a dict with ``ids`` and ``metadatas`` keys.
|
||||
|
||||
Raises:
|
||||
CollisionError: when a drawer_id maps to two or more distinct
|
||||
``(source_file, chunk_index)`` tuples in the union of
|
||||
incoming and existing rows.
|
||||
"""
|
||||
if not proposed:
|
||||
return
|
||||
|
||||
# Build incoming map: drawer_id -> set of metadata key tuples.
|
||||
# Using a set collapses duplicate-metadata cases (same chunk twice
|
||||
# in the batch) without flagging them as collisions.
|
||||
incoming: dict[str, set[tuple]] = defaultdict(set)
|
||||
for drawer_id, meta in proposed:
|
||||
incoming[drawer_id].add(_metadata_key(meta))
|
||||
|
||||
# Query existing rows for any incoming id. ChromaDB's get(ids=...)
|
||||
# returns only the rows whose ids are present; missing ids are
|
||||
# silently absent from the result, which is what we want.
|
||||
incoming_ids = list(incoming.keys())
|
||||
result = collection.get(ids=incoming_ids, include=["metadatas"])
|
||||
existing_ids: list = result["ids"] if hasattr(result, "__getitem__") else []
|
||||
existing_metas: list = result["metadatas"] if existing_ids else []
|
||||
|
||||
# Merge existing metadata into the incoming map. A real collision is
|
||||
# a drawer_id whose incoming + existing metadata key tuples are not
|
||||
# all the same.
|
||||
for drawer_id, meta in zip(existing_ids, existing_metas):
|
||||
incoming[drawer_id].add(_metadata_key(meta or {}))
|
||||
|
||||
collisions = {did: keys for did, keys in incoming.items() if len(keys) > 1}
|
||||
if collisions:
|
||||
raise CollisionError(_format_collisions(collisions))
|
||||
|
||||
|
||||
def _format_collisions(collisions: dict[str, set[tuple]]) -> str:
|
||||
"""Render a CollisionError message that enumerates every colliding
|
||||
drawer_id and the metadata tuples producing it."""
|
||||
lines = [
|
||||
f"Pre-mining collision scan detected {len(collisions)} "
|
||||
f"colliding drawer_id{'s' if len(collisions) != 1 else ''}:",
|
||||
]
|
||||
for drawer_id, keys in sorted(collisions.items()):
|
||||
lines.append(f" {drawer_id}:")
|
||||
for key in sorted(keys, key=lambda k: tuple(str(part) for part in k)):
|
||||
if len(key) == 1:
|
||||
lines.append(f" source_file={key[0]!r}")
|
||||
else:
|
||||
lines.append(f" source_file={key[0]!r}, chunk_index={key[1]!r}")
|
||||
lines.append(
|
||||
"Each colliding drawer_id would cause the second ChromaDB upsert "
|
||||
"to silently overwrite the first. Fix the upstream chunker / "
|
||||
"miner to emit distinct keys, or investigate the SHA-256 hash "
|
||||
"collision."
|
||||
)
|
||||
return "\n".join(lines)
|
||||
+411
-3
@@ -11,6 +11,14 @@ from datetime import date, datetime
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
|
||||
from .write_routing import (
|
||||
ResolvedWriteRoutingPolicy,
|
||||
RoutingPolicyCandidate,
|
||||
WriteRoutingError,
|
||||
WriteRoutingPolicy,
|
||||
resolve_write_routing_policy,
|
||||
)
|
||||
|
||||
# ── Input validation ──────────────────────────────────────────────────────────
|
||||
# Shared sanitizers for wing/room/entity names. Prevents path traversal,
|
||||
# excessively long strings, and special characters that could cause issues
|
||||
@@ -31,14 +39,35 @@ def strip_lone_surrogates(text: str) -> str:
|
||||
return _LONE_SURROGATE_RE.sub("�", text)
|
||||
|
||||
|
||||
# Tool output mined from real transcripts routinely embeds a NUL character
|
||||
# (U+0000) — e.g. captured Bash output where a reader raced a background
|
||||
# writer, or genuine binary/NUL-delimited command output. A document
|
||||
# containing one is otherwise valid, well-formed text (unlike a lone
|
||||
# surrogate, which is invalid UTF-8), but handing it to ChromaDB's
|
||||
# SQLite/FTS5 layer can corrupt the FTS5 inverted index for the *whole*
|
||||
# collection (``PRAGMA quick_check`` reports "malformed inverted index for
|
||||
# FTS5 table"), not just fail to store that one document. Stripping it
|
||||
# before it reaches the chromadb client is the same defense-in-depth this
|
||||
# module already applies to lone surrogates (#1235) — sanitize input we
|
||||
# don't control before it reaches a datastore we don't control.
|
||||
def strip_nul_bytes(text: str) -> str:
|
||||
"""Replace embedded NUL characters with U+FFFD before ChromaDB storage."""
|
||||
return text.replace("\x00", "�")
|
||||
|
||||
|
||||
def normalize_wing_name(name: str) -> str:
|
||||
"""Lower-case + collapse separators (`-`, ` `) to `_` for wing slugs.
|
||||
|
||||
The same rule is applied by ``init`` when persisting `topics_by_wing`
|
||||
and when writing `mempalace.yaml`, so the miner's lookup matches at
|
||||
mine time regardless of the source dirname.
|
||||
|
||||
Leading/trailing separators are stripped so a path-encoded dirname like
|
||||
``-home-user-proj`` yields ``home_user_proj`` rather than a leading-
|
||||
underscore slug that ``sanitize_name`` (and thus the MCP write tools)
|
||||
would reject.
|
||||
"""
|
||||
return name.lower().replace(" ", "_").replace("-", "_")
|
||||
return name.lower().replace(" ", "_").replace("-", "_").strip("_")
|
||||
|
||||
|
||||
def sanitize_name(value: str, field_name: str = "name") -> str:
|
||||
@@ -191,6 +220,44 @@ def sanitize_content(value: str, max_length: int = 100_000) -> str:
|
||||
|
||||
DEFAULT_PALACE_PATH = os.path.expanduser("~/.mempalace/palace")
|
||||
DEFAULT_COLLECTION_NAME = "mempalace_drawers"
|
||||
DEFAULT_BACKEND = "chroma"
|
||||
DEFAULT_MILVUS_CONSISTENCY_LEVEL = "Strong"
|
||||
_MILVUS_CONSISTENCY_LEVELS = {
|
||||
"strong": "Strong",
|
||||
"session": "Session",
|
||||
"bounded": "Bounded",
|
||||
"eventually": "Eventually",
|
||||
}
|
||||
|
||||
# How many timestamped palace backups to retain before the oldest are
|
||||
# pruned. Applies to the accumulating backups written by ``mempalace
|
||||
# migrate`` and ``mempalace repair max-seq-id`` — see
|
||||
# ``MempalaceConfig.max_backups``.
|
||||
DEFAULT_MAX_BACKUPS = 10
|
||||
|
||||
|
||||
def normalize_milvus_consistency_level(value) -> str:
|
||||
raw = str(value).strip() if value else DEFAULT_MILVUS_CONSISTENCY_LEVEL
|
||||
normalized = _MILVUS_CONSISTENCY_LEVELS.get(raw.lower())
|
||||
if normalized:
|
||||
return normalized
|
||||
allowed = ", ".join(_MILVUS_CONSISTENCY_LEVELS.values())
|
||||
raise ValueError(f"milvus_consistency_level must be one of: {allowed}")
|
||||
|
||||
|
||||
def sqlite_read_uri(db_path: str) -> str:
|
||||
"""Return a read-only ``file:`` URI for ``sqlite3.connect(..., uri=True)``.
|
||||
|
||||
A bare ``f"file:{db_path}?mode=ro"`` mis-parses paths containing spaces or
|
||||
other URI-reserved characters — common in real home directories (a Windows
|
||||
user folder like ``First Last``, many macOS paths). ``pathname2url``
|
||||
percent-encodes the path and normalizes separators so the database opens on
|
||||
every platform.
|
||||
"""
|
||||
from urllib.request import pathname2url
|
||||
|
||||
db_path = os.fspath(db_path)
|
||||
return f"file:{pathname2url(db_path)}?mode=ro"
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
@@ -283,18 +350,26 @@ class MempalaceConfig:
|
||||
Load order: env vars > config file > defaults.
|
||||
"""
|
||||
|
||||
def __init__(self, config_dir=None):
|
||||
def __init__(self, config_dir=None, palace_path=None):
|
||||
"""Initialize config.
|
||||
|
||||
Args:
|
||||
config_dir: Override config directory (useful for testing).
|
||||
Defaults to ~/.mempalace.
|
||||
palace_path: Explicit palace data directory. This is primarily
|
||||
used by CLI operations that received ``--palace``;
|
||||
it takes precedence over environment and file config.
|
||||
"""
|
||||
self._config_dir = (
|
||||
Path(config_dir) if config_dir else Path(os.path.expanduser("~/.mempalace"))
|
||||
)
|
||||
self._config_file = self._config_dir / "config.json"
|
||||
self._people_map_file = self._config_dir / "people_map.json"
|
||||
self._palace_path_override = (
|
||||
os.path.abspath(os.path.expanduser(str(palace_path)))
|
||||
if palace_path is not None
|
||||
else None
|
||||
)
|
||||
self._file_config = {}
|
||||
|
||||
if self._config_file.exists():
|
||||
@@ -307,24 +382,169 @@ class MempalaceConfig:
|
||||
@property
|
||||
def palace_path(self):
|
||||
"""Path to the memory palace data directory."""
|
||||
if self._palace_path_override is not None:
|
||||
return self._palace_path_override
|
||||
env_val = os.environ.get("MEMPALACE_PALACE_PATH") or os.environ.get("MEMPAL_PALACE_PATH")
|
||||
if env_val:
|
||||
# Normalize: expand ~ and collapse .. to match the CLI --palace
|
||||
# code path (mcp_server.py:62) and prevent surprise redirection
|
||||
# when the env var contains unresolved components.
|
||||
return os.path.abspath(os.path.expanduser(env_val))
|
||||
return self._file_config.get("palace_path", DEFAULT_PALACE_PATH)
|
||||
return os.path.expanduser(self._file_config.get("palace_path", DEFAULT_PALACE_PATH))
|
||||
|
||||
@property
|
||||
def tunnel_file(self):
|
||||
"""Path to the tunnel file, sibling of palace_path."""
|
||||
return os.path.join(os.path.dirname(self.palace_path), "tunnels.json")
|
||||
|
||||
@property
|
||||
def hallway_file(self):
|
||||
"""Path to the hallway file, sibling of palace_path.
|
||||
|
||||
Mirrors ``tunnel_file`` so within-wing hallway state is scoped to the
|
||||
configured palace and survives palace rebuilds (it does not live in
|
||||
ChromaDB which can be recreated). Prior to this property the path was
|
||||
hardcoded under ``~/.mempalace/hallways.json`` and multiple palaces on
|
||||
one host silently shared one file (see ``hallways._legacy_hallway_file``).
|
||||
"""
|
||||
return os.path.join(os.path.dirname(self.palace_path), "hallways.json")
|
||||
|
||||
@property
|
||||
def collection_name(self):
|
||||
"""ChromaDB collection name."""
|
||||
return self._file_config.get("collection_name", DEFAULT_COLLECTION_NAME)
|
||||
|
||||
@property
|
||||
def backend(self):
|
||||
"""Storage backend name.
|
||||
|
||||
Read from ``config.json`` first, then ``MEMPALACE_BACKEND``, then
|
||||
``"chroma"`` for backwards compatibility with existing palaces.
|
||||
"""
|
||||
cfg_val = self._file_config.get("backend")
|
||||
if cfg_val:
|
||||
return str(cfg_val).strip().lower()
|
||||
env_val = os.environ.get("MEMPALACE_BACKEND")
|
||||
if env_val:
|
||||
return env_val.strip().lower()
|
||||
return DEFAULT_BACKEND
|
||||
|
||||
@property
|
||||
def qdrant_url(self):
|
||||
"""Qdrant endpoint for the opt-in ``qdrant`` backend.
|
||||
|
||||
Defaults to localhost so selecting Qdrant never silently sends memory
|
||||
to a remote service. Users can point at a LAN or cloud endpoint via
|
||||
config or ``MEMPALACE_QDRANT_URL`` when they deliberately choose that.
|
||||
"""
|
||||
env_val = os.environ.get("MEMPALACE_QDRANT_URL")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
return str(self._file_config.get("qdrant_url", "http://localhost:6333")).strip()
|
||||
|
||||
@property
|
||||
def qdrant_api_key(self):
|
||||
"""API key for the opt-in ``qdrant`` backend, if configured."""
|
||||
env_val = os.environ.get("MEMPALACE_QDRANT_API_KEY")
|
||||
if env_val:
|
||||
return env_val
|
||||
value = self._file_config.get("qdrant_api_key")
|
||||
return str(value) if value else None
|
||||
|
||||
@property
|
||||
def qdrant_namespace(self):
|
||||
"""Optional Qdrant collection namespace/prefix."""
|
||||
env_val = os.environ.get("MEMPALACE_QDRANT_NAMESPACE")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
value = self._file_config.get("qdrant_namespace")
|
||||
return str(value).strip() if value else None
|
||||
|
||||
@property
|
||||
def qdrant_timeout(self):
|
||||
"""Qdrant HTTP timeout in seconds."""
|
||||
env_val = os.environ.get("MEMPALACE_QDRANT_TIMEOUT")
|
||||
raw = env_val if env_val is not None else self._file_config.get("qdrant_timeout", 10.0)
|
||||
try:
|
||||
timeout = float(raw)
|
||||
except (TypeError, ValueError):
|
||||
timeout = 10.0
|
||||
return timeout if timeout > 0 else 10.0
|
||||
|
||||
@property
|
||||
def milvus_uri(self):
|
||||
"""Milvus endpoint for the opt-in ``milvus`` backend.
|
||||
|
||||
Defaults to ``None`` so selecting Milvus uses per-palace Milvus Lite at
|
||||
``<palace>/milvus.db``. Set this only to deliberately use a shared
|
||||
Milvus server, Zilliz Cloud, or a custom local Lite file.
|
||||
"""
|
||||
env_val = os.environ.get("MEMPALACE_MILVUS_URI")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
value = self._file_config.get("milvus_uri")
|
||||
return str(value).strip() if value else None
|
||||
|
||||
@property
|
||||
def milvus_token(self):
|
||||
"""Token for the opt-in ``milvus`` backend, if configured."""
|
||||
env_val = os.environ.get("MEMPALACE_MILVUS_TOKEN")
|
||||
if env_val:
|
||||
return env_val
|
||||
value = self._file_config.get("milvus_token")
|
||||
return str(value) if value else None
|
||||
|
||||
@property
|
||||
def milvus_db_name(self):
|
||||
"""Optional Milvus database name for the opt-in ``milvus`` backend."""
|
||||
env_val = os.environ.get("MEMPALACE_MILVUS_DB_NAME")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
value = self._file_config.get("milvus_db_name")
|
||||
return str(value).strip() if value else None
|
||||
|
||||
@property
|
||||
def milvus_namespace(self):
|
||||
"""Optional Milvus collection namespace/prefix."""
|
||||
env_val = os.environ.get("MEMPALACE_MILVUS_NAMESPACE")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
value = self._file_config.get("milvus_namespace")
|
||||
return str(value).strip() if value else None
|
||||
|
||||
@property
|
||||
def milvus_consistency_level(self):
|
||||
"""Milvus read consistency level for the opt-in ``milvus`` backend."""
|
||||
env_val = os.environ.get("MEMPALACE_MILVUS_CONSISTENCY_LEVEL")
|
||||
if env_val:
|
||||
return normalize_milvus_consistency_level(env_val)
|
||||
value = self._file_config.get("milvus_consistency_level")
|
||||
return normalize_milvus_consistency_level(value)
|
||||
|
||||
@property
|
||||
def pgvector_dsn(self):
|
||||
"""Postgres DSN for the opt-in ``pgvector`` backend.
|
||||
|
||||
Defaults to a localhost DSN so selecting pgvector never silently sends
|
||||
memory to a remote database. Point at a LAN or cloud Postgres via config
|
||||
or ``MEMPALACE_PGVECTOR_DSN`` only when deliberately chosen.
|
||||
"""
|
||||
env_val = os.environ.get("MEMPALACE_PGVECTOR_DSN")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
return str(
|
||||
self._file_config.get("pgvector_dsn", "postgresql://localhost:5432/mempalace")
|
||||
).strip()
|
||||
|
||||
@property
|
||||
def pgvector_namespace(self):
|
||||
"""Optional pgvector table namespace/prefix for multi-tenant isolation."""
|
||||
env_val = os.environ.get("MEMPALACE_PGVECTOR_NAMESPACE")
|
||||
if env_val:
|
||||
return env_val.strip()
|
||||
value = self._file_config.get("pgvector_namespace")
|
||||
return str(value).strip() if value else None
|
||||
|
||||
@property
|
||||
def people_map(self):
|
||||
"""Mapping of name variants to canonical names."""
|
||||
@@ -540,6 +760,37 @@ class MempalaceConfig:
|
||||
return env_val.strip().lower()
|
||||
return str(self._file_config.get("embedding_model", "minilm")).strip().lower()
|
||||
|
||||
@property
|
||||
def embedding_threads(self) -> int:
|
||||
"""Cap on the embedder's ONNX Runtime intra-op thread pool (#1068).
|
||||
|
||||
ChromaDB's ONNX embedder builds its ``InferenceSession`` with no thread
|
||||
cap, so the intra-op pool defaults to the physical core count and a
|
||||
background ``mine`` pins every core — stacked Stop-hook fires turn into
|
||||
thermal events. ``OMP_NUM_THREADS`` is inert here (ORT owns its own
|
||||
pool), so the cap is applied via ``SessionOptions`` in
|
||||
:mod:`mempalace.embedding`.
|
||||
|
||||
Read from env ``MEMPALACE_EMBEDDING_THREADS`` first, then
|
||||
``embedding_threads`` in ``config.json``. Semantics:
|
||||
|
||||
- unset / ``"auto"`` → half the logical CPUs (min 1), so a background
|
||||
mine leaves the machine usable out of the box.
|
||||
- a positive integer → exactly that many intra-op threads.
|
||||
- ``0`` or negative → uncapped: ORT's default (physical core count),
|
||||
for users who want maximum indexing throughput.
|
||||
"""
|
||||
raw = os.environ.get("MEMPALACE_EMBEDDING_THREADS")
|
||||
if raw is None:
|
||||
raw = self._file_config.get("embedding_threads")
|
||||
if raw is None or str(raw).strip().lower() in ("", "auto"):
|
||||
return max(1, (os.cpu_count() or 2) // 2)
|
||||
try:
|
||||
val = int(str(raw).strip())
|
||||
except (TypeError, ValueError):
|
||||
return max(1, (os.cpu_count() or 2) // 2)
|
||||
return val if val > 0 else 0
|
||||
|
||||
def set_embedding_model(self, model: str) -> None:
|
||||
"""Persist the embedding-model choice to ``config.json``.
|
||||
|
||||
@@ -560,6 +811,24 @@ class MempalaceConfig:
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
|
||||
def set_backend(self, backend: str) -> None:
|
||||
"""Persist the storage backend choice to ``config.json``."""
|
||||
backend = str(backend).strip().lower()
|
||||
from .backends import get_backend_class
|
||||
|
||||
get_backend_class(backend)
|
||||
self._file_config["backend"] = backend
|
||||
self._config_dir.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
with open(self._config_file, "w", encoding="utf-8") as f:
|
||||
json.dump(self._file_config, f, indent=2, ensure_ascii=False)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
self._config_file.chmod(0o600)
|
||||
except (OSError, NotImplementedError):
|
||||
pass
|
||||
|
||||
@property
|
||||
def topic_tunnel_min_count(self):
|
||||
"""Minimum number of overlapping confirmed topics required to create
|
||||
@@ -586,6 +855,36 @@ class MempalaceConfig:
|
||||
parsed = 1
|
||||
return max(1, parsed)
|
||||
|
||||
@property
|
||||
def max_backups(self) -> int:
|
||||
"""Number of timestamped palace backups to retain before pruning.
|
||||
|
||||
Applies to the accumulating, timestamped backups created by
|
||||
``mempalace migrate`` (``<palace>.pre-migrate.<timestamp>``) and
|
||||
``mempalace repair max-seq-id``
|
||||
(``chroma.sqlite3.max-seq-id-backup-<timestamp>``). Each of those
|
||||
commands writes a fresh full-size copy every run and historically
|
||||
never deleted the old ones, so on a machine that mines or repairs on
|
||||
a schedule the backup set could silently grow until it filled the
|
||||
disk. After each backup is written, copies beyond this count (oldest
|
||||
first) are removed.
|
||||
|
||||
Reads ``MEMPALACE_MAX_BACKUPS`` env first, then ``max_backups`` in
|
||||
``config.json``, then the default of ``10``. A value of ``0`` disables
|
||||
pruning and keeps every backup (use when an external retention policy
|
||||
manages cleanup). Negative or non-numeric values fall back to the
|
||||
default rather than crashing migrate/repair.
|
||||
"""
|
||||
env_val = os.environ.get("MEMPALACE_MAX_BACKUPS")
|
||||
if env_val is not None:
|
||||
coerced = self._try_coerce_int(env_val, minimum=0)
|
||||
if coerced is not None:
|
||||
return coerced
|
||||
coerced = self._try_coerce_int(
|
||||
self._file_config.get("max_backups", DEFAULT_MAX_BACKUPS), minimum=0
|
||||
)
|
||||
return DEFAULT_MAX_BACKUPS if coerced is None else coerced
|
||||
|
||||
@property
|
||||
def hook_silent_save(self):
|
||||
"""Whether the stop hook saves directly (True) or blocks for MCP calls (False)."""
|
||||
@@ -596,6 +895,115 @@ class MempalaceConfig:
|
||||
"""Whether the stop hook shows a desktop notification via notify-send."""
|
||||
return self._file_config.get("hooks", {}).get("desktop_toast", False)
|
||||
|
||||
def resolve_write_routing(self, scope: str) -> ResolvedWriteRoutingPolicy:
|
||||
"""Resolve the configured write policy for ``hooks`` or ``cli``.
|
||||
|
||||
Precedence is:
|
||||
|
||||
1. scope-specific environment variable;
|
||||
2. global environment variable;
|
||||
3. legacy hook environment variable;
|
||||
4. scope-specific config value;
|
||||
5. global config value;
|
||||
6. legacy hook config value;
|
||||
7. ``direct``.
|
||||
|
||||
This foundation does not change current hook or CLI behavior. The
|
||||
policy-aware consumers are introduced by follow-up PRs.
|
||||
"""
|
||||
|
||||
normalized_scope = str(scope).strip().lower()
|
||||
env_names = {
|
||||
"hooks": "MEMPALACE_HOOK_WRITE_ROUTING",
|
||||
"cli": "MEMPALACE_CLI_WRITE_ROUTING",
|
||||
}
|
||||
|
||||
if normalized_scope not in env_names:
|
||||
raise WriteRoutingError("write routing scope must be 'hooks' or 'cli'")
|
||||
|
||||
routing_config = self._file_config.get("write_routing", {})
|
||||
if routing_config is None:
|
||||
routing_config = {}
|
||||
|
||||
if not isinstance(routing_config, dict):
|
||||
raise WriteRoutingError("config write_routing must be an object")
|
||||
|
||||
candidates = [
|
||||
RoutingPolicyCandidate(
|
||||
env_names[normalized_scope],
|
||||
os.environ.get(env_names[normalized_scope]),
|
||||
),
|
||||
RoutingPolicyCandidate(
|
||||
"MEMPALACE_WRITE_ROUTING",
|
||||
os.environ.get("MEMPALACE_WRITE_ROUTING"),
|
||||
),
|
||||
]
|
||||
|
||||
if normalized_scope == "hooks":
|
||||
candidates.append(
|
||||
RoutingPolicyCandidate(
|
||||
"MEMPALACE_HOOKS_DAEMON (legacy)",
|
||||
os.environ.get("MEMPALACE_HOOKS_DAEMON"),
|
||||
legacy_boolean=True,
|
||||
)
|
||||
)
|
||||
|
||||
candidates.extend(
|
||||
[
|
||||
RoutingPolicyCandidate(
|
||||
f"config write_routing.{normalized_scope}",
|
||||
routing_config.get(normalized_scope),
|
||||
),
|
||||
RoutingPolicyCandidate(
|
||||
"config write_routing.default",
|
||||
routing_config.get("default"),
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
if normalized_scope == "hooks":
|
||||
hooks_config = self._file_config.get("hooks", {})
|
||||
if hooks_config is None:
|
||||
hooks_config = {}
|
||||
|
||||
if not isinstance(hooks_config, dict):
|
||||
raise WriteRoutingError("config hooks must be an object")
|
||||
|
||||
candidates.append(
|
||||
RoutingPolicyCandidate(
|
||||
"config hooks.daemon (legacy)",
|
||||
hooks_config.get("daemon"),
|
||||
legacy_boolean=True,
|
||||
)
|
||||
)
|
||||
|
||||
return resolve_write_routing_policy(candidates)
|
||||
|
||||
@property
|
||||
def hook_write_routing(self) -> WriteRoutingPolicy:
|
||||
"""Resolved future routing policy for hook-triggered writes."""
|
||||
|
||||
return self.resolve_write_routing("hooks").policy
|
||||
|
||||
@property
|
||||
def cli_write_routing(self) -> WriteRoutingPolicy:
|
||||
"""Resolved future routing policy for routine CLI writes."""
|
||||
|
||||
return self.resolve_write_routing("cli").policy
|
||||
|
||||
@property
|
||||
def hook_use_daemon(self):
|
||||
"""Whether hooks should submit save/mine work to the opt-in daemon."""
|
||||
env_val = os.environ.get("MEMPALACE_HOOKS_DAEMON")
|
||||
if env_val is not None:
|
||||
return env_val.lower() in ("true", "1", "yes", "on")
|
||||
value = self._file_config.get("hooks", {}).get("daemon", False)
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.lower() in ("true", "1", "yes", "on")
|
||||
return value == 1
|
||||
|
||||
def set_hook_setting(self, key: str, value: bool):
|
||||
"""Update a hook setting and write config to disk."""
|
||||
if "hooks" not in self._file_config:
|
||||
|
||||
+253
-62
@@ -10,14 +10,18 @@ Same palace as project mining. Different ingest strategy.
|
||||
|
||||
import os
|
||||
import sys
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import stat
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from collections import defaultdict
|
||||
from typing import Optional
|
||||
|
||||
from .collision_scan import assert_no_collisions
|
||||
from .ids import ID_RECIPE, make_convo_drawer_id, make_convo_sentinel_id
|
||||
from .normalize import normalize
|
||||
from .entities import entities_metadata
|
||||
from .palace import (
|
||||
NORMALIZE_VERSION,
|
||||
SKIP_DIRS,
|
||||
@@ -61,6 +65,16 @@ CONVO_EXTENSIONS = {
|
||||
".jsonl",
|
||||
}
|
||||
|
||||
# Directories inside conversation sources that never hold conversations.
|
||||
# ``tool-results``: Claude Code pages large tool outputs to
|
||||
# ``<session>/tool-results/*.txt`` inside ``~/.claude/projects/<slug>/``.
|
||||
# They are raw machine dumps referenced from the transcript JSONL — mining
|
||||
# them stores megabytes of command output as "memories" (field measurement:
|
||||
# 12.8k drawers from tool-results files on one palace; a single file
|
||||
# produced 3.6k). Extends the generic SKIP_DIRS set for the convo scanner
|
||||
# only — project mining semantics are unchanged.
|
||||
CONVO_SKIP_DIRS = SKIP_DIRS | {"tool-results"}
|
||||
|
||||
MIN_CHUNK_SIZE = 30
|
||||
CHUNK_SIZE = 800 # chars per drawer — align with miner.py
|
||||
_LINE_GROUP_SIZE = 25 # lines per fallback group when no paragraph breaks
|
||||
@@ -76,30 +90,67 @@ MAX_FILE_SIZE = 500 * 1024 * 1024 # 500 MB — skip files larger than this.
|
||||
# use also scales with source size.
|
||||
|
||||
|
||||
def _path_within_root(path: Path, root: Path) -> bool:
|
||||
try:
|
||||
path.expanduser().resolve().relative_to(root.expanduser().resolve())
|
||||
return True
|
||||
except (OSError, ValueError):
|
||||
return False
|
||||
|
||||
|
||||
def _is_regular_source_file(filepath: Path, root: Path) -> bool:
|
||||
if not _path_within_root(filepath, root):
|
||||
return False
|
||||
flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0)
|
||||
fd = -1
|
||||
try:
|
||||
fd = os.open(filepath, flags)
|
||||
st = os.fstat(fd)
|
||||
return stat.S_ISREG(st.st_mode) and st.st_size <= MAX_FILE_SIZE
|
||||
except OSError:
|
||||
return False
|
||||
finally:
|
||||
if fd != -1:
|
||||
try:
|
||||
os.close(fd)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _register_file(collection, source_file: str, wing: str, agent: str, extract_mode: str):
|
||||
"""Write a sentinel so file_already_mined() returns True for 0-chunk files.
|
||||
|
||||
Without this, files that normalize to nothing or produce zero chunks are
|
||||
re-read and re-processed on every mine run because nothing was written to
|
||||
ChromaDB on the first pass.
|
||||
|
||||
Stamps source_mtime like every real drawer does, so a file that later
|
||||
grows past the min-chunk-size floor (e.g. a short session that gets
|
||||
extended) is correctly detected as changed on the next mine instead of
|
||||
being skipped forever by this sentinel.
|
||||
"""
|
||||
sentinel_key = f"{source_file}:{extract_mode}"
|
||||
sentinel_id = f"_reg_{hashlib.sha256(sentinel_key.encode()).hexdigest()[:24]}"
|
||||
try:
|
||||
source_mtime = os.path.getmtime(source_file)
|
||||
except OSError:
|
||||
source_mtime = None
|
||||
sentinel_id = make_convo_sentinel_id(source_file, extract_mode)
|
||||
meta = {
|
||||
"wing": wing,
|
||||
"room": "_registry",
|
||||
"source_file": source_file,
|
||||
"added_by": agent,
|
||||
"filed_at": datetime.now().isoformat(),
|
||||
"ingest_mode": "registry",
|
||||
"extract_mode": extract_mode,
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
"id_recipe": ID_RECIPE,
|
||||
}
|
||||
if source_mtime is not None:
|
||||
meta["source_mtime"] = source_mtime
|
||||
collection.upsert(
|
||||
documents=[f"[registry] {source_file}"],
|
||||
ids=[sentinel_id],
|
||||
metadatas=[
|
||||
{
|
||||
"wing": wing,
|
||||
"room": "_registry",
|
||||
"source_file": source_file,
|
||||
"added_by": agent,
|
||||
"filed_at": datetime.now().isoformat(),
|
||||
"ingest_mode": "registry",
|
||||
"extract_mode": extract_mode,
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
}
|
||||
],
|
||||
metadatas=[meta],
|
||||
)
|
||||
|
||||
|
||||
@@ -192,11 +243,15 @@ def _chunk_by_exchange(lines: list, chunk_size: int, min_chunk_size: int) -> lis
|
||||
next_line = lines[i]
|
||||
if next_line.strip().startswith(">") or next_line.strip().startswith("---"):
|
||||
break
|
||||
if next_line.strip():
|
||||
ai_lines.append(next_line.strip())
|
||||
# Preserve the line as-is — blank lines and indentation carry meaning
|
||||
# (paragraph breaks, list/code structure) and must survive verbatim.
|
||||
ai_lines.append(next_line)
|
||||
i += 1
|
||||
|
||||
ai_response = " ".join(ai_lines)
|
||||
# Join on newline (not space) so line structure, blank lines, and
|
||||
# indentation reach the drawer unchanged. Trim only trailing blank
|
||||
# lines produced by the loop stopping at the next `>` turn.
|
||||
ai_response = "\n".join(ai_lines).rstrip("\n")
|
||||
content = f"{user_turn}\n{ai_response}" if ai_response else user_turn
|
||||
|
||||
_emit_bounded(chunks, content, chunk_size, min_chunk_size)
|
||||
@@ -350,7 +405,7 @@ def scan_convos(convo_dir: str) -> list:
|
||||
convo_path = Path(convo_dir).expanduser().resolve()
|
||||
files = []
|
||||
for root, dirs, filenames in os.walk(convo_path):
|
||||
dirs[:] = [d for d in dirs if d not in SKIP_DIRS]
|
||||
dirs[:] = [d for d in dirs if d not in CONVO_SKIP_DIRS]
|
||||
for filename in filenames:
|
||||
if filename.endswith(".meta.json"):
|
||||
continue
|
||||
@@ -364,10 +419,30 @@ def scan_convos(convo_dir: str) -> list:
|
||||
except OSError:
|
||||
pass
|
||||
continue
|
||||
# Skip files exceeding size limit, or those whose stat() raises
|
||||
# (permission denied, racing delete, broken symlink that
|
||||
# survived the earlier is_symlink check). Both branches log
|
||||
# to stderr to match the SKIP: (symlink) line above; silent
|
||||
# drops at this gate were the original #923 complaint.
|
||||
try:
|
||||
if filepath.stat().st_size > MAX_FILE_SIZE:
|
||||
file_size = filepath.stat().st_size
|
||||
if file_size > MAX_FILE_SIZE:
|
||||
print(
|
||||
f" SKIP: {filepath.name} ({file_size / (1024 * 1024):.1f} MB)"
|
||||
f" exceeds {MAX_FILE_SIZE // (1024 * 1024)} MB limit",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
except OSError:
|
||||
except OSError as exc:
|
||||
# Prefer ``exc.strerror`` so the path isn't duplicated in
|
||||
# the output (see the matching comment in
|
||||
# ``miner.scan_project``).
|
||||
print(
|
||||
f" SKIP: {filepath.name} (stat error: {exc.strerror or exc})",
|
||||
file=sys.stderr,
|
||||
)
|
||||
continue
|
||||
if not _is_regular_source_file(filepath, convo_path):
|
||||
continue
|
||||
files.append(filepath)
|
||||
return files
|
||||
@@ -378,12 +453,51 @@ def scan_convos(convo_dir: str) -> list:
|
||||
# =============================================================================
|
||||
|
||||
|
||||
def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extract_mode):
|
||||
def _extract_authored_at(filepath):
|
||||
"""Most-recent message timestamp in a transcript, used as the drawer's authored date.
|
||||
|
||||
Both Claude Code and Codex JSONL transcripts carry a top-level ISO-8601
|
||||
``timestamp`` on each line. We take the max so ``authored_at`` reflects when the
|
||||
content was actually written, independent of when it was mined (``filed_at``).
|
||||
This restores chronology: a session from days ago keeps its real date even when
|
||||
re-mined today, instead of every drawer collapsing to ingest time. Returns None
|
||||
for formats without per-line timestamps (e.g. plain ``.md``).
|
||||
"""
|
||||
path = Path(filepath)
|
||||
if path.suffix != ".jsonl":
|
||||
return None
|
||||
latest = None
|
||||
try:
|
||||
with path.open(encoding="utf-8", errors="ignore") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
ts = json.loads(line).get("timestamp")
|
||||
except (ValueError, TypeError, AttributeError):
|
||||
continue
|
||||
# ISO-8601 timestamps are strings; guard against a non-string
|
||||
# ``timestamp`` so a malformed line can't raise TypeError on compare.
|
||||
if isinstance(ts, str) and (latest is None or ts > latest):
|
||||
latest = ts
|
||||
except OSError:
|
||||
return None
|
||||
return latest
|
||||
|
||||
|
||||
def _file_chunks_locked(
|
||||
collection, source_file, chunks, wing, room, agent, extract_mode, authored_at=None
|
||||
):
|
||||
"""Lock the source file, purge stale drawers, and upsert fresh chunks.
|
||||
|
||||
Combines the per-file serialization that prevents concurrent agents from
|
||||
duplicating work (via mine_lock) with the normalize-version rebuild
|
||||
contract (purge-before-insert so pre-v2 drawers don't survive).
|
||||
duplicating work (via mine_lock) with the rebuild contract
|
||||
(purge-before-insert so stale drawers never survive) that fires on
|
||||
either a normalize-version bump OR a changed/grown source file (mtime
|
||||
differs from what's stored) -- transcripts are not assumed immutable,
|
||||
since a Claude Code session keeps appending to its own file while
|
||||
active and /compact or /clear can rewrite one in place.
|
||||
|
||||
Returns (drawers_added, room_counts_delta, skipped).
|
||||
"""
|
||||
@@ -391,14 +505,15 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
|
||||
drawers_added = 0
|
||||
with mine_lock(source_file):
|
||||
# Re-check after lock — another agent may have just finished this file
|
||||
# at the current schema. A stale-version hit here returns False, so we
|
||||
# at the current schema/mtime. A stale hit here returns False, so we
|
||||
# still fall through to the purge+rebuild path below.
|
||||
if file_already_mined(collection, source_file, extract_mode=extract_mode):
|
||||
if file_already_mined(collection, source_file, check_mtime=True, extract_mode=extract_mode):
|
||||
return 0, room_counts_delta, True
|
||||
|
||||
# Purge stale drawers first. When the normalize schema bumps,
|
||||
# file_already_mined() returned False for pre-v2 drawers — clean
|
||||
# them out so the source doesn't end up with mixed old/new drawers.
|
||||
# Purge stale drawers first. Fires both on a normalize-schema bump
|
||||
# (file_already_mined() returned False for pre-v2 drawers) and on a
|
||||
# changed/grown transcript (mtime differs) — clean them out so the
|
||||
# source doesn't end up with mixed old/new drawers.
|
||||
try:
|
||||
delete_ids = _source_file_delete_ids(collection, source_file, extract_mode)
|
||||
if delete_ids:
|
||||
@@ -411,6 +526,10 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
|
||||
# one filed_at per source file so all transcript drawers share an
|
||||
# ingest timestamp.
|
||||
filed_at = datetime.now().isoformat()
|
||||
try:
|
||||
source_mtime = os.path.getmtime(source_file)
|
||||
except OSError:
|
||||
source_mtime = None
|
||||
for batch_start in range(0, len(chunks), DRAWER_UPSERT_BATCH_SIZE):
|
||||
batch_docs: list = []
|
||||
batch_ids: list = []
|
||||
@@ -419,27 +538,30 @@ def _file_chunks_locked(collection, source_file, chunks, wing, room, agent, extr
|
||||
chunk_room = chunk.get("memory_type", room) if extract_mode == "general" else room
|
||||
if extract_mode == "general":
|
||||
room_counts_delta[chunk_room] += 1
|
||||
drawer_key = f"{source_file}:{extract_mode}:{chunk['chunk_index']}"
|
||||
drawer_id = (
|
||||
f"drawer_{wing}_{chunk_room}_"
|
||||
f"{hashlib.sha256(drawer_key.encode()).hexdigest()[:24]}"
|
||||
drawer_id = make_convo_drawer_id(
|
||||
wing, chunk_room, source_file, extract_mode, chunk["chunk_index"]
|
||||
)
|
||||
batch_docs.append(chunk["content"])
|
||||
batch_ids.append(drawer_id)
|
||||
batch_metas.append(
|
||||
{
|
||||
"wing": wing,
|
||||
"room": chunk_room,
|
||||
"hall": _detect_hall_cached(chunk["content"]),
|
||||
"source_file": source_file,
|
||||
"chunk_index": chunk["chunk_index"],
|
||||
"added_by": agent,
|
||||
"filed_at": filed_at,
|
||||
"ingest_mode": "convos",
|
||||
"extract_mode": extract_mode,
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
}
|
||||
)
|
||||
meta = {
|
||||
"wing": wing,
|
||||
"room": chunk_room,
|
||||
"hall": _detect_hall_cached(chunk["content"]),
|
||||
"source_file": source_file,
|
||||
"chunk_index": chunk["chunk_index"],
|
||||
"added_by": agent,
|
||||
"filed_at": filed_at,
|
||||
"entities": entities_metadata(chunk["content"]),
|
||||
"authored_at": authored_at if authored_at is not None else filed_at,
|
||||
"ingest_mode": "convos",
|
||||
"extract_mode": extract_mode,
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
"id_recipe": ID_RECIPE,
|
||||
}
|
||||
if source_mtime is not None:
|
||||
meta["source_mtime"] = source_mtime
|
||||
batch_metas.append(meta)
|
||||
assert_no_collisions(list(zip(batch_ids, batch_metas)), collection)
|
||||
try:
|
||||
collection.upsert(
|
||||
documents=batch_docs,
|
||||
@@ -482,6 +604,28 @@ def _is_ai_tool_path(path: Path) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def _is_unchanged_since_last_mine(source_file: str, mined_mtimes: dict) -> bool:
|
||||
"""True iff source_file was mined at the current schema AND its on-disk
|
||||
mtime still matches what was stored -- the mtime-aware replacement for
|
||||
"we've seen this source_file before" (transcripts are not immutable).
|
||||
|
||||
False (re-mine) whenever the file isn't in mined_mtimes at all, its
|
||||
stored mtime is None (never recorded -- pre-mtime-tracking drawer, or
|
||||
getmtime failed when it was written), or getmtime fails right now
|
||||
(treat as changed rather than silently trusting stale data).
|
||||
"""
|
||||
if source_file not in mined_mtimes:
|
||||
return False
|
||||
stored_mtime = mined_mtimes[source_file]
|
||||
if stored_mtime is None:
|
||||
return False
|
||||
try:
|
||||
current_mtime = os.path.getmtime(source_file)
|
||||
except OSError:
|
||||
return False
|
||||
return abs(stored_mtime - current_mtime) < 0.001
|
||||
|
||||
|
||||
def _resolve_wing(convo_path: Path, wing: Optional[str]) -> str:
|
||||
"""Determine the destination wing for ``mine_convos``.
|
||||
|
||||
@@ -561,6 +705,22 @@ def mine_convos(
|
||||
)
|
||||
|
||||
|
||||
def _compute_hallways_for_wing_safe(wing, collection, drawers_filed, config=None):
|
||||
"""Auto-populate the associative graph from the entities just mined.
|
||||
|
||||
Best-effort: hallway computation must never fail an otherwise-good mine, and is
|
||||
skipped when nothing new was filed.
|
||||
"""
|
||||
if drawers_filed <= 0:
|
||||
return
|
||||
try:
|
||||
from .hallways import compute_hallways_for_wing
|
||||
|
||||
compute_hallways_for_wing(wing, col=collection, config=config)
|
||||
except Exception as exc:
|
||||
print(f" (hallways skipped: {exc})")
|
||||
|
||||
|
||||
def _mine_convos_impl(
|
||||
convo_dir: str,
|
||||
palace_path: str,
|
||||
@@ -572,7 +732,7 @@ def _mine_convos_impl(
|
||||
):
|
||||
from .config import MempalaceConfig
|
||||
|
||||
palace_config = MempalaceConfig()
|
||||
palace_config = MempalaceConfig(palace_path=palace_path)
|
||||
cfg_chunk_size = palace_config.chunk_size
|
||||
# Only override convo_miner's MIN_CHUNK_SIZE when the user has set
|
||||
# min_chunk_size explicitly. min_chunk_size_explicit returns the
|
||||
@@ -589,15 +749,14 @@ def _mine_convos_impl(
|
||||
wing = _resolve_wing(convo_path, wing)
|
||||
|
||||
files = scan_convos(convo_dir)
|
||||
if limit > 0:
|
||||
files = files[:limit]
|
||||
|
||||
print(f"\n{'=' * 55}")
|
||||
print(" MemPalace Mine — Conversations")
|
||||
print(f"{'=' * 55}")
|
||||
print(f" Wing: {wing}")
|
||||
print(f" Source: {convo_path}")
|
||||
print(f" Files: {len(files)}")
|
||||
limit_suffix = f" (limit: {limit} new)" if limit > 0 else ""
|
||||
print(f" Files: {len(files)}{limit_suffix}")
|
||||
print(f" Palace: {palace_path}")
|
||||
if dry_run:
|
||||
print(" DRY RUN — nothing will be filed")
|
||||
@@ -605,24 +764,39 @@ def _mine_convos_impl(
|
||||
|
||||
collection = get_collection(palace_path) if not dry_run else None
|
||||
|
||||
# Bulk pre-fetch already-mined set in one paginated pass instead of
|
||||
# `len(files)` separate WHERE-source_file queries. On a 150k-drawer
|
||||
# palace each per-file query costs ~2s, so a 2000-file sweep used to
|
||||
# spend >1h just deciding to skip. prefetch_mined_set() does the same
|
||||
# decisions in a single scan; loop body becomes an O(1) set check.
|
||||
mined_set: set[str] = (
|
||||
prefetch_mined_set(collection, extract_mode=extract_mode) if not dry_run else set()
|
||||
# Bulk pre-fetch already-mined source_file -> stored mtime in one
|
||||
# paginated pass instead of `len(files)` separate WHERE-source_file
|
||||
# queries. On a 150k-drawer palace each per-file query costs ~2s, so a
|
||||
# 2000-file sweep used to spend >1h just deciding to skip.
|
||||
# prefetch_mined_set() does the same decisions in a single scan; loop
|
||||
# body becomes an O(1) dict lookup + a cheap local mtime comparison.
|
||||
mined_mtimes: dict = (
|
||||
prefetch_mined_set(collection, extract_mode=extract_mode) if not dry_run else {}
|
||||
)
|
||||
|
||||
total_drawers = 0
|
||||
files_mined = 0
|
||||
files_skipped = 0
|
||||
files_processed = 0
|
||||
room_counts = defaultdict(int)
|
||||
|
||||
for i, filepath in enumerate(files, 1):
|
||||
files_processed = i
|
||||
source_file = str(filepath)
|
||||
|
||||
# Skip if already filed at current NORMALIZE_VERSION
|
||||
if not dry_run and source_file in mined_set:
|
||||
# Skip only if already filed at the current NORMALIZE_VERSION AND
|
||||
# unchanged on disk since. Transcripts are NOT assumed immutable:
|
||||
# a Claude Code session keeps appending to the same file while
|
||||
# active, and /compact or /clear can rewrite one in place -- so
|
||||
# "we've seen this source_file before" alone is not sufficient.
|
||||
# Falling through re-mines: _file_chunks_locked purges this
|
||||
# source_file's stale drawers before inserting fresh ones, so this
|
||||
# never leaves duplicates behind.
|
||||
if not dry_run and _is_unchanged_since_last_mine(source_file, mined_mtimes):
|
||||
files_skipped += 1
|
||||
continue
|
||||
|
||||
if not _is_regular_source_file(filepath, Path(convo_dir).expanduser().resolve()):
|
||||
files_skipped += 1
|
||||
continue
|
||||
|
||||
@@ -679,6 +853,9 @@ def _mine_convos_impl(
|
||||
room_counts[c.get("memory_type", "general")] += 1
|
||||
else:
|
||||
room_counts[room] += 1
|
||||
files_mined += 1
|
||||
if limit > 0 and files_mined >= limit:
|
||||
break
|
||||
continue
|
||||
|
||||
if extract_mode != "general":
|
||||
@@ -687,7 +864,14 @@ def _mine_convos_impl(
|
||||
# Lock + purge stale + file fresh chunks. Lock serializes concurrent
|
||||
# agents; purge removes pre-v2 drawers so the schema bump applies.
|
||||
drawers_added, room_delta, skipped = _file_chunks_locked(
|
||||
collection, source_file, chunks, wing, room, agent, extract_mode
|
||||
collection,
|
||||
source_file,
|
||||
chunks,
|
||||
wing,
|
||||
room,
|
||||
agent,
|
||||
extract_mode,
|
||||
authored_at=_extract_authored_at(filepath),
|
||||
)
|
||||
if skipped:
|
||||
files_skipped += 1
|
||||
@@ -696,14 +880,21 @@ def _mine_convos_impl(
|
||||
room_counts[r] += n
|
||||
|
||||
total_drawers += drawers_added
|
||||
files_mined += 1
|
||||
print(f" + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}")
|
||||
if limit > 0 and files_mined >= limit:
|
||||
break
|
||||
|
||||
if not dry_run:
|
||||
# Compute hallways before the FTS5 validation: the latter opens a direct sqlite
|
||||
# connection to the Chroma DB, which can invalidate the live collection handle on
|
||||
# some Chroma builds and make the hallway fetch fail.
|
||||
_compute_hallways_for_wing_safe(wing, collection, total_drawers, config=palace_config)
|
||||
_validate_palace_fts5_after_mine(palace_path)
|
||||
|
||||
print(f"\n{'=' * 55}")
|
||||
print(" Done.")
|
||||
print(f" Files processed: {len(files) - files_skipped}")
|
||||
print(f" Files processed: {files_processed - files_skipped}")
|
||||
print(f" Files skipped (already filed): {files_skipped}")
|
||||
print(f" Drawers filed: {total_drawers}")
|
||||
if room_counts:
|
||||
|
||||
+1164
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,68 @@
|
||||
{
|
||||
"schema_version": 1,
|
||||
"description": "Multi-word product/system names that must be detected atomically — not decomposed into their constituent words. When any of these compounds appears in mined content, the entity detector counts the COMPOUND, not the parts. The single-word detection pass that runs after this compound pre-pass would otherwise decompose 'Claude Code' into 'Claude' + 'Code', and the COCA filter (Tier 2) would then drop 'Code' as a content word while attributing usage incorrectly to 'Claude' alone.",
|
||||
"source": "Curated 2026-05-24 from common AI/development product names that mempalace users frequently mention. Each entry is a multi-word name (two-or-more whitespace-separated tokens, or hyphenated). Single-word product names like 'ChatGPT', 'Cursor', 'OrbStack' are NOT included — they have no decomposition risk and are handled by the existing single-word regex.",
|
||||
"case_handling": "Matching is case-insensitive. Canonical form (the entry below) is what gets counted. Caller must use re.IGNORECASE when matching.",
|
||||
"tier_2_interaction": "This file complements coca_content_words.json. Tier 2 filters single-word content nouns. Tier 3 protects multi-word product names. Run THIS pre-pass FIRST so the COCA filter doesn't see the decomposed words.",
|
||||
"compounds": [
|
||||
"Claude Code",
|
||||
"Claude Desktop",
|
||||
"Claude Sonnet",
|
||||
"Claude Opus",
|
||||
"Claude Haiku",
|
||||
"Claude Sonnet 4.5",
|
||||
"Claude Sonnet 4.6",
|
||||
"Claude Opus 4.5",
|
||||
"Claude Opus 4.6",
|
||||
"Claude Opus 4.7",
|
||||
"Gemini Code Assist",
|
||||
"Gemini 1.5 Pro",
|
||||
"Gemini 2.5 Pro",
|
||||
"Gemini 2.5 Flash",
|
||||
"Gemini Pro",
|
||||
"Gemini Flash",
|
||||
"GitHub Copilot",
|
||||
"GitHub Copilot CLI",
|
||||
"Microsoft Copilot",
|
||||
"Visual Studio Code",
|
||||
"Visual Studio",
|
||||
"IntelliJ IDEA",
|
||||
"Android Studio",
|
||||
"Sublime Text",
|
||||
"Replit Agent",
|
||||
"Docker Desktop",
|
||||
"Docker Compose",
|
||||
"GitHub Actions",
|
||||
"GitHub Pages",
|
||||
"GitHub Codespaces",
|
||||
"GitHub Pull Request",
|
||||
"Hugging Face",
|
||||
"Stack Overflow",
|
||||
"Stack Exchange",
|
||||
"Hacker News",
|
||||
"Y Combinator",
|
||||
"Google Cloud Platform",
|
||||
"Amazon Web Services",
|
||||
"Microsoft Azure",
|
||||
"Azure DevOps",
|
||||
"Notion AI",
|
||||
"OpenAI API",
|
||||
"Anthropic Console",
|
||||
"GPT-4",
|
||||
"GPT-4o",
|
||||
"GPT-5",
|
||||
"Llama 3",
|
||||
"Llama 3.1",
|
||||
"Mistral Small",
|
||||
"Mistral Large",
|
||||
"Microsoft Word",
|
||||
"Microsoft Excel",
|
||||
"Microsoft PowerPoint",
|
||||
"Google Docs",
|
||||
"Google Sheets",
|
||||
"Google Drive",
|
||||
"Google Cloud",
|
||||
"Apple Watch",
|
||||
"Apple TV"
|
||||
]
|
||||
}
|
||||
+6
-4
@@ -7,7 +7,9 @@ accumulate. This module finds drawers from the same source_file that
|
||||
are too similar (cosine distance < threshold), keeps the longest/richest
|
||||
version, and deletes the rest.
|
||||
|
||||
No API calls — uses ChromaDB's built-in embedding similarity.
|
||||
Uses the configured storage backend's similarity search. With the default
|
||||
local backends (Chroma, sqlite_exact) this stays on-machine with no external
|
||||
calls; with a remote backend (e.g. Qdrant) it issues queries to that backend.
|
||||
|
||||
Usage (standalone):
|
||||
python -m mempalace.dedup # dedup all
|
||||
@@ -27,7 +29,7 @@ import os
|
||||
import time
|
||||
from collections import defaultdict
|
||||
|
||||
from .backends.chroma import ChromaBackend
|
||||
from .palace import get_collection
|
||||
|
||||
|
||||
COLLECTION_NAME = "mempalace_drawers"
|
||||
@@ -130,7 +132,7 @@ def dedup_source_group(col, drawer_ids, threshold=DEFAULT_THRESHOLD, dry_run=Tru
|
||||
def show_stats(palace_path=None):
|
||||
"""Show duplication statistics without making changes."""
|
||||
palace_path = palace_path or _get_palace_path()
|
||||
col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME)
|
||||
col = get_collection(palace_path, COLLECTION_NAME)
|
||||
|
||||
groups = get_source_groups(col)
|
||||
|
||||
@@ -162,7 +164,7 @@ def dedup_palace(
|
||||
print(" MemPalace Deduplicator")
|
||||
print(f"{'=' * 55}")
|
||||
|
||||
col = ChromaBackend().get_collection(palace_path, COLLECTION_NAME)
|
||||
col = get_collection(palace_path, COLLECTION_NAME)
|
||||
|
||||
print(f" Palace: {palace_path}")
|
||||
print(f" Drawers: {col.count():,}")
|
||||
|
||||
+252
-55
@@ -32,6 +32,8 @@ rather than hard-failing — mining must still work on a laptop without CUDA.
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -56,6 +58,10 @@ _AUTO_ORDER = [
|
||||
]
|
||||
|
||||
_EF_CACHE: dict = {}
|
||||
# Check-then-construct on the cache must be atomic: without it, two threads
|
||||
# resolving the same key each keep their own EF instance, and each instance
|
||||
# later lazy-loads its own copy of the model.
|
||||
_EF_CACHE_LOCK = threading.Lock()
|
||||
_WARNED: set = set()
|
||||
|
||||
|
||||
@@ -107,6 +113,35 @@ def _resolve_providers(device: str) -> tuple[list, str]:
|
||||
return (requested, device)
|
||||
|
||||
|
||||
def _intra_op_session_options(intra_op_num_threads: int):
|
||||
"""Build ORT ``SessionOptions`` capping the intra-op thread pool (#1068).
|
||||
|
||||
Returns ``None`` when ``intra_op_num_threads <= 0`` so the caller leaves
|
||||
ORT at its default (≈ physical core count). ChromaDB's embedder ignores
|
||||
``OMP_NUM_THREADS`` — ORT owns its own intra-op pool, settable only via
|
||||
``SessionOptions`` at session construction — so a cap has to be threaded
|
||||
through here rather than via the environment.
|
||||
"""
|
||||
if not intra_op_num_threads or intra_op_num_threads <= 0:
|
||||
return None
|
||||
import onnxruntime as ort
|
||||
|
||||
so = ort.SessionOptions()
|
||||
so.intra_op_num_threads = intra_op_num_threads
|
||||
return so
|
||||
|
||||
|
||||
def _resolve_intra_op_threads() -> int:
|
||||
"""Read the configured ORT intra-op thread cap (``0`` = uncapped, #1068)."""
|
||||
try:
|
||||
from .config import MempalaceConfig
|
||||
|
||||
return MempalaceConfig().embedding_threads
|
||||
except Exception:
|
||||
logger.debug("embedding_threads resolution failed; leaving ORT default", exc_info=True)
|
||||
return 0
|
||||
|
||||
|
||||
def _build_ef_class():
|
||||
"""Subclass ``ONNXMiniLM_L6_V2`` with name ``"default"``.
|
||||
|
||||
@@ -117,13 +152,51 @@ def _build_ef_class():
|
||||
palaces created with ``DefaultEmbeddingFunction`` *and* palaces we
|
||||
create ourselves, with the same GPU-capable ``preferred_providers``.
|
||||
"""
|
||||
from functools import cached_property
|
||||
|
||||
from chromadb.utils.embedding_functions import ONNXMiniLM_L6_V2
|
||||
|
||||
class _MempalaceONNX(ONNXMiniLM_L6_V2):
|
||||
def __init__(self, preferred_providers=None, intra_op_num_threads=0):
|
||||
super().__init__(preferred_providers=preferred_providers)
|
||||
self._intra_op_num_threads = intra_op_num_threads
|
||||
|
||||
@staticmethod
|
||||
def name() -> str:
|
||||
return "default"
|
||||
|
||||
@cached_property
|
||||
def model(self):
|
||||
# Upstream builds the InferenceSession with no intra-op thread cap,
|
||||
# so ORT defaults its pool to the physical core count and a
|
||||
# background mine pins every core (#1068). Rebuild the session the
|
||||
# same way upstream does (same SessionOptions, same CoreML pruning,
|
||||
# same model path) but with our cap applied. If upstream's
|
||||
# internals shift, fall back to its uncapped build so embedding
|
||||
# still works.
|
||||
cap = getattr(self, "_intra_op_num_threads", 0)
|
||||
if not cap or cap <= 0:
|
||||
return super().model
|
||||
try:
|
||||
ort = self.ort
|
||||
providers = self._preferred_providers or ort.get_available_providers()
|
||||
providers = [p for p in providers if p != "CoreMLExecutionProvider"]
|
||||
so = ort.SessionOptions()
|
||||
so.log_severity_level = 3
|
||||
so.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
so.intra_op_num_threads = cap
|
||||
return ort.InferenceSession(
|
||||
os.path.join(self.DOWNLOAD_PATH, self.EXTRACTED_FOLDER_NAME, "model.onnx"),
|
||||
providers=providers,
|
||||
sess_options=so,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"thread-capped ORT session build failed; using ORT defaults",
|
||||
exc_info=True,
|
||||
)
|
||||
return super().model
|
||||
|
||||
return _MempalaceONNX
|
||||
|
||||
|
||||
@@ -135,6 +208,16 @@ _EMBEDDINGGEMMA_ONNX = "model_quantized.onnx"
|
||||
_EMBEDDINGGEMMA_PREFIX = "task: sentence similarity | query: "
|
||||
_EMBEDDINGGEMMA_DIM = 384 # Matryoshka truncation — first 384 dims of the 768
|
||||
_EMBEDDINGGEMMA_MAX_LEN = 2048
|
||||
# Default docs per session.run. The ONNX graph has no internal batching,
|
||||
# so one unchunked run over a repair-scale batch (5000 docs, repair.py/
|
||||
# cli.py) allocates attention buffers that grow with batch size and
|
||||
# superlinearly with padded length (score tensors are batch x heads x
|
||||
# len^2 per layer), and the kernel OOM-kills the process (#1770). 32
|
||||
# matches the internal batch size of chromadb's ONNXMiniLM_L6_V2, whose
|
||||
# chunked _forward survives the same call sites. embeddinggemma's
|
||||
# sentence_embedding output is attention-masked, so sub-batch padding
|
||||
# does not change any row's vector.
|
||||
_EMBEDDINGGEMMA_BATCH_SIZE = 32
|
||||
|
||||
|
||||
class EmbeddinggemmaONNX:
|
||||
@@ -158,70 +241,123 @@ class EmbeddinggemmaONNX:
|
||||
# when switching models. Keep it stable.
|
||||
return "embeddinggemma_300m"
|
||||
|
||||
def __init__(self, preferred_providers=None):
|
||||
def __init__(
|
||||
self,
|
||||
preferred_providers=None,
|
||||
batch_size: int = _EMBEDDINGGEMMA_BATCH_SIZE,
|
||||
intra_op_num_threads: int = 0,
|
||||
):
|
||||
if batch_size < 1:
|
||||
raise ValueError(f"batch_size must be >= 1, got {batch_size}")
|
||||
self._providers = (
|
||||
list(preferred_providers) if preferred_providers else ["CPUExecutionProvider"]
|
||||
)
|
||||
self._batch_size = batch_size
|
||||
self._intra_op_num_threads = intra_op_num_threads
|
||||
self._session = None
|
||||
self._tokenizer = None
|
||||
self._np = None
|
||||
self._output_idx = None
|
||||
# Instances are shared across threads via _EF_CACHE; serialize the
|
||||
# one-time model load so concurrent cold calls cannot build (and
|
||||
# transiently hold) two full model sessions.
|
||||
self._load_lock = threading.Lock()
|
||||
|
||||
def _lazy_load(self) -> None:
|
||||
if self._session is not None:
|
||||
return
|
||||
try:
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
from huggingface_hub import hf_hub_download
|
||||
from tokenizers import Tokenizer
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"EmbeddinggemmaONNX requires huggingface_hub, tokenizers, and "
|
||||
"numpy — these ship with mempalace core, so this error usually "
|
||||
"means one was uninstalled or pinned to an incompatible version. "
|
||||
"Reinstall with: pip install --upgrade --force-reinstall mempalace"
|
||||
) from e
|
||||
with self._load_lock:
|
||||
if self._session is not None:
|
||||
return
|
||||
try:
|
||||
import numpy as np
|
||||
import onnxruntime as ort
|
||||
from huggingface_hub import hf_hub_download
|
||||
from tokenizers import Tokenizer
|
||||
except ImportError as e:
|
||||
raise ImportError(
|
||||
"EmbeddinggemmaONNX requires huggingface_hub, tokenizers, and "
|
||||
"numpy — these ship with mempalace core, so this error usually "
|
||||
"means one was uninstalled or pinned to an incompatible version. "
|
||||
"Reinstall with: pip install --upgrade --force-reinstall mempalace"
|
||||
) from e
|
||||
|
||||
logger.info(
|
||||
"Downloading %s/%s (cached after first run)…",
|
||||
_EMBEDDINGGEMMA_REPO,
|
||||
_EMBEDDINGGEMMA_ONNX,
|
||||
)
|
||||
model_path = hf_hub_download(
|
||||
_EMBEDDINGGEMMA_REPO, subfolder="onnx", filename=_EMBEDDINGGEMMA_ONNX
|
||||
)
|
||||
tok_path = hf_hub_download(_EMBEDDINGGEMMA_REPO, filename="tokenizer.json")
|
||||
logger.info(
|
||||
"Downloading %s/%s (cached after first run)…",
|
||||
_EMBEDDINGGEMMA_REPO,
|
||||
_EMBEDDINGGEMMA_ONNX,
|
||||
)
|
||||
model_path = hf_hub_download(
|
||||
_EMBEDDINGGEMMA_REPO, subfolder="onnx", filename=_EMBEDDINGGEMMA_ONNX
|
||||
)
|
||||
hf_hub_download(
|
||||
_EMBEDDINGGEMMA_REPO, subfolder="onnx", filename=_EMBEDDINGGEMMA_ONNX + "_data"
|
||||
)
|
||||
tok_path = hf_hub_download(_EMBEDDINGGEMMA_REPO, filename="tokenizer.json")
|
||||
|
||||
self._session = ort.InferenceSession(model_path, providers=self._providers)
|
||||
out_names = [o.name for o in self._session.get_outputs()]
|
||||
# Model card: sentence_embedding is the pooled output (last_hidden_state
|
||||
# is the per-token output we don't want).
|
||||
self._output_idx = (
|
||||
out_names.index("sentence_embedding") if "sentence_embedding" in out_names else 1
|
||||
)
|
||||
session = ort.InferenceSession(
|
||||
model_path,
|
||||
sess_options=_intra_op_session_options(self._intra_op_num_threads),
|
||||
providers=self._providers,
|
||||
)
|
||||
out_names = [o.name for o in session.get_outputs()]
|
||||
# Model card: sentence_embedding is the pooled output (last_hidden_state
|
||||
# is the per-token output we don't want).
|
||||
output_idx = (
|
||||
out_names.index("sentence_embedding") if "sentence_embedding" in out_names else 1
|
||||
)
|
||||
|
||||
tokenizer = Tokenizer.from_file(tok_path)
|
||||
tokenizer.enable_padding()
|
||||
tokenizer.enable_truncation(max_length=_EMBEDDINGGEMMA_MAX_LEN)
|
||||
self._tokenizer = tokenizer
|
||||
self._np = np
|
||||
tokenizer = Tokenizer.from_file(tok_path)
|
||||
tokenizer.enable_padding()
|
||||
tokenizer.enable_truncation(max_length=_EMBEDDINGGEMMA_MAX_LEN)
|
||||
self._output_idx = output_idx
|
||||
self._tokenizer = tokenizer
|
||||
self._np = np
|
||||
# Session is assigned last: the unlocked fast path above treats a
|
||||
# non-None session as "fully loaded", so every other attribute
|
||||
# must already be in place when it becomes visible.
|
||||
self._session = session
|
||||
|
||||
def __call__(self, input): # noqa: A002 — ChromaDB EF protocol uses `input`
|
||||
def __call__(self, input: str | list[str] | None) -> list[list[float]]: # noqa: A002 — ChromaDB EF protocol
|
||||
if isinstance(input, str):
|
||||
# A bare string would be iterated character by character below,
|
||||
# silently producing one garbage vector per character.
|
||||
input = [input]
|
||||
if input is None or len(input) == 0:
|
||||
# None or zero docs: nothing to embed; skip the lazy model
|
||||
# download. len() over truthiness so an array-like documents
|
||||
# sequence is not rejected by ambiguous-truth-value semantics.
|
||||
return []
|
||||
self._lazy_load()
|
||||
np = self._np
|
||||
texts = [_EMBEDDINGGEMMA_PREFIX + t for t in input]
|
||||
encs = self._tokenizer.encode_batch(texts)
|
||||
input_ids = np.asarray([e.ids for e in encs], dtype=np.int64)
|
||||
attention_mask = np.asarray([e.attention_mask for e in encs], dtype=np.int64)
|
||||
outputs = self._session.run(
|
||||
None, {"input_ids": input_ids, "attention_mask": attention_mask}
|
||||
)
|
||||
sent_emb = outputs[self._output_idx][:, :_EMBEDDINGGEMMA_DIM]
|
||||
# L2-normalize so cosine similarity == dot product (matches what the
|
||||
# MTEB methodology assumes; ChromaDB's distance is configured for it).
|
||||
norms = np.linalg.norm(sent_emb, axis=1, keepdims=True) + 1e-12
|
||||
return (sent_emb / norms).tolist()
|
||||
embeddings: list[list[float]] = []
|
||||
# Tokenize and run per sub-batch, not over the whole input: padding
|
||||
# is to the longest sequence in the sub-batch, and the ONNX runtime
|
||||
# only ever holds batch_size rows of attention buffers at a time
|
||||
# (#1770).
|
||||
for start in range(0, len(input), self._batch_size):
|
||||
chunk = input[start : start + self._batch_size]
|
||||
texts = [_EMBEDDINGGEMMA_PREFIX + t for t in chunk]
|
||||
encs = self._tokenizer.encode_batch(texts)
|
||||
input_ids = np.asarray([e.ids for e in encs], dtype=np.int64)
|
||||
attention_mask = np.asarray([e.attention_mask for e in encs], dtype=np.int64)
|
||||
outputs = self._session.run(
|
||||
None, {"input_ids": input_ids, "attention_mask": attention_mask}
|
||||
)
|
||||
sent_emb = outputs[self._output_idx][:, :_EMBEDDINGGEMMA_DIM]
|
||||
# L2-normalize so cosine similarity == dot product (matches what the
|
||||
# MTEB methodology assumes; ChromaDB's distance is configured for it).
|
||||
norms = np.linalg.norm(sent_emb, axis=1, keepdims=True) + 1e-12
|
||||
embeddings.extend((sent_emb / norms).tolist())
|
||||
return embeddings
|
||||
|
||||
def embed_query(self, input: list[str]) -> list[list[float]]: # noqa: A002 — ChromaDB EF protocol
|
||||
"""Embed query documents (ChromaDB EF protocol)."""
|
||||
return self(input)
|
||||
|
||||
def embed_documents(self, input: list[str]) -> list[list[float]]: # noqa: A002
|
||||
"""Embed a batch of documents (ChromaDB EF protocol)."""
|
||||
return self(input)
|
||||
|
||||
|
||||
def get_embedding_function(device: Optional[str] = None, model: Optional[str] = None):
|
||||
@@ -243,18 +379,23 @@ def get_embedding_function(device: Optional[str] = None, model: Optional[str] =
|
||||
|
||||
providers, effective = _resolve_providers(device)
|
||||
cache_key = (model, tuple(providers))
|
||||
cached = _EF_CACHE.get(cache_key)
|
||||
cached = _EF_CACHE.get(cache_key) # lock-free fast path; dict.get is GIL-atomic
|
||||
if cached is not None:
|
||||
return cached
|
||||
with _EF_CACHE_LOCK:
|
||||
cached = _EF_CACHE.get(cache_key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
if model == "embeddinggemma":
|
||||
ef = EmbeddinggemmaONNX(preferred_providers=providers)
|
||||
else:
|
||||
# Default: minilm (or anything we don't recognize — back-compat win).
|
||||
ef_cls = _build_ef_class()
|
||||
ef = ef_cls(preferred_providers=providers)
|
||||
threads = _resolve_intra_op_threads()
|
||||
if model == "embeddinggemma":
|
||||
ef = EmbeddinggemmaONNX(preferred_providers=providers, intra_op_num_threads=threads)
|
||||
else:
|
||||
# Default: minilm (or anything we don't recognize — back-compat win).
|
||||
ef_cls = _build_ef_class()
|
||||
ef = ef_cls(preferred_providers=providers, intra_op_num_threads=threads)
|
||||
|
||||
_EF_CACHE[cache_key] = ef
|
||||
_EF_CACHE[cache_key] = ef
|
||||
logger.info(
|
||||
"Embedding function initialized (model=%s device=%s providers=%s)",
|
||||
model,
|
||||
@@ -276,3 +417,59 @@ def describe_device(device: Optional[str] = None) -> str:
|
||||
device = MempalaceConfig().embedding_device
|
||||
_, effective = _resolve_providers(device)
|
||||
return effective
|
||||
|
||||
|
||||
# Probed vector widths, keyed by resolved model name. Populated once per
|
||||
# process the first time an identity is resolved for a model.
|
||||
_DIM_CACHE: dict = {}
|
||||
|
||||
|
||||
def current_model_name(model: Optional[str] = None) -> str:
|
||||
"""Resolve the canonical embedder model name (cheap, no model load).
|
||||
|
||||
This is the configured ``embedding_model`` (``"minilm"`` /
|
||||
``"embeddinggemma"`` / ...), not the embedding function's internal
|
||||
``name()`` (which is spoofed to ``"default"`` for ChromaDB compatibility).
|
||||
"""
|
||||
if model is not None:
|
||||
return str(model).strip().lower()
|
||||
from .config import MempalaceConfig
|
||||
|
||||
return MempalaceConfig().embedding_model
|
||||
|
||||
|
||||
def probe_dimension(device: Optional[str] = None, model: Optional[str] = None) -> int:
|
||||
"""Return the embedder's output dimension by embedding a short probe.
|
||||
|
||||
Model-agnostic — works for any model without a hardcoded table — and
|
||||
cached per resolved model name so the probe is paid at most once per
|
||||
process. Returns ``0`` if the probe fails (treated as "dimension unknown"
|
||||
by the identity check, so a probe failure never blocks normal operation).
|
||||
"""
|
||||
name = current_model_name(model)
|
||||
cached = _DIM_CACHE.get(name)
|
||||
if cached is not None:
|
||||
return cached
|
||||
try:
|
||||
ef = get_embedding_function(device=device, model=model)
|
||||
vectors = ef(input=["probe"])
|
||||
dim = len(vectors[0]) if vectors and vectors[0] is not None else 0
|
||||
except Exception:
|
||||
logger.debug("Embedding dimension probe failed for model=%s", name, exc_info=True)
|
||||
dim = 0
|
||||
_DIM_CACHE[name] = dim
|
||||
return dim
|
||||
|
||||
|
||||
def get_embedder_identity(device: Optional[str] = None, model: Optional[str] = None):
|
||||
"""Resolve the current embedder identity (RFC 001).
|
||||
|
||||
``model_name`` from config (cheap); ``dimension`` from a cached one-time
|
||||
probe. Returns an :class:`~mempalace.backends.base.EmbedderIdentity`.
|
||||
"""
|
||||
from .backends.base import EmbedderIdentity
|
||||
|
||||
return EmbedderIdentity(
|
||||
model_name=current_model_name(model),
|
||||
dimension=probe_dimension(device=device, model=model),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""No-LLM structural entity extraction for the associative graph.
|
||||
|
||||
Pulls deterministic, *structural* tokens from text — author-quoted code spans, URLs,
|
||||
file paths, qualified identifiers, and CamelCase symbols — to populate the ``entities``
|
||||
drawer-metadata field that hallways/tunnels consume. Structural-only by design: no
|
||||
wordlists, no NLP models, no domain vocabulary, so it stays language-neutral and
|
||||
predictable, and biases to precision (only tokens that are unambiguously "a thing being
|
||||
referred to") over recall.
|
||||
|
||||
The output format matches what ``hallways._parse_entities`` expects: a ``;``-joined string.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
# Author-quoted code spans are the highest-signal structural marker: `foo`, `obj.method()`.
|
||||
_BACKTICK = re.compile(r"`([^`\n]{2,64})`")
|
||||
# URLs.
|
||||
_URL = re.compile(r"https?://[^\s)>\]}\"']+")
|
||||
# Paths with a separator and a short extension: rag/foo.py, a/b/c.tsx.
|
||||
_PATH = re.compile(r"\b[\w.-]+/[\w./-]*\.[A-Za-z][A-Za-z0-9]{0,4}\b")
|
||||
# Qualified dotted identifiers; each segment starts with a letter and is >=2 chars, so
|
||||
# "1.2.3", "e.g", and "i.e" are excluded: module.func, pkg.Class.method.
|
||||
_QUALIFIED = re.compile(r"\b[A-Za-z][A-Za-z0-9_]+(?:\.[A-Za-z][A-Za-z0-9_]+)+\b")
|
||||
# CamelCase with >=2 humps — strongly code-specific: ChromaBackend, MemoryStack.
|
||||
_CAMEL = re.compile(r"\b[A-Z][a-z0-9]+(?:[A-Z][a-z0-9]+)+\b")
|
||||
# snake_case (must contain an underscore, so it can't match plain English): do_thing,
|
||||
# _extract_authored_at. The optional leading/trailing `_?` matches dunder-style names
|
||||
# whose underscore would otherwise fall outside the `\b` boundary (`_` is a word char).
|
||||
_SNAKE = re.compile(r"\b_?[a-z][a-z0-9]*(?:_[a-z0-9]+)+_?\b")
|
||||
|
||||
_MAX_ENTITIES = 24
|
||||
_MIN_LEN = 2
|
||||
_MAX_LEN = 64
|
||||
|
||||
|
||||
def _clean(token):
|
||||
# `;` is the entities-metadata separator, so it must never survive inside an entity
|
||||
# (e.g. a URL query string or a backtick span) or it would split the field.
|
||||
return token.replace(";", " ").strip().strip("`.,:()[]{}<>\"'").strip()
|
||||
|
||||
|
||||
def extract_structural_entities(text, max_entities=_MAX_ENTITIES):
|
||||
"""Return up to ``max_entities`` structural entities from ``text``.
|
||||
|
||||
Deterministic and order-stable: entities are ranked by occurrence count (ties broken
|
||||
by first appearance), deduplicated case-insensitively, preserving the first-seen
|
||||
surface form.
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
counts = {}
|
||||
order = {}
|
||||
seq = 0
|
||||
for pattern in (_BACKTICK, _URL, _PATH, _QUALIFIED, _CAMEL, _SNAKE):
|
||||
for match in pattern.finditer(text):
|
||||
token = _clean(match.group(1) if pattern is _BACKTICK else match.group(0))
|
||||
if not (_MIN_LEN <= len(token) <= _MAX_LEN):
|
||||
continue
|
||||
key = token.lower()
|
||||
if key not in counts:
|
||||
counts[key] = 0
|
||||
order[key] = (seq, token)
|
||||
seq += 1
|
||||
counts[key] += 1
|
||||
ranked = sorted(order, key=lambda k: (-counts[k], order[k][0]))
|
||||
return [order[k][1] for k in ranked[:max_entities]]
|
||||
|
||||
|
||||
def entities_metadata(text, max_entities=_MAX_ENTITIES):
|
||||
"""``;``-joined entity string for drawer metadata, or ``""`` when none are found."""
|
||||
return ";".join(extract_structural_entities(text, max_entities=max_entities))
|
||||
@@ -78,6 +78,98 @@ def _get_coca_filter() -> frozenset[str]:
|
||||
return frozenset()
|
||||
|
||||
|
||||
# ==================== KNOWN-SYSTEMS COMPOUND LEXICON (Tier 3 linguistics cleanup) ====================
|
||||
#
|
||||
# Multi-word product / system names that must be detected atomically — NOT
|
||||
# decomposed into their constituent words. When "Claude Code" appears in
|
||||
# content, the entity detector counts the compound, not the parts. Without
|
||||
# this pre-pass, the single-word loop would split "Claude Code" into
|
||||
# "Claude" + "Code", and the COCA filter (Tier 2) would drop "Code" as a
|
||||
# content word — leaving "Claude" alone with the wrong attribution.
|
||||
#
|
||||
# Data file: ``mempalace/data/known_systems.json``. Loaded once on first
|
||||
# call via ``_get_known_systems``. Matching is case-insensitive with word
|
||||
# boundaries.
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _get_known_systems() -> tuple[tuple[str, "re.Pattern[str]"], ...]:
|
||||
"""Return the known-systems compound tuple — pairs of (canonical name,
|
||||
pre-compiled case-insensitive word-bounded regex).
|
||||
|
||||
Loads ``mempalace/data/known_systems.json`` on first call, compiles a
|
||||
regex for each valid compound, and caches the resulting tuple of
|
||||
pairs. Subsequent calls are O(1) and skip both the disk read AND the
|
||||
regex compilation. Returns an empty tuple if the data file is missing
|
||||
or malformed — extraction behavior then degrades gracefully
|
||||
(compounds detected only by the existing multi-word regex) rather
|
||||
than crashing.
|
||||
|
||||
Entries are sorted by length descending so the compound matcher
|
||||
prefers longer matches first (e.g. "Visual Studio Code" wins over
|
||||
a hypothetical "Visual Studio" if both were in the lexicon).
|
||||
"""
|
||||
data_path = Path(__file__).parent / "data" / "known_systems.json"
|
||||
try:
|
||||
raw = json.loads(data_path.read_text(encoding="utf-8"))
|
||||
compounds = raw.get("compounds", [])
|
||||
valid = [c for c in compounds if isinstance(c, str) and c.strip()]
|
||||
# Sort by length descending so longest-match-wins during the
|
||||
# pre-pass scan (longer compounds get masked first, so a shorter
|
||||
# compound contained within a longer one doesn't double-count).
|
||||
sorted_compounds = sorted(valid, key=len, reverse=True)
|
||||
|
||||
compiled: list[tuple[str, re.Pattern[str]]] = []
|
||||
for c in sorted_compounds:
|
||||
# Word-boundary, case-insensitive. Compound may contain
|
||||
# hyphens or spaces — re.escape handles special chars; word
|
||||
# boundaries on each side prevent partial-word matches
|
||||
# (e.g. "GPT-4" must not match "GPT-40").
|
||||
pattern = r"(?<!\w)" + re.escape(c) + r"(?!\w)"
|
||||
try:
|
||||
compiled.append((c, re.compile(pattern, re.IGNORECASE)))
|
||||
except re.error:
|
||||
continue
|
||||
return tuple(compiled)
|
||||
except (OSError, json.JSONDecodeError, AttributeError, TypeError):
|
||||
return ()
|
||||
|
||||
|
||||
def _apply_known_systems_prepass(text: str) -> tuple[str, dict[str, int]]:
|
||||
"""Scan ``text`` for known-systems compounds, return a working copy
|
||||
with matched spans masked to whitespace plus a dict of detected
|
||||
compound counts.
|
||||
|
||||
Returning the counts (instead of mutating a caller-supplied container)
|
||||
lets the three call sites (``extract_candidates`` at init-time,
|
||||
``palace.build_closet_lines`` at closet construction, and
|
||||
``miner._extract_entities_for_metadata`` at per-drawer tagging) use
|
||||
whichever container shape they already maintain.
|
||||
|
||||
Compounds are matched case-insensitively with word boundaries; the
|
||||
canonical (lexicon) casing is what gets counted, regardless of how
|
||||
the compound appears in the source text. Regexes are pre-compiled
|
||||
once in ``_get_known_systems`` so this function does no compilation.
|
||||
"""
|
||||
compounds = _get_known_systems()
|
||||
if not compounds:
|
||||
return text, {}
|
||||
working = text
|
||||
compound_counts: dict[str, int] = {}
|
||||
for compound, rx in compounds:
|
||||
matches = list(rx.finditer(working))
|
||||
if not matches:
|
||||
continue
|
||||
compound_counts[compound] = compound_counts.get(compound, 0) + len(matches)
|
||||
# Mask matched spans with spaces so the subsequent regex passes
|
||||
# don't re-decompose. Replacing right-to-left keeps earlier
|
||||
# indices stable.
|
||||
for m in reversed(matches):
|
||||
start, end = m.span()
|
||||
working = working[:start] + (" " * (end - start)) + working[end:]
|
||||
return working, compound_counts
|
||||
|
||||
|
||||
# ==================== LANGUAGE-AWARE PATTERN LOADING ====================
|
||||
|
||||
|
||||
@@ -123,6 +215,8 @@ PROSE_EXTENSIONS = {
|
||||
".md",
|
||||
".rst",
|
||||
".csv",
|
||||
".tex",
|
||||
".bib",
|
||||
}
|
||||
|
||||
READABLE_EXTENSIONS = {
|
||||
@@ -194,13 +288,21 @@ def extract_candidates(text: str, languages=("en",)) -> dict:
|
||||
|
||||
counts: defaultdict = defaultdict(int)
|
||||
|
||||
# Tier 3 — known-systems compound pre-pass. Find compound product names
|
||||
# ("Claude Code", "GitHub Copilot", ...) FIRST and mask them out of the
|
||||
# working text so the subsequent single-word + multi-word loops don't
|
||||
# re-decompose them into their constituent tokens.
|
||||
working_text, compound_counts = _apply_known_systems_prepass(text)
|
||||
for compound, n in compound_counts.items():
|
||||
counts[compound] += n
|
||||
|
||||
# Single-word candidates — one pre-wrapped pattern per language
|
||||
for wrapped_pat in patterns["candidate_patterns"]:
|
||||
try:
|
||||
rx = re.compile(wrapped_pat)
|
||||
except re.error:
|
||||
continue
|
||||
for word in rx.findall(text):
|
||||
for word in rx.findall(working_text):
|
||||
wl = word.lower()
|
||||
if wl in stopwords:
|
||||
continue
|
||||
@@ -214,13 +316,16 @@ def extract_candidates(text: str, languages=("en",)) -> dict:
|
||||
continue
|
||||
counts[word] += 1
|
||||
|
||||
# Multi-word candidates — one pre-wrapped pattern per language
|
||||
# Multi-word candidates — one pre-wrapped pattern per language.
|
||||
# Runs against the working_text (compounds already masked) so an
|
||||
# unknown two-word phrase like "Jane Smith" still gets caught by
|
||||
# the regex without competing with known compounds.
|
||||
for wrapped_pat in patterns["multi_word_patterns"]:
|
||||
try:
|
||||
rx = re.compile(wrapped_pat)
|
||||
except re.error:
|
||||
continue
|
||||
for phrase in rx.findall(text):
|
||||
for phrase in rx.findall(working_text):
|
||||
if any(w.lower() in stopwords for w in phrase.split()):
|
||||
continue
|
||||
counts[phrase] += 1
|
||||
|
||||
+21
-10
@@ -82,6 +82,8 @@ from .palace import (
|
||||
# mempalace.format_miner.<name>. Lazy imports inside functions would not
|
||||
# expose these as attributes of this module, breaking the test seams.
|
||||
from .config import MempalaceConfig, normalize_wing_name
|
||||
from .collision_scan import assert_no_collisions
|
||||
from .ids import ID_RECIPE, make_drawer_id_from_chunk
|
||||
from .miner import (
|
||||
_compute_topic_tunnels_for_wing,
|
||||
chunk_text,
|
||||
@@ -505,7 +507,7 @@ def scan_formats(directory: Union[Path, str]) -> list[Path]:
|
||||
|
||||
|
||||
def _print_mine_summary(
|
||||
files: list,
|
||||
files_seen: int,
|
||||
files_with_text: int,
|
||||
files_skipped: int,
|
||||
files_errored: int,
|
||||
@@ -521,7 +523,7 @@ def _print_mine_summary(
|
||||
print(f"\n{'=' * 55}")
|
||||
print(" Summary")
|
||||
print(f"{'-' * 55}")
|
||||
print(f" Files seen: {len(files)}")
|
||||
print(f" Files seen: {files_seen}")
|
||||
print(f" Files extracted: {files_with_text}")
|
||||
print(f" Files skipped: {files_skipped}")
|
||||
print(f" Files errored: {files_errored}")
|
||||
@@ -640,8 +642,7 @@ def _file_chunks_locked(
|
||||
batch_ids: list = []
|
||||
batch_metas: list = []
|
||||
for chunk in chunks[batch_start : batch_start + DRAWER_UPSERT_BATCH_SIZE]:
|
||||
key = (source_file + str(chunk["chunk_index"])).encode()
|
||||
drawer_id = f"drawer_{wing}_{room}_{hashlib.sha256(key).hexdigest()[:24]}"
|
||||
drawer_id = make_drawer_id_from_chunk(wing, room, source_file, chunk["chunk_index"])
|
||||
content = chunk["content"]
|
||||
meta: dict = {
|
||||
"wing": wing,
|
||||
@@ -654,6 +655,7 @@ def _file_chunks_locked(
|
||||
"extract_mode": "format",
|
||||
"normalize_version": NORMALIZE_VERSION,
|
||||
"hall": detect_hall(content),
|
||||
"id_recipe": ID_RECIPE,
|
||||
}
|
||||
if source_mtime is not None:
|
||||
meta["source_mtime"] = source_mtime
|
||||
@@ -674,6 +676,7 @@ def _file_chunks_locked(
|
||||
batch_docs.append(content)
|
||||
batch_ids.append(drawer_id)
|
||||
batch_metas.append(meta)
|
||||
assert_no_collisions(list(zip(batch_ids, batch_metas)), collection)
|
||||
try:
|
||||
collection.upsert(
|
||||
documents=batch_docs,
|
||||
@@ -747,7 +750,7 @@ def mine_formats(
|
||||
# min_chunk_size) are now threaded through chunk_text below, so users
|
||||
# who customized their config see the effect in format-mode mining.
|
||||
# Per PR #1555 review (Gemini #3).
|
||||
palace_config = MempalaceConfig()
|
||||
palace_config = MempalaceConfig(palace_path=palace_path)
|
||||
|
||||
format_path = Path(format_dir).expanduser().resolve()
|
||||
if not wing:
|
||||
@@ -786,9 +789,11 @@ def mine_formats(
|
||||
files: list = []
|
||||
collection = None
|
||||
total_drawers = 0
|
||||
files_mined = 0
|
||||
files_skipped = 0
|
||||
files_with_text = 0
|
||||
files_errored = 0
|
||||
files_processed = 0
|
||||
status_counts: dict = defaultdict(int)
|
||||
|
||||
try:
|
||||
@@ -796,15 +801,14 @@ def mine_formats(
|
||||
# ``~/docs`` and relative inputs work consistently. Per PR #1555 review
|
||||
# (Copilot #10).
|
||||
files = scan_formats(format_path)
|
||||
if limit > 0:
|
||||
files = files[:limit]
|
||||
|
||||
print(f"\n{'=' * 55}")
|
||||
print(" MemPalace Mine — Format extraction")
|
||||
print(f"{'=' * 55}")
|
||||
print(f" Wing: {wing}")
|
||||
print(f" Source: {format_path}")
|
||||
print(f" Files: {len(files)}")
|
||||
limit_suffix = f" (limit: {limit} new)" if limit > 0 else ""
|
||||
print(f" Files: {len(files)}{limit_suffix}")
|
||||
print(f" Palace: {palace_path}")
|
||||
if dry_run:
|
||||
print(" DRY RUN — nothing will be filed")
|
||||
@@ -813,6 +817,7 @@ def mine_formats(
|
||||
collection = get_collection(palace_path) if not dry_run else None
|
||||
|
||||
for i, filepath in enumerate(files, 1):
|
||||
files_processed = i
|
||||
source_file = str(filepath)
|
||||
|
||||
# Per-file try/except so one bad file can't crash the whole mine.
|
||||
@@ -873,6 +878,9 @@ def mine_formats(
|
||||
if dry_run:
|
||||
print(f" [DRY RUN] {filepath.name} → {len(chunks)} drawers")
|
||||
total_drawers += len(chunks)
|
||||
files_mined += 1
|
||||
if limit > 0 and files_mined >= limit:
|
||||
break
|
||||
continue
|
||||
|
||||
drawers_added, skipped = _file_chunks_locked(
|
||||
@@ -890,7 +898,10 @@ def mine_formats(
|
||||
continue
|
||||
|
||||
total_drawers += drawers_added
|
||||
files_mined += 1
|
||||
print(f" + [{i:4}/{len(files)}] {filepath.name[:50]:50} +{drawers_added}")
|
||||
if limit > 0 and files_mined >= limit:
|
||||
break
|
||||
except Exception as exc:
|
||||
# Log and continue — one malformed file shouldn't kill the
|
||||
# whole mine. Mirrors miner.py's per-file recovery.
|
||||
@@ -934,7 +945,7 @@ def mine_formats(
|
||||
# skipped quietly.
|
||||
if not dry_run:
|
||||
try:
|
||||
tunnels_added = _compute_topic_tunnels_for_wing(wing)
|
||||
tunnels_added = _compute_topic_tunnels_for_wing(wing, config=palace_config)
|
||||
if tunnels_added:
|
||||
print(f"\n Topic tunnels: +{tunnels_added} cross-wing link(s)")
|
||||
except Exception as exc:
|
||||
@@ -970,7 +981,7 @@ def mine_formats(
|
||||
logger.debug("mine_formats: _cleanup_mine_pid_file failed", exc_info=True)
|
||||
|
||||
_print_mine_summary(
|
||||
files=files,
|
||||
files_seen=files_processed,
|
||||
files_with_text=files_with_text,
|
||||
files_skipped=files_skipped,
|
||||
files_errored=files_errored,
|
||||
|
||||
+97
-34
@@ -46,10 +46,11 @@ from .dynamics import initialize_dynamics_fields
|
||||
|
||||
logger = logging.getLogger("mempalace_hallways")
|
||||
|
||||
# Persistence target. Mirrors ``palace_graph._TUNNEL_FILE`` so the storage
|
||||
# pattern is uniform across the two related primitives. Tests override
|
||||
# this via ``monkeypatch.setattr(hallways, "_HALLWAY_FILE", tmp_path/...)``.
|
||||
_HALLWAY_FILE = os.path.join(os.path.expanduser("~"), ".mempalace", "hallways.json")
|
||||
# Persistence target is resolved through ``_get_hallway_file`` below, which
|
||||
# mirrors ``palace_graph._get_tunnel_file`` (the 3.3.6 palace-scoped pattern)
|
||||
# so the storage layout is uniform across the two related primitives. Tests
|
||||
# should monkey-patch ``_get_hallway_file`` and ``_legacy_hallway_file`` rather
|
||||
# than poking a module-level constant.
|
||||
|
||||
_SCHEMA_VERSION = 1
|
||||
|
||||
@@ -62,36 +63,72 @@ __all__ = [
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# Persistence — JSON file at _HALLWAY_FILE, restricted perms (0600) on POSIX
|
||||
# Persistence — JSON file resolved from MempalaceConfig.hallway_file,
|
||||
# restricted perms (0600) on POSIX. Pre-3.3.6 behavior (hardcoded
|
||||
# ~/.mempalace/hallways.json) is kept only as a one-time orphan detection
|
||||
# fallback, matching the palace_graph tunnel-file migration pattern.
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _load_hallways() -> list[dict]:
|
||||
"""Read all hallway records. Returns ``[]`` if the file is missing or corrupt."""
|
||||
if not os.path.exists(_HALLWAY_FILE):
|
||||
def _get_hallway_file(config=None) -> str:
|
||||
"""Return the path to the hallways.json file, derived from MempalaceConfig.palace_path."""
|
||||
from .config import MempalaceConfig
|
||||
|
||||
config = config or MempalaceConfig()
|
||||
return config.hallway_file
|
||||
|
||||
|
||||
def _legacy_hallway_file() -> str:
|
||||
"""The pre-palace-scoped hardcoded path. Kept only for one-time orphan detection."""
|
||||
return os.path.join(os.path.expanduser("~"), ".mempalace", "hallways.json")
|
||||
|
||||
|
||||
def _load_hallways(config=None) -> list[dict]:
|
||||
"""Read all hallway records. Returns ``[]`` if the file is missing or corrupt.
|
||||
|
||||
Backwards-compatibility: prior to this migration the hallway file was
|
||||
hardcoded at ``~/.mempalace/hallways.json`` regardless of the configured
|
||||
palace_path. If the configured hallway file is missing but a legacy file
|
||||
exists at a different path, log a one-line warning naming both paths so
|
||||
users can move the file manually. We do NOT auto-migrate — auto-merging
|
||||
hallway state across two locations is too magical for a bugfix and risks
|
||||
clobbering newer data. Same posture as ``palace_graph._load_tunnels``.
|
||||
"""
|
||||
current_hallway_file = _get_hallway_file(config)
|
||||
if os.path.exists(current_hallway_file):
|
||||
try:
|
||||
with open(current_hallway_file, encoding="utf-8") as f:
|
||||
raw = json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
logger.debug("hallways: load failed, treating as empty", exc_info=True)
|
||||
return []
|
||||
if isinstance(raw, dict) and "hallways" in raw:
|
||||
return raw.get("hallways") or []
|
||||
if isinstance(raw, list):
|
||||
return raw
|
||||
return []
|
||||
try:
|
||||
with open(_HALLWAY_FILE, encoding="utf-8") as f:
|
||||
raw = json.load(f)
|
||||
except (OSError, json.JSONDecodeError):
|
||||
logger.debug("hallways: load failed, treating as empty", exc_info=True)
|
||||
return []
|
||||
if isinstance(raw, dict) and "hallways" in raw:
|
||||
return raw.get("hallways") or []
|
||||
if isinstance(raw, list):
|
||||
return raw
|
||||
|
||||
legacy = _legacy_hallway_file()
|
||||
if legacy != current_hallway_file and os.path.exists(legacy):
|
||||
logger.warning(
|
||||
"Legacy hallways file at '%s' is being ignored; configured location is '%s'. "
|
||||
"Move or copy the legacy file to the configured path to recover its hallways.",
|
||||
legacy,
|
||||
current_hallway_file,
|
||||
)
|
||||
return []
|
||||
|
||||
|
||||
def _save_hallways(hallways: list[dict]) -> None:
|
||||
"""Atomically persist hallway records to _HALLWAY_FILE.
|
||||
def _save_hallways(hallways: list[dict], config=None) -> None:
|
||||
"""Atomically persist hallway records to the configured hallway file.
|
||||
|
||||
Uses an os.replace temp-file dance so a crash mid-write doesn't
|
||||
corrupt the file. POSIX permission is restricted to 0600 because
|
||||
hallways reveal within-wing entity connections that the user may
|
||||
not want world-readable.
|
||||
"""
|
||||
directory = os.path.dirname(_HALLWAY_FILE)
|
||||
hallway_file = _get_hallway_file(config)
|
||||
directory = os.path.dirname(hallway_file)
|
||||
os.makedirs(directory, exist_ok=True)
|
||||
payload = {
|
||||
"schema_version": _SCHEMA_VERSION,
|
||||
@@ -106,7 +143,7 @@ def _save_hallways(hallways: list[dict]) -> None:
|
||||
except OSError:
|
||||
# Non-POSIX systems may not support chmod; not fatal.
|
||||
pass
|
||||
os.replace(tmp_path, _HALLWAY_FILE)
|
||||
os.replace(tmp_path, hallway_file)
|
||||
except Exception:
|
||||
try:
|
||||
os.unlink(tmp_path)
|
||||
@@ -162,6 +199,7 @@ def compute_hallways_for_wing(
|
||||
wing: str,
|
||||
col=None,
|
||||
min_count: int = 2,
|
||||
config=None,
|
||||
) -> list[dict]:
|
||||
"""Compute entity-pair hallways for one wing.
|
||||
|
||||
@@ -179,7 +217,12 @@ def compute_hallways_for_wing(
|
||||
|
||||
Args:
|
||||
wing: wing name to scan.
|
||||
col: ChromaDB collection — must support ``.get(where=..., include=...)``.
|
||||
col: ChromaDB collection — must support ``.count()`` and paginated
|
||||
``.get(limit=..., offset=..., include=...)``. The fetch is filtered
|
||||
to ``wing`` client-side rather than via ``.get(where={"wing": ...})``,
|
||||
which binds one SQL variable per matched id and overflows SQLite's
|
||||
``SQLITE_MAX_VARIABLE_NUMBER`` on large wings (#1619). Fake
|
||||
collections and alternate backends must implement this shape.
|
||||
If ``None``, returns ``[]`` (caller didn't supply a backing
|
||||
store, so nothing to compute against). Tests pass a controlled
|
||||
MagicMock.
|
||||
@@ -187,6 +230,10 @@ def compute_hallways_for_wing(
|
||||
hallway between two entities. Default 2 — single co-occurrences
|
||||
are noise (entities mentioned together once in one drawer);
|
||||
two or more is a real signal. Clamped to ``>=1``.
|
||||
config: Optional ``MempalaceConfig`` selecting the palace-scoped
|
||||
hallway sidecar. Callers using an explicit palace path must pass
|
||||
the matching config so derived graph state cannot leak into the
|
||||
default palace.
|
||||
|
||||
Returns:
|
||||
List of hallway dicts created for this wing. Records for other
|
||||
@@ -198,16 +245,32 @@ def compute_hallways_for_wing(
|
||||
|
||||
min_count = max(1, int(min_count))
|
||||
|
||||
# 1. Query drawers for this wing.
|
||||
# 1. Query drawers for this wing. Paginate the fetch and filter to the
|
||||
# wing client-side: a single get(where={"wing": wing}) binds one SQL
|
||||
# variable per matched id and overflows SQLite's
|
||||
# SQLITE_MAX_VARIABLE_NUMBER (32766) on wings > ~32k drawers (#1619).
|
||||
# Mirrors the established pagination in miner.status / palace /
|
||||
# palace_graph.
|
||||
metadatas: list = []
|
||||
try:
|
||||
results = col.get(where={"wing": wing}, include=["metadatas"])
|
||||
total = col.count()
|
||||
batch_size = 5000
|
||||
offset = 0
|
||||
while offset < total:
|
||||
batch = col.get(limit=batch_size, offset=offset, include=["metadatas"])
|
||||
batch_metas = (batch or {}).get("metadatas") or []
|
||||
if not batch_metas:
|
||||
break
|
||||
metadatas.extend(
|
||||
m for m in batch_metas if isinstance(m, dict) and m.get("wing") == wing
|
||||
)
|
||||
offset += len(batch_metas)
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"compute_hallways_for_wing: collection.get failed for %s", wing, exc_info=True
|
||||
"compute_hallways_for_wing: collection fetch failed for %s", wing, exc_info=True
|
||||
)
|
||||
return []
|
||||
|
||||
metadatas = (results or {}).get("metadatas") or []
|
||||
if not metadatas:
|
||||
return []
|
||||
|
||||
@@ -252,7 +315,7 @@ def compute_hallways_for_wing(
|
||||
# across recomputes. Without this preservation, every mine wipes
|
||||
# the connection weights accumulated through use — defeating the
|
||||
# living-connection layer entirely.
|
||||
existing = _load_hallways()
|
||||
existing = _load_hallways(config)
|
||||
existing_dynamics_lookup: dict = {}
|
||||
for h in existing:
|
||||
if h.get("wing") != wing:
|
||||
@@ -302,7 +365,7 @@ def compute_hallways_for_wing(
|
||||
|
||||
# 4. Persist — preserve other-wing records, replace this wing's records.
|
||||
preserved_other_wings = [h for h in existing if h.get("wing") != wing]
|
||||
_save_hallways(preserved_other_wings + created)
|
||||
_save_hallways(preserved_other_wings + created, config)
|
||||
|
||||
return created
|
||||
|
||||
@@ -312,19 +375,19 @@ def compute_hallways_for_wing(
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def list_hallways(wing: Optional[str] = None) -> list[dict]:
|
||||
def list_hallways(wing: Optional[str] = None, config=None) -> list[dict]:
|
||||
"""List hallway records. Filter by ``wing`` if specified."""
|
||||
all_hallways = _load_hallways()
|
||||
all_hallways = _load_hallways(config)
|
||||
if wing is None:
|
||||
return list(all_hallways)
|
||||
return [h for h in all_hallways if h.get("wing") == wing]
|
||||
|
||||
|
||||
def delete_hallway(hallway_id: str) -> bool:
|
||||
def delete_hallway(hallway_id: str, config=None) -> bool:
|
||||
"""Remove one hallway record by id. Returns True if a record was removed."""
|
||||
hallways = _load_hallways()
|
||||
hallways = _load_hallways(config)
|
||||
filtered = [h for h in hallways if h.get("id") != hallway_id]
|
||||
if len(filtered) == len(hallways):
|
||||
return False
|
||||
_save_hallways(filtered)
|
||||
_save_hallways(filtered, config)
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,159 @@
|
||||
"""Compatibility helpers for legacy shell hooks.
|
||||
|
||||
The shell hooks intentionally stay small and portable, but parsing Claude
|
||||
hook JSON and counting UTF-8 JSONL transcripts is safer in Python than in
|
||||
inline shell snippets. This module centralizes that behavior for both
|
||||
hooks/mempal_save_hook.sh and hooks/mempal_precompact_hook.sh.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
_SESSION_ID_RE = re.compile(r"[^a-zA-Z0-9_-]")
|
||||
_CONTROL_CHARS_RE = re.compile(r"[\x00\r\n]")
|
||||
|
||||
|
||||
def sanitize_session_id(session_id: object) -> str:
|
||||
"""Keep session ids safe for state-file names."""
|
||||
sanitized = _SESSION_ID_RE.sub("", str(session_id or ""))
|
||||
return sanitized or "unknown"
|
||||
|
||||
|
||||
def normalize_transcript_path(path: object) -> str:
|
||||
r"""Normalize a hook transcript path without destroying Windows paths.
|
||||
|
||||
Claude Code on Windows sends paths like:
|
||||
|
||||
C:\Users\me\.claude\projects\<project>\<session>.jsonl
|
||||
|
||||
The old shell sanitizer removed both the drive-letter colon and
|
||||
backslashes. That turned a valid transcript path into a nonexistent path.
|
||||
For transcript paths, we only remove control characters that would break
|
||||
newline-delimited shell parsing, and normalize backslashes to forward
|
||||
slashes so Git Bash can still address the same Windows file.
|
||||
"""
|
||||
|
||||
normalized = str(path or "").replace("\\", "/")
|
||||
return _CONTROL_CHARS_RE.sub("", normalized)
|
||||
|
||||
|
||||
def _stop_hook_active(value: object) -> str:
|
||||
"""Return the exact boolean string expected by the shell hook."""
|
||||
if value is True:
|
||||
return "True"
|
||||
if str(value).strip().lower() in ("true", "1", "yes"):
|
||||
return "True"
|
||||
return "False"
|
||||
|
||||
|
||||
def parse_stop_payload(payload: dict) -> tuple[str, str, str]:
|
||||
return (
|
||||
sanitize_session_id(payload.get("session_id", "")),
|
||||
_stop_hook_active(payload.get("stop_hook_active", False)),
|
||||
normalize_transcript_path(payload.get("transcript_path", "")),
|
||||
)
|
||||
|
||||
|
||||
def parse_precompact_payload(payload: dict) -> tuple[str, str]:
|
||||
return (
|
||||
sanitize_session_id(payload.get("session_id", "")),
|
||||
normalize_transcript_path(payload.get("transcript_path", "")),
|
||||
)
|
||||
|
||||
|
||||
def count_human_messages(path: str) -> int:
|
||||
"""Count user messages in a Claude transcript JSONL file.
|
||||
|
||||
Claude transcripts are UTF-8. Windows Python defaults to cp1252 in many
|
||||
environments, so the encoding must be explicit. Invalid bytes are ignored
|
||||
to match the hooks' fail-soft behavior.
|
||||
"""
|
||||
|
||||
count = 0
|
||||
with open(path, encoding="utf-8", errors="ignore") as fh:
|
||||
for line in fh:
|
||||
try:
|
||||
entry = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
msg = entry.get("message", {})
|
||||
if not isinstance(msg, dict) or msg.get("role") != "user":
|
||||
continue
|
||||
|
||||
content = msg.get("content", "")
|
||||
if isinstance(content, str) and "<command-message>" in content:
|
||||
continue
|
||||
|
||||
count += 1
|
||||
|
||||
return count
|
||||
|
||||
|
||||
def _load_stdin_json() -> dict:
|
||||
raw = sys.stdin.read()
|
||||
|
||||
# Empty stdin is a legitimate hook state. Treat it as an empty payload so
|
||||
# the sentinel is printed and the shell fail-loud guard does not spam disk.
|
||||
if raw == "":
|
||||
return {}
|
||||
|
||||
# For non-empty malformed input, intentionally let json.loads raise.
|
||||
# The shell hooks capture this stderr in last_python_err.log and, because
|
||||
# no sentinel is printed, write a bounded copy of the raw payload to
|
||||
# last_input.log. That fail-loud contract is pinned by
|
||||
# tests/test_hooks_bash_compat.py.
|
||||
data = json.loads(raw)
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise TypeError(f"hook input must be a JSON object, got {type(data).__name__}")
|
||||
|
||||
return data
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
argv = list(sys.argv[1:] if argv is None else argv)
|
||||
if not argv:
|
||||
print(
|
||||
"usage: python -m mempalace.hook_shell <parse-stop|parse-precompact|count-human-messages>",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
command = argv[0]
|
||||
|
||||
if command == "parse-stop":
|
||||
session_id, stop_hook_active, transcript_path = parse_stop_payload(_load_stdin_json())
|
||||
print("__MEMPAL_PARSE_OK__")
|
||||
print(session_id)
|
||||
print(stop_hook_active)
|
||||
print(transcript_path)
|
||||
return 0
|
||||
|
||||
if command == "parse-precompact":
|
||||
session_id, transcript_path = parse_precompact_payload(_load_stdin_json())
|
||||
print("__MEMPAL_PARSE_OK__")
|
||||
print(session_id)
|
||||
print(transcript_path)
|
||||
return 0
|
||||
|
||||
if command == "count-human-messages":
|
||||
if len(argv) != 2:
|
||||
print("count-human-messages requires a transcript path", file=sys.stderr)
|
||||
return 2
|
||||
try:
|
||||
print(count_human_messages(argv[1]))
|
||||
except Exception:
|
||||
print(0)
|
||||
return 0
|
||||
|
||||
print(f"unknown hook_shell command: {command}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
+341
-18
@@ -1,8 +1,8 @@
|
||||
"""
|
||||
Hook logic for MemPalace — Python implementation of session-start, stop, and precompact hooks.
|
||||
Hook logic for MemPalace — Python implementation of session-start, stop, session-end, and precompact hooks.
|
||||
|
||||
Reads JSON from stdin, outputs JSON to stdout.
|
||||
Supported hooks: session-start, stop, precompact
|
||||
Supported hooks: session-start, stop, session-end, precompact
|
||||
Supported harnesses: claude-code, codex (extensible to cursor, gemini, etc.)
|
||||
"""
|
||||
|
||||
@@ -25,7 +25,7 @@ PALACE_ROOT = Path.home() / ".mempalace"
|
||||
|
||||
|
||||
def _detached_popen_kwargs() -> dict:
|
||||
"""Kwargs that fully detach a Popen child so the hook process can exit.
|
||||
"""Kwargs that give a Popen child a hidden console so the hook can exit.
|
||||
|
||||
Without these, Windows holds the parent open until the child closes the
|
||||
inherited stdout/stderr handles — manifesting as "Stop hook hangs" at
|
||||
@@ -36,7 +36,7 @@ def _detached_popen_kwargs() -> dict:
|
||||
kwargs: dict = {"stdin": subprocess.DEVNULL, "close_fds": True}
|
||||
if os.name == "nt":
|
||||
flags = 0
|
||||
for name in ("DETACHED_PROCESS", "CREATE_NEW_PROCESS_GROUP", "CREATE_BREAKAWAY_FROM_JOB"):
|
||||
for name in ("CREATE_NO_WINDOW", "CREATE_NEW_PROCESS_GROUP", "CREATE_BREAKAWAY_FROM_JOB"):
|
||||
flags |= getattr(subprocess, name, 0)
|
||||
if flags:
|
||||
kwargs["creationflags"] = flags
|
||||
@@ -509,6 +509,73 @@ def _spawn_mine(cmd: list) -> None:
|
||||
pass
|
||||
|
||||
|
||||
def _hooks_daemon_enabled() -> bool:
|
||||
try:
|
||||
return MempalaceConfig().hook_use_daemon is True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _daemon_mine_dedupe_key(source: str, mode: str) -> str:
|
||||
try:
|
||||
source_key = str(Path(source).expanduser().resolve())
|
||||
except OSError:
|
||||
source_key = str(Path(source).expanduser())
|
||||
return f"hook:mine:{mode}:{source_key}"
|
||||
|
||||
|
||||
def _daemon_available() -> bool:
|
||||
"""True iff a daemon is already running for the configured palace.
|
||||
|
||||
This is a fast localhost health check, not a spawn: the 500ms hook budget
|
||||
forbids auto-starting a python subprocess from a hook (cold start is
|
||||
~15s). Daemon mode for hooks requires the user to have started the daemon
|
||||
explicitly via `mempalace daemon start`; when it isn't up, hooks fall back
|
||||
to the existing direct (in-process / spawn) path instead of blocking.
|
||||
"""
|
||||
from .daemon import HOOK_PROBE_TIMEOUT, get_client_if_running
|
||||
|
||||
try:
|
||||
return (
|
||||
get_client_if_running(MempalaceConfig().palace_path, health_timeout=HOOK_PROBE_TIMEOUT)
|
||||
is not None
|
||||
)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
def _submit_daemon_job(
|
||||
kind: str,
|
||||
payload: dict,
|
||||
*,
|
||||
dedupe_key: str = None,
|
||||
priority: int = 0,
|
||||
wait: bool = False,
|
||||
timeout: float = 60.0,
|
||||
):
|
||||
"""Submit to an already-running daemon. Never auto-starts (see _daemon_available).
|
||||
|
||||
Raises DaemonError on a real failure (job rejected, timeout, daemon died
|
||||
mid-submit). Callers must NOT fall back to the direct path on such errors —
|
||||
the daemon may already have accepted the job, and re-running it would
|
||||
duplicate verbatim content. Only an absent daemon (handled by the caller's
|
||||
_daemon_available() precheck) should fall back.
|
||||
"""
|
||||
from .daemon import submit_job
|
||||
|
||||
palace_path = MempalaceConfig().palace_path
|
||||
return submit_job(
|
||||
kind,
|
||||
payload,
|
||||
palace_path=palace_path,
|
||||
dedupe_key=dedupe_key,
|
||||
priority=priority,
|
||||
wait=wait,
|
||||
auto_start=False,
|
||||
timeout=timeout,
|
||||
)
|
||||
|
||||
|
||||
def _maybe_auto_ingest():
|
||||
"""Background-mine MEMPAL_DIR (project files) if set.
|
||||
|
||||
@@ -527,9 +594,26 @@ def _maybe_auto_ingest():
|
||||
return
|
||||
for mine_dir, mode in targets:
|
||||
try:
|
||||
if _hooks_daemon_enabled() and _daemon_available():
|
||||
try:
|
||||
_submit_daemon_job(
|
||||
"mine",
|
||||
{"source": mine_dir, "mode": mode, "agent": "mempalace"},
|
||||
dedupe_key=_daemon_mine_dedupe_key(mine_dir, mode),
|
||||
wait=False,
|
||||
)
|
||||
except Exception as exc:
|
||||
# Daemon accepted context — don't fall back (would double-mine).
|
||||
_log(f"Daemon mine submission failed: {exc}")
|
||||
continue
|
||||
_spawn_mine([_mempalace_python(), "-m", "mempalace", "mine", mine_dir, "--mode", mode])
|
||||
except OSError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
# Non-daemon spawn path failed. Hooks must never crash the user's
|
||||
# shell — log and continue. Do not label this a daemon failure: the
|
||||
# daemon block above handles its own errors with its own message.
|
||||
_log(f"mine hook failed: {exc}")
|
||||
|
||||
|
||||
def _mine_sync():
|
||||
@@ -546,6 +630,22 @@ def _mine_sync():
|
||||
log_path = STATE_DIR / "hook.log"
|
||||
for mine_dir, mode in targets:
|
||||
try:
|
||||
if _hooks_daemon_enabled() and _daemon_available():
|
||||
try:
|
||||
job = _submit_daemon_job(
|
||||
"mine",
|
||||
{"source": mine_dir, "mode": mode, "agent": "mempalace"},
|
||||
dedupe_key=_daemon_mine_dedupe_key(mine_dir, mode),
|
||||
wait=True,
|
||||
timeout=60,
|
||||
)
|
||||
result = job.get("result") or {}
|
||||
if job.get("state") != "succeeded" or not result.get("success", True):
|
||||
_log(f"Daemon sync mine failed: {result.get('error', job.get('error'))}")
|
||||
except Exception as exc:
|
||||
# Daemon accepted context — don't fall back (would double-mine).
|
||||
_log(f"Daemon sync mine submission failed: {exc}")
|
||||
continue
|
||||
with open(log_path, "a") as log_f:
|
||||
subprocess.run(
|
||||
[
|
||||
@@ -560,9 +660,19 @@ def _mine_sync():
|
||||
stdout=log_f,
|
||||
stderr=log_f,
|
||||
timeout=60,
|
||||
# Windows: hide the conhost window this sync mine would
|
||||
# otherwise flash on every fire. Mirrors the async paths
|
||||
# (_spawn_mine / _desktop_toast) via _detached_popen_kwargs().
|
||||
# 0 is a no-op off-Windows.
|
||||
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0),
|
||||
)
|
||||
except (OSError, subprocess.TimeoutExpired):
|
||||
pass
|
||||
except Exception as exc:
|
||||
# Non-daemon sync spawn path failed. Hooks must never crash the
|
||||
# user's shell — log and continue (not a daemon failure; the daemon
|
||||
# block above handles its own errors).
|
||||
_log(f"mine hook failed: {exc}")
|
||||
|
||||
|
||||
def _desktop_toast(body: str, title: str = "MemPalace"):
|
||||
@@ -651,12 +761,16 @@ def _save_diary_direct(
|
||||
session_id: str,
|
||||
wing: str = "",
|
||||
toast: bool = False,
|
||||
*,
|
||||
agent_name: str,
|
||||
) -> dict:
|
||||
"""Write a diary checkpoint by calling the tool function directly (no MCP roundtrip).
|
||||
|
||||
If `wing` is set, the entry lands in that wing (typically the project wing
|
||||
derived from the transcript path). Otherwise falls back to `tool_diary_write`'s
|
||||
default of `wing_session-hook`.
|
||||
The entry is filed under `agent_name` so the agent that later calls
|
||||
`mempalace_diary_read(agent_name=...)` discovers it (#1693). If `wing` is
|
||||
set, the entry lands in that wing (typically the project wing derived from
|
||||
the transcript path); a `diary_read` with an empty wing spans every wing
|
||||
the agent wrote to, so project-derived wings stay discoverable.
|
||||
|
||||
Returns {"count": N, "themes": [...]} on success, {"count": 0} on failure.
|
||||
"""
|
||||
@@ -676,10 +790,45 @@ def _save_diary_direct(
|
||||
)
|
||||
|
||||
try:
|
||||
if _hooks_daemon_enabled() and _daemon_available():
|
||||
try:
|
||||
job = _submit_daemon_job(
|
||||
"diary_write",
|
||||
{
|
||||
"agent_name": agent_name,
|
||||
"entry": entry,
|
||||
"topic": "checkpoint",
|
||||
"wing": wing,
|
||||
},
|
||||
priority=10,
|
||||
wait=True,
|
||||
timeout=30,
|
||||
)
|
||||
except Exception as exc:
|
||||
# Daemon accepted context — don't fall back (would double-write).
|
||||
_log(f"Daemon diary checkpoint failed: {exc}")
|
||||
return {"count": 0}
|
||||
result = job.get("result") or {}
|
||||
if job.get("state") == "succeeded" and result.get("success"):
|
||||
_log(f"Diary checkpoint saved: {result.get('entry_id', '?')}")
|
||||
try:
|
||||
ack_file = STATE_DIR / "last_checkpoint"
|
||||
ack_file.write_text(
|
||||
json.dumps({"msgs": len(messages), "ts": now.isoformat()}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
except OSError:
|
||||
pass
|
||||
if toast:
|
||||
_desktop_toast(f"Checkpoint saved - {len(messages)} messages archived")
|
||||
return {"count": len(messages), "themes": themes}
|
||||
_log(f"Daemon diary checkpoint failed: {result.get('error', job.get('error'))}")
|
||||
return {"count": 0}
|
||||
|
||||
from .mcp_server import tool_diary_write
|
||||
|
||||
result = tool_diary_write(
|
||||
agent_name="session-hook",
|
||||
agent_name=agent_name,
|
||||
entry=entry,
|
||||
topic="checkpoint",
|
||||
wing=wing,
|
||||
@@ -707,8 +856,13 @@ def _save_diary_direct(
|
||||
|
||||
def _ingest_transcript(transcript_path: str):
|
||||
"""Mine a Claude Code session transcript into the palace as a conversation."""
|
||||
path = Path(transcript_path).expanduser()
|
||||
if not path.is_file() or path.stat().st_size < 100:
|
||||
path = _validate_transcript_path(transcript_path)
|
||||
if path is None:
|
||||
return
|
||||
try:
|
||||
if not path.is_file() or path.stat().st_size < 100:
|
||||
return
|
||||
except OSError:
|
||||
return
|
||||
|
||||
try:
|
||||
@@ -717,6 +871,25 @@ def _ingest_transcript(transcript_path: str):
|
||||
return
|
||||
|
||||
try:
|
||||
if _hooks_daemon_enabled() and _daemon_available():
|
||||
try:
|
||||
_submit_daemon_job(
|
||||
"mine",
|
||||
{
|
||||
"source": str(path.parent),
|
||||
"mode": "convos",
|
||||
"wing": "sessions",
|
||||
"agent": "mempalace",
|
||||
},
|
||||
dedupe_key=_daemon_mine_dedupe_key(str(path.parent), "convos"),
|
||||
wait=False,
|
||||
)
|
||||
_log(f"Transcript ingest submitted to daemon: {path.name}")
|
||||
except Exception as exc:
|
||||
# Daemon accepted context — don't fall back (would double-mine).
|
||||
_log(f"Daemon transcript ingest failed: {exc}")
|
||||
return
|
||||
|
||||
# Route through ``_spawn_mine`` so the per-target PID guard kicks
|
||||
# in here too — repeated Stop/PreCompact fires for the same
|
||||
# transcript should not stack up parallel ingest mines.
|
||||
@@ -736,11 +909,30 @@ def _ingest_transcript(transcript_path: str):
|
||||
_log(f"Transcript ingest started: {path.name}")
|
||||
except OSError:
|
||||
pass
|
||||
except Exception as exc:
|
||||
# Non-daemon ingest spawn path failed. Hooks must never crash the
|
||||
# user's shell — log and continue (not a daemon failure; the daemon
|
||||
# block above handles its own errors).
|
||||
_log(f"transcript ingest hook failed: {exc}")
|
||||
|
||||
|
||||
SUPPORTED_HARNESSES = {"claude-code", "codex"}
|
||||
|
||||
|
||||
def _diary_agent_for_harness(harness: str) -> str:
|
||||
"""Return the diary ``agent_name`` a session in ``harness`` reads under.
|
||||
|
||||
Stop-hook checkpoints must be filed beside the agent's own entries so
|
||||
``mempalace_diary_read(agent_name=...)`` surfaces them. The old code filed
|
||||
them under a fixed ``"session-hook"`` identity that no reader ever queried,
|
||||
hiding every checkpoint (#1693). A ``claude-code`` session reads its diary
|
||||
as ``"claude"``; every other harness already reads under its own name, so
|
||||
returning the harness name keeps a newly supported harness discoverable
|
||||
instead of silently invisible again.
|
||||
"""
|
||||
return "claude" if harness == "claude-code" else harness
|
||||
|
||||
|
||||
def _parse_harness_input(data: dict, harness: str) -> dict:
|
||||
"""Parse stdin JSON according to the harness type."""
|
||||
if harness not in SUPPORTED_HARNESSES:
|
||||
@@ -769,6 +961,26 @@ _ENCODED_PARENT_PREFIXES = (
|
||||
)
|
||||
|
||||
|
||||
def _safe_wing_slug(name: str) -> str:
|
||||
"""Normalize a project directory name into a wing slug ``sanitize_name`` accepts.
|
||||
|
||||
Builds on the historical space/hyphen handling: map characters outside
|
||||
``sanitize_name``'s set to ``_`` while keeping ``.`` and ``'`` so existing wings
|
||||
for names like ``my.app`` are preserved (renaming a wing would orphan diary
|
||||
entries already filed under it), collapse ``..`` which the validator rejects as
|
||||
path traversal, trim edge separators it won't accept, and cap the length so
|
||||
``wing_<slug>`` stays within ``sanitize_name``'s 128-character limit. Without
|
||||
this a folder containing e.g. a leading ``+`` produced ``wing_+project``, which
|
||||
``sanitize_name`` rejects — silently breaking diary auto-save. Falls back to
|
||||
``sessions`` when a name reduces to nothing (e.g. ``+``).
|
||||
"""
|
||||
slug = name.lower().replace(" ", "_").replace("-", "_")
|
||||
slug = re.sub(r"[^\w.']+", "_", slug)
|
||||
slug = re.sub(r"\.{2,}", ".", slug)
|
||||
slug = slug[:120].strip("_.'")
|
||||
return slug or "sessions"
|
||||
|
||||
|
||||
def _wing_from_jsonl_cwd(transcript_path: str) -> Optional[str]:
|
||||
"""Read ``cwd`` from the first JSONL line that records it.
|
||||
|
||||
@@ -802,8 +1014,7 @@ def _wing_from_jsonl_cwd(transcript_path: str) -> Optional[str]:
|
||||
continue
|
||||
project = cwd_norm.rsplit("/", 1)[-1]
|
||||
if project:
|
||||
slug = project.lower().replace(" ", "_").replace("-", "_")
|
||||
return f"wing_{slug}"
|
||||
return f"wing_{_safe_wing_slug(project)}"
|
||||
except OSError:
|
||||
pass
|
||||
return None
|
||||
@@ -860,15 +1071,12 @@ def _wing_from_transcript_path(transcript_path: str) -> str:
|
||||
if encoded.startswith(prefix):
|
||||
encoded = encoded[len(prefix) :]
|
||||
break
|
||||
project = encoded.lower().replace(" ", "_").replace("-", "_")
|
||||
if project:
|
||||
return f"wing_{project}"
|
||||
return f"wing_{_safe_wing_slug(encoded)}"
|
||||
|
||||
# 3. Legacy — explicit -Projects-<name> segment
|
||||
match = re.search(r"-Projects-([^/]+?)(?:/|$)", normalized)
|
||||
if match:
|
||||
project = match.group(1).lower().replace(" ", "_").replace("-", "_")
|
||||
return f"wing_{project}"
|
||||
return f"wing_{_safe_wing_slug(match.group(1))}"
|
||||
|
||||
# 4. Default
|
||||
return "wing_sessions"
|
||||
@@ -942,7 +1150,11 @@ def hook_stop(data: dict, harness: str):
|
||||
result = {"count": 0}
|
||||
if transcript_path:
|
||||
result = _save_diary_direct(
|
||||
transcript_path, session_id, wing=project_wing, toast=toast
|
||||
transcript_path,
|
||||
session_id,
|
||||
wing=project_wing,
|
||||
toast=toast,
|
||||
agent_name=_diary_agent_for_harness(harness),
|
||||
)
|
||||
_ingest_transcript(transcript_path)
|
||||
_maybe_auto_ingest()
|
||||
@@ -999,6 +1211,116 @@ def hook_session_start(data: dict, harness: str):
|
||||
_output({})
|
||||
|
||||
|
||||
def _clear_session_last_save(session_id: str) -> None:
|
||||
"""Drop the per-session save marker once a session has ended.
|
||||
|
||||
``hook_stop`` writes ``{session_id}_last_save`` but never had a clean-exit
|
||||
cleanup path, so the marker lingered. The session is over by the time
|
||||
``hook_session_end`` runs, so removing it here keeps ``hook_state/`` from
|
||||
accumulating dead markers. OS errors (including a missing marker, since
|
||||
``FileNotFoundError`` is an ``OSError``) are swallowed — this is best-effort
|
||||
cleanup, never a reason to fail the hook.
|
||||
"""
|
||||
try:
|
||||
(STATE_DIR / f"{session_id}_last_save").unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def hook_session_end(data: dict, harness: str):
|
||||
"""Session end hook: one final flush when a session exits cleanly.
|
||||
|
||||
Closes the gap (#1341) where a session that never crosses ``SAVE_INTERVAL``
|
||||
on ``Stop`` and never triggers ``PreCompact`` exits with nothing saved —
|
||||
the common case for short, useful sessions.
|
||||
|
||||
Why background instead of mine inline: Claude Code's hooks reference
|
||||
documents a default SessionEnd timeout of 1.5 seconds, and "timeouts set on
|
||||
plugin-provided hooks do not raise the budget"
|
||||
(https://code.claude.com/docs/en/hooks). A cold ``mempalace`` start alone
|
||||
exceeds 1.5s, so this handler must never mine in the hook foreground. The
|
||||
shell wrapper backgrounds it and returns immediately; the heavy capture is
|
||||
spawned *detached* via ``_ingest_transcript`` / ``_maybe_auto_ingest`` (both
|
||||
route through ``_spawn_mine`` / ``_detached_popen_kwargs``). On POSIX that
|
||||
detached child reliably outlives the session (verified). On Windows only the
|
||||
mine grandchild (spawned with detached-process flags) is designed to break
|
||||
away from the session; the backgrounded hook process and the in-process
|
||||
diary write are best-effort there (no Windows CI coverage yet). This
|
||||
honors the "background everything / hooks under 500ms" budget. SessionEnd
|
||||
has no decision control, so this only ever saves; it never emits a block
|
||||
payload.
|
||||
"""
|
||||
if not _palace_root_exists():
|
||||
_output({})
|
||||
return
|
||||
|
||||
# Parse inside the try so a malformed payload (e.g. non-dict stdin that
|
||||
# makes _parse_harness_input raise) still runs the finally cleanup below.
|
||||
session_id = "unknown"
|
||||
try:
|
||||
parsed = _parse_harness_input(data, harness)
|
||||
session_id = parsed["session_id"]
|
||||
transcript_path = parsed["transcript_path"]
|
||||
|
||||
# Read config defensively (mirror hook_stop): a corrupt or unreadable
|
||||
# config must not lose the final save, so default to auto-save on and
|
||||
# toasts off rather than crashing the hook.
|
||||
try:
|
||||
config = MempalaceConfig()
|
||||
auto_save = config.hooks_auto_save
|
||||
toast = config.hook_desktop_toast
|
||||
except Exception:
|
||||
auto_save = True
|
||||
toast = False
|
||||
|
||||
# Respect auto_save config toggle (clean opt-out)
|
||||
if not auto_save:
|
||||
_output({})
|
||||
return
|
||||
|
||||
_log(f"SESSION END for session {session_id}")
|
||||
|
||||
# Validate the harness-provided transcript path before touching it
|
||||
# (extension + ".." traversal check), mirroring the read path that
|
||||
# already runs through _validate_transcript_path. A rejected path skips
|
||||
# the transcript captures but still lets the independent MEMPAL_DIR mine
|
||||
# run.
|
||||
valid_transcript = ""
|
||||
if transcript_path:
|
||||
try:
|
||||
validated = _validate_transcript_path(transcript_path)
|
||||
except OSError:
|
||||
validated = None
|
||||
if validated is None:
|
||||
_log(f"WARNING: transcript_path rejected by validator: {transcript_path!r}")
|
||||
else:
|
||||
valid_transcript = str(validated)
|
||||
|
||||
# Flush. The diary checkpoint (in-process ChromaDB write) runs FIRST,
|
||||
# before any detached mine is spawned, so it never contends for the
|
||||
# palace lock; this handler is already backgrounded by the wrapper, so it
|
||||
# is not under the SessionEnd budget and has time to finish. The detached
|
||||
# transcript ingest follows; re-mining a transcript ``Stop`` already
|
||||
# captured is a near no-op (deterministic convo IDs + ``file_already_mined``
|
||||
# short-circuit + upsert). ``reason`` is intentionally not branched on:
|
||||
# every clean-exit reason (incl. ``/clear`` / ``resume``) warrants the
|
||||
# flush. Order matches ``hook_stop``.
|
||||
if valid_transcript:
|
||||
_save_diary_direct(
|
||||
valid_transcript,
|
||||
session_id,
|
||||
wing=_wing_from_transcript_path(valid_transcript),
|
||||
toast=toast,
|
||||
agent_name=_diary_agent_for_harness(harness),
|
||||
)
|
||||
_ingest_transcript(valid_transcript)
|
||||
_maybe_auto_ingest()
|
||||
|
||||
_output({})
|
||||
finally:
|
||||
_clear_session_last_save(session_id)
|
||||
|
||||
|
||||
def hook_precompact(data: dict, harness: str):
|
||||
"""Precompact hook: mine transcript synchronously, then allow compaction.
|
||||
|
||||
@@ -1042,6 +1364,7 @@ def run_hook(hook_name: str, harness: str):
|
||||
hooks = {
|
||||
"session-start": hook_session_start,
|
||||
"stop": hook_stop,
|
||||
"session-end": hook_session_end,
|
||||
"precompact": hook_precompact,
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user