commit e5287289f054d3b7cc2ce7659832a9220bd64c6e Author: xiaowei Date: Sun May 24 16:58:20 2026 +0800 Initial commit: agent-memory-skill (linear decay, tier stratification) diff --git a/.memory-config.json b/.memory-config.json new file mode 100644 index 0000000..867079c --- /dev/null +++ b/.memory-config.json @@ -0,0 +1,12 @@ +{ + "tiers": {"active": 7, "warm": 21, "cold": 60}, + "decay_rate": 0.015, + "relevance_floor": 0.1, + "skip_patterns": ["_index.md", "MOC-*.md"], + "type_inference": { + "crm/": "crm", + "leads/": "lead", + "contacts/": "contact" + }, + "use_git_dates": true +} diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..3127e52 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Serge Shima + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/README.md b/README.md new file mode 100644 index 0000000..de8213a --- /dev/null +++ b/README.md @@ -0,0 +1,345 @@ +# agent-memory-skill + +[![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE) +[![Python 3.10+](https://img.shields.io/badge/Python-3.10+-green.svg)](https://python.org) +[![No Dependencies](https://img.shields.io/badge/Dependencies-Zero-brightgreen.svg)](#) + +**Memory system for AI agents based on the Ebbinghaus forgetting curve.** + +One Python file. Zero dependencies. Works with any directory of markdown files. + +``` + Memory Strength + 1.0 ┤████████████████████████████████████████████████ ← just accessed + │ ╲ + 0.9 ┤ ╲ ← Day 7: active + │ ╲ + 0.8 ┤ ╲ + │ ╲ + 0.7 ┤ ╲ ← Day 21: warm + │ ╲ + 0.6 ┤ ╲ + │ ╲ + 0.5 ┤ ╲ ← Day 33: cold + │ ╲ + 0.4 ┤ ╲ + │ ╲ + 0.3 ┤ ╲ + │ ╲ + 0.2 ┤ ╲ + │ ╲ + 0.1 ┤─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─ ─floor ← Day 60+: archive + │ + 0.0 ┤ + └──┬──────┬──────────────┬────────────┬──────────→ Days + 0 7 21 60 + + active warm cold archive +``` + +--- + +## The Problem + +AI agents forget everything between sessions. Two common solutions — and why they don't work: + +**"Dump everything into context"** — Loads entire knowledge bases per turn. Burns 20K+ tokens before the conversation starts. No prioritization: a 2-month-old contact gets the same weight as yesterday's hot lead. + +**"Vector search everything"** — Retrieves by semantic similarity but has no temporal awareness. A card accessed yesterday and a card untouched for 90 days look identical. No concept of "fading" or "forgetting." No serendipity. + +Human memory does something different: it **decays over time**, **strengthens with use**, and occasionally **surprises you** with random connections. This skill brings that to AI agents. + +--- + +## The Solution — 3-Layer Memory + +``` +┌─────────────────────────────────────────────────────────┐ +│ Layer 1: HOT CONTEXT always loaded, <4KB │ +│ └── State file — active focus, blockers, reminders │ +├─────────────────────────────────────────────────────────┤ +│ Layer 2: SEARCHABLE VAULT on-demand, unlimited │ +│ └── Cards with YAML frontmatter — one file per entity │ +│ Tiers: core → active → warm → cold → archive │ +├─────────────────────────────────────────────────────────┤ +│ Layer 3: ARCHIVE deep/creative only │ +│ └── Old logs, completed projects, cold contacts │ +│ Still searchable, excluded from default queries │ +└─────────────────────────────────────────────────────────┘ +``` + +**Rule:** Each fact lives in ONE place. If it's in a card, don't duplicate it in the state file. + +See [docs/architecture.md](docs/architecture.md) for full design rationale. + +--- + +## Forgetting Curve + +Inspired by Hermann Ebbinghaus (1885). Each card has a `relevance` score (0.0–1.0) that decays linearly: + +``` +relevance = max(0.1, 1.0 - days × 0.015) +``` + +### Tier Assignment + +| Tier | Days Since Access | Relevance | Behavior | +|------|-------------------|-----------|----------| +| **core** | manual | 1.0 | Never auto-demoted. Identity, security, pricing. | +| **active** | 0–7 | 1.0–0.90 | Searched in all modes. Hot context. | +| **warm** | 8–21 | 0.89–0.69 | Default search. Gradually fading. | +| **cold** | 22–60 | 0.68–0.10 | Deep search only. Mostly forgotten. | +| **archive** | 60+ | 0.10 (floor) | Creative mode or explicit recall. | + +### Graduated Touch (Spaced Repetition) + +Unlike a simple "reset to top," `touch` promotes **one tier at a time**: + +``` +archive → cold → warm → active → active (refresh) +``` + +Multiple reads = stronger memory. Natural spaced repetition without manual scheduling. + +--- + +## Search Modes + +| Mode | Tiers Searched | When to Use | Token Cost | +|------|---------------|-------------|------------| +| **heartbeat** | core + active | Quick status checks, monitoring | ~2K | +| **normal** | active + warm | Most questions, task execution | ~5K | +| **deep** | all tiers | Strategy, "find everything about X" | ~15K | +| **creative** | random cold+archive | Brainstorming, ideation, "what if" | ~3K | + +### Creative Mode — The Highlight Feature + +Human creativity comes from random associations — shower thoughts, serendipitous encounters, dreams. Creative mode simulates this by pulling **random forgotten cards** back into working memory. + +```bash +python3 memory-engine.py creative 5 vault/ +``` + +``` + creative recall — 5 random cards: + [cold] Cloud Migration Strategy + projects/cloud-migration.md (r=0.25, last=2026-01-05) + [archive] React Native Performance Notes + notes/rn-performance.md (r=0.1, last=2025-11-20) + ... + + read these cards and look for unexpected connections to your current task +``` + +The randomness is the feature, not a bug. + +See [docs/search-protocols.md](docs/search-protocols.md) for detailed protocols. + +--- + +## Quick Start + +```bash +# 1. Copy to your project +curl -O https://raw.githubusercontent.com/smixs/agent-memory-skill/main/memory-engine.py + +# 2. Scan your vault +python3 memory-engine.py scan vault/ + +# 3. Initialize cards (add YAML frontmatter) +python3 memory-engine.py init vault/ --dry-run # preview first +python3 memory-engine.py init vault/ # apply + +# 4. Run decay (update relevance scores) +python3 memory-engine.py decay vault/ + +# 5. Creative recall (surface forgotten cards) +python3 memory-engine.py creative 5 vault/ +``` + +--- + +## All Commands + +| Command | Description | Example | +|---------|-------------|---------| +| `scan` | Analyze files, report stats (no changes) | `memory-engine.py scan vault/` | +| `init` | Add YAML frontmatter to files missing it | `memory-engine.py init vault/ --dry-run` | +| `decay` | Update relevance scores and tiers | `memory-engine.py decay vault/` | +| `touch` | Promote card one tier up (graduated recall) | `memory-engine.py touch vault/crm/acme.md` | +| `creative` | Random N cards from cold/archive tiers | `memory-engine.py creative 5 vault/` | +| `daily` | Bootstrap and decay daily files (YYYY-MM-DD.md) | `memory-engine.py daily vault/daily/` | +| `stats` | Show tier distribution and health metrics | `memory-engine.py stats vault/` | +| `config` | Generate default `.memory-config.json` | `memory-engine.py config vault/` | + +### Global Options + +| Flag | Description | +|------|-------------| +| `--dry-run` | Preview changes without writing files | +| `--verbose` | Show per-file details | +| `--config ` | Custom config file path | + +--- + +## YAML Schema + +### Required Fields + +```yaml +--- +type: crm # crm | lead | contact | project | personal | daily | note +description: >- # One-line search snippet + Cloud provider, enterprise tier +tags: [cloud, enterprise] # 2-5 freeform tags +status: active # active | draft | pending | done | inactive +--- +``` + +### Auto-Managed Fields (by memory-engine.py) + +```yaml +last_accessed: 2026-02-20 # When card was last read/touched +relevance: 0.85 # 0.0-1.0, decays over time +tier: active # core | active | warm | cold | archive +``` + +### Full Example + +```yaml +--- +type: crm +description: >- + Cloud infrastructure provider, enterprise tier, renewal Q2 2026 +tags: [cloud, enterprise, renewal] +status: active +industry: IT +region: US +created: 2026-01-15 +updated: 2026-02-20 +last_accessed: 2026-02-20 +relevance: 0.85 +tier: active +--- + +# Acme Cloud Corp + +## Overview +- **Industry:** Cloud Infrastructure +- **Contact:** Jane Smith, VP Sales +``` + +See [docs/yaml-schema.md](docs/yaml-schema.md) for the full schema reference. + +--- + +## Configuration + +Generate a default config: + +```bash +python3 memory-engine.py config vault/ +``` + +Creates `.memory-config.json`: + +```json +{ + "tiers": {"active": 7, "warm": 21, "cold": 60}, + "decay_rate": 0.015, + "relevance_floor": 0.1, + "skip_patterns": ["_index.md"], + "type_inference": {"crm/": "crm", "leads/": "lead"}, + "use_git_dates": true +} +``` + +### Domain-Specific Tuning + +| Domain | Active | Warm | Cold | Rate | Why | +|--------|--------|------|------|------|-----| +| Sales/CRM | 3 | 10 | 30 | 0.025 | Fast-moving, deals expire quickly | +| Default | 7 | 21 | 60 | 0.015 | Balanced for general use | +| Research | 14 | 45 | 120 | 0.008 | Knowledge stays relevant longer | + +### Config Fields + +| Field | Type | Default | Description | +|-------|------|---------|-------------| +| `tiers.active` | int | 7 | Days threshold for active tier | +| `tiers.warm` | int | 21 | Days threshold for warm tier | +| `tiers.cold` | int | 60 | Days threshold for cold tier | +| `decay_rate` | float | 0.015 | Relevance loss per day | +| `relevance_floor` | float | 0.1 | Minimum relevance (never reaches 0) | +| `skip_patterns` | list | `["_index.md"]` | Glob patterns to skip | +| `type_inference` | object | `{}` | Path → type mapping for auto-inference | +| `use_git_dates` | bool | true | Use git log for date resolution | + +--- + +## Integration with Claude Code + +### As a Skill + +Copy the files into your project's skill directory: + +``` +.claude/skills/agent-memory/ +├── SKILL.md # Skill definition (Claude reads this) +├── scripts/ +│ └── memory-engine.py # Engine +└── references/ + ├── architecture.md + ├── search-protocols.md + └── yaml-schema.md +``` + +Claude Code will automatically discover and use the skill based on `SKILL.md`. + +### Daily Decay via Cron + +Add to your crontab for automatic forgetting: + +```bash +# Run decay every day at midnight +0 0 * * * cd /path/to/vault && python3 memory-engine.py decay . +``` + +Or via systemd timer for more reliability. + +### Full System + +For the complete voice-first AI assistant (Telegram → Obsidian + Todoist), see [agent-second-brain](https://github.com/smixs/agent-second-brain). + +--- + +## How It Compares + +| | Dump All | Vector DB | **agent-memory** | +|---|---|---|---| +| **Context cost** | High (all cards every turn) | Medium (top-K results) | Low (tiered loading) | +| **Temporal awareness** | None | None | Built-in (decay + tiers) | +| **Serendipity** | None | None | Creative mode | +| **Dependencies** | None | Embedding model + DB | None (pure Python) | +| **Setup** | Copy files | Deploy infra | Copy one file | +| **Spaced repetition** | No | No | Graduated touch | +| **Works offline** | Yes | Needs model API | Yes | + +--- + +## Anti-Patterns + +| Don't | Do Instead | +|-------|------------| +| Load all cards into context | Search on demand, filter by tier | +| Store same fact in 3 files | One card per entity, reference via links | +| Delete old files to "save space" | Let tier decay handle visibility | +| Touch every card during bulk ops | Only touch on meaningful read/update | +| Build elaborate review systems | Let decay handle it — unused = fades | + +--- + +## License + +[MIT](LICENSE) diff --git a/SKILL.md b/SKILL.md new file mode 100644 index 0000000..3e83eb2 --- /dev/null +++ b/SKILL.md @@ -0,0 +1,143 @@ +# Agent Memory Management + +Memory system with automatic decay, tiered search, and creative recall. +Works with any directory of markdown files. + +## Quick Start + +### 1. Scan existing files +```bash +python3 memory-engine.py scan +``` +Reports: file count, YAML coverage, size, recommendations. + +### 2. Bootstrap YAML frontmatter +```bash +python3 memory-engine.py init [--dry-run] +``` +Adds `relevance`, `last_accessed`, `tier` to files missing frontmatter. +Infers `type` from directory path. Infers dates from YAML fields, git log, or file mtime. + +### 3. Run decay +```bash +python3 memory-engine.py decay [--dry-run] +``` +Updates all cards: recalculates relevance scores and reassigns tiers. +Schedule daily via cron for automatic forgetting. + +### 4. Touch on read +```bash +python3 memory-engine.py touch +``` +Promotes card one tier up (graduated recall). Multiple reads = stronger memory. + +### 5. Creative recall +```bash +python3 memory-engine.py creative +``` +Random sample from cold/archive tiers. Read these cards and look for unexpected connections. + +### 6. Health check +```bash +python3 memory-engine.py stats +``` +Shows tier distribution, context budget, stale card count. + +## Core Concepts + +### Three-Layer Architecture + +| Layer | What | Size Target | Loaded | +|-------|------|-------------|--------| +| Hot context | State file (volatile focus, blockers) | <4KB | Every turn | +| Searchable vault | Cards with YAML, one per entity | Unlimited | On demand | +| Archive | Old logs, completed work | Unlimited | Deep/creative only | + +**Rule:** Each fact lives in ONE place. If it's in a card, don't also put it in the state file. +See [docs/architecture.md](docs/architecture.md) for full design rationale. + +### Forgetting Curve + +Cards have `relevance: 0.0-1.0` that decays linearly over time: +- Day 0: 1.0 (just accessed) +- Day 7: 0.90 → tier: active +- Day 21: 0.69 → tier: warm +- Day 33: 0.50 → tier: cold +- Day 60+: 0.10 (floor) → tier: archive + +`core` tier is manual-only — for identity, security, pricing. Never auto-demoted. + +### Tier-Aware Search + +| Mode | Tiers searched | When | +|------|---------------|------| +| heartbeat | core + active | Quick checks, monitoring | +| normal | active + warm | Most questions | +| deep | all tiers | Strategy, complex analysis | +| creative | random cold+archive | Brainstorming, ideation | + +See [docs/search-protocols.md](docs/search-protocols.md) for detailed protocols. + +### YAML Frontmatter + +Minimum required fields (managed by engine): +```yaml +--- +relevance: 0.85 +last_accessed: 2026-02-25 +tier: active +--- +``` + +See [docs/yaml-schema.md](docs/yaml-schema.md) for full schema with domain-specific fields. + +## Daily Files (Episodic Memory) + +Daily files (`YYYY-MM-DD.md`) are the agent's episodic memory — what happened each day. +They follow the same decay system as vault cards. **Never delete them.** + +### Lifecycle + +``` +Day 0: Created (end of day cron or manual) → tier: active, relevance: 1.0 +Day 1-7: Auto-loaded at session start (today+yesterday) → tier: active +Day 8-21: Searchable but not auto-loaded → tier: warm +Day 22-60: Deep search only → tier: cold +Day 60+: Creative mode or explicit recall → tier: archive +``` + +## Configuration + +Generate default config: +```bash +python3 memory-engine.py config +``` + +Creates `.memory-config.json`: +```json +{ + "tiers": {"active": 7, "warm": 21, "cold": 60}, + "decay_rate": 0.015, + "relevance_floor": 0.1, + "skip_patterns": ["_index.md"], + "type_inference": {"crm/": "crm", "leads/": "lead"}, + "use_git_dates": true +} +``` + +Adjust tier thresholds and decay rate to match your domain's natural rhythm. +Fast-moving domains (sales): tighter thresholds (active=3, warm=10, cold=30). +Slow domains (research): wider thresholds (active=14, warm=45, cold=120). + +## Anti-Patterns + +| Don't | Do Instead | +|-------|------------| +| Load all contacts into state file | Keep them in vault cards, search on demand | +| Create "knowledge graph" that duplicates vault | Vault IS the graph. Use index files for navigation | +| Store same fact in 3 files | One card per entity, reference via links | +| Delete old daily files to "save space" | Keep all dailies, let tier decay handle visibility | +| Auto-load all daily files at session start | Load only today + yesterday; search older on demand | +| Search all 500 cards for every question | Check index first, filter by tier, then search | +| Touch every card during bulk operations | Only touch on meaningful read/update | +| Build elaborate review systems | Let decay handle it — if you don't use it, it fades | diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..8eb76bd --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,192 @@ +# Memory Architecture + +## The Problem + +AI agents forget everything between sessions. Common solutions (dump everything into context, vector-search everything) create bloat and noise. Human memory works differently: it has layers, it decays, and it surprises you with random connections. + +## Design Principles + +### Single Source of Truth (DRY) +Each fact lives in ONE place. If a contact exists in a CRM card, don't also list them in a "key contacts" section of your state file. Instead, reference them: `see vault/crm/alice.md`. + +**Violation test:** Search for any person/project name. If it appears in 3+ files with substantive detail, you have a DRY violation. + +### Context is a Shared Resource (KISS) +Every byte loaded per turn costs tokens. An agent loading 40KB of context files is burning ~10,000 tokens before the conversation starts. Measure your context budget: +- State file (volatile): target <4KB +- Config files: target <15KB total +- Per-turn overhead = sum of all always-loaded files + +### Build Only What You Use (YAGNI) +Don't create "knowledge graphs," "daily digests," or "weekly reviews" unless you actively query them. If a cron job generates a report nobody reads, it's waste. + +## Three-Layer Architecture + +``` +Layer 1: HOT CONTEXT (always loaded, <4KB) + └── State file — volatile: active focus, blockers, reminders + NOT: contacts, history, reference, tools + +Layer 2: SEARCHABLE VAULT (on-demand, unlimited) + └── Cards with YAML frontmatter — one file per entity + Searched via: semantic search, grep, graph traversal + Organized by: domain directories + index files (MOC) + +Layer 3: ARCHIVE (deep search only) + └── Old daily logs, completed projects, cold contacts + Still searchable but excluded from default queries +``` + +### Layer 1: Hot Context + +This is what's loaded every turn. Must be ruthlessly slim. + +**Include:** Current week's focus, active blockers, 3-5 pending items, security rules, navigation hints. + +**Exclude:** Contact lists (use vault), tool docs (separate file), event history (daily files), anything searchable. + +**Audit:** If your state file exceeds 80 lines, something belongs in Layer 2. + +### Layer 2: Searchable Vault + +One markdown file per entity (person, project, company, idea). YAML frontmatter enables filtering and automation. + +**Directory structure:** +``` +vault/ +├── crm/ # contacts and companies +│ ├── clients/ +│ ├── leads/ +│ └── personal/ +├── projects/ # active and past projects +├── MOC/ # Maps of Content (index files) +└── .graph/ # computed graph data +``` + +**Why one file per entity:** Enables granular decay, individual relevance scoring, and precise search results. A single "contacts.md" with 200 entries can't decay — the whole file is either loaded or not. + +### Layer 2b: Daily Files (Episodic Memory) + +Daily logs (`YYYY-MM-DD.md`) sit between vault and archive. They capture the raw narrative of each day — conversations, decisions, reasoning, context that doesn't fit neatly into entity cards. + +**Never delete daily files.** The decay system handles visibility: +- Active (0-7 days): loaded at session start +- Warm (8-21 days): searchable, not auto-loaded +- Cold (22-60 days): deep search only +- Archive (60+): creative mode, explicit queries + +**Why keep everything:** +- Disk: 365 daily files ≈ 4MB. Irrelevant cost. +- Context: knowledge graphs capture entities but lose reasoning and tone. +- Search: semantic search finds old dailies just as well as new ones. +- Source rebuild: re-reading 100+ messages is far more expensive than keeping a 10KB summary. + +**Compression:** If a daily file exceeds 20KB, extract key facts into vault cards and trim the daily to a 20-line summary. Keep the YAML frontmatter intact. + +### Layer 3: Archive + +One-off analysis reports. Completed project retrospectives. Generated artifacts that served a temporary purpose. These are still searchable but excluded from default queries. Daily files are NOT archive — they stay in their directory with tier-based visibility. + +## Forgetting Curve + +Inspired by Ebbinghaus (1885): memory strength decays over time without reinforcement. + +### Relevance Score + +Each card has `relevance: 0.0-1.0` in its frontmatter. Decays linearly: + +``` +relevance = max(floor, 1.0 - days_since_access × rate) + +Default: rate=0.015, floor=0.1 +→ After 7 days: 0.90 +→ After 21 days: 0.69 +→ After 33 days: 0.50 +→ After 60 days: 0.10 (floor) +``` + +### Tier Assignment + +Based on days since `last_accessed`: + +| Tier | Days | Description | +|------|------|-------------| +| core | manual | Never auto-assigned. Identity, security, pricing. | +| active | 0-7 | Hot context. Searched in all modes. | +| warm | 8-21 | Default search radius. Gradually fading. | +| cold | 22-60 | Deep search only. Mostly forgotten. | +| archive | 60+ | Creative mode or explicit recall only. | + +### Touch Protocol + +When an agent reads or references a card, it should promote the card: +```bash +python3 memory-engine.py touch +``` +Each touch promotes one tier up (graduated recall), not straight to top. +Multiple reads = stronger memory — natural spaced repetition. + +**When to touch:** +- Agent reads card content to answer a question +- Agent updates card with new information +- User explicitly mentions the entity + +**When NOT to touch:** +- Card appears in search results but isn't opened +- Automated scan (decay script itself) +- Bulk operations + +## Search Protocols + +Different tasks need different search depths: + +### Heartbeat Mode (fast checks) +Search: core + active only. +Use for: quick status checks, routine monitoring, simple questions. +Cost: minimal — only hot cards in scope. + +### Normal Mode (default) +Search: core + active + warm. +Use for: most questions, task execution, lookups. +Cost: moderate — includes fading but recent cards. + +### Deep Mode (complex tasks) +Search: all tiers. +Use for: strategy, complex analysis, "find everything about X." +Cost: high — full vault scan. + +### Creative Mode (divergent thinking) +Method: random sample from cold + archive tiers. +Use for: brainstorming, finding unexpected connections, "what if" scenarios. +Not semantic search — deliberately random to surface forgotten associations. + +```bash +python3 memory-engine.py creative 5 vault/ +``` + +## Multi-Agent Shared Memory + +When multiple agents share a vault: + +1. **Shared YAML schema** — all agents use same frontmatter fields +2. **Each agent touches on read** — keeps decay accurate across agents +3. **One vault, one truth** — don't fork the vault per agent +4. **Conflict resolution** — last write wins for metadata; append-only for history sections + +## Context Budget Calculator + +Measure your actual per-turn cost: + +``` +Always-loaded files: + STATE.md _____ bytes + CONFIG.md _____ bytes + IDENTITY.md _____ bytes + RULES.md _____ bytes + ───────────────────────── + Total context: _____ bytes ÷ 4 ≈ _____ tokens + +Target: <25KB (6,000 tokens) for always-loaded context +``` + +Every 1KB saved = ~250 tokens freed per turn for actual conversation. diff --git a/docs/search-protocols.md b/docs/search-protocols.md new file mode 100644 index 0000000..b173f7f --- /dev/null +++ b/docs/search-protocols.md @@ -0,0 +1,118 @@ +# Search Protocols + +## Protocol Selection + +Before searching, classify the task: + +| Signal | Mode | Search Radius | +|--------|------|---------------| +| Quick status check, monitoring | heartbeat | core + active | +| Normal question, lookup | normal | active + warm | +| "Find everything about X" | deep | all tiers | +| Strategy, complex analysis | deep | all tiers | +| Brainstorm, "what if", ideation | creative | random from cold+archive | + +## Search Order (All Modes) + +1. **Index files first** — MOC or `_index.md` files are pre-built navigation. Check them before searching. +2. **Follow links** — If an index points to `[[crm/acme]]`, read the card directly. Don't search for "Acme." +3. **Semantic search** — Only when indexes don't cover the query. +4. **Grep** — Last resort for exact strings, IDs, phone numbers. + +## Tier Filtering + +### Pre-filter with grep +```bash +# Find all active cards +grep -rl "^tier: active" vault/crm/ + +# Find all cold+ cards for deep search +grep -rl "^tier: \(cold\|archive\)" vault/crm/ + +# Find high-relevance cards +grep -l "^relevance: 0\.9" vault/crm/ +``` + +### Combine with semantic search +1. Grep for tier → get file list +2. Read relevant files → answer question + +This is cheaper than searching all 400+ cards semantically. + +## Heartbeat Protocol + +Heartbeats are frequent, automated checks. Minimize cost: + +1. Read state file (always loaded) +2. Check only `core` + `active` tier cards matching current projects +3. Skip warm/cold/archive entirely +4. Total reads: 0-5 files per heartbeat + +## Normal Protocol + +Default for user questions: + +1. Check index/MOC for direct link +2. If not found: semantic search with default radius (active + warm) +3. Read top 3-5 results +4. Touch any card you read: `memory-engine.py touch ` + +## Deep Protocol + +For complex, multi-step analysis: + +1. Check indexes +2. Semantic search across ALL tiers (no filtering) +3. Read all relevant results (up to 10-15 files) +4. Cross-reference cards for connections +5. Touch all read cards + +## Creative Protocol + +Deliberately non-directed. Goal: surface unexpected connections. + +```bash +python3 memory-engine.py creative 5 vault/ +``` + +1. Get 5 random cards from cold/archive +2. Read each one +3. For each card, ask: "How could this connect to my current task?" +4. If a connection exists → touch the card (promotes it back) +5. If no connection → leave it (continues to decay) + +### When to use creative mode +- Stuck on a problem +- Looking for new angles +- User says "brainstorm" or "what if" +- Exploring forgotten knowledge +- Weekly reflection sessions + +### What makes it work +Human creativity often comes from random associations — shower thoughts, dreams, serendipitous encounters. Creative mode simulates this by pulling forgotten cards back into working memory. The randomness is the feature, not a bug. + +## Touch Discipline + +Touching a card resets its decay. Over-touching defeats the purpose. + +**Touch when:** +- You read the card's content to answer a question +- You update the card with new information +- User explicitly discusses the entity + +**Don't touch when:** +- Card appears in search results but you didn't open it +- Running automated scans or reports +- Bulk migration or formatting changes +- Card mentioned in passing but not substantively used + +## Cost Awareness + +| Mode | Typical reads | Token cost | +|------|---------------|------------| +| Heartbeat | 0-5 files | ~2K tokens | +| Normal | 3-8 files | ~5K tokens | +| Deep | 10-20 files | ~15K tokens | +| Creative | 5 files | ~3K tokens | + +Optimize by reading index files first — they're cheap navigation that prevents expensive full-text searches. diff --git a/docs/yaml-schema.md b/docs/yaml-schema.md new file mode 100644 index 0000000..a5582cd --- /dev/null +++ b/docs/yaml-schema.md @@ -0,0 +1,165 @@ +# YAML Frontmatter Schema + +## Canonical Schema (all card types) + +Every card MUST have this structure. Fields marked (auto) are managed by memory-engine.py. +Fields marked (required) must be written by the agent when creating a card. + +```yaml +--- +# ── identity ── +type: crm # (required) crm | lead | contact | project | personal | daily | note +description: >- # (required) One-line summary. What is this card about? + Cloud infrastructure provider, enterprise tier, renewal Q2 2026 + +# ── classification ── +tags: [cloud, enterprise, renewal] # (required) 2-5 freeform tags for grep filtering +status: active # (required) universal: active|draft|pending|done|inactive; CRM-only: prospect|negotiation|won|lost +industry: SaaS # (optional) For CRM/leads +region: US # (optional) ISO country codes +source: referral # (optional) How this entity entered the system +priority: High # (optional) High | Medium | Low + +# ── ownership ── +owner: agent # (optional) Who owns this relationship +responsible: agent # (optional) Who is doing the work + +# ── dates ── +created: 2026-01-15 # (recommended) When card was first created +updated: 2026-02-20 # (recommended) When content was last meaningfully changed + +# ── deal tracking ── +deal_status: negotiation # (optional) For active deals +deal_deadline: 2026-03-15 # (optional) Deal close date + +# ── memory system (auto) ── +last_accessed: 2026-02-25 # (auto) When card was last read/touched +relevance: 0.85 # (auto) 0.0-1.0, decays over time +tier: active # (auto) core | active | warm | cold | archive +--- +``` + +## Required Fields Explained + +### `description` (string, one line) +The single most important field for search quality. Write a concise summary that answers: +"If someone searches for this entity, what should they see in results?" + +Good: `"Cloud infrastructure provider, enterprise tier, renewal Q2 2026"` +Bad: `"contact"` (too vague) +Bad: (empty — defeats the purpose of the entire system) + +### `tags` (list, 2-5 items) +Cross-cutting labels for fast grep filtering. Use lowercase, hyphens. +```yaml +tags: [hot-lead, ai-training, enterprise, follow-up] +``` +Search: `grep -rl "hot-lead" vault/crm/` + +### `type` (enum) +| Type | When | +|------|------| +| crm | Existing client/company | +| lead | Potential client | +| contact | Person (not a lead/client) | +| project | Active or past project | +| personal | Family, friends | +| daily | Daily log file | +| note | Everything else | + +### `status` (enum, normalized) +Domain-specific lifecycle. NOT the same as `tier` (which is memory-system lifecycle). + +**Universal (all card types):** + +| Status | Meaning | +|--------|---------| +| `active` | Currently relevant, in use, engaged | +| `draft` | Work in progress, not finalized | +| `pending` | Waiting for external input or decision | +| `done` | Completed, kept for reference | +| `inactive` | Was active, went quiet or outdated | + +**CRM-specific (only for type: crm, lead, client):** + +| Status | Meaning | +|--------|---------| +| `prospect` | Identified lead, no deep engagement yet | +| `negotiation` | Proposal sent, in talks | +| `won` | Deal closed positively | +| `lost` | Rejected, didn't pursue | + +**ONLY these 9 values.** No mixed case, no free-text. + +Typical lifecycle by type: +- **crm/lead:** prospect → active → negotiation → won/lost +- **project:** draft → active → done +- **contact:** active → inactive +- **note/knowledge:** draft → active → inactive (outdated) +- **personal:** active → inactive +- **daily:** no status needed (has `date` field) + +When in doubt: CRM → `prospect`, everything else → `active`. + +## Memory System Fields (auto-managed) + +### `relevance` (float, 0.0-1.0) +Computed by decay engine. Do not manually edit unless marking as `core`. +- 1.0 = just accessed +- 0.5 = ~33 days old +- 0.1 = floor (60+ days) + +### `tier` (enum) +Computed by decay engine based on `last_accessed`. +- `core` — only manually assigned, never auto-demoted. Use for: identity, security rules, pricing, critical reference. +- `active` — 0-7 days since access. Searched in all modes. +- `warm` — 8-21 days. Searched in normal+ modes. +- `cold` — 22-60 days. Deep search only. +- `archive` — 60+ days. Creative mode or explicit queries. + +### `last_accessed` (ISO date) +Updated by `touch` command (graduated: +1 tier per touch). + +## Agent Protocol for New Cards + +When creating ANY new card: +1. ALWAYS include: `type`, `description`, `tags`, `status` +2. Write `description` as if it's a search result snippet — concise, informative +3. Add 2-5 `tags` that cross-cut directory structure +4. Run `memory-engine.py touch ` after creation +5. The engine will auto-add `relevance`, `last_accessed`, `tier` on next decay run + +## Field Semantics + +### `type` (string) +Auto-inferred from directory path if not present. Configurable via `type_inference` in `.memory-config.json`. + +### `tags` (list) +Freeform. Useful for cross-cutting concerns that don't fit directory structure. +```yaml +tags: [hot-lead, ai-training, enterprise, follow-up] +``` +Search: `grep -rl "hot-lead" vault/crm/` + +## Configuration + +Type inference mapping in `.memory-config.json`: +```json +{ + "type_inference": { + "crm/clients/": "crm", + "crm/leads/": "lead", + "contacts/": "contact", + "projects/": "project" + } +} +``` + +When `memory-engine.py init` encounters a file without `type`, it checks the file path against these patterns. + +## Frontmatter Tips + +1. **Don't duplicate content.** If the H1 heading says "# Acme Corp", you don't need `title: Acme Corp` — the engine infers it. +2. **Tags > nested directories.** A flat `crm/` with tags is more flexible than `crm/hot/enterprise/ai/`. +3. **Status is domain-specific.** The memory system uses `tier` for lifecycle; `status` is for your business logic (active/won/lost/churned). +4. **`updated` vs `last_accessed`**: `updated` = when content changed; `last_accessed` = when anyone read it. Both matter for decay; the engine uses whichever is most recent. diff --git a/examples/sample-card.md b/examples/sample-card.md new file mode 100644 index 0000000..f684273 --- /dev/null +++ b/examples/sample-card.md @@ -0,0 +1,30 @@ +--- +type: crm +description: >- + Cloud infrastructure provider, enterprise tier, renewal Q2 2026 +tags: [cloud, enterprise, renewal] +status: active +industry: IT +region: US +created: 2026-01-15 +updated: 2026-02-20 +last_accessed: 2026-02-20 +relevance: 0.85 +tier: active +--- + +# Acme Cloud Corp + +## Overview +- **Industry:** Cloud Infrastructure +- **Contact:** Jane Smith, VP Sales + +## Active Deals +### Enterprise Renewal +- **Status:** negotiation +- **Value:** $XXK +- **Deadline:** 2026-06-30 + +## History +- 2026-01-15: Initial contact at CloudExpo +- 2026-02-10: Demo completed, positive feedback diff --git a/memory-engine.py b/memory-engine.py new file mode 100644 index 0000000..972be97 --- /dev/null +++ b/memory-engine.py @@ -0,0 +1,645 @@ +#!/usr/bin/env python3 +""" +memory-engine.py — Universal memory management for AI agents. + +Implements Ebbinghaus-inspired forgetting curve with tiered recall. +Works with any directory of markdown files. + +Commands: + scan [dir] Analyze files, report stats (no changes) + init [dir] Add YAML frontmatter to files missing it + decay [dir] Update relevance scores and tiers + touch Reset file to active (on read/use) + creative [dir] Random N cards from cold/archive tiers + stats [dir] Show tier distribution and health metrics + +Options: + --config Config JSON (default: .memory-config.json in target dir) + --dry-run Preview changes without writing + --verbose Show per-file details + +Config (.memory-config.json): +{ + "tiers": { + "active": 7, // days threshold + "warm": 21, + "cold": 60 + // beyond cold = archive + }, + "decay_rate": 0.015, // relevance loss per day (linear) + "relevance_floor": 0.1, // minimum relevance + "skip_patterns": ["_index.md", "MOC-*"], + "type_inference": { + "crm/": "crm", + "leads/": "lead", + "personal/": "personal" + }, + "use_git_dates": true +} +""" + +import os +import re +import sys +import json +import random +import subprocess +from datetime import datetime, date +from pathlib import Path +from typing import Optional + +# ─── defaults ─────────────────────────────────────────────────── + +DEFAULT_CONFIG = { + "tiers": {"active": 7, "warm": 21, "cold": 60}, + "decay_rate": 0.015, + "relevance_floor": 0.1, + "skip_patterns": ["_index.md"], + "type_inference": {}, + "use_git_dates": True, +} + +TODAY = date.today() + +# ─── YAML frontmatter parsing ────────────────────────────────── + +def parse_frontmatter(content: str) -> tuple[dict, str, bool]: + """Parse YAML frontmatter. Returns (fields, body, had_yaml).""" + if content.startswith("---\n"): + end = content.find("\n---\n", 4) + if end != -1: + yaml_block = content[4:end] + body = content[end + 5:] + fields = {} + for line in yaml_block.split("\n"): + if ":" in line: + key, _, val = line.partition(":") + fields[key.strip()] = val.strip() + return fields, body, True + return {}, content, False + + +def build_frontmatter(fields: dict, field_order: list[str] | None = None) -> str: + """Build YAML frontmatter from dict, preserving field order.""" + if field_order is None: + field_order = [ + "type", "title", "description", "tags", + "industry", "source", "priority", "status", "region", + "owner", "responsible", "domain", "related", "client", + "deal_status", "deal_deadline", "deadline", + "created", "updated", "last_accessed", "relevance", "tier", + ] + lines = [] + used = set() + for key in field_order: + if key in fields: + lines.append(f"{key}: {fields[key]}") + used.add(key) + for key, val in fields.items(): + if key not in used: + lines.append(f"{key}: {val}") + return "---\n" + "\n".join(lines) + "\n---\n" + + +# ─── date resolution ─────────────────────────────────────────── + +def get_git_date(filepath: Path) -> date | None: + """Last git commit date for file.""" + try: + result = subprocess.run( + ["git", "log", "-1", "--format=%aI", "--", str(filepath)], + capture_output=True, text=True, + cwd=filepath.parent, timeout=5, + ) + if result.returncode == 0 and result.stdout.strip(): + return date.fromisoformat(result.stdout.strip()[:10]) + except Exception: + pass + return None + + +def get_best_date(fields: dict, filepath: Path, use_git: bool = True) -> date: + """Most recent date from YAML fields, git history, or file mtime.""" + candidates = [] + for field in ["last_accessed", "updated", "created"]: + val = fields.get(field, "") + try: + candidates.append(date.fromisoformat(val[:10])) + except (ValueError, IndexError): + continue + if use_git: + git_date = get_git_date(filepath) + if git_date: + candidates.append(git_date) + if not candidates: + mtime = os.path.getmtime(filepath) + candidates.append(date.fromtimestamp(mtime)) + return max(candidates) + + +# ─── core logic ───────────────────────────────────────────────── + +def calc_relevance(days: int, rate: float, floor: float) -> float: + """Linear decay with floor.""" + return round(max(floor, 1.0 - days * rate), 2) + + +def calc_tier(days: int, tiers: dict, current_tier: str = "") -> str: + """Assign tier based on days since last access.""" + if current_tier == "core": + return "core" # never auto-demote core + sorted_tiers = sorted(tiers.items(), key=lambda x: x[1]) + for tier_name, threshold in sorted_tiers: + if days <= threshold: + return tier_name + return "archive" + + +def infer_type(filepath: Path, type_map: dict) -> str: + """Infer card type from path using configurable mapping.""" + path_str = str(filepath) + for pattern, card_type in type_map.items(): + if pattern in path_str: + return card_type + return "note" + + +def should_skip(filepath: Path, patterns: list[str]) -> bool: + """Check if file matches skip patterns.""" + import fnmatch + name = filepath.name + for pattern in patterns: + if fnmatch.fnmatch(name, pattern): + return True + return False + + +def infer_title(body: str) -> str: + """Extract title from first H1 heading.""" + for line in body.split("\n")[:10]: + if line.startswith("# "): + return line[2:].strip() + return "" + + +# ─── config ───────────────────────────────────────────────────── + +def load_config(target_dir: Path, config_path: str | None = None) -> dict: + """Load config from file or return defaults.""" + if config_path: + p = Path(config_path) + else: + p = target_dir / ".memory-config.json" + if p.exists(): + with open(p) as f: + user = json.load(f) + # Merge with defaults + config = {**DEFAULT_CONFIG, **user} + config["tiers"] = {**DEFAULT_CONFIG["tiers"], **user.get("tiers", {})} + return config + return DEFAULT_CONFIG.copy() + + +def save_default_config(target_dir: Path): + """Write default config file.""" + p = target_dir / ".memory-config.json" + with open(p, "w") as f: + json.dump(DEFAULT_CONFIG, f, indent=2) + print(f" wrote default config: {p}") + + +# ─── commands ─────────────────────────────────────────────────── + +def find_cards(target_dir: Path, config: dict) -> list[Path]: + """Find all markdown files, respecting skip patterns.""" + cards = sorted(target_dir.rglob("*.md")) + return [c for c in cards if c.exists() and not c.is_symlink() and not should_skip(c, config["skip_patterns"])] + + +def cmd_scan(target_dir: Path, config: dict, verbose: bool = False): + """Analyze files without changes.""" + cards = find_cards(target_dir, config) + with_yaml = 0 + without_yaml = 0 + has_relevance = 0 + has_tier = 0 + total_bytes = 0 + + for card in cards: + content = card.read_text(encoding="utf-8", errors="replace") + total_bytes += len(content.encode("utf-8")) + fields, body, had_yaml = parse_frontmatter(content) + if had_yaml: + with_yaml += 1 + else: + without_yaml += 1 + if "relevance" in fields: + has_relevance += 1 + if "tier" in fields: + has_tier += 1 + if verbose and not had_yaml: + print(f" no yaml: {card.relative_to(target_dir)}") + + print(f"\n scan results for {target_dir}:") + print(f" total files: {len(cards)}") + print(f" with yaml: {with_yaml}") + print(f" without yaml: {without_yaml}") + print(f" has relevance: {has_relevance}") + print(f" has tier: {has_tier}") + print(f" total size: {total_bytes / 1024:.0f} KB") + print(f" avg file size: {total_bytes / max(len(cards), 1) / 1024:.1f} KB") + + if without_yaml: + print(f"\n → run 'init' to add YAML frontmatter to {without_yaml} files") + if not has_relevance: + print(f" → run 'decay' to add relevance scores and tiers") + + +def cmd_init(target_dir: Path, config: dict, dry_run: bool = False, verbose: bool = False): + """Add YAML frontmatter to files missing it.""" + cards = find_cards(target_dir, config) + added = 0 + + for card in cards: + content = card.read_text(encoding="utf-8", errors="replace") + fields, body, had_yaml = parse_frontmatter(content) + if had_yaml: + continue + + # Create minimal frontmatter + new_fields = { + "type": infer_type(card, config["type_inference"]), + } + title = infer_title(body) + if title: + new_fields["title"] = title + + ref_date = get_best_date({}, card, config["use_git_dates"]) + new_fields["last_accessed"] = ref_date.isoformat() + days = max(0, (TODAY - ref_date).days) + new_fields["relevance"] = str(calc_relevance(days, config["decay_rate"], config["relevance_floor"])) + new_fields["tier"] = calc_tier(days, config["tiers"]) + + new_content = build_frontmatter(new_fields) + body + if not dry_run: + card.write_text(new_content, encoding="utf-8") + added += 1 + if verbose: + print(f" {'[dry] ' if dry_run else ''}init: {card.relative_to(target_dir)} → {new_fields['tier']}") + + print(f"\n {'DRY RUN — ' if dry_run else ''}init results:") + print(f" files processed: {len(cards)}") + print(f" frontmatter added: {added}") + + +def cmd_decay(target_dir: Path, config: dict, dry_run: bool = False, verbose: bool = False): + """Update relevance and tiers based on time decay.""" + cards = find_cards(target_dir, config) + results = [] + + for card in cards: + content = card.read_text(encoding="utf-8", errors="replace") + fields, body, had_yaml = parse_frontmatter(content) + + ref_date = get_best_date(fields, card, config["use_git_dates"]) + days = max(0, (TODAY - ref_date).days) + + old_tier = fields.get("tier", "") + new_relevance = calc_relevance(days, config["decay_rate"], config["relevance_floor"]) + new_tier = calc_tier(days, config["tiers"], old_tier) + + if "last_accessed" not in fields: + fields["last_accessed"] = ref_date.isoformat() + if "type" not in fields: + fields["type"] = infer_type(card, config["type_inference"]) + + fields["relevance"] = str(new_relevance) + fields["tier"] = new_tier + + new_content = build_frontmatter(fields) + body + changed = new_content != content + + if changed and not dry_run: + card.write_text(new_content, encoding="utf-8") + + results.append({ + "path": str(card.relative_to(target_dir)), + "days": days, + "relevance": new_relevance, + "tier": new_tier, + "changed": changed, + }) + + if verbose and changed: + print(f" {'[dry] ' if dry_run else ''}{card.relative_to(target_dir)}: {old_tier or '?'}→{new_tier} r={new_relevance}") + + # Stats + tiers = {} + changed_count = sum(1 for r in results if r["changed"]) + for r in results: + tiers[r["tier"]] = tiers.get(r["tier"], 0) + 1 + avg_rel = sum(r["relevance"] for r in results) / max(len(results), 1) + + print(f"\n {'DRY RUN — ' if dry_run else ''}decay results:") + print(f" total: {len(results)}, changed: {changed_count}") + print(f" avg relevance: {avg_rel:.2f}") + for tier in ["core", "active", "warm", "cold", "archive"]: + count = tiers.get(tier, 0) + bar = "█" * (count // 3) + if count: + print(f" {tier:8s}: {count:4d} {bar}") + + +def cmd_touch(filepath: str, config: dict): + """Promote a file one tier up (graduated recall). + + archive → cold → warm → active → active (refresh) + Each touch promotes one level, not straight to top. + Natural spaced repetition: multiple reads = stronger memory. + """ + p = Path(filepath) + if not p.exists(): + print(f" error: {filepath} not found") + sys.exit(1) + + content = p.read_text(encoding="utf-8", errors="replace") + fields, body, had_yaml = parse_frontmatter(content) + + if fields.get("tier") == "core": + fields["last_accessed"] = TODAY.isoformat() + fields["relevance"] = "1.0" + new_content = build_frontmatter(fields) + body + p.write_text(new_content, encoding="utf-8") + print(f" touched: {filepath} → core (refreshed)") + return + + tiers_cfg = config["tiers"] + # Promotion targets: set last_accessed to midpoint of next-higher tier + # archive → cold: midpoint of cold range + # cold → warm: midpoint of warm range + # warm → active: midpoint of active range + # active → active: today (refresh) + cold_threshold = tiers_cfg.get("cold", 60) + warm_threshold = tiers_cfg.get("warm", 21) + active_threshold = tiers_cfg.get("active", 7) + + current_tier = fields.get("tier", "archive") + if current_tier == "archive": + # Promote to cold: set last_accessed to midpoint of cold range + target_days = (warm_threshold + cold_threshold) // 2 + new_tier = "cold" + elif current_tier == "cold": + # Promote to warm: midpoint of warm range + target_days = (active_threshold + warm_threshold) // 2 + new_tier = "warm" + elif current_tier == "warm": + # Promote to active: midpoint of active range + target_days = active_threshold // 2 + new_tier = "active" + else: + # Already active: refresh to today + target_days = 0 + new_tier = "active" + + from datetime import timedelta + new_date = TODAY - timedelta(days=target_days) + new_relevance = calc_relevance(target_days, config["decay_rate"], config["relevance_floor"]) + + fields["last_accessed"] = new_date.isoformat() + fields["relevance"] = str(new_relevance) + fields["tier"] = new_tier + + new_content = build_frontmatter(fields) + body + p.write_text(new_content, encoding="utf-8") + print(f" touched: {filepath} → {current_tier}→{new_tier}, relevance={new_relevance}") + + +def cmd_creative(n: int, target_dir: Path, config: dict): + """Random sample from cold/archive tiers for divergent thinking.""" + cards = find_cards(target_dir, config) + cold_cards = [] + + for card in cards: + content = card.read_text(encoding="utf-8", errors="replace") + fields, body, had_yaml = parse_frontmatter(content) + tier = fields.get("tier", "") + if tier in ("cold", "archive", "warm"): + title = fields.get("title", "") or infer_title(body) + cold_cards.append({ + "path": str(card.relative_to(target_dir)), + "tier": tier, + "relevance": fields.get("relevance", "?"), + "title": title, + "last_accessed": fields.get("last_accessed", "?"), + }) + + if not cold_cards: + print(" no cold/archive cards found — memory is too fresh") + return + + sample = random.sample(cold_cards, min(n, len(cold_cards))) + print(f"\n creative recall — {len(sample)} random cards:") + for card in sample: + print(f" [{card['tier']}] {card['title'] or card['path']}") + print(f" {card['path']} (r={card['relevance']}, last={card['last_accessed']})") + print(f"\n read these cards and look for unexpected connections to your current task") + + +def cmd_daily(target_dir: Path, config: dict, dry_run: bool = False, verbose: bool = False): + """Bootstrap and decay daily files (YYYY-MM-DD.md pattern).""" + date_pattern = re.compile(r"^\d{4}-\d{2}-\d{2}\.md$") + daily_files = sorted( + f for f in target_dir.rglob("*.md") + if date_pattern.match(f.name) + ) + + if not daily_files: + print(f" no daily files (YYYY-MM-DD.md) found in {target_dir}") + return + + results = [] + for f in daily_files: + content = f.read_text(encoding="utf-8", errors="replace") + fields, body, had_yaml = parse_frontmatter(content) + + # Extract date from filename + file_date = date.fromisoformat(f.stem) + days = max(0, (TODAY - file_date).days) + + # Use file_date as reference (not git/mtime — daily files are date-intrinsic) + la = fields.get("last_accessed", "") + try: + last_acc = date.fromisoformat(la[:10]) + # Use most recent of: file_date, last_accessed + ref_days = max(0, (TODAY - max(file_date, last_acc)).days) + except (ValueError, IndexError): + ref_days = days + + new_relevance = calc_relevance(ref_days, config["decay_rate"], config["relevance_floor"]) + old_tier = fields.get("tier", "") + new_tier = calc_tier(ref_days, config["tiers"], old_tier) + + fields["type"] = "daily" + fields["date"] = file_date.isoformat() + if "last_accessed" not in fields: + fields["last_accessed"] = file_date.isoformat() + fields["relevance"] = str(new_relevance) + fields["tier"] = new_tier + + new_content = build_frontmatter(fields) + body + changed = new_content != content + + if changed and not dry_run: + f.write_text(new_content, encoding="utf-8") + + results.append({ + "file": f.name, + "date": file_date.isoformat(), + "days": ref_days, + "relevance": new_relevance, + "tier": new_tier, + "changed": changed, + }) + + if verbose: + print(f" {'[dry] ' if dry_run else ''}{f.name}: {old_tier or '?'}→{new_tier} r={new_relevance} ({ref_days}d)") + + # Summary + tiers = {} + for r in results: + tiers[r["tier"]] = tiers.get(r["tier"], 0) + 1 + changed_count = sum(1 for r in results if r["changed"]) + + print(f"\n {'DRY RUN — ' if dry_run else ''}daily results:") + print(f" files: {len(results)}, changed: {changed_count}") + for tier in ["active", "warm", "cold", "archive"]: + count = tiers.get(tier, 0) + if count: + dates = [r["date"] for r in results if r["tier"] == tier] + print(f" {tier:8s}: {count:3d} ({dates[0]}..{dates[-1]})") + + +def cmd_stats(target_dir: Path, config: dict): + """Show comprehensive memory health stats.""" + cards = find_cards(target_dir, config) + tiers = {} + total_bytes = 0 + stale_count = 0 + no_yaml = 0 + + for card in cards: + content = card.read_text(encoding="utf-8", errors="replace") + total_bytes += len(content.encode("utf-8")) + fields, body, had_yaml = parse_frontmatter(content) + if not had_yaml: + no_yaml += 1 + tier = fields.get("tier", "unknown") + tiers[tier] = tiers.get(tier, 0) + 1 + try: + la = date.fromisoformat(fields.get("last_accessed", "")[:10]) + if (TODAY - la).days > 90: + stale_count += 1 + except (ValueError, IndexError): + pass + + print(f"\n memory health — {target_dir}") + print(f" {'─' * 40}") + print(f" total cards: {len(cards)}") + print(f" total size: {total_bytes / 1024:.0f} KB") + print(f" without yaml: {no_yaml}") + print(f" stale (>90 days): {stale_count}") + print(f" {'─' * 40}") + print(f" tier distribution:") + for tier in ["core", "active", "warm", "cold", "archive", "unknown"]: + count = tiers.get(tier, 0) + if count: + pct = count / len(cards) * 100 + bar = "█" * int(pct / 2) + print(f" {tier:8s}: {count:4d} ({pct:4.1f}%) {bar}") + + # Context budget estimate (assuming ~4 chars per token) + active_bytes = 0 + for card in cards: + content = card.read_text(encoding="utf-8", errors="replace") + fields, _, _ = parse_frontmatter(content) + if fields.get("tier") in ("core", "active"): + active_bytes += len(content.encode("utf-8")) + print(f" {'─' * 40}") + print(f" active context: {active_bytes / 1024:.0f} KB (~{active_bytes // 4:,} tokens)") + print(f" total context: {total_bytes / 1024:.0f} KB (~{total_bytes // 4:,} tokens)") + + +# ─── main ─────────────────────────────────────────────────────── + +def main(): + args = sys.argv[1:] + if not args or args[0] in ("-h", "--help"): + print(__doc__) + sys.exit(0) + + cmd = args[0] + dry_run = "--dry-run" in args + verbose = "--verbose" in args + + # Find config path + config_path = None + if "--config" in args: + idx = args.index("--config") + config_path = args[idx + 1] if idx + 1 < len(args) else None + + # Find target directory (first non-flag argument after command) + target = None + for a in args[1:]: + if not a.startswith("-") and a != config_path: + target = a + break + + if cmd == "touch": + if not target: + print(" error: touch requires a file path") + sys.exit(1) + config = load_config(Path(target).parent, config_path) + cmd_touch(target, config) + return + + if cmd == "creative": + n = 5 + creative_dir = None + for a in args[1:]: + if not a.startswith("-"): + try: + n = int(a) + except ValueError: + creative_dir = a + target_dir = Path(creative_dir) if creative_dir else Path(".") + config = load_config(target_dir, config_path) + cmd_creative(n, target_dir, config) + return + + target_dir = Path(target) if target else Path(".") + if not target_dir.is_dir(): + print(f" error: {target_dir} is not a directory") + sys.exit(1) + + config = load_config(target_dir, config_path) + + if cmd == "scan": + cmd_scan(target_dir, config, verbose) + elif cmd == "init": + cmd_init(target_dir, config, dry_run, verbose) + elif cmd == "decay": + cmd_decay(target_dir, config, dry_run, verbose) + elif cmd == "daily": + cmd_daily(target_dir, config, dry_run, verbose) + elif cmd == "stats": + cmd_stats(target_dir, config) + elif cmd == "config": + save_default_config(target_dir) + else: + print(f" unknown command: {cmd}") + print(" commands: scan, init, decay, daily, touch, creative, stats, config") + sys.exit(1) + + +if __name__ == "__main__": + main()