From a10cdcd197747f371b7879a32c2cc3f77471e9c2 Mon Sep 17 00:00:00 2001 From: Elliot Chen <2340896+cyfyifanchen@users.noreply.github.com> Date: Tue, 16 Jun 2026 21:46:17 +0800 Subject: [PATCH] chore(release): prepare EverOS 1.0.1 (#290) --- CHANGELOG.md | 27 ++++++- CITATION.md | 2 +- QUICKSTART.md | 6 +- docs/architecture.md | 15 ++-- docs/openapi.json | 12 ++-- docs/overview.md | 5 +- pyproject.toml | 8 +-- src/everos/core/errors.py | 10 +++ .../core/persistence/markdown/writer.py | 30 ++++++-- src/everos/entrypoints/api/app.py | 3 +- src/everos/entrypoints/api/routes/memorize.py | 28 +++++--- .../test_markdown/test_writer.py | 72 +++++++++++++++++++ .../test_api/test_app_metadata.py | 11 +++ .../test_memorize_route_validation.py | 71 ++++++++++++++++++ uv.lock | 2 +- 15 files changed, 264 insertions(+), 38 deletions(-) create mode 100644 tests/unit/test_entrypoints/test_api/test_app_metadata.py create mode 100644 tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f0b2720..6fabd11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,30 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 _Unreleased changes on `dev` will be listed here._ +## [1.0.1] - 2026-06-16 + +### Security + +- **Path-traversal hardening for caller-supplied identifiers.** `sender_id` + now carries the same path-safety guard as `app_id` / `project_id`: a + character whitelist plus rejection of the `.` / `..` tokens. The whitelist + admits `@` and `+` so email-style ids and plus-addressing still pass. +- **Defense-in-depth write containment.** `MarkdownWriter` now rejects any + write target that resolves outside the configured memory root before reading, + creating parent directories, or writing files. The API layer maps this + backstop error to HTTP 400. + +### Documentation + +- Add a multimodal usage guide and correct the multimodal error semantics + after end-to-end verification. +- Document the upcoming Knowledge Wiki and idle/offline Reflection/Dreaming + roadmap in the README and documentation set. +- Rename outdated algorithm-library references to `everalgo` across docs and + code comments; no code identifiers changed. +- Fix accuracy drift found in a documentation audit; reflect the `everalgo` + packages being published and the v1.0.0 stable status. + ## [1.0.0] - 2026-06-03 First public release of EverOS — a Markdown-first memory extraction framework @@ -36,5 +60,6 @@ for AI agents. - **Decoupled algorithms** — memory extraction algorithms live in the standalone `everalgo-*` libraries published on PyPI. -[Unreleased]: https://github.com/EverMind-AI/EverOS/compare/v1.0.0...HEAD +[Unreleased]: https://github.com/EverMind-AI/EverOS/compare/v1.0.1...HEAD +[1.0.1]: https://github.com/EverMind-AI/EverOS/releases/tag/v1.0.1 [1.0.0]: https://github.com/EverMind-AI/EverOS/releases/tag/v1.0.0 diff --git a/CITATION.md b/CITATION.md index 99f40f5..e797b0c 100644 --- a/CITATION.md +++ b/CITATION.md @@ -33,7 +33,7 @@ To cite the software itself: ``` EverOS: md-first memory extraction framework for AI agents -Version: 1.0.0 +Version: 1.0.1 URL: https://github.com/EverMind-AI/EverOS License: Apache 2.0 ``` diff --git a/QUICKSTART.md b/QUICKSTART.md index d250f46..ad243eb 100644 --- a/QUICKSTART.md +++ b/QUICKSTART.md @@ -296,8 +296,10 @@ LLM → metrics) before exiting. HTTP 415; PDF / image / audio / HTML still work. - **Filter DSL and search modes** — `/search` supports a filter DSL (`AND` / `OR` / scalar predicates) and four methods (`HYBRID` / - `KEYWORD` / `VECTOR` / `AGENTIC`). See the OpenAPI schema served at - `/docs`. + `KEYWORD` / `VECTOR` / `AGENTIC`). The OpenAPI docs UI is served at + `/docs` only when the server runs with `ENV=DEV`; the default (`prod`) + serves the API without the docs UI. The schema also lives at + [docs/openapi.json](docs/openapi.json). - **Architecture** — see [docs/architecture.md](docs/architecture.md) for the DDD layering and cascade design, and [docs/storage_layout.md](docs/storage_layout.md) for the on-disk diff --git a/docs/architecture.md b/docs/architecture.md index d9384f2..901227b 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -111,10 +111,10 @@ External message User query │ ▼ -1. service.retrieve +1. service.search │ ▼ -2. memory.search.hybrid single LanceDB query = +2. memory.search (hybrid) single LanceDB query = BM25 + vector ANN + scalar filter │ ▼ @@ -191,12 +191,15 @@ protection (L1 read-only / L2 system / L3 business / L4 user). ## everalgo boundary -[`everalgo`](https://github.com/EverMind-AI/EverAlgo) is a separate Python library (published as the `everalgo-*` PyPI packages) holding **only memory extraction algorithms**: +`everalgo` is a set of PyPI-published packages (`everalgo-core`, +`everalgo-boundary`, `everalgo-user-memory`, `everalgo-agent-memory`, +`everalgo-rank`, plus the optional `everalgo-parser` extra), imported under +the `everalgo` namespace, holding **only memory extraction algorithms**: -- `everalgo.parser` — multi-modal parsing +- `everalgo.parser` — multi-modal parsing (optional `[multimodal]` extra) - `everalgo.user_memory` — ConvMemCell / Episode / Foresight / AtomicFact / Profile extractors - `everalgo.agent_memory` — AgentMemCell / Case / Skill extractors -- `everalgo.knowledge` — file-to-knowledge +- `everalgo.boundary` / `everalgo.rank` — boundary detection / fusion + rerank everalgo is: @@ -204,7 +207,7 @@ everalgo is: - **No I/O** — does not touch md files / LanceDB / SQLite - **No prompts inline** — receives `PromptSlot` parameter, project supplies defaults -This boundary lets everalgo be reused across product forms (this open-source build, EverMind Cloud, OpenClaw plugins, etc.). +This boundary lets everalgo be reused across product forms (this open-source build, EverOS Cloud, OpenClaw plugins, etc.). ## Further reading diff --git a/docs/openapi.json b/docs/openapi.json index 0ae6608..1d4a3b4 100644 --- a/docs/openapi.json +++ b/docs/openapi.json @@ -3,7 +3,7 @@ "info": { "title": "everos", "description": "md-first memory extraction framework", - "version": "0.1.0" + "version": "1.0.1" }, "paths": { "/health": { @@ -827,7 +827,7 @@ "type": "string", "maxLength": 128, "minLength": 1, - "pattern": "^[a-zA-Z0-9_.-]+$", + "pattern": "^[a-zA-Z0-9_.@+-]+$", "title": "App Id", "default": "default" }, @@ -835,7 +835,7 @@ "type": "string", "maxLength": 128, "minLength": 1, - "pattern": "^[a-zA-Z0-9_.-]+$", + "pattern": "^[a-zA-Z0-9_.@+-]+$", "title": "Project Id", "default": "default" }, @@ -868,7 +868,7 @@ "type": "string", "maxLength": 128, "minLength": 1, - "pattern": "^[a-zA-Z0-9_.-]+$", + "pattern": "^[a-zA-Z0-9_.@+-]+$", "title": "App Id", "default": "default" }, @@ -876,7 +876,7 @@ "type": "string", "maxLength": 128, "minLength": 1, - "pattern": "^[a-zA-Z0-9_.-]+$", + "pattern": "^[a-zA-Z0-9_.@+-]+$", "title": "Project Id", "default": "default" } @@ -891,7 +891,9 @@ "properties": { "sender_id": { "type": "string", + "maxLength": 128, "minLength": 1, + "pattern": "^[a-zA-Z0-9_.@+-]+$", "title": "Sender Id" }, "sender_name": { diff --git a/docs/overview.md b/docs/overview.md index 9ba7336..038f1b8 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -43,7 +43,7 @@ User trust comes from physical visibility — the user can `cat` / `vim` / `grep ### 3. Algorithm-orchestration separation -[`everalgo`](https://github.com/EverMind-AI/EverAlgo) (a separate library, published as the `everalgo-*` PyPI packages) holds the extraction algorithms (MemCell extraction, Episode generation, Profile evolution). EverOS calls everalgo via the PromptSlot interface; everalgo knows nothing about storage. +`everalgo` (a set of separate PyPI packages — `everalgo-core` / `-boundary` / `-user-memory` / `-agent-memory` / `-rank`, plus the optional `-parser` extra) holds the extraction algorithms (MemCell extraction, Episode generation, Profile evolution). EverOS calls everalgo via the PromptSlot interface; everalgo knows nothing about storage. This boundary lets the same algorithm power both this open-source lightweight version and other product forms. @@ -82,4 +82,5 @@ Strict single-direction dependency, enforced by `import-linter` in CI. ## Status -**Alpha — v0.1.0 in active development**. Core API may change before v1.0. +**Stable (v1.0.1)** — Released on PyPI; the v1 API is stable. Development +continues on `dev` toward v1.1. diff --git a/pyproject.toml b/pyproject.toml index 9a453a0..0715b99 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "everos" -version = "1.0.0" +version = "1.0.1" description = "EverOS — local-first markdown memory framework for AI agents and user chats; lightweight, dev-friendly, small-team" license = {text = "Apache-2.0"} readme = "README.md" @@ -10,7 +10,7 @@ authors = [ ] keywords = ["memory", "ai-agent", "markdown", "lancedb", "rag", "everos"] classifiers = [ - "Development Status :: 3 - Alpha", + "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: Apache Software License", "Programming Language :: Python :: 3", @@ -83,8 +83,8 @@ packages = ["src/everos"] # things like CLAUDE.md, .claude/, CI configs, or IDE settings. New top-level # files default to NOT-shipped — you have to opt them in here. # -# exclude is kept as belt-and-suspenders for build artefacts that CI may -# generate inside the project tree (e.g. UV_CACHE_DIR=.uv-cache). +# exclude is kept as belt-and-suspenders for build artefacts that CI generates +# inside the project tree (UV_CACHE_DIR=.uv-cache in .gitlab-ci.yml). [tool.hatch.build.targets.sdist] include = [ "/src", diff --git a/src/everos/core/errors.py b/src/everos/core/errors.py index ce18ac6..650da2f 100644 --- a/src/everos/core/errors.py +++ b/src/everos/core/errors.py @@ -9,6 +9,16 @@ not import ``memory`` directly). from __future__ import annotations +class PathTraversalError(Exception): + """A write target resolved outside the configured memory root. + + Raised by the markdown writer as a defense-in-depth backstop: any + caller-supplied identifier that becomes a path segment is validated at + the DTO layer, but this containment check does not depend on every such + id being sanitized upstream. The API layer maps it to HTTP 400. + """ + + class MultimodalError(Exception): """Base for multimodal-parsing errors meant to reach the caller. diff --git a/src/everos/core/persistence/markdown/writer.py b/src/everos/core/persistence/markdown/writer.py index 82fab25..066772b 100644 --- a/src/everos/core/persistence/markdown/writer.py +++ b/src/everos/core/persistence/markdown/writer.py @@ -48,6 +48,8 @@ from typing import Any import anyio +from everos.core.errors import PathTraversalError + from ..memory_root import MemoryRoot from .entries import EntryId from .frontmatter import dump_frontmatter @@ -57,9 +59,10 @@ from .reader import MarkdownReader class MarkdownWriter: """Atomic writer for markdown files inside a memory-root. - The ``memory_root`` reference is held to enable future enforcement that - targets stay within the configured root; current writes do not depend on - it for the rename itself (same-dir temp file). + The ``memory_root`` reference anchors a containment check: every write + target must resolve inside ``memory_root.root``. This is defense-in-depth + against path traversal via caller-supplied identifiers that become path + segments. """ def __init__(self, memory_root: MemoryRoot) -> None: @@ -96,19 +99,31 @@ class MarkdownWriter: self._path_locks[key] = lock return lock + def _ensure_within_root(self, target: Path) -> Path: + """Reject a write target that resolves outside the memory root.""" + root = self._memory_root.root + resolved = target.resolve() + if not resolved.is_relative_to(root): + raise PathTraversalError( + f"write target escapes the memory root: {resolved} not under {root}" + ) + return resolved + async def write(self, path: Path, content: str) -> Path: """Atomically write ``content`` to ``path``. Steps: - 1. ``mkdir -p`` the parent directory. - 2. Write to ``/..tmp.``. - 3. ``flush`` + ``fsync`` the temp file. - 4. ``os.replace`` the temp file onto ``path`` (atomic on POSIX). + 1. Assert the target resolves inside the memory root. + 2. ``mkdir -p`` the parent directory. + 3. Write to ``/..tmp.``. + 4. ``flush`` + ``fsync`` the temp file. + 5. ``os.replace`` the temp file onto ``path`` (atomic on POSIX). Returns: ``path`` (resolved as written). """ target = Path(path) + self._ensure_within_root(target) await anyio.Path(target.parent).mkdir(parents=True, exist_ok=True) tmp = target.parent / f".{target.name}.tmp.{uuid.uuid4().hex}" try: @@ -224,6 +239,7 @@ class MarkdownWriter: breaks the safety contract. """ target = Path(path) + self._ensure_within_root(target) # 1. Load existing markdown (or initialise empty). if await anyio.Path(target).is_file(): diff --git a/src/everos/entrypoints/api/app.py b/src/everos/entrypoints/api/app.py index 8e728fa..20c0484 100644 --- a/src/everos/entrypoints/api/app.py +++ b/src/everos/entrypoints/api/app.py @@ -12,6 +12,7 @@ from fastapi import FastAPI, HTTPException from fastapi.exceptions import RequestValidationError from fastapi.middleware.cors import CORSMiddleware +from everos import __version__ from everos.core.lifespan import ( LifespanProvider, MetricsLifespanProvider, @@ -88,7 +89,7 @@ def create_app( app = FastAPI( title="everos", - version="0.1.0", + version=__version__, description="md-first memory extraction framework", lifespan=build_lifespan(lifespan_providers), docs_url="/docs" if enable_docs else None, diff --git a/src/everos/entrypoints/api/routes/memorize.py b/src/everos/entrypoints/api/routes/memorize.py index 392fde9..b3681de 100644 --- a/src/everos/entrypoints/api/routes/memorize.py +++ b/src/everos/entrypoints/api/routes/memorize.py @@ -15,7 +15,7 @@ from typing import Annotated, Any, Literal from fastapi import APIRouter, HTTPException, Request from pydantic import AfterValidator, BaseModel, ConfigDict, Field -from everos.core.errors import MultimodalError +from everos.core.errors import MultimodalError, PathTraversalError from everos.core.observability.tracing import gen_request_id from everos.service import memorize @@ -23,12 +23,17 @@ router = APIRouter(prefix="/api/v1/memory", tags=["memory"]) # ── Path-safe identifier ──────────────────────────────────────────────────── -# ``app_id`` / ``project_id`` become directory segments under the memory -# root, so they must reject ``.`` and ``..`` (path traversal). The basic -# character whitelist is enforced via ``pattern`` (pydantic_core uses the -# Rust regex engine, which does NOT support lookaround), and the two -# reserved tokens are filtered out with a follow-up ``AfterValidator``. -_PATH_SAFE_CHARSET = r"^[a-zA-Z0-9_.-]+$" +# ``app_id`` / ``project_id`` / ``sender_id`` all become directory segments +# under the memory root (``sender_id`` flows through to ``owner_id``), so they +# must reject ``.`` and ``..`` (path traversal). The basic character whitelist +# is enforced via ``pattern`` (pydantic_core uses the Rust regex engine, which +# does NOT support lookaround), and the two reserved tokens are filtered out +# with a follow-up ``AfterValidator``. +# +# ``@`` and ``+`` are admitted so real-world ids survive (email-style ids and +# plus-addressing). Path separators and NUL stay out of the whitelist, while +# the markdown writer's root-containment check is the final backstop. +_PATH_SAFE_CHARSET = r"^[a-zA-Z0-9_.@+-]+$" _PATH_TRAVERSAL_TOKENS = frozenset({".", ".."}) @@ -70,7 +75,12 @@ class ContentItemDTO(BaseModel): class MessageItemDTO(BaseModel): - sender_id: str = Field(..., min_length=1) + sender_id: PathSafeId = Field( + ..., + min_length=1, + max_length=128, + pattern=_PATH_SAFE_CHARSET, + ) sender_name: str | None = None role: Literal["user", "assistant", "tool"] timestamp: int = Field( @@ -150,6 +160,8 @@ async def add_memory( result = await memorize(req.model_dump()) except MultimodalError as exc: raise HTTPException(status_code=415, detail=str(exc)) from exc + except PathTraversalError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc return SuccessEnvelope( request_id=request_id, data=AddResponseData( diff --git a/tests/unit/test_core/test_persistence/test_markdown/test_writer.py b/tests/unit/test_core/test_persistence/test_markdown/test_writer.py index 6468def..cd2365b 100644 --- a/tests/unit/test_core/test_persistence/test_markdown/test_writer.py +++ b/tests/unit/test_core/test_persistence/test_markdown/test_writer.py @@ -8,6 +8,7 @@ from unittest.mock import patch import pytest +from everos.core.errors import PathTraversalError from everos.core.persistence import ( EntryId, MarkdownReader, @@ -227,3 +228,74 @@ async def test_append_entry_round_trip_with_reader(tmp_path: Path) -> None: for i, e in enumerate(parsed.entries): assert e.id == f"umc_20260422_{i + 1:08d}" assert e.body == f"content {i}" + + +# ── memory-root containment (path-traversal defense in depth) ────────────── + + +async def test_write_rejects_target_escaping_root(tmp_path: Path) -> None: + root = tmp_path / "memory_root" + root.mkdir() + writer = MarkdownWriter(MemoryRoot(root)) + escaping = root / "users" / ".." / ".." / ".." / "ESCAPED" / "f.md" + + with pytest.raises(PathTraversalError): + await writer.write(escaping, "x") + + assert not (tmp_path / "ESCAPED").exists() + + +async def test_write_markdown_rejects_escaping_target(tmp_path: Path) -> None: + root = tmp_path / "memory_root" + root.mkdir() + writer = MarkdownWriter(MemoryRoot(root)) + escaping = root / ".." / "ESCAPED.md" + + with pytest.raises(PathTraversalError): + await writer.write_markdown(escaping, body="x") + assert not (tmp_path / "ESCAPED.md").exists() + + +async def test_append_entry_rejects_escaping_target(tmp_path: Path) -> None: + root = tmp_path / "memory_root" + root.mkdir() + writer = MarkdownWriter(MemoryRoot(root)) + escaping = root / "users" / ".." / ".." / "ESCAPED" / "log.md" + + with pytest.raises(PathTraversalError): + await writer.append_entry( + escaping, + entry_body="x", + entry_id=EntryId(prefix="umc", date=dt.date(2026, 4, 22), seq=1), + ) + assert not (tmp_path / "ESCAPED").exists() + + +async def test_append_entry_does_not_read_out_of_root_file(tmp_path: Path) -> None: + root = tmp_path / "memory_root" + root.mkdir() + secret = tmp_path / "secret.md" + secret.write_text("---\ntop: secret\n---\nbody\n", encoding="utf-8") + writer = MarkdownWriter(MemoryRoot(root)) + escaping = root / "users" / ".." / ".." / "secret.md" + + with ( + patch.object(MarkdownReader, "read", wraps=MarkdownReader.read) as spy, + pytest.raises(PathTraversalError), + ): + await writer.append_entry( + escaping, + entry_body="x", + entry_id=EntryId(prefix="umc", date=dt.date(2026, 4, 22), seq=1), + ) + spy.assert_not_called() + assert secret.read_text(encoding="utf-8") == "---\ntop: secret\n---\nbody\n" + + +async def test_write_allows_target_inside_root(tmp_path: Path) -> None: + root = tmp_path / "memory_root" + root.mkdir() + writer = MarkdownWriter(MemoryRoot(root)) + target = root / "users" / "u1" / "episodes" / "e.md" + written = await writer.write(target, "hello\n") + assert written.read_text(encoding="utf-8") == "hello\n" diff --git a/tests/unit/test_entrypoints/test_api/test_app_metadata.py b/tests/unit/test_entrypoints/test_api/test_app_metadata.py new file mode 100644 index 0000000..0a6df21 --- /dev/null +++ b/tests/unit/test_entrypoints/test_api/test_app_metadata.py @@ -0,0 +1,11 @@ +"""FastAPI metadata must stay aligned with package metadata.""" + +from __future__ import annotations + +from everos import __version__ +from everos.entrypoints.api.app import create_app + + +def test_openapi_info_version_matches_package_version() -> None: + app = create_app(lifespan_providers=[]) + assert app.openapi()["info"]["version"] == __version__ diff --git a/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py b/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py new file mode 100644 index 0000000..234c49a --- /dev/null +++ b/tests/unit/test_entrypoints/test_api/test_routes/test_memorize_route_validation.py @@ -0,0 +1,71 @@ +"""DTO-layer path-safety validation for ``POST /api/v1/memory/add``.""" + +from __future__ import annotations + +import pytest +from pydantic import ValidationError + +from everos.entrypoints.api.routes.memorize import ( + MemorizeAddRequest, + MessageItemDTO, +) + + +def _message(sender_id: str) -> MessageItemDTO: + return MessageItemDTO( + sender_id=sender_id, + role="user", + timestamp=1_700_000_000_000, + content="x", + ) + + +@pytest.mark.parametrize( + "bad_sender_id", + [ + "../../../../etc", + "..", + ".", + "a/b", + "a/../b", + "with space", + "", + ], +) +def test_message_item_rejects_unsafe_sender_id(bad_sender_id: str) -> None: + with pytest.raises(ValidationError): + _message(bad_sender_id) + + +@pytest.mark.parametrize( + "good_sender_id", + [ + "u1", + "u_jason", + "user-123", + "a.b_c-1", + "default", + "user@example.com", + "user+tag", + "user+tag@example.com", + ], +) +def test_message_item_accepts_path_safe_sender_id(good_sender_id: str) -> None: + assert _message(good_sender_id).sender_id == good_sender_id + + +def test_add_request_rejects_traversal_sender_id_in_messages() -> None: + with pytest.raises(ValidationError): + MemorizeAddRequest( + session_id="s1", + app_id="default", + project_id="default", + messages=[ + { + "sender_id": "../../../../ESCAPED", + "role": "user", + "timestamp": 1_700_000_000_000, + "content": "secret", + } + ], + ) diff --git a/uv.lock b/uv.lock index a662c5a..584612a 100644 --- a/uv.lock +++ b/uv.lock @@ -545,7 +545,7 @@ wheels = [ [[package]] name = "everos" -version = "1.0.0" +version = "1.0.1" source = { editable = "." } dependencies = [ { name = "aiosqlite" },