feat: add fts support (#408)

Add BM25-based full-text search with CJK (jieba) tokenization, supporting query_string and match_string syntax, phrase queries, boolean operators (AND/OR/NOT/MUST), and hybrid retrieval with existing vector search. ## Core - BitPacked posting format with block-max WAND pruning - Tokenizer pipeline: jieba (cut/cut_for_search/hmm/full), whitespace, lowercase, with extensible pipeline composition - Query parser: boolean operators, phrase queries, field scoping, boost, MUST(+) modifier inside OR (ES query_string semantics) - AST rewriter: dedup repeated terms with linear boost aggregation, flatten same-type composites, canonicalize OR-with-must_not into AND wrapper, empty-node propagation, contradiction detection - FTS reduce/merge integrated into Optimize compaction - Multi-segment score-descending sort - Auto-register bundled jieba dict on SDK import ## Performance - Block-max WAND with cached block_max_info_for (single binary search) - AVX2/SSE bitpacked encoding with cross-arch scalar fallback - MultiGet for batch posting retrieval and phrase position verification - HashSkipList memtable for posting writes - PinnableSlice zero-copy reads - Filter pushdown into composite iterators (Disjunction/Conjunction/Phrase) - Candidate-driven (brute-force) evaluation for selective invert filters - Precomputed BM25 IDF weights, cached SIMD dispatch pointers - Shortest-list anchor for phrase position matching - Single-open per-term posting iterator ## Query - Tokenize query terms through the same pipeline as indexing - EmptyNode for zero-token queries (all stop-words / punctuation) - Backslash unescape after lexing in query parser - Schema allows collections without vector fields (FTS-only use case) - Create/Drop Index validates supported index types - FTS fields disallowed in SQL filter expressions ## Bindings - C API: fts query params, brute-force ratio config - Python SDK: FTS search, jieba dict auto-registration ## Internals - Bypass cppjieba::Jieba to drop KeywordExtractor (~12MB fewer required files) - Hide tokenizer pipeline from public header (Pimpl-style FtsState) - ListColumnFamilies to avoid double-open on segment load - Reorganized fts_column into tokenizer/, posting/, iterator/ subdirs
2026-06-01 15:02:54 +08:00
parent 74beb2a828
commit 02bfb31cf5
154 changed files with 22646 additions and 223 deletions
@@ -0,0 +1,188 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""End-to-end tests for FTS-only collections (no vector field).
+
+The schema validation rule "must have at least one vector field" has been
+lifted; these tests pin the new behavior so insert / query / delete /
+optimize all work on a vector-less collection.
+"""
+
+from __future__ import annotations
+
+import pytest
+import zvec
+from zvec import (
+    Collection,
+    CollectionOption,
+    DataType,
+    Doc,
+    FieldSchema,
+    FtsIndexParam,
+    OptimizeOption,
+)
+from zvec.model.param.query import Fts, Query
+
+
+# ==================== Fixtures ====================
+
+
+@pytest.fixture(scope="function")
+def fts_collection(tmp_path_factory) -> Collection:
+    """FTS-only collection: a STRING field for forward + an FTS-indexed STRING."""
+    temp_dir = tmp_path_factory.mktemp("zvec_fts_only")
+    collection_path = temp_dir / "fts_collection"
+
+    schema = zvec.CollectionSchema(
+        name="fts_only",
+        fields=[
+            FieldSchema("title", DataType.STRING, nullable=False),
+            FieldSchema(
+                "content",
+                DataType.STRING,
+                nullable=False,
+                index_param=FtsIndexParam(
+                    tokenizer_name="standard",
+                    filters=["lowercase"],
+                ),
+            ),
+        ],
+        # vectors omitted on purpose — schema validation must accept this.
+    )
+
+    coll = zvec.create_and_open(
+        path=str(collection_path),
+        schema=schema,
+        option=CollectionOption(read_only=False, enable_mmap=True),
+    )
+    assert coll is not None
+
+    try:
+        yield coll
+    finally:
+        try:
+            coll.destroy()
+        except Exception as e:
+            print(f"Warning: failed to destroy collection: {e}")
+
+
+def _make_docs() -> list[Doc]:
+    """5-doc corpus where 4 contain 'hello' and doc 4 is the only outlier."""
+    return [
+        Doc(id="pk_0", fields={"title": "intro", "content": "hello world"}),
+        Doc(id="pk_1", fields={"title": "guide", "content": "hello foo bar"}),
+        Doc(id="pk_2", fields={"title": "tips", "content": "hello baz"}),
+        Doc(id="pk_3", fields={"title": "more", "content": "hello hello"}),
+        Doc(id="pk_4", fields={"title": "other", "content": "nothing relevant"}),
+    ]
+
+
+def _fts_query(coll: Collection, term: str) -> list[Doc]:
+    """Run a single-term FTS match query against the `content` field."""
+    return coll.query(
+        queries=Query(field_name="content", fts=Fts(match_string=term)),
+        topk=10,
+    )
+
+
+# ==================== Tests ====================
+
+
+class TestFtsOnlyCollectionSchema:
+    def test_create_and_open_without_vectors(self, fts_collection: Collection):
+        """Schema with zero vector fields must be accepted by validate()."""
+        assert fts_collection.schema.name == "fts_only"
+        assert {f.name for f in fts_collection.schema.fields} == {"title", "content"}
+        # Empty vectors is the whole point of the test.
+        assert list(fts_collection.schema.vectors) == []
+        assert fts_collection.stats.doc_count == 0
+
+    def test_create_schema_omitting_vectors_kwarg(self):
+        """Constructing CollectionSchema without `vectors=` argument is valid."""
+        schema = zvec.CollectionSchema(
+            name="bare_fts",
+            fields=[
+                FieldSchema(
+                    "content",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=FtsIndexParam(),
+                ),
+            ],
+        )
+        assert list(schema.vectors) == []
+        assert {f.name for f in schema.fields} == {"content"}
+
+
+class TestFtsOnlyCollectionLifecycle:
+    def test_insert_and_fts_query(self, fts_collection: Collection):
+        """FTS-only collection supports insert + FTS query end-to-end."""
+        results = fts_collection.insert(_make_docs())
+        assert all(r.ok() for r in results)
+        assert fts_collection.stats.doc_count == 5
+
+        hits = _fts_query(fts_collection, "hello")
+        assert len(hits) == 4
+        assert {doc.id for doc in hits} == {"pk_0", "pk_1", "pk_2", "pk_3"}
+
+        # Term that nothing in the surviving corpus contains.
+        assert _fts_query(fts_collection, "missing_term_xyz") == []
+
+    def test_delete_then_query(self, fts_collection: Collection):
+        """Tombstone filter must drop deleted docs from FTS results."""
+        fts_collection.insert(_make_docs())
+        statuses = fts_collection.delete(["pk_0", "pk_4"])
+        assert all(s.ok() for s in statuses)
+        assert fts_collection.stats.doc_count == 3
+
+        hits = _fts_query(fts_collection, "hello")
+        assert len(hits) == 3
+        assert {doc.id for doc in hits} == {"pk_1", "pk_2", "pk_3"}
+        # pk_4's unique term is filtered out post-delete.
+        assert _fts_query(fts_collection, "nothing") == []
+
+    def test_optimize_rebuilds_fts(self, fts_collection: Collection):
+        """Optimize with >30% deletes triggers ReduceFts; recall unchanged."""
+        fts_collection.insert(_make_docs())
+        # 40% delete ratio — above COMPACT_DELETE_RATIO_THRESHOLD=0.3, so
+        # build_compact_task picks the rebuild path and ReduceFts runs.
+        fts_collection.delete(["pk_0", "pk_4"])
+
+        before = {doc.id for doc in _fts_query(fts_collection, "hello")}
+        assert before == {"pk_1", "pk_2", "pk_3"}
+
+        fts_collection.optimize(option=OptimizeOption())
+        assert fts_collection.stats.doc_count == 3
+
+        after = {doc.id for doc in _fts_query(fts_collection, "hello")}
+        assert after == before
+        assert _fts_query(fts_collection, "nothing") == []
+
+
+class TestFtsOnlyCollectionQueryValidation:
+    def test_vector_query_rejected(self, fts_collection: Collection):
+        """Vector query on a no-vector collection must raise."""
+        with pytest.raises(ValueError, match="vector or id"):
+            fts_collection.query(
+                queries=Query(field_name="content", vector=[0.1, 0.2, 0.3]),
+                topk=5,
+            )
+
+    def test_id_query_rejected(self, fts_collection: Collection):
+        """ID-based query on a no-vector collection must raise."""
+        fts_collection.insert(_make_docs()[:1])
+        with pytest.raises(ValueError, match="vector or id"):
+            fts_collection.query(
+                queries=Query(field_name="content", id="pk_0"),
+                topk=5,
+            )
@@ -0,0 +1,158 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Tests for FTS (Full-Text Search) query support in the Python SDK."""
+
+import pickle
+
+import pytest
+
+from zvec.model.param.query import Fts, Query
+
+
+class TestFtsQueryValidation:
+    """Test FTS parameter validation in Query dataclass."""
+
+    def test_fts_query_string_only(self):
+        """Query with only query_string in Fts should be valid."""
+        q = Query(
+            field_name="content", fts=Fts(query_string='+hello -world "exact phrase"')
+        )
+        q._validate()
+        assert q.fts.query_string == '+hello -world "exact phrase"'
+        assert q.fts.match_string is None
+        assert q.has_fts() is True
+
+    def test_fts_match_string_only(self):
+        """Query with only match_string in Fts should be valid."""
+        q = Query(field_name="content", fts=Fts(match_string="machine learning"))
+        q._validate()
+        assert q.fts.match_string == "machine learning"
+        assert q.fts.query_string is None
+        assert q.has_fts() is True
+
+    def test_fts_query_string_and_match_string_mutually_exclusive(self):
+        """Cannot provide both query_string and match_string in Fts."""
+        q = Query(
+            field_name="content",
+            fts=Fts(query_string="+hello", match_string="hello world"),
+        )
+        with pytest.raises(ValueError, match="mutually exclusive"):
+            q._validate()
+
+    def test_no_fts(self):
+        """Query without FTS fields should have has_fts() == False."""
+        q = Query(field_name="embedding", vector=[0.1, 0.2, 0.3])
+        assert q.has_fts() is False
+
+    def test_vector_and_fts_mutually_exclusive(self):
+        """Cannot combine vector search with FTS in a single Query."""
+        q = Query(
+            field_name="embedding",
+            vector=[0.1, 0.2, 0.3],
+            fts=Fts(match_string="deep learning"),
+        )
+        with pytest.raises(ValueError, match="Cannot combine fts with vector search"):
+            q._validate()
+
+    def test_fts_without_vector_or_id(self):
+        """Query with only FTS (no vector, no id) should be valid."""
+        q = Query(field_name="content", fts=Fts(query_string="hello"))
+        q._validate()
+        assert q.has_vector() is False
+        assert q.has_id() is False
+        assert q.has_fts() is True
+
+
+class TestFtsQueryBinding:
+    """Test FTS binding layer (_Fts)."""
+
+    def test_import_fts_query(self):
+        """_Fts should be importable from _zvec.param."""
+        from _zvec.param import _Fts
+
+        fts = _Fts()
+        assert fts.query_string == ""
+        assert fts.match_string == ""
+
+    def test_fts_query_set_fields(self):
+        """Setting fields on _Fts should work."""
+        from _zvec.param import _Fts
+
+        fts = _Fts()
+        fts.query_string = "+hello -world"
+        assert fts.query_string == "+hello -world"
+
+        fts2 = _Fts()
+        fts2.match_string = "machine learning"
+        assert fts2.match_string == "machine learning"
+
+    def test_fts_query_pickle(self):
+        """_Fts should support pickling."""
+        from _zvec.param import _Fts
+
+        fts = _Fts()
+        fts.query_string = "+vector search"
+        fts.match_string = ""
+
+        data = pickle.dumps(fts)
+        restored = pickle.loads(data)
+        assert restored.query_string == "+vector search"
+        assert restored.match_string == ""
+
+    def test_vector_query_fts_field(self):
+        """_VectorQuery should have fts field."""
+        from _zvec.param import _Fts, _VectorQuery
+
+        vq = _VectorQuery()
+        # fts should be None by default (optional)
+        assert vq.fts is None
+
+        # set fts
+        fts = _Fts()
+        fts.query_string = "hello"
+        vq.fts = fts
+        assert vq.fts is not None
+        assert vq.fts.query_string == "hello"
+
+    def test_vector_query_pickle_with_fts(self):
+        """_VectorQuery with fts should survive pickling."""
+        from _zvec.param import _Fts, _VectorQuery
+
+        vq = _VectorQuery()
+        vq.topk = 10
+        vq.field_name = "embedding"
+        fts = _Fts()
+        fts.match_string = "test query"
+        vq.fts = fts
+
+        data = pickle.dumps(vq)
+        restored = pickle.loads(data)
+        assert restored.topk == 10
+        assert restored.field_name == "embedding"
+        assert restored.fts is not None
+        assert restored.fts.match_string == "test query"
+
+    def test_vector_query_pickle_without_fts(self):
+        """_VectorQuery without fts should survive pickling."""
+        from _zvec.param import _VectorQuery
+
+        vq = _VectorQuery()
+        vq.topk = 5
+        vq.field_name = "vec"
+
+        data = pickle.dumps(vq)
+        restored = pickle.loads(data)
+        assert restored.topk == 5
+        assert restored.field_name == "vec"
+        assert restored.fts is None
@@ -0,0 +1,201 @@
+# Copyright 2025-present the zvec project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""End-to-end: jieba FTS works without any user configuration.
+
+`import zvec` is supposed to register the wheel-bundled jieba dict
+directory via `set_default_jieba_dict_dir`. With that in place a user can
+declare an FTS field with `tokenizer_name="jieba"`, leave `extra_params`
+empty, and Chinese full-text search just works.
+
+Falls back to GTEST_SKIP-equivalent when running against a build that did
+not bundle the dict (e.g., source-tree dev install without the install
+step). In that case CI will rely on the C++ unit tests instead.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+
+import pytest
+import zvec
+from zvec import (
+    Collection,
+    CollectionOption,
+    DataType,
+    Doc,
+    FieldSchema,
+    FtsIndexParam,
+)
+from zvec.model.param.query import Fts, Query
+
+
+def _bundled_dict_dir() -> str:
+    """Path zvec.__init__ would have registered; empty when not bundled."""
+    return zvec.get_default_jieba_dict_dir()
+
+
+def _bundled_dict_files_exist() -> bool:
+    """Whether the registered default actually contains the dict files.
+
+    `importlib.resources` happily returns a path even when the data dir was
+    not installed (e.g. source-tree dev runs); only an installed wheel has
+    the files on disk.
+    """
+    import os
+
+    base = _bundled_dict_dir()
+    if not base:
+        return False
+    return os.path.isfile(os.path.join(base, "jieba.dict.utf8")) and os.path.isfile(
+        os.path.join(base, "hmm_model.utf8")
+    )
+
+
+@pytest.fixture(scope="module", autouse=True)
+def _require_bundled_dict():
+    if not _bundled_dict_files_exist():
+        pytest.skip(
+            "Bundled jieba dict not found at zvec/data/jieba_dict/ — "
+            "this test requires an installed wheel (not a source-tree dev "
+            "build without the install step).",
+        )
+
+
+@pytest.fixture(scope="function")
+def jieba_collection(tmp_path_factory) -> Collection:
+    """FTS-only collection using jieba tokenizer and no explicit dict path."""
+    # env-var shadows GlobalConfig in the priority chain.
+    if os.environ.get("ZVEC_JIEBA_DICT_DIR"):
+        pytest.skip("ZVEC_JIEBA_DICT_DIR shadows the bundled default")
+    temp_dir = tmp_path_factory.mktemp("zvec_jieba_default")
+    collection_path = temp_dir / "fts_jieba"
+
+    schema = zvec.CollectionSchema(
+        name="fts_jieba_default",
+        fields=[
+            FieldSchema("title", DataType.STRING, nullable=False),
+            FieldSchema(
+                "content",
+                DataType.STRING,
+                nullable=False,
+                # Deliberately omit extra_params — the bundled default must
+                # be picked up via GlobalConfig.jieba_dict_dir.
+                index_param=FtsIndexParam(
+                    tokenizer_name="jieba",
+                    filters=["lowercase"],
+                ),
+            ),
+        ],
+    )
+
+    coll = zvec.create_and_open(
+        path=str(collection_path),
+        schema=schema,
+        option=CollectionOption(read_only=False, enable_mmap=True),
+    )
+    assert coll is not None
+    try:
+        yield coll
+    finally:
+        try:
+            coll.destroy()
+        except Exception as e:
+            print(f"Warning: failed to destroy collection: {e}")
+
+
+def test_jieba_works_without_explicit_dict_path(jieba_collection: Collection):
+    """User opens collection, inserts CJK doc, searches — no init() / no
+    extra_params / no env var / no manual setter call. Just `import zvec`."""
+    docs = [
+        Doc(id="pk_1", fields={"title": "t1", "content": "中华人民共和国成立"}),
+        Doc(id="pk_2", fields={"title": "t2", "content": "无关文档"}),
+    ]
+    insert_results = jieba_collection.insert(docs)
+    assert all(r.ok() for r in insert_results)
+
+    hits = jieba_collection.query(
+        queries=Query(field_name="content", fts=Fts(match_string="中华")),
+        topk=10,
+    )
+    ids = {doc.id for doc in hits}
+    assert "pk_1" in ids
+    assert "pk_2" not in ids
+
+
+def test_default_dict_dir_is_registered_on_import():
+    """Sanity check: zvec.__init__ registered a non-empty default."""
+    assert _bundled_dict_dir() != ""
+
+
+def test_user_can_override_default_at_runtime():
+    """zvec.set_default_jieba_dict_dir can be called any time to override."""
+    saved = zvec.get_default_jieba_dict_dir()
+    try:
+        zvec.set_default_jieba_dict_dir("/tmp/zvec/jieba-override")
+        assert zvec.get_default_jieba_dict_dir() == "/tmp/zvec/jieba-override"
+    finally:
+        zvec.set_default_jieba_dict_dir(saved)
+
+
+@pytest.mark.skipif(
+    sys.platform == "win32",
+    reason="os.environ writes may not propagate across CRT to zvec.pyd",
+)
+def test_env_var_overrides_global_config(monkeypatch, tmp_path_factory):
+    """ZVEC_JIEBA_DICT_DIR beats GlobalConfig in jieba's resolution chain."""
+    bundled = _bundled_dict_dir()
+    monkeypatch.setenv("ZVEC_JIEBA_DICT_DIR", bundled)
+    saved_global = zvec.get_default_jieba_dict_dir()
+    try:
+        zvec.set_default_jieba_dict_dir("/zvec/intentionally/missing/global")
+
+        temp_dir = tmp_path_factory.mktemp("zvec_jieba_env")
+        schema = zvec.CollectionSchema(
+            name="fts_jieba_env",
+            fields=[
+                FieldSchema("title", DataType.STRING, nullable=False),
+                FieldSchema(
+                    "content",
+                    DataType.STRING,
+                    nullable=False,
+                    index_param=FtsIndexParam(
+                        tokenizer_name="jieba",
+                        filters=["lowercase"],
+                    ),
+                ),
+            ],
+        )
+        coll = zvec.create_and_open(
+            path=str(temp_dir / "fts_jieba_env"),
+            schema=schema,
+            option=CollectionOption(read_only=False, enable_mmap=True),
+        )
+        assert coll is not None
+        try:
+            results = coll.insert(
+                [
+                    Doc(id="pk_1", fields={"title": "t", "content": "搜索引擎技术"}),
+                ]
+            )
+            assert all(r.ok() for r in results)
+            hits = coll.query(
+                queries=Query(field_name="content", fts=Fts(match_string="搜索")),
+                topk=10,
+            )
+            assert {d.id for d in hits} == {"pk_1"}
+        finally:
+            coll.destroy()
+    finally:
+        zvec.set_default_jieba_dict_dir(saved_global)
@@ -259,7 +259,9 @@ class TestNoVectorQueryExecutor:
    def test_do_validate_with_queries(self):
        schema = MockCollectionSchema()
        executor = NoVectorQueryExecutor(schema)
-        ctx = QueryContext(topk=10, queries=[Query(field_name="test")])
+        ctx = QueryContext(
+            topk=10, queries=[Query(field_name="test", vector=[0.1, 0.2, 0.3])]
+        )

        with pytest.raises(
            ValueError, match="Collection does not support query with vector or id"