Files
zvec/python/tests/test_fts_query.py
T
egolearner 02bfb31cf5 feat: add fts support (#408)
Add BM25-based full-text search with CJK (jieba) tokenization, supporting
  query_string and match_string syntax, phrase queries, boolean operators
  (AND/OR/NOT/MUST), and hybrid retrieval with existing vector search.

  ## Core
  - BitPacked posting format with block-max WAND pruning
  - Tokenizer pipeline: jieba (cut/cut_for_search/hmm/full), whitespace,
    lowercase, with extensible pipeline composition
  - Query parser: boolean operators, phrase queries, field scoping,
    boost, MUST(+) modifier inside OR (ES query_string semantics)
  - AST rewriter: dedup repeated terms with linear boost aggregation,
    flatten same-type composites, canonicalize OR-with-must_not into AND
    wrapper, empty-node propagation, contradiction detection
  - FTS reduce/merge integrated into Optimize compaction
  - Multi-segment score-descending sort
  - Auto-register bundled jieba dict on SDK import

  ## Performance
  - Block-max WAND with cached block_max_info_for (single binary search)
  - AVX2/SSE bitpacked encoding with cross-arch scalar fallback
  - MultiGet for batch posting retrieval and phrase position verification
  - HashSkipList memtable for posting writes
  - PinnableSlice zero-copy reads
  - Filter pushdown into composite iterators (Disjunction/Conjunction/Phrase)
  - Candidate-driven (brute-force) evaluation for selective invert filters
  - Precomputed BM25 IDF weights, cached SIMD dispatch pointers
  - Shortest-list anchor for phrase position matching
  - Single-open per-term posting iterator

  ## Query
  - Tokenize query terms through the same pipeline as indexing
  - EmptyNode for zero-token queries (all stop-words / punctuation)
  - Backslash unescape after lexing in query parser
  - Schema allows collections without vector fields (FTS-only use case)
  - Create/Drop Index validates supported index types
  - FTS fields disallowed in SQL filter expressions

  ## Bindings
  - C API: fts query params, brute-force ratio config
  - Python SDK: FTS search, jieba dict auto-registration

  ## Internals
  - Bypass cppjieba::Jieba to drop KeywordExtractor (~12MB fewer required files)
  - Hide tokenizer pipeline from public header (Pimpl-style FtsState)
  - ListColumnFamilies to avoid double-open on segment load
  - Reorganized fts_column into tokenizer/, posting/, iterator/ subdirs
2026-06-01 15:02:54 +08:00

159 lines
5.2 KiB
Python

# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for FTS (Full-Text Search) query support in the Python SDK."""
import pickle
import pytest
from zvec.model.param.query import Fts, Query
class TestFtsQueryValidation:
"""Test FTS parameter validation in Query dataclass."""
def test_fts_query_string_only(self):
"""Query with only query_string in Fts should be valid."""
q = Query(
field_name="content", fts=Fts(query_string='+hello -world "exact phrase"')
)
q._validate()
assert q.fts.query_string == '+hello -world "exact phrase"'
assert q.fts.match_string is None
assert q.has_fts() is True
def test_fts_match_string_only(self):
"""Query with only match_string in Fts should be valid."""
q = Query(field_name="content", fts=Fts(match_string="machine learning"))
q._validate()
assert q.fts.match_string == "machine learning"
assert q.fts.query_string is None
assert q.has_fts() is True
def test_fts_query_string_and_match_string_mutually_exclusive(self):
"""Cannot provide both query_string and match_string in Fts."""
q = Query(
field_name="content",
fts=Fts(query_string="+hello", match_string="hello world"),
)
with pytest.raises(ValueError, match="mutually exclusive"):
q._validate()
def test_no_fts(self):
"""Query without FTS fields should have has_fts() == False."""
q = Query(field_name="embedding", vector=[0.1, 0.2, 0.3])
assert q.has_fts() is False
def test_vector_and_fts_mutually_exclusive(self):
"""Cannot combine vector search with FTS in a single Query."""
q = Query(
field_name="embedding",
vector=[0.1, 0.2, 0.3],
fts=Fts(match_string="deep learning"),
)
with pytest.raises(ValueError, match="Cannot combine fts with vector search"):
q._validate()
def test_fts_without_vector_or_id(self):
"""Query with only FTS (no vector, no id) should be valid."""
q = Query(field_name="content", fts=Fts(query_string="hello"))
q._validate()
assert q.has_vector() is False
assert q.has_id() is False
assert q.has_fts() is True
class TestFtsQueryBinding:
"""Test FTS binding layer (_Fts)."""
def test_import_fts_query(self):
"""_Fts should be importable from _zvec.param."""
from _zvec.param import _Fts
fts = _Fts()
assert fts.query_string == ""
assert fts.match_string == ""
def test_fts_query_set_fields(self):
"""Setting fields on _Fts should work."""
from _zvec.param import _Fts
fts = _Fts()
fts.query_string = "+hello -world"
assert fts.query_string == "+hello -world"
fts2 = _Fts()
fts2.match_string = "machine learning"
assert fts2.match_string == "machine learning"
def test_fts_query_pickle(self):
"""_Fts should support pickling."""
from _zvec.param import _Fts
fts = _Fts()
fts.query_string = "+vector search"
fts.match_string = ""
data = pickle.dumps(fts)
restored = pickle.loads(data)
assert restored.query_string == "+vector search"
assert restored.match_string == ""
def test_vector_query_fts_field(self):
"""_VectorQuery should have fts field."""
from _zvec.param import _Fts, _VectorQuery
vq = _VectorQuery()
# fts should be None by default (optional)
assert vq.fts is None
# set fts
fts = _Fts()
fts.query_string = "hello"
vq.fts = fts
assert vq.fts is not None
assert vq.fts.query_string == "hello"
def test_vector_query_pickle_with_fts(self):
"""_VectorQuery with fts should survive pickling."""
from _zvec.param import _Fts, _VectorQuery
vq = _VectorQuery()
vq.topk = 10
vq.field_name = "embedding"
fts = _Fts()
fts.match_string = "test query"
vq.fts = fts
data = pickle.dumps(vq)
restored = pickle.loads(data)
assert restored.topk == 10
assert restored.field_name == "embedding"
assert restored.fts is not None
assert restored.fts.match_string == "test query"
def test_vector_query_pickle_without_fts(self):
"""_VectorQuery without fts should survive pickling."""
from _zvec.param import _VectorQuery
vq = _VectorQuery()
vq.topk = 5
vq.field_name = "vec"
data = pickle.dumps(vq)
restored = pickle.loads(data)
assert restored.topk == 5
assert restored.field_name == "vec"
assert restored.fts is None