feat: add fts support (#408)

Add BM25-based full-text search with CJK (jieba) tokenization, supporting
  query_string and match_string syntax, phrase queries, boolean operators
  (AND/OR/NOT/MUST), and hybrid retrieval with existing vector search.

  ## Core
  - BitPacked posting format with block-max WAND pruning
  - Tokenizer pipeline: jieba (cut/cut_for_search/hmm/full), whitespace,
    lowercase, with extensible pipeline composition
  - Query parser: boolean operators, phrase queries, field scoping,
    boost, MUST(+) modifier inside OR (ES query_string semantics)
  - AST rewriter: dedup repeated terms with linear boost aggregation,
    flatten same-type composites, canonicalize OR-with-must_not into AND
    wrapper, empty-node propagation, contradiction detection
  - FTS reduce/merge integrated into Optimize compaction
  - Multi-segment score-descending sort
  - Auto-register bundled jieba dict on SDK import

  ## Performance
  - Block-max WAND with cached block_max_info_for (single binary search)
  - AVX2/SSE bitpacked encoding with cross-arch scalar fallback
  - MultiGet for batch posting retrieval and phrase position verification
  - HashSkipList memtable for posting writes
  - PinnableSlice zero-copy reads
  - Filter pushdown into composite iterators (Disjunction/Conjunction/Phrase)
  - Candidate-driven (brute-force) evaluation for selective invert filters
  - Precomputed BM25 IDF weights, cached SIMD dispatch pointers
  - Shortest-list anchor for phrase position matching
  - Single-open per-term posting iterator

  ## Query
  - Tokenize query terms through the same pipeline as indexing
  - EmptyNode for zero-token queries (all stop-words / punctuation)
  - Backslash unescape after lexing in query parser
  - Schema allows collections without vector fields (FTS-only use case)
  - Create/Drop Index validates supported index types
  - FTS fields disallowed in SQL filter expressions

  ## Bindings
  - C API: fts query params, brute-force ratio config
  - Python SDK: FTS search, jieba dict auto-registration

  ## Internals
  - Bypass cppjieba::Jieba to drop KeywordExtractor (~12MB fewer required files)
  - Hide tokenizer pipeline from public header (Pimpl-style FtsState)
  - ListColumnFamilies to avoid double-open on segment load
  - Reorganized fts_column into tokenizer/, posting/, iterator/ subdirs
This commit is contained in:
egolearner
2026-06-01 15:02:54 +08:00
committed by GitHub
parent 74beb2a828
commit 02bfb31cf5
154 changed files with 22646 additions and 223 deletions
+27 -1
View File
@@ -21,6 +21,24 @@ if TYPE_CHECKING:
from importlib.metadata import PackageNotFoundError
# Register the wheel-bundled jieba dict dir so `import zvec` alone makes
# the jieba FTS tokenizer usable. Users can still override via
# zvec.init(jieba_dict_dir=...), zvec.set_default_jieba_dict_dir(...),
# ZVEC_JIEBA_DICT_DIR, or per-field FtsIndexParam.extra_params.
try:
from importlib.resources import files as _resource_files
from _zvec import (
get_default_jieba_dict_dir,
set_default_jieba_dict_dir,
)
set_default_jieba_dict_dir(str(_resource_files("zvec").joinpath("data/jieba_dict")))
except Exception:
# Custom builds without bundled dict; users must configure explicitly.
pass
# ==============================
# Public API — grouped by category
# ==============================
@@ -56,11 +74,14 @@ from .model.collection import Collection
from .model.doc import Doc
# —— Query & index parameters ——
# —— FTS params (C++ binding) ——
from .model.param import (
AddColumnOption,
AlterColumnOption,
CollectionOption,
FlatIndexParam,
FtsIndexParam,
FtsQueryParam,
HnswIndexParam,
HnswQueryParam,
HnswRabitqIndexParam,
@@ -73,7 +94,7 @@ from .model.param import (
VamanaIndexParam,
VamanaQueryParam,
)
from .model.param.query import Query, VectorQuery
from .model.param.query import Fts, Query, VectorQuery
# —— Schema & field definitions ——
from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
@@ -101,6 +122,8 @@ __all__ = [
"create_and_open",
"init",
"open",
"set_default_jieba_dict_dir",
"get_default_jieba_dict_dir",
# Core classes
"Collection",
"Doc",
@@ -112,6 +135,9 @@ __all__ = [
# Parameters
"Query",
"VectorQuery",
"Fts",
"FtsIndexParam",
"FtsQueryParam",
"InvertIndexParam",
"HnswIndexParam",
"HnswRabitqIndexParam",
+37 -12
View File
@@ -20,7 +20,7 @@ from typing import Optional, Union, final
import numpy as np
from _zvec import _Collection, _MultiQuery
from _zvec.param import _SubQuery, _VectorQuery
from _zvec.param import _Fts, _SubQuery, _VectorQuery
from ..extension import ReRanker, RrfReRanker, WeightedReRanker
from ..model.convert import convert_to_py_doc
@@ -141,6 +141,14 @@ class QueryExecutor(ABC):
core_vector.output_fields = ctx.output_fields
return core_vector
def _do_build_fts_query(self, query: Query, core_vector: _VectorQuery) -> None:
"""Set FTS query on core_vector if the query has FTS parameters."""
if query.has_fts():
fts = _Fts()
fts.query_string = query.fts.query_string or ""
fts.match_string = query.fts.match_string or ""
core_vector.fts = fts
def _do_build_query_with_vector(
self, ctx: QueryContext, query: Query, collection: _Collection
) -> _VectorQuery:
@@ -149,25 +157,34 @@ class QueryExecutor(ABC):
if query.param:
core_vector.query_params = query.param
vector_schema = (
self._schema.vector(query.field_name) if query else self._schema.vectors[0]
)
if vector_schema is None:
raise ValueError("No vector field found")
# set FTS query if provided
self._do_build_fts_query(query, core_vector)
# set output_fields
core_vector.output_fields = ctx.output_fields
vector_schema = None
if query.has_vector() or query.has_id():
vector_schema = (
self._schema.vector(query.field_name)
if query
else self._schema.vectors[0]
)
if vector_schema is None:
raise ValueError("No vector field found")
# set vector
if query.has_vector():
vec_data = query.vector
else:
elif query.has_id():
fetched = collection.Fetch([query.id])
doc = next(iter(fetched.values()))
if not doc:
return core_vector
vec_data = doc.get_any(vector_schema.name, vector_schema.data_type)
else:
return core_vector
target_dtype = DTYPE_MAP.get(vector_schema.data_type.value)
core_vector.set_vector(
@@ -243,13 +260,21 @@ class NoVectorQueryExecutor(QueryExecutor):
super().__init__(schema)
def _do_validate(self, ctx: QueryContext) -> None:
if len(ctx.queries) > 0:
raise ValueError("Collection does not support query with vector or id")
for query in ctx.queries:
if query.has_vector() or query.has_id():
raise ValueError("Collection does not support query with vector or id")
query._validate()
def _do_build(
self, ctx: QueryContext, _collection: _Collection
self, ctx: QueryContext, collection: _Collection
) -> list[_VectorQuery]:
return [self._do_build_query_wo_vector(ctx)]
if len(ctx.queries) == 0:
return [self._do_build_query_wo_vector(ctx)]
# FTS-only branch in _do_build_query_with_vector skips vector resolution.
return [
self._do_build_query_with_vector(ctx, query, collection)
for query in ctx.queries
]
class SingleVectorQueryExecutor(NoVectorQueryExecutor):
+2 -1
View File
@@ -15,7 +15,7 @@ from __future__ import annotations
from .collection import Collection
from .doc import Doc
from .param.query import Query, VectorQuery
from .param.query import Fts, Query, VectorQuery
from .schema.collection_schema import CollectionSchema
from .schema.field_schema import FieldSchema
@@ -24,6 +24,7 @@ __all__ = [
"CollectionSchema",
"Doc",
"FieldSchema",
"Fts",
"Query",
"VectorQuery",
]
+4
View File
@@ -18,6 +18,8 @@ from _zvec.param import (
AlterColumnOption,
CollectionOption,
FlatIndexParam,
FtsIndexParam,
FtsQueryParam,
HnswIndexParam,
HnswQueryParam,
HnswRabitqIndexParam,
@@ -36,6 +38,8 @@ __all__ = [
"AlterColumnOption",
"CollectionOption",
"FlatIndexParam",
"FtsIndexParam",
"FtsQueryParam",
"HnswIndexParam",
"HnswQueryParam",
"HnswRabitqIndexParam",
+50 -7
View File
@@ -20,26 +20,42 @@ from typing import Optional, Union
from ...common import VectorType
from . import HnswQueryParam, HnswRabitqQueryParam, IVFQueryParam
__all__ = ["Query", "VectorQuery"]
__all__ = ["Fts", "Query", "VectorQuery"]
@dataclass(frozen=True)
class Fts:
"""Full-text search query parameters.
Attributes:
query_string (Optional[str]): FTS query expression
(e.g. '+vector -slow "exact phrase"'). Mutually exclusive with match_string.
match_string (Optional[str]): Natural language match string,
tokenized and combined using the default operator.
Mutually exclusive with query_string.
"""
query_string: Optional[str] = None
match_string: Optional[str] = None
@dataclass(frozen=True)
class Query:
"""Represents a search query for a specific field in a collection.
A `Query` can be constructed using either a document ID (to look up
its vector) or an explicit vector. It may optionally include index-specific
query parameters to control search behavior (e.g., `ef` for HNSW, `nprobe` for IVF).
A `Query` can be constructed for either vector search or full-text search,
but not both simultaneously.
Exactly one of `id` or `vector` should be provided. If both are given,
behavior is implementation-defined (typically `id` takes precedence).
For vector search, provide `id` or `vector` (and optionally `param`).
For FTS, provide `fts`.
Attributes:
field_name (str): Name of the field to query.
id (Optional[str], optional): Document ID to fetch vector from. Default is None.
vector (VectorType, optional): Explicit query vector. Default is None.
param (Optional[Union[HnswQueryParam, IVFQueryParam]], optional):
Index-specific query parameters. Default is None.
Index-specific query parameters for vector search. Default is None.
fts (Optional[Fts], optional): Full-text search parameters. Default is None.
Examples:
>>> import zvec
@@ -51,12 +67,18 @@ class Query:
... vector=[0.1, 0.2, 0.3],
... param=HnswQueryParam(ef=300)
... )
>>> # FTS query
>>> q3 = zvec.Query(
... field_name="content",
... fts=Fts(match_string="machine learning")
... )
"""
field_name: str
id: Optional[str] = None
vector: VectorType = None
param: Optional[Union[HnswQueryParam, HnswRabitqQueryParam, IVFQueryParam]] = None
fts: Optional[Fts] = None
def has_id(self) -> bool:
"""Check if the query is based on a document ID.
@@ -74,11 +96,32 @@ class Query:
"""
return self.vector is not None and len(self.vector) > 0
def has_fts(self) -> bool:
"""Check if the query contains an FTS (full-text search) condition.
Returns:
bool: True if `fts` is set with a query_string or match_string.
"""
if self.fts is not None:
return bool(self.fts.query_string) or bool(self.fts.match_string)
return False
def _validate(self) -> None:
if self.field_name is None:
raise ValueError("Field name cannot be empty")
if self.id and self.vector:
raise ValueError("Cannot provide both id and vector")
if self.has_fts() and (
self.has_vector() or self.has_id() or self.param is not None
):
raise ValueError(
"Cannot combine fts with vector search fields (id/vector/param) in a single Query"
)
if self.fts is not None and self.fts.query_string and self.fts.match_string:
raise ValueError(
"Cannot provide both query_string and match_string in Fts; "
"they are mutually exclusive"
)
class VectorQuery(Query):
+20
View File
@@ -38,7 +38,9 @@ def init(
optimize_threads: Optional[int] = None,
invert_to_forward_scan_ratio: Optional[float] = None,
brute_force_by_keys_ratio: Optional[float] = None,
fts_brute_force_by_keys_ratio: Optional[float] = None,
memory_limit_mb: Optional[int] = None,
jieba_dict_dir: Optional[str] = None,
) -> None:
"""Initialize Zvec with configuration options.
@@ -88,11 +90,25 @@ def init(
Threshold to use brute-force key lookup over index.
Lower → prefer index; higher → prefer brute-force.
Range: [0.0, 1.0]. Default: ``0.1``.
fts_brute_force_by_keys_ratio (Optional[float], optional):
Threshold to switch FTS scan from posting-driven to
candidate-driven (brute-force) when the invert filter is
highly selective. Independent from ``brute_force_by_keys_ratio``
because per-candidate FTS cost is higher.
Range: [0.0, 1.0]. Default: ``0.05``.
memory_limit_mb (Optional[int], optional):
Soft memory cap in MB. Zvec may throttle or fail operations
approaching this limit.
If ``None``, inferred from cgroup memory limit * 0.8 (e.g., in Docker).
Must be > 0 if provided.
jieba_dict_dir (Optional[str], optional):
Override the default directory containing ``jieba.dict.utf8`` and
``hmm_model.utf8`` for the jieba FTS tokenizer. When ``None``, the
value previously registered by ``zvec.set_default_jieba_dict_dir``
(called automatically on ``import zvec`` to point at the wheel's
bundled dict) is preserved. JiebaTokenizer also honors the
``ZVEC_JIEBA_DICT_DIR`` environment variable and per-field
``FtsIndexParam.extra_params.jieba_dict_dir`` ahead of this value.
Raises:
RuntimeError: If Zvec is already initialized.
@@ -157,8 +173,12 @@ def init(
config_dict["invert_to_forward_scan_ratio"] = invert_to_forward_scan_ratio
if brute_force_by_keys_ratio is not None:
config_dict["brute_force_by_keys_ratio"] = brute_force_by_keys_ratio
if fts_brute_force_by_keys_ratio is not None:
config_dict["fts_brute_force_by_keys_ratio"] = fts_brute_force_by_keys_ratio
if memory_limit_mb is not None:
config_dict["memory_limit_mb"] = memory_limit_mb
if jieba_dict_dir is not None:
config_dict["jieba_dict_dir"] = jieba_dict_dir
Initialize(config_dict)