feat: add fts support (#408)
Add BM25-based full-text search with CJK (jieba) tokenization, supporting
query_string and match_string syntax, phrase queries, boolean operators
(AND/OR/NOT/MUST), and hybrid retrieval with existing vector search.
## Core
- BitPacked posting format with block-max WAND pruning
- Tokenizer pipeline: jieba (cut/cut_for_search/hmm/full), whitespace,
lowercase, with extensible pipeline composition
- Query parser: boolean operators, phrase queries, field scoping,
boost, MUST(+) modifier inside OR (ES query_string semantics)
- AST rewriter: dedup repeated terms with linear boost aggregation,
flatten same-type composites, canonicalize OR-with-must_not into AND
wrapper, empty-node propagation, contradiction detection
- FTS reduce/merge integrated into Optimize compaction
- Multi-segment score-descending sort
- Auto-register bundled jieba dict on SDK import
## Performance
- Block-max WAND with cached block_max_info_for (single binary search)
- AVX2/SSE bitpacked encoding with cross-arch scalar fallback
- MultiGet for batch posting retrieval and phrase position verification
- HashSkipList memtable for posting writes
- PinnableSlice zero-copy reads
- Filter pushdown into composite iterators (Disjunction/Conjunction/Phrase)
- Candidate-driven (brute-force) evaluation for selective invert filters
- Precomputed BM25 IDF weights, cached SIMD dispatch pointers
- Shortest-list anchor for phrase position matching
- Single-open per-term posting iterator
## Query
- Tokenize query terms through the same pipeline as indexing
- EmptyNode for zero-token queries (all stop-words / punctuation)
- Backslash unescape after lexing in query parser
- Schema allows collections without vector fields (FTS-only use case)
- Create/Drop Index validates supported index types
- FTS fields disallowed in SQL filter expressions
## Bindings
- C API: fts query params, brute-force ratio config
- Python SDK: FTS search, jieba dict auto-registration
## Internals
- Bypass cppjieba::Jieba to drop KeywordExtractor (~12MB fewer required files)
- Hide tokenizer pipeline from public header (Pimpl-style FtsState)
- ListColumnFamilies to avoid double-open on segment load
- Reorganized fts_column into tokenizer/, posting/, iterator/ subdirs
This commit is contained in:
+27
-1
@@ -21,6 +21,24 @@ if TYPE_CHECKING:
|
||||
from importlib.metadata import PackageNotFoundError
|
||||
|
||||
|
||||
# Register the wheel-bundled jieba dict dir so `import zvec` alone makes
|
||||
# the jieba FTS tokenizer usable. Users can still override via
|
||||
# zvec.init(jieba_dict_dir=...), zvec.set_default_jieba_dict_dir(...),
|
||||
# ZVEC_JIEBA_DICT_DIR, or per-field FtsIndexParam.extra_params.
|
||||
try:
|
||||
from importlib.resources import files as _resource_files
|
||||
|
||||
from _zvec import (
|
||||
get_default_jieba_dict_dir,
|
||||
set_default_jieba_dict_dir,
|
||||
)
|
||||
|
||||
set_default_jieba_dict_dir(str(_resource_files("zvec").joinpath("data/jieba_dict")))
|
||||
except Exception:
|
||||
# Custom builds without bundled dict; users must configure explicitly.
|
||||
pass
|
||||
|
||||
|
||||
# ==============================
|
||||
# Public API — grouped by category
|
||||
# ==============================
|
||||
@@ -56,11 +74,14 @@ from .model.collection import Collection
|
||||
from .model.doc import Doc
|
||||
|
||||
# —— Query & index parameters ——
|
||||
# —— FTS params (C++ binding) ——
|
||||
from .model.param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
FlatIndexParam,
|
||||
FtsIndexParam,
|
||||
FtsQueryParam,
|
||||
HnswIndexParam,
|
||||
HnswQueryParam,
|
||||
HnswRabitqIndexParam,
|
||||
@@ -73,7 +94,7 @@ from .model.param import (
|
||||
VamanaIndexParam,
|
||||
VamanaQueryParam,
|
||||
)
|
||||
from .model.param.query import Query, VectorQuery
|
||||
from .model.param.query import Fts, Query, VectorQuery
|
||||
|
||||
# —— Schema & field definitions ——
|
||||
from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
|
||||
@@ -101,6 +122,8 @@ __all__ = [
|
||||
"create_and_open",
|
||||
"init",
|
||||
"open",
|
||||
"set_default_jieba_dict_dir",
|
||||
"get_default_jieba_dict_dir",
|
||||
# Core classes
|
||||
"Collection",
|
||||
"Doc",
|
||||
@@ -112,6 +135,9 @@ __all__ = [
|
||||
# Parameters
|
||||
"Query",
|
||||
"VectorQuery",
|
||||
"Fts",
|
||||
"FtsIndexParam",
|
||||
"FtsQueryParam",
|
||||
"InvertIndexParam",
|
||||
"HnswIndexParam",
|
||||
"HnswRabitqIndexParam",
|
||||
|
||||
@@ -20,7 +20,7 @@ from typing import Optional, Union, final
|
||||
|
||||
import numpy as np
|
||||
from _zvec import _Collection, _MultiQuery
|
||||
from _zvec.param import _SubQuery, _VectorQuery
|
||||
from _zvec.param import _Fts, _SubQuery, _VectorQuery
|
||||
|
||||
from ..extension import ReRanker, RrfReRanker, WeightedReRanker
|
||||
from ..model.convert import convert_to_py_doc
|
||||
@@ -141,6 +141,14 @@ class QueryExecutor(ABC):
|
||||
core_vector.output_fields = ctx.output_fields
|
||||
return core_vector
|
||||
|
||||
def _do_build_fts_query(self, query: Query, core_vector: _VectorQuery) -> None:
|
||||
"""Set FTS query on core_vector if the query has FTS parameters."""
|
||||
if query.has_fts():
|
||||
fts = _Fts()
|
||||
fts.query_string = query.fts.query_string or ""
|
||||
fts.match_string = query.fts.match_string or ""
|
||||
core_vector.fts = fts
|
||||
|
||||
def _do_build_query_with_vector(
|
||||
self, ctx: QueryContext, query: Query, collection: _Collection
|
||||
) -> _VectorQuery:
|
||||
@@ -149,25 +157,34 @@ class QueryExecutor(ABC):
|
||||
if query.param:
|
||||
core_vector.query_params = query.param
|
||||
|
||||
vector_schema = (
|
||||
self._schema.vector(query.field_name) if query else self._schema.vectors[0]
|
||||
)
|
||||
|
||||
if vector_schema is None:
|
||||
raise ValueError("No vector field found")
|
||||
# set FTS query if provided
|
||||
self._do_build_fts_query(query, core_vector)
|
||||
|
||||
# set output_fields
|
||||
core_vector.output_fields = ctx.output_fields
|
||||
|
||||
vector_schema = None
|
||||
if query.has_vector() or query.has_id():
|
||||
vector_schema = (
|
||||
self._schema.vector(query.field_name)
|
||||
if query
|
||||
else self._schema.vectors[0]
|
||||
)
|
||||
|
||||
if vector_schema is None:
|
||||
raise ValueError("No vector field found")
|
||||
|
||||
# set vector
|
||||
if query.has_vector():
|
||||
vec_data = query.vector
|
||||
else:
|
||||
elif query.has_id():
|
||||
fetched = collection.Fetch([query.id])
|
||||
doc = next(iter(fetched.values()))
|
||||
if not doc:
|
||||
return core_vector
|
||||
vec_data = doc.get_any(vector_schema.name, vector_schema.data_type)
|
||||
else:
|
||||
return core_vector
|
||||
|
||||
target_dtype = DTYPE_MAP.get(vector_schema.data_type.value)
|
||||
core_vector.set_vector(
|
||||
@@ -243,13 +260,21 @@ class NoVectorQueryExecutor(QueryExecutor):
|
||||
super().__init__(schema)
|
||||
|
||||
def _do_validate(self, ctx: QueryContext) -> None:
|
||||
if len(ctx.queries) > 0:
|
||||
raise ValueError("Collection does not support query with vector or id")
|
||||
for query in ctx.queries:
|
||||
if query.has_vector() or query.has_id():
|
||||
raise ValueError("Collection does not support query with vector or id")
|
||||
query._validate()
|
||||
|
||||
def _do_build(
|
||||
self, ctx: QueryContext, _collection: _Collection
|
||||
self, ctx: QueryContext, collection: _Collection
|
||||
) -> list[_VectorQuery]:
|
||||
return [self._do_build_query_wo_vector(ctx)]
|
||||
if len(ctx.queries) == 0:
|
||||
return [self._do_build_query_wo_vector(ctx)]
|
||||
# FTS-only branch in _do_build_query_with_vector skips vector resolution.
|
||||
return [
|
||||
self._do_build_query_with_vector(ctx, query, collection)
|
||||
for query in ctx.queries
|
||||
]
|
||||
|
||||
|
||||
class SingleVectorQueryExecutor(NoVectorQueryExecutor):
|
||||
|
||||
@@ -15,7 +15,7 @@ from __future__ import annotations
|
||||
|
||||
from .collection import Collection
|
||||
from .doc import Doc
|
||||
from .param.query import Query, VectorQuery
|
||||
from .param.query import Fts, Query, VectorQuery
|
||||
from .schema.collection_schema import CollectionSchema
|
||||
from .schema.field_schema import FieldSchema
|
||||
|
||||
@@ -24,6 +24,7 @@ __all__ = [
|
||||
"CollectionSchema",
|
||||
"Doc",
|
||||
"FieldSchema",
|
||||
"Fts",
|
||||
"Query",
|
||||
"VectorQuery",
|
||||
]
|
||||
|
||||
@@ -18,6 +18,8 @@ from _zvec.param import (
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
FlatIndexParam,
|
||||
FtsIndexParam,
|
||||
FtsQueryParam,
|
||||
HnswIndexParam,
|
||||
HnswQueryParam,
|
||||
HnswRabitqIndexParam,
|
||||
@@ -36,6 +38,8 @@ __all__ = [
|
||||
"AlterColumnOption",
|
||||
"CollectionOption",
|
||||
"FlatIndexParam",
|
||||
"FtsIndexParam",
|
||||
"FtsQueryParam",
|
||||
"HnswIndexParam",
|
||||
"HnswQueryParam",
|
||||
"HnswRabitqIndexParam",
|
||||
|
||||
@@ -20,26 +20,42 @@ from typing import Optional, Union
|
||||
from ...common import VectorType
|
||||
from . import HnswQueryParam, HnswRabitqQueryParam, IVFQueryParam
|
||||
|
||||
__all__ = ["Query", "VectorQuery"]
|
||||
__all__ = ["Fts", "Query", "VectorQuery"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Fts:
|
||||
"""Full-text search query parameters.
|
||||
|
||||
Attributes:
|
||||
query_string (Optional[str]): FTS query expression
|
||||
(e.g. '+vector -slow "exact phrase"'). Mutually exclusive with match_string.
|
||||
match_string (Optional[str]): Natural language match string,
|
||||
tokenized and combined using the default operator.
|
||||
Mutually exclusive with query_string.
|
||||
"""
|
||||
|
||||
query_string: Optional[str] = None
|
||||
match_string: Optional[str] = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Query:
|
||||
"""Represents a search query for a specific field in a collection.
|
||||
|
||||
A `Query` can be constructed using either a document ID (to look up
|
||||
its vector) or an explicit vector. It may optionally include index-specific
|
||||
query parameters to control search behavior (e.g., `ef` for HNSW, `nprobe` for IVF).
|
||||
A `Query` can be constructed for either vector search or full-text search,
|
||||
but not both simultaneously.
|
||||
|
||||
Exactly one of `id` or `vector` should be provided. If both are given,
|
||||
behavior is implementation-defined (typically `id` takes precedence).
|
||||
For vector search, provide `id` or `vector` (and optionally `param`).
|
||||
For FTS, provide `fts`.
|
||||
|
||||
Attributes:
|
||||
field_name (str): Name of the field to query.
|
||||
id (Optional[str], optional): Document ID to fetch vector from. Default is None.
|
||||
vector (VectorType, optional): Explicit query vector. Default is None.
|
||||
param (Optional[Union[HnswQueryParam, IVFQueryParam]], optional):
|
||||
Index-specific query parameters. Default is None.
|
||||
Index-specific query parameters for vector search. Default is None.
|
||||
fts (Optional[Fts], optional): Full-text search parameters. Default is None.
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
@@ -51,12 +67,18 @@ class Query:
|
||||
... vector=[0.1, 0.2, 0.3],
|
||||
... param=HnswQueryParam(ef=300)
|
||||
... )
|
||||
>>> # FTS query
|
||||
>>> q3 = zvec.Query(
|
||||
... field_name="content",
|
||||
... fts=Fts(match_string="machine learning")
|
||||
... )
|
||||
"""
|
||||
|
||||
field_name: str
|
||||
id: Optional[str] = None
|
||||
vector: VectorType = None
|
||||
param: Optional[Union[HnswQueryParam, HnswRabitqQueryParam, IVFQueryParam]] = None
|
||||
fts: Optional[Fts] = None
|
||||
|
||||
def has_id(self) -> bool:
|
||||
"""Check if the query is based on a document ID.
|
||||
@@ -74,11 +96,32 @@ class Query:
|
||||
"""
|
||||
return self.vector is not None and len(self.vector) > 0
|
||||
|
||||
def has_fts(self) -> bool:
|
||||
"""Check if the query contains an FTS (full-text search) condition.
|
||||
|
||||
Returns:
|
||||
bool: True if `fts` is set with a query_string or match_string.
|
||||
"""
|
||||
if self.fts is not None:
|
||||
return bool(self.fts.query_string) or bool(self.fts.match_string)
|
||||
return False
|
||||
|
||||
def _validate(self) -> None:
|
||||
if self.field_name is None:
|
||||
raise ValueError("Field name cannot be empty")
|
||||
if self.id and self.vector:
|
||||
raise ValueError("Cannot provide both id and vector")
|
||||
if self.has_fts() and (
|
||||
self.has_vector() or self.has_id() or self.param is not None
|
||||
):
|
||||
raise ValueError(
|
||||
"Cannot combine fts with vector search fields (id/vector/param) in a single Query"
|
||||
)
|
||||
if self.fts is not None and self.fts.query_string and self.fts.match_string:
|
||||
raise ValueError(
|
||||
"Cannot provide both query_string and match_string in Fts; "
|
||||
"they are mutually exclusive"
|
||||
)
|
||||
|
||||
|
||||
class VectorQuery(Query):
|
||||
|
||||
@@ -38,7 +38,9 @@ def init(
|
||||
optimize_threads: Optional[int] = None,
|
||||
invert_to_forward_scan_ratio: Optional[float] = None,
|
||||
brute_force_by_keys_ratio: Optional[float] = None,
|
||||
fts_brute_force_by_keys_ratio: Optional[float] = None,
|
||||
memory_limit_mb: Optional[int] = None,
|
||||
jieba_dict_dir: Optional[str] = None,
|
||||
) -> None:
|
||||
"""Initialize Zvec with configuration options.
|
||||
|
||||
@@ -88,11 +90,25 @@ def init(
|
||||
Threshold to use brute-force key lookup over index.
|
||||
Lower → prefer index; higher → prefer brute-force.
|
||||
Range: [0.0, 1.0]. Default: ``0.1``.
|
||||
fts_brute_force_by_keys_ratio (Optional[float], optional):
|
||||
Threshold to switch FTS scan from posting-driven to
|
||||
candidate-driven (brute-force) when the invert filter is
|
||||
highly selective. Independent from ``brute_force_by_keys_ratio``
|
||||
because per-candidate FTS cost is higher.
|
||||
Range: [0.0, 1.0]. Default: ``0.05``.
|
||||
memory_limit_mb (Optional[int], optional):
|
||||
Soft memory cap in MB. Zvec may throttle or fail operations
|
||||
approaching this limit.
|
||||
If ``None``, inferred from cgroup memory limit * 0.8 (e.g., in Docker).
|
||||
Must be > 0 if provided.
|
||||
jieba_dict_dir (Optional[str], optional):
|
||||
Override the default directory containing ``jieba.dict.utf8`` and
|
||||
``hmm_model.utf8`` for the jieba FTS tokenizer. When ``None``, the
|
||||
value previously registered by ``zvec.set_default_jieba_dict_dir``
|
||||
(called automatically on ``import zvec`` to point at the wheel's
|
||||
bundled dict) is preserved. JiebaTokenizer also honors the
|
||||
``ZVEC_JIEBA_DICT_DIR`` environment variable and per-field
|
||||
``FtsIndexParam.extra_params.jieba_dict_dir`` ahead of this value.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If Zvec is already initialized.
|
||||
@@ -157,8 +173,12 @@ def init(
|
||||
config_dict["invert_to_forward_scan_ratio"] = invert_to_forward_scan_ratio
|
||||
if brute_force_by_keys_ratio is not None:
|
||||
config_dict["brute_force_by_keys_ratio"] = brute_force_by_keys_ratio
|
||||
if fts_brute_force_by_keys_ratio is not None:
|
||||
config_dict["fts_brute_force_by_keys_ratio"] = fts_brute_force_by_keys_ratio
|
||||
if memory_limit_mb is not None:
|
||||
config_dict["memory_limit_mb"] = memory_limit_mb
|
||||
if jieba_dict_dir is not None:
|
||||
config_dict["jieba_dict_dir"] = jieba_dict_dir
|
||||
|
||||
Initialize(config_dict)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user