Add BM25-based full-text search with CJK (jieba) tokenization, supporting
query_string and match_string syntax, phrase queries, boolean operators
(AND/OR/NOT/MUST), and hybrid retrieval with existing vector search.
## Core
- BitPacked posting format with block-max WAND pruning
- Tokenizer pipeline: jieba (cut/cut_for_search/hmm/full), whitespace,
lowercase, with extensible pipeline composition
- Query parser: boolean operators, phrase queries, field scoping,
boost, MUST(+) modifier inside OR (ES query_string semantics)
- AST rewriter: dedup repeated terms with linear boost aggregation,
flatten same-type composites, canonicalize OR-with-must_not into AND
wrapper, empty-node propagation, contradiction detection
- FTS reduce/merge integrated into Optimize compaction
- Multi-segment score-descending sort
- Auto-register bundled jieba dict on SDK import
## Performance
- Block-max WAND with cached block_max_info_for (single binary search)
- AVX2/SSE bitpacked encoding with cross-arch scalar fallback
- MultiGet for batch posting retrieval and phrase position verification
- HashSkipList memtable for posting writes
- PinnableSlice zero-copy reads
- Filter pushdown into composite iterators (Disjunction/Conjunction/Phrase)
- Candidate-driven (brute-force) evaluation for selective invert filters
- Precomputed BM25 IDF weights, cached SIMD dispatch pointers
- Shortest-list anchor for phrase position matching
- Single-open per-term posting iterator
## Query
- Tokenize query terms through the same pipeline as indexing
- EmptyNode for zero-token queries (all stop-words / punctuation)
- Backslash unescape after lexing in query parser
- Schema allows collections without vector fields (FTS-only use case)
- Create/Drop Index validates supported index types
- FTS fields disallowed in SQL filter expressions
## Bindings
- C API: fts query params, brute-force ratio config
- Python SDK: FTS search, jieba dict auto-registration
## Internals
- Bypass cppjieba::Jieba to drop KeywordExtractor (~12MB fewer required files)
- Hide tokenizer pipeline from public header (Pimpl-style FtsState)
- ListColumnFamilies to avoid double-open on segment load
- Reorganized fts_column into tokenizer/, posting/, iterator/ subdirs
200 lines
4.8 KiB
Python
200 lines
4.8 KiB
Python
# Copyright 2025-present the zvec project
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
from __future__ import annotations
|
|
|
|
import sys
|
|
from typing import TYPE_CHECKING
|
|
|
|
if TYPE_CHECKING:
|
|
from importlib.metadata import PackageNotFoundError
|
|
|
|
|
|
# Register the wheel-bundled jieba dict dir so `import zvec` alone makes
|
|
# the jieba FTS tokenizer usable. Users can still override via
|
|
# zvec.init(jieba_dict_dir=...), zvec.set_default_jieba_dict_dir(...),
|
|
# ZVEC_JIEBA_DICT_DIR, or per-field FtsIndexParam.extra_params.
|
|
try:
|
|
from importlib.resources import files as _resource_files
|
|
|
|
from _zvec import (
|
|
get_default_jieba_dict_dir,
|
|
set_default_jieba_dict_dir,
|
|
)
|
|
|
|
set_default_jieba_dict_dir(str(_resource_files("zvec").joinpath("data/jieba_dict")))
|
|
except Exception:
|
|
# Custom builds without bundled dict; users must configure explicitly.
|
|
pass
|
|
|
|
|
|
# ==============================
|
|
# Public API — grouped by category
|
|
# ==============================
|
|
|
|
from . import model as model
|
|
|
|
# —— Extensions ——
|
|
from .extension import (
|
|
BM25EmbeddingFunction,
|
|
DefaultLocalDenseEmbedding,
|
|
DefaultLocalReRanker,
|
|
DefaultLocalSparseEmbedding,
|
|
DenseEmbeddingFunction,
|
|
OpenAIDenseEmbedding,
|
|
OpenAIFunctionBase,
|
|
QwenDenseEmbedding,
|
|
QwenFunctionBase,
|
|
QwenReRanker,
|
|
QwenSparseEmbedding,
|
|
ReRanker,
|
|
RrfReRanker,
|
|
SentenceTransformerFunctionBase,
|
|
SparseEmbeddingFunction,
|
|
WeightedReRanker,
|
|
)
|
|
|
|
# —— Typing ——
|
|
from .model import param as param
|
|
from .model import schema as schema
|
|
|
|
# —— Core data structures ——
|
|
from .model.collection import Collection
|
|
from .model.doc import Doc
|
|
|
|
# —— Query & index parameters ——
|
|
# —— FTS params (C++ binding) ——
|
|
from .model.param import (
|
|
AddColumnOption,
|
|
AlterColumnOption,
|
|
CollectionOption,
|
|
FlatIndexParam,
|
|
FtsIndexParam,
|
|
FtsQueryParam,
|
|
HnswIndexParam,
|
|
HnswQueryParam,
|
|
HnswRabitqIndexParam,
|
|
HnswRabitqQueryParam,
|
|
IndexOption,
|
|
InvertIndexParam,
|
|
IVFIndexParam,
|
|
IVFQueryParam,
|
|
OptimizeOption,
|
|
VamanaIndexParam,
|
|
VamanaQueryParam,
|
|
)
|
|
from .model.param.query import Fts, Query, VectorQuery
|
|
|
|
# —— Schema & field definitions ——
|
|
from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
|
|
|
|
# —— tools ——
|
|
from .tool import require_module
|
|
from .typing import (
|
|
DataType,
|
|
IndexType,
|
|
MetricType,
|
|
QuantizeType,
|
|
Status,
|
|
StatusCode,
|
|
)
|
|
from .typing.enum import LogLevel, LogType
|
|
|
|
# —— lifecycle ——
|
|
from .zvec import create_and_open, init, open
|
|
|
|
# ==============================
|
|
# Public interface declaration
|
|
# ==============================
|
|
__all__ = [
|
|
# Zvec functions
|
|
"create_and_open",
|
|
"init",
|
|
"open",
|
|
"set_default_jieba_dict_dir",
|
|
"get_default_jieba_dict_dir",
|
|
# Core classes
|
|
"Collection",
|
|
"Doc",
|
|
# Schema
|
|
"CollectionSchema",
|
|
"FieldSchema",
|
|
"VectorSchema",
|
|
"CollectionStats",
|
|
# Parameters
|
|
"Query",
|
|
"VectorQuery",
|
|
"Fts",
|
|
"FtsIndexParam",
|
|
"FtsQueryParam",
|
|
"InvertIndexParam",
|
|
"HnswIndexParam",
|
|
"HnswRabitqIndexParam",
|
|
"FlatIndexParam",
|
|
"IVFIndexParam",
|
|
"CollectionOption",
|
|
"IndexOption",
|
|
"OptimizeOption",
|
|
"AddColumnOption",
|
|
"AlterColumnOption",
|
|
"HnswQueryParam",
|
|
"HnswRabitqQueryParam",
|
|
"IVFQueryParam",
|
|
"VamanaIndexParam",
|
|
"VamanaQueryParam",
|
|
# Extensions
|
|
"DenseEmbeddingFunction",
|
|
"SparseEmbeddingFunction",
|
|
"QwenFunctionBase",
|
|
"OpenAIFunctionBase",
|
|
"SentenceTransformerFunctionBase",
|
|
"ReRanker",
|
|
"DefaultLocalDenseEmbedding",
|
|
"DefaultLocalSparseEmbedding",
|
|
"BM25EmbeddingFunction",
|
|
"OpenAIDenseEmbedding",
|
|
"QwenDenseEmbedding",
|
|
"QwenSparseEmbedding",
|
|
"RrfReRanker",
|
|
"WeightedReRanker",
|
|
"DefaultLocalReRanker",
|
|
"QwenReRanker",
|
|
# Typing
|
|
"DataType",
|
|
"MetricType",
|
|
"QuantizeType",
|
|
"IndexType",
|
|
"LogLevel",
|
|
"LogType",
|
|
"Status",
|
|
"StatusCode",
|
|
# Tools
|
|
"require_module",
|
|
]
|
|
|
|
# ==============================
|
|
# Version handling
|
|
# ==============================
|
|
__version__: str
|
|
|
|
try:
|
|
from importlib.metadata import version
|
|
except ImportError:
|
|
from importlib_metadata import version # Python < 3.8
|
|
|
|
try:
|
|
__version__ = version("zvec")
|
|
except Exception:
|
|
__version__ = "unknown"
|