Files
zvec/python/zvec/__init__.py
T
egolearner 02bfb31cf5 feat: add fts support (#408)
Add BM25-based full-text search with CJK (jieba) tokenization, supporting
  query_string and match_string syntax, phrase queries, boolean operators
  (AND/OR/NOT/MUST), and hybrid retrieval with existing vector search.

  ## Core
  - BitPacked posting format with block-max WAND pruning
  - Tokenizer pipeline: jieba (cut/cut_for_search/hmm/full), whitespace,
    lowercase, with extensible pipeline composition
  - Query parser: boolean operators, phrase queries, field scoping,
    boost, MUST(+) modifier inside OR (ES query_string semantics)
  - AST rewriter: dedup repeated terms with linear boost aggregation,
    flatten same-type composites, canonicalize OR-with-must_not into AND
    wrapper, empty-node propagation, contradiction detection
  - FTS reduce/merge integrated into Optimize compaction
  - Multi-segment score-descending sort
  - Auto-register bundled jieba dict on SDK import

  ## Performance
  - Block-max WAND with cached block_max_info_for (single binary search)
  - AVX2/SSE bitpacked encoding with cross-arch scalar fallback
  - MultiGet for batch posting retrieval and phrase position verification
  - HashSkipList memtable for posting writes
  - PinnableSlice zero-copy reads
  - Filter pushdown into composite iterators (Disjunction/Conjunction/Phrase)
  - Candidate-driven (brute-force) evaluation for selective invert filters
  - Precomputed BM25 IDF weights, cached SIMD dispatch pointers
  - Shortest-list anchor for phrase position matching
  - Single-open per-term posting iterator

  ## Query
  - Tokenize query terms through the same pipeline as indexing
  - EmptyNode for zero-token queries (all stop-words / punctuation)
  - Backslash unescape after lexing in query parser
  - Schema allows collections without vector fields (FTS-only use case)
  - Create/Drop Index validates supported index types
  - FTS fields disallowed in SQL filter expressions

  ## Bindings
  - C API: fts query params, brute-force ratio config
  - Python SDK: FTS search, jieba dict auto-registration

  ## Internals
  - Bypass cppjieba::Jieba to drop KeywordExtractor (~12MB fewer required files)
  - Hide tokenizer pipeline from public header (Pimpl-style FtsState)
  - ListColumnFamilies to avoid double-open on segment load
  - Reorganized fts_column into tokenizer/, posting/, iterator/ subdirs
2026-06-01 15:02:54 +08:00

200 lines
4.8 KiB
Python

# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import sys
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from importlib.metadata import PackageNotFoundError
# Register the wheel-bundled jieba dict dir so `import zvec` alone makes
# the jieba FTS tokenizer usable. Users can still override via
# zvec.init(jieba_dict_dir=...), zvec.set_default_jieba_dict_dir(...),
# ZVEC_JIEBA_DICT_DIR, or per-field FtsIndexParam.extra_params.
try:
from importlib.resources import files as _resource_files
from _zvec import (
get_default_jieba_dict_dir,
set_default_jieba_dict_dir,
)
set_default_jieba_dict_dir(str(_resource_files("zvec").joinpath("data/jieba_dict")))
except Exception:
# Custom builds without bundled dict; users must configure explicitly.
pass
# ==============================
# Public API — grouped by category
# ==============================
from . import model as model
# —— Extensions ——
from .extension import (
BM25EmbeddingFunction,
DefaultLocalDenseEmbedding,
DefaultLocalReRanker,
DefaultLocalSparseEmbedding,
DenseEmbeddingFunction,
OpenAIDenseEmbedding,
OpenAIFunctionBase,
QwenDenseEmbedding,
QwenFunctionBase,
QwenReRanker,
QwenSparseEmbedding,
ReRanker,
RrfReRanker,
SentenceTransformerFunctionBase,
SparseEmbeddingFunction,
WeightedReRanker,
)
# —— Typing ——
from .model import param as param
from .model import schema as schema
# —— Core data structures ——
from .model.collection import Collection
from .model.doc import Doc
# —— Query & index parameters ——
# —— FTS params (C++ binding) ——
from .model.param import (
AddColumnOption,
AlterColumnOption,
CollectionOption,
FlatIndexParam,
FtsIndexParam,
FtsQueryParam,
HnswIndexParam,
HnswQueryParam,
HnswRabitqIndexParam,
HnswRabitqQueryParam,
IndexOption,
InvertIndexParam,
IVFIndexParam,
IVFQueryParam,
OptimizeOption,
VamanaIndexParam,
VamanaQueryParam,
)
from .model.param.query import Fts, Query, VectorQuery
# —— Schema & field definitions ——
from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
# —— tools ——
from .tool import require_module
from .typing import (
DataType,
IndexType,
MetricType,
QuantizeType,
Status,
StatusCode,
)
from .typing.enum import LogLevel, LogType
# —— lifecycle ——
from .zvec import create_and_open, init, open
# ==============================
# Public interface declaration
# ==============================
__all__ = [
# Zvec functions
"create_and_open",
"init",
"open",
"set_default_jieba_dict_dir",
"get_default_jieba_dict_dir",
# Core classes
"Collection",
"Doc",
# Schema
"CollectionSchema",
"FieldSchema",
"VectorSchema",
"CollectionStats",
# Parameters
"Query",
"VectorQuery",
"Fts",
"FtsIndexParam",
"FtsQueryParam",
"InvertIndexParam",
"HnswIndexParam",
"HnswRabitqIndexParam",
"FlatIndexParam",
"IVFIndexParam",
"CollectionOption",
"IndexOption",
"OptimizeOption",
"AddColumnOption",
"AlterColumnOption",
"HnswQueryParam",
"HnswRabitqQueryParam",
"IVFQueryParam",
"VamanaIndexParam",
"VamanaQueryParam",
# Extensions
"DenseEmbeddingFunction",
"SparseEmbeddingFunction",
"QwenFunctionBase",
"OpenAIFunctionBase",
"SentenceTransformerFunctionBase",
"ReRanker",
"DefaultLocalDenseEmbedding",
"DefaultLocalSparseEmbedding",
"BM25EmbeddingFunction",
"OpenAIDenseEmbedding",
"QwenDenseEmbedding",
"QwenSparseEmbedding",
"RrfReRanker",
"WeightedReRanker",
"DefaultLocalReRanker",
"QwenReRanker",
# Typing
"DataType",
"MetricType",
"QuantizeType",
"IndexType",
"LogLevel",
"LogType",
"Status",
"StatusCode",
# Tools
"require_module",
]
# ==============================
# Version handling
# ==============================
__version__: str
try:
from importlib.metadata import version
except ImportError:
from importlib_metadata import version # Python < 3.8
try:
__version__ = version("zvec")
except Exception:
__version__ = "unknown"