feat: add fts support (#408)

Add BM25-based full-text search with CJK (jieba) tokenization, supporting
  query_string and match_string syntax, phrase queries, boolean operators
  (AND/OR/NOT/MUST), and hybrid retrieval with existing vector search.

  ## Core
  - BitPacked posting format with block-max WAND pruning
  - Tokenizer pipeline: jieba (cut/cut_for_search/hmm/full), whitespace,
    lowercase, with extensible pipeline composition
  - Query parser: boolean operators, phrase queries, field scoping,
    boost, MUST(+) modifier inside OR (ES query_string semantics)
  - AST rewriter: dedup repeated terms with linear boost aggregation,
    flatten same-type composites, canonicalize OR-with-must_not into AND
    wrapper, empty-node propagation, contradiction detection
  - FTS reduce/merge integrated into Optimize compaction
  - Multi-segment score-descending sort
  - Auto-register bundled jieba dict on SDK import

  ## Performance
  - Block-max WAND with cached block_max_info_for (single binary search)
  - AVX2/SSE bitpacked encoding with cross-arch scalar fallback
  - MultiGet for batch posting retrieval and phrase position verification
  - HashSkipList memtable for posting writes
  - PinnableSlice zero-copy reads
  - Filter pushdown into composite iterators (Disjunction/Conjunction/Phrase)
  - Candidate-driven (brute-force) evaluation for selective invert filters
  - Precomputed BM25 IDF weights, cached SIMD dispatch pointers
  - Shortest-list anchor for phrase position matching
  - Single-open per-term posting iterator

  ## Query
  - Tokenize query terms through the same pipeline as indexing
  - EmptyNode for zero-token queries (all stop-words / punctuation)
  - Backslash unescape after lexing in query parser
  - Schema allows collections without vector fields (FTS-only use case)
  - Create/Drop Index validates supported index types
  - FTS fields disallowed in SQL filter expressions

  ## Bindings
  - C API: fts query params, brute-force ratio config
  - Python SDK: FTS search, jieba dict auto-registration

  ## Internals
  - Bypass cppjieba::Jieba to drop KeywordExtractor (~12MB fewer required files)
  - Hide tokenizer pipeline from public header (Pimpl-style FtsState)
  - ListColumnFamilies to avoid double-open on segment load
  - Reorganized fts_column into tokenizer/, posting/, iterator/ subdirs
This commit is contained in:
egolearner
2026-06-01 15:02:54 +08:00
committed by GitHub
parent 74beb2a828
commit 02bfb31cf5
154 changed files with 22646 additions and 223 deletions
@@ -177,6 +177,24 @@ void ZVecPyConfig::Initialize(pybind11::module_ &m) {
data.brute_force_by_keys_ratio = static_cast<float>(v);
}
// set fts_brute_force_by_keys_ratio
if (has_key(config_dict, "fts_brute_force_by_keys_ratio")) {
auto v =
get_if<double>(config_dict, "fts_brute_force_by_keys_ratio").value();
if (v < 0.0 || v > 1.0) {
throw py::value_error(
"fts_brute_force_by_keys_ratio must be in [0.0, 1.0]");
}
data.fts_brute_force_by_keys_ratio = static_cast<float>(v);
}
// jieba_dict_dir: optional override of the SDK-registered default.
// Empty value is a no-op (Initialize preserves the SDK default).
if (has_key(config_dict, "jieba_dict_dir")) {
data.jieba_dict_dir =
get_if<std::string>(config_dict, "jieba_dict_dir").value();
}
// initialize (contains validate)
Status status = GlobalConfig::Instance().Initialize(data);
if (!status.ok()) {
@@ -184,6 +202,21 @@ void ZVecPyConfig::Initialize(pybind11::module_ &m) {
}
return py::none();
});
// Process-wide setter, independent of Initialize(); called by __init__.py
// on import to register the wheel-bundled dict path.
m.def(
"set_default_jieba_dict_dir",
[](const std::string &dir) {
GlobalConfig::Instance().set_default_jieba_dict_dir(dir);
},
pybind11::arg("dir"),
"Register the process-wide default jieba dict directory.");
m.def(
"get_default_jieba_dict_dir",
[]() -> std::string { return GlobalConfig::Instance().jieba_dict_dir(); },
"Read the currently registered default jieba dict directory.");
}
+193 -6
View File
@@ -36,6 +36,8 @@ static std::string index_type_to_string(const IndexType type) {
return "HNSW_RABITQ";
case IndexType::VAMANA:
return "VAMANA";
case IndexType::FTS:
return "FTS";
default:
return "UNDEFINED";
}
@@ -252,6 +254,88 @@ Note: Prefix search is always enabled regardless of this setting.
t[1].cast<bool>());
}));
// binding fts index params
py::class_<FtsIndexParams, IndexParams, std::shared_ptr<FtsIndexParams>>
fts_index_params(m, "FtsIndexParam", R"pbdoc(
Parameters for configuring a full-text search (FTS) index.
Controls the tokenizer pipeline used during indexing and querying.
Attributes:
type (IndexType): Always ``IndexType.FTS``.
tokenizer_name (str): Name of the tokenizer (e.g., "standard", "jieba").
Default is "standard".
filters (list[str]): List of token filter names applied after tokenization.
Default is ["lowercase"].
extra_params (str): Additional parameters passed to the tokenizer.
Default is "".
Examples:
>>> params = FtsIndexParam(tokenizer_name="jieba", filters=["lowercase"])
>>> print(params.tokenizer_name)
jieba
)pbdoc");
fts_index_params
.def(py::init<std::string, std::vector<std::string>, std::string>(),
py::arg("tokenizer_name") = "standard",
py::arg("filters") = std::vector<std::string>{"lowercase"},
py::arg("extra_params") = "",
R"pbdoc(
Constructs an FtsIndexParam instance.
Args:
tokenizer_name (str, optional): Tokenizer name. Defaults to "standard".
filters (list[str], optional): Token filter names. Defaults to ["lowercase"].
extra_params (str, optional): Extra tokenizer parameters. Defaults to "".
)pbdoc")
.def_property_readonly("tokenizer_name", &FtsIndexParams::tokenizer_name,
"str: Name of the tokenizer.")
.def_property_readonly("filters", &FtsIndexParams::filters,
"list[str]: Token filter names.")
.def_property_readonly("extra_params", &FtsIndexParams::extra_params,
"str: Additional tokenizer parameters.")
.def(
"to_dict",
[](const FtsIndexParams &self) -> py::dict {
py::dict dict;
dict["type"] = index_type_to_string(self.type());
dict["tokenizer_name"] = self.tokenizer_name();
dict["filters"] = self.filters();
dict["extra_params"] = self.extra_params();
return dict;
},
"Convert to dictionary with all fields")
.def("__repr__",
[](const FtsIndexParams &self) -> std::string {
std::string filters_str = "[";
for (size_t i = 0; i < self.filters().size(); ++i) {
if (i > 0) {
filters_str += ",";
}
filters_str += "\"" + self.filters()[i] + "\"";
}
filters_str += "]";
return "{"
"\"type\":\"" +
index_type_to_string(self.type()) +
"\", \"tokenizer_name\":\"" + self.tokenizer_name() +
"\", \"filters\":" + filters_str + ", \"extra_params\":\"" +
self.extra_params() + "\"}";
})
.def(py::pickle(
[](const FtsIndexParams &self) {
return py::make_tuple(self.tokenizer_name(), self.filters(),
self.extra_params());
},
[](py::tuple t) {
if (t.size() != 3) {
throw std::runtime_error("Invalid state for FtsIndexParams");
}
return std::make_shared<FtsIndexParams>(
t[0].cast<std::string>(), t[1].cast<std::vector<std::string>>(),
t[2].cast<std::string>());
}));
// binding base vector index params
py::class_<VectorIndexParams, IndexParams, std::shared_ptr<VectorIndexParams>>
vector_params(m, "VectorIndexParam", R"pbdoc(
@@ -1103,6 +1187,64 @@ Args:
obj->set_is_using_refiner(t[3].cast<bool>());
return obj;
}));
// binding fts query params
py::class_<FtsQueryParams, QueryParams, std::shared_ptr<FtsQueryParams>>
fts_query_params(m, "FtsQueryParam", R"pbdoc(
Query parameters for full-text search (FTS) index.
Controls the default boolean operator used to combine adjacent bare terms
in a query string.
Attributes:
type (IndexType): Always ``IndexType.FTS``.
default_operator (str): Default boolean operator for adjacent bare terms.
Supported values (case-insensitive): "OR" (default), "AND".
Examples:
>>> params = FtsQueryParam(default_operator="AND")
>>> print(params.default_operator)
AND
)pbdoc");
fts_query_params
.def(py::init([](const std::string &default_operator) {
auto params = std::make_shared<FtsQueryParams>();
if (!default_operator.empty()) {
params->set_default_operator(default_operator);
}
return params;
}),
py::arg("default_operator") = "",
R"pbdoc(
Constructs an FtsQueryParam instance.
Args:
default_operator (str, optional): Default boolean operator for adjacent
bare terms. Supported: "OR", "AND". Defaults to "" (uses engine default).
)pbdoc")
.def_property_readonly("default_operator",
&FtsQueryParams::default_operator,
"str: Default boolean operator for bare terms.")
.def("__repr__",
[](const FtsQueryParams &self) -> std::string {
return "{"
"\"type\":\"" +
index_type_to_string(self.type()) +
"\", \"default_operator\":\"" + self.default_operator() +
"\"}";
})
.def(py::pickle(
[](const FtsQueryParams &self) {
return py::make_tuple(self.default_operator());
},
[](py::tuple t) {
if (t.size() != 1) {
throw std::runtime_error("Invalid state for FtsQueryParams");
}
auto obj = std::make_shared<FtsQueryParams>();
obj->set_default_operator(t[0].cast<std::string>());
return obj;
}));
}
void ZVecPyParams::bind_options(py::module_ &m) { // binding collection options
@@ -1373,6 +1515,24 @@ Args:
}
void ZVecPyParams::bind_vector_query(py::module_ &m) {
// bind Fts
py::class_<FtsClause>(m, "_Fts")
.def(py::init<>())
.def_readwrite("query_string", &FtsClause::query_string_)
.def_readwrite("match_string", &FtsClause::match_string_)
.def(py::pickle(
[](const FtsClause &self) {
return py::make_tuple(self.query_string_, self.match_string_);
},
[](py::tuple t) {
if (t.size() != 2)
throw std::runtime_error("Invalid pickle data for Fts");
FtsClause obj{};
obj.query_string_ = t[0].cast<std::string>();
obj.match_string_ = t[1].cast<std::string>();
return obj;
}));
// Bind SubQuery (used by MultiQuery)
py::class_<SubQuery>(m, "_SubQuery")
.def(py::init<>())
@@ -1409,6 +1569,23 @@ void ZVecPyParams::bind_vector_query(py::module_ &m) {
s.target_.query_params_ = std::move(p);
})
.def_readwrite("output_fields", &SearchQuery::output_fields_)
.def_property(
"fts",
[](const SearchQuery &self) -> py::object {
const auto *fc = self.target_.get_fts_clause();
if (fc != nullptr) {
return py::cast(*fc);
}
return py::none();
},
[](SearchQuery &self, const py::object &obj) {
if (obj.is_none()) {
// Clearing FTS resets the target to an empty vector clause.
self.target_.clause_ = VectorClause{};
} else {
self.target_.clause_ = obj.cast<FtsClause>();
}
})
// vector
.def("set_vector",
[](SearchQuery &self, const FieldSchema &field_schema,
@@ -1614,6 +1791,7 @@ void ZVecPyParams::bind_vector_query(py::module_ &m) {
.def(py::pickle(
[](const SearchQuery &self) {
const VectorClause *vc = self.target_.get_vector_clause();
const auto *fc = self.target_.get_fts_clause();
return py::make_tuple(self.topk_, self.target_.field_name_,
vc ? vc->query_vector_ : std::string(),
vc ? vc->sparse_indices_ : std::string(),
@@ -1622,22 +1800,31 @@ void ZVecPyParams::bind_vector_query(py::module_ &m) {
self.output_fields_,
self.target_.query_params_
? py::cast(self.target_.query_params_)
: py::none());
: py::none(),
fc ? py::cast(*fc) : py::none());
},
[](py::tuple t) {
if (t.size() != 9)
if (t.size() != 10)
throw std::runtime_error("Invalid pickle data for _VectorQuery");
SearchQuery obj{};
obj.topk_ = t[0].cast<int>();
obj.target_.field_name_ = t[1].cast<std::string>();
obj.target_.clause_ =
VectorClause{t[2].cast<std::string>(), t[3].cast<std::string>(),
t[4].cast<std::string>()};
// A vector clause and an FTS clause are mutually exclusive in the
// variant target; restore whichever the pickle carried.
if (!t[9].is_none()) {
obj.target_.clause_ = t[9].cast<FtsClause>();
} else {
obj.target_.clause_ = VectorClause{t[2].cast<std::string>(),
t[3].cast<std::string>(),
t[4].cast<std::string>()};
}
obj.filter_ = t[5].cast<std::string>();
obj.include_vector_ = t[6].cast<bool>();
obj.output_fields_ = t[7].cast<std::vector<std::string>>();
if (!t[7].is_none()) {
obj.output_fields_ = t[7].cast<std::vector<std::string>>();
}
if (!t[8].is_none()) {
obj.target_.query_params_ = t[8].cast<QueryParams::Ptr>();
}