feat: add fts support (#408)
Add BM25-based full-text search with CJK (jieba) tokenization, supporting
query_string and match_string syntax, phrase queries, boolean operators
(AND/OR/NOT/MUST), and hybrid retrieval with existing vector search.
## Core
- BitPacked posting format with block-max WAND pruning
- Tokenizer pipeline: jieba (cut/cut_for_search/hmm/full), whitespace,
lowercase, with extensible pipeline composition
- Query parser: boolean operators, phrase queries, field scoping,
boost, MUST(+) modifier inside OR (ES query_string semantics)
- AST rewriter: dedup repeated terms with linear boost aggregation,
flatten same-type composites, canonicalize OR-with-must_not into AND
wrapper, empty-node propagation, contradiction detection
- FTS reduce/merge integrated into Optimize compaction
- Multi-segment score-descending sort
- Auto-register bundled jieba dict on SDK import
## Performance
- Block-max WAND with cached block_max_info_for (single binary search)
- AVX2/SSE bitpacked encoding with cross-arch scalar fallback
- MultiGet for batch posting retrieval and phrase position verification
- HashSkipList memtable for posting writes
- PinnableSlice zero-copy reads
- Filter pushdown into composite iterators (Disjunction/Conjunction/Phrase)
- Candidate-driven (brute-force) evaluation for selective invert filters
- Precomputed BM25 IDF weights, cached SIMD dispatch pointers
- Shortest-list anchor for phrase position matching
- Single-open per-term posting iterator
## Query
- Tokenize query terms through the same pipeline as indexing
- EmptyNode for zero-token queries (all stop-words / punctuation)
- Backslash unescape after lexing in query parser
- Schema allows collections without vector fields (FTS-only use case)
- Create/Drop Index validates supported index types
- FTS fields disallowed in SQL filter expressions
## Bindings
- C API: fts query params, brute-force ratio config
- Python SDK: FTS search, jieba dict auto-registration
## Internals
- Bypass cppjieba::Jieba to drop KeywordExtractor (~12MB fewer required files)
- Hide tokenizer pipeline from public header (Pimpl-style FtsState)
- ListColumnFamilies to avoid double-open on segment load
- Reorganized fts_column into tokenizer/, posting/, iterator/ subdirs
This commit is contained in:
@@ -177,6 +177,24 @@ void ZVecPyConfig::Initialize(pybind11::module_ &m) {
|
||||
data.brute_force_by_keys_ratio = static_cast<float>(v);
|
||||
}
|
||||
|
||||
// set fts_brute_force_by_keys_ratio
|
||||
if (has_key(config_dict, "fts_brute_force_by_keys_ratio")) {
|
||||
auto v =
|
||||
get_if<double>(config_dict, "fts_brute_force_by_keys_ratio").value();
|
||||
if (v < 0.0 || v > 1.0) {
|
||||
throw py::value_error(
|
||||
"fts_brute_force_by_keys_ratio must be in [0.0, 1.0]");
|
||||
}
|
||||
data.fts_brute_force_by_keys_ratio = static_cast<float>(v);
|
||||
}
|
||||
|
||||
// jieba_dict_dir: optional override of the SDK-registered default.
|
||||
// Empty value is a no-op (Initialize preserves the SDK default).
|
||||
if (has_key(config_dict, "jieba_dict_dir")) {
|
||||
data.jieba_dict_dir =
|
||||
get_if<std::string>(config_dict, "jieba_dict_dir").value();
|
||||
}
|
||||
|
||||
// initialize (contains validate)
|
||||
Status status = GlobalConfig::Instance().Initialize(data);
|
||||
if (!status.ok()) {
|
||||
@@ -184,6 +202,21 @@ void ZVecPyConfig::Initialize(pybind11::module_ &m) {
|
||||
}
|
||||
return py::none();
|
||||
});
|
||||
|
||||
// Process-wide setter, independent of Initialize(); called by __init__.py
|
||||
// on import to register the wheel-bundled dict path.
|
||||
m.def(
|
||||
"set_default_jieba_dict_dir",
|
||||
[](const std::string &dir) {
|
||||
GlobalConfig::Instance().set_default_jieba_dict_dir(dir);
|
||||
},
|
||||
pybind11::arg("dir"),
|
||||
"Register the process-wide default jieba dict directory.");
|
||||
|
||||
m.def(
|
||||
"get_default_jieba_dict_dir",
|
||||
[]() -> std::string { return GlobalConfig::Instance().jieba_dict_dir(); },
|
||||
"Read the currently registered default jieba dict directory.");
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -36,6 +36,8 @@ static std::string index_type_to_string(const IndexType type) {
|
||||
return "HNSW_RABITQ";
|
||||
case IndexType::VAMANA:
|
||||
return "VAMANA";
|
||||
case IndexType::FTS:
|
||||
return "FTS";
|
||||
default:
|
||||
return "UNDEFINED";
|
||||
}
|
||||
@@ -252,6 +254,88 @@ Note: Prefix search is always enabled regardless of this setting.
|
||||
t[1].cast<bool>());
|
||||
}));
|
||||
|
||||
// binding fts index params
|
||||
py::class_<FtsIndexParams, IndexParams, std::shared_ptr<FtsIndexParams>>
|
||||
fts_index_params(m, "FtsIndexParam", R"pbdoc(
|
||||
Parameters for configuring a full-text search (FTS) index.
|
||||
|
||||
Controls the tokenizer pipeline used during indexing and querying.
|
||||
|
||||
Attributes:
|
||||
type (IndexType): Always ``IndexType.FTS``.
|
||||
tokenizer_name (str): Name of the tokenizer (e.g., "standard", "jieba").
|
||||
Default is "standard".
|
||||
filters (list[str]): List of token filter names applied after tokenization.
|
||||
Default is ["lowercase"].
|
||||
extra_params (str): Additional parameters passed to the tokenizer.
|
||||
Default is "".
|
||||
|
||||
Examples:
|
||||
>>> params = FtsIndexParam(tokenizer_name="jieba", filters=["lowercase"])
|
||||
>>> print(params.tokenizer_name)
|
||||
jieba
|
||||
)pbdoc");
|
||||
fts_index_params
|
||||
.def(py::init<std::string, std::vector<std::string>, std::string>(),
|
||||
py::arg("tokenizer_name") = "standard",
|
||||
py::arg("filters") = std::vector<std::string>{"lowercase"},
|
||||
py::arg("extra_params") = "",
|
||||
R"pbdoc(
|
||||
Constructs an FtsIndexParam instance.
|
||||
|
||||
Args:
|
||||
tokenizer_name (str, optional): Tokenizer name. Defaults to "standard".
|
||||
filters (list[str], optional): Token filter names. Defaults to ["lowercase"].
|
||||
extra_params (str, optional): Extra tokenizer parameters. Defaults to "".
|
||||
)pbdoc")
|
||||
.def_property_readonly("tokenizer_name", &FtsIndexParams::tokenizer_name,
|
||||
"str: Name of the tokenizer.")
|
||||
.def_property_readonly("filters", &FtsIndexParams::filters,
|
||||
"list[str]: Token filter names.")
|
||||
.def_property_readonly("extra_params", &FtsIndexParams::extra_params,
|
||||
"str: Additional tokenizer parameters.")
|
||||
.def(
|
||||
"to_dict",
|
||||
[](const FtsIndexParams &self) -> py::dict {
|
||||
py::dict dict;
|
||||
dict["type"] = index_type_to_string(self.type());
|
||||
dict["tokenizer_name"] = self.tokenizer_name();
|
||||
dict["filters"] = self.filters();
|
||||
dict["extra_params"] = self.extra_params();
|
||||
return dict;
|
||||
},
|
||||
"Convert to dictionary with all fields")
|
||||
.def("__repr__",
|
||||
[](const FtsIndexParams &self) -> std::string {
|
||||
std::string filters_str = "[";
|
||||
for (size_t i = 0; i < self.filters().size(); ++i) {
|
||||
if (i > 0) {
|
||||
filters_str += ",";
|
||||
}
|
||||
filters_str += "\"" + self.filters()[i] + "\"";
|
||||
}
|
||||
filters_str += "]";
|
||||
return "{"
|
||||
"\"type\":\"" +
|
||||
index_type_to_string(self.type()) +
|
||||
"\", \"tokenizer_name\":\"" + self.tokenizer_name() +
|
||||
"\", \"filters\":" + filters_str + ", \"extra_params\":\"" +
|
||||
self.extra_params() + "\"}";
|
||||
})
|
||||
.def(py::pickle(
|
||||
[](const FtsIndexParams &self) {
|
||||
return py::make_tuple(self.tokenizer_name(), self.filters(),
|
||||
self.extra_params());
|
||||
},
|
||||
[](py::tuple t) {
|
||||
if (t.size() != 3) {
|
||||
throw std::runtime_error("Invalid state for FtsIndexParams");
|
||||
}
|
||||
return std::make_shared<FtsIndexParams>(
|
||||
t[0].cast<std::string>(), t[1].cast<std::vector<std::string>>(),
|
||||
t[2].cast<std::string>());
|
||||
}));
|
||||
|
||||
// binding base vector index params
|
||||
py::class_<VectorIndexParams, IndexParams, std::shared_ptr<VectorIndexParams>>
|
||||
vector_params(m, "VectorIndexParam", R"pbdoc(
|
||||
@@ -1103,6 +1187,64 @@ Args:
|
||||
obj->set_is_using_refiner(t[3].cast<bool>());
|
||||
return obj;
|
||||
}));
|
||||
|
||||
// binding fts query params
|
||||
py::class_<FtsQueryParams, QueryParams, std::shared_ptr<FtsQueryParams>>
|
||||
fts_query_params(m, "FtsQueryParam", R"pbdoc(
|
||||
Query parameters for full-text search (FTS) index.
|
||||
|
||||
Controls the default boolean operator used to combine adjacent bare terms
|
||||
in a query string.
|
||||
|
||||
Attributes:
|
||||
type (IndexType): Always ``IndexType.FTS``.
|
||||
default_operator (str): Default boolean operator for adjacent bare terms.
|
||||
Supported values (case-insensitive): "OR" (default), "AND".
|
||||
|
||||
Examples:
|
||||
>>> params = FtsQueryParam(default_operator="AND")
|
||||
>>> print(params.default_operator)
|
||||
AND
|
||||
)pbdoc");
|
||||
fts_query_params
|
||||
.def(py::init([](const std::string &default_operator) {
|
||||
auto params = std::make_shared<FtsQueryParams>();
|
||||
if (!default_operator.empty()) {
|
||||
params->set_default_operator(default_operator);
|
||||
}
|
||||
return params;
|
||||
}),
|
||||
py::arg("default_operator") = "",
|
||||
R"pbdoc(
|
||||
Constructs an FtsQueryParam instance.
|
||||
|
||||
Args:
|
||||
default_operator (str, optional): Default boolean operator for adjacent
|
||||
bare terms. Supported: "OR", "AND". Defaults to "" (uses engine default).
|
||||
)pbdoc")
|
||||
.def_property_readonly("default_operator",
|
||||
&FtsQueryParams::default_operator,
|
||||
"str: Default boolean operator for bare terms.")
|
||||
.def("__repr__",
|
||||
[](const FtsQueryParams &self) -> std::string {
|
||||
return "{"
|
||||
"\"type\":\"" +
|
||||
index_type_to_string(self.type()) +
|
||||
"\", \"default_operator\":\"" + self.default_operator() +
|
||||
"\"}";
|
||||
})
|
||||
.def(py::pickle(
|
||||
[](const FtsQueryParams &self) {
|
||||
return py::make_tuple(self.default_operator());
|
||||
},
|
||||
[](py::tuple t) {
|
||||
if (t.size() != 1) {
|
||||
throw std::runtime_error("Invalid state for FtsQueryParams");
|
||||
}
|
||||
auto obj = std::make_shared<FtsQueryParams>();
|
||||
obj->set_default_operator(t[0].cast<std::string>());
|
||||
return obj;
|
||||
}));
|
||||
}
|
||||
|
||||
void ZVecPyParams::bind_options(py::module_ &m) { // binding collection options
|
||||
@@ -1373,6 +1515,24 @@ Args:
|
||||
}
|
||||
|
||||
void ZVecPyParams::bind_vector_query(py::module_ &m) {
|
||||
// bind Fts
|
||||
py::class_<FtsClause>(m, "_Fts")
|
||||
.def(py::init<>())
|
||||
.def_readwrite("query_string", &FtsClause::query_string_)
|
||||
.def_readwrite("match_string", &FtsClause::match_string_)
|
||||
.def(py::pickle(
|
||||
[](const FtsClause &self) {
|
||||
return py::make_tuple(self.query_string_, self.match_string_);
|
||||
},
|
||||
[](py::tuple t) {
|
||||
if (t.size() != 2)
|
||||
throw std::runtime_error("Invalid pickle data for Fts");
|
||||
FtsClause obj{};
|
||||
obj.query_string_ = t[0].cast<std::string>();
|
||||
obj.match_string_ = t[1].cast<std::string>();
|
||||
return obj;
|
||||
}));
|
||||
|
||||
// Bind SubQuery (used by MultiQuery)
|
||||
py::class_<SubQuery>(m, "_SubQuery")
|
||||
.def(py::init<>())
|
||||
@@ -1409,6 +1569,23 @@ void ZVecPyParams::bind_vector_query(py::module_ &m) {
|
||||
s.target_.query_params_ = std::move(p);
|
||||
})
|
||||
.def_readwrite("output_fields", &SearchQuery::output_fields_)
|
||||
.def_property(
|
||||
"fts",
|
||||
[](const SearchQuery &self) -> py::object {
|
||||
const auto *fc = self.target_.get_fts_clause();
|
||||
if (fc != nullptr) {
|
||||
return py::cast(*fc);
|
||||
}
|
||||
return py::none();
|
||||
},
|
||||
[](SearchQuery &self, const py::object &obj) {
|
||||
if (obj.is_none()) {
|
||||
// Clearing FTS resets the target to an empty vector clause.
|
||||
self.target_.clause_ = VectorClause{};
|
||||
} else {
|
||||
self.target_.clause_ = obj.cast<FtsClause>();
|
||||
}
|
||||
})
|
||||
// vector
|
||||
.def("set_vector",
|
||||
[](SearchQuery &self, const FieldSchema &field_schema,
|
||||
@@ -1614,6 +1791,7 @@ void ZVecPyParams::bind_vector_query(py::module_ &m) {
|
||||
.def(py::pickle(
|
||||
[](const SearchQuery &self) {
|
||||
const VectorClause *vc = self.target_.get_vector_clause();
|
||||
const auto *fc = self.target_.get_fts_clause();
|
||||
return py::make_tuple(self.topk_, self.target_.field_name_,
|
||||
vc ? vc->query_vector_ : std::string(),
|
||||
vc ? vc->sparse_indices_ : std::string(),
|
||||
@@ -1622,22 +1800,31 @@ void ZVecPyParams::bind_vector_query(py::module_ &m) {
|
||||
self.output_fields_,
|
||||
self.target_.query_params_
|
||||
? py::cast(self.target_.query_params_)
|
||||
: py::none());
|
||||
: py::none(),
|
||||
fc ? py::cast(*fc) : py::none());
|
||||
},
|
||||
[](py::tuple t) {
|
||||
if (t.size() != 9)
|
||||
if (t.size() != 10)
|
||||
throw std::runtime_error("Invalid pickle data for _VectorQuery");
|
||||
|
||||
SearchQuery obj{};
|
||||
obj.topk_ = t[0].cast<int>();
|
||||
obj.target_.field_name_ = t[1].cast<std::string>();
|
||||
obj.target_.clause_ =
|
||||
VectorClause{t[2].cast<std::string>(), t[3].cast<std::string>(),
|
||||
t[4].cast<std::string>()};
|
||||
// A vector clause and an FTS clause are mutually exclusive in the
|
||||
// variant target; restore whichever the pickle carried.
|
||||
if (!t[9].is_none()) {
|
||||
obj.target_.clause_ = t[9].cast<FtsClause>();
|
||||
} else {
|
||||
obj.target_.clause_ = VectorClause{t[2].cast<std::string>(),
|
||||
t[3].cast<std::string>(),
|
||||
t[4].cast<std::string>()};
|
||||
}
|
||||
obj.filter_ = t[5].cast<std::string>();
|
||||
obj.include_vector_ = t[6].cast<bool>();
|
||||
obj.output_fields_ = t[7].cast<std::vector<std::string>>();
|
||||
|
||||
if (!t[7].is_none()) {
|
||||
obj.output_fields_ = t[7].cast<std::vector<std::string>>();
|
||||
}
|
||||
if (!t[8].is_none()) {
|
||||
obj.target_.query_params_ = t[8].cast<QueryParams::Ptr>();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user