feat: add hnsw-rabitq support (#69)

* feat: add hnsw-rabitq

* undefine transform

* support config rotator type

* support sample_count

* fix ut

* refactor: update interface

* refactor: update interface

* update rabitq index params

* fix interface update

* fix searcher test

* fix streamer test

* streamer support bf and add more ut

* rm env check

* add collection ut

* add schema check

* add rabitq query param binding

* add integration test

* fix local_builder

* cleanup dist calculator

* add RaBitQ-Library submodule

* cleanup rabitq converter/reformer

* add files

* disable build on mac

* disable Feature_Optimize_HNSW_RABITQ

* disable python/tests/test_collection_hnsw_rabitq.py:13

* check avx2/avx512

* fix refine

* fix mac ci

* add dimension check

* fix mac ci

* fix ci

* add missing lib

* fix centroids selection for Cosine/InnerProduct

* rename hnsw-rabitq to hnsw_rabitq

* address comments

* fix compile

* fix rabitqlib name

* fix mac compile

* check avx2/avx512 support

* runtime check again compile-time

* check AUTO_DETECT_ARCH

* rm debug log

* fix

* rabitq use avx2 by default

* add missing file

* fix search_bf/group_by dist

* address comments

* fix typo

* address comments

* rabitq support disable id_map

* address comments
This commit is contained in:
egolearner
2026-03-19 17:21:42 +08:00
committed by GitHub
parent 432de52c5f
commit e5ba11b6fe
96 changed files with 14500 additions and 92 deletions
+2
View File
@@ -24,6 +24,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
$<TARGET_FILE:core_knn_flat_static>
$<TARGET_FILE:core_knn_flat_sparse_static>
$<TARGET_FILE:core_knn_hnsw_static>
$<TARGET_FILE:core_knn_hnsw_rabitq_static>
$<TARGET_FILE:core_knn_hnsw_sparse_static>
$<TARGET_FILE:core_knn_ivf_static>
$<TARGET_FILE:core_knn_cluster_static>
@@ -42,6 +43,7 @@ elseif (APPLE)
-Wl,-force_load,$<TARGET_FILE:core_knn_flat_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_flat_sparse_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_hnsw_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_hnsw_rabitq_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_hnsw_sparse_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_ivf_static>
-Wl,-force_load,$<TARGET_FILE:core_knn_cluster_static>
+176 -2
View File
@@ -31,6 +31,8 @@ static std::string index_type_to_string(const IndexType type) {
return "IVF";
case IndexType::HNSW:
return "HNSW";
case IndexType::HNSW_RABITQ:
return "HNSW_RABITQ";
default:
return "UNDEFINED";
}
@@ -59,6 +61,8 @@ static std::string quantize_type_to_string(const QuantizeType type) {
return "INT4";
case QuantizeType::FP16:
return "FP16";
case QuantizeType::RABITQ:
return "RABITQ";
default:
return "UNDEFINED";
}
@@ -376,6 +380,105 @@ Examples:
t[3].cast<QuantizeType>());
}));
// binding hnsw rabitq index params
py::class_<HnswRabitqIndexParams, VectorIndexParams,
std::shared_ptr<HnswRabitqIndexParams>>
hnsw_rabitq_params(m, "HnswRabitqIndexParam", R"pbdoc(
Parameters for configuring an HNSW (Hierarchical Navigable Small World) index with RabitQ quantization.
HNSW is a graph-based approximate nearest neighbor search index. RabitQ is a
quantization method that provides high compression with minimal accuracy loss.
Attributes:
metric_type (MetricType): Distance metric used for similarity computation.
Default is ``MetricType.IP`` (inner product).
m (int): Number of bi-directional links created for every new element
during construction. Higher values improve accuracy but increase
memory usage and construction time. Default is 50.
ef_construction (int): Size of the dynamic candidate list for nearest
neighbors during index construction. Larger values yield better
graph quality at the cost of slower build time. Default is 500.
Examples:
>>> from zvec.typing import MetricType
>>> params = HnswRabitqIndexParam(
... metric_type=MetricType.COSINE,
... m=16,
... ef_construction=200
... )
>>> print(params)
{'metric_type': 'COSINE', 'm': 16, 'ef_construction': 200}
)pbdoc");
hnsw_rabitq_params
.def(py::init<MetricType, int, int, int, int, int>(),
py::arg("metric_type") = MetricType::IP,
py::arg("total_bits") = core_interface::kDefaultRabitqTotalBits,
py::arg("num_clusters") = core_interface::kDefaultRabitqNumClusters,
py::arg("m") = core_interface::kDefaultHnswNeighborCnt,
py::arg("ef_construction") =
core_interface::kDefaultHnswEfConstruction,
py::arg("sample_count") = 0)
.def_property_readonly("m", &HnswRabitqIndexParams::m,
"int: Maximum number of neighbors per node.")
.def_property_readonly(
"ef_construction", &HnswRabitqIndexParams::ef_construction,
"int: Candidate list size during index construction.")
.def_property_readonly("total_bits", &HnswRabitqIndexParams::total_bits,
"int: Total bits for RabitQ quantization.")
.def_property_readonly("num_clusters",
&HnswRabitqIndexParams::num_clusters,
"int: Number of clusters for RabitQ.")
.def_property_readonly("sample_count",
&HnswRabitqIndexParams::sample_count,
"int: Sample count for RabitQ training.")
.def(
"to_dict",
[](const HnswRabitqIndexParams &self) -> py::dict {
py::dict dict;
dict["type"] = index_type_to_string(self.type());
dict["metric_type"] = metric_type_to_string(self.metric_type());
dict["quantize_type"] =
quantize_type_to_string(self.quantize_type());
dict["total_bits"] = self.total_bits();
dict["num_clusters"] = self.num_clusters();
dict["sample_count"] = self.sample_count();
dict["m"] = self.m();
dict["ef_construction"] = self.ef_construction();
return dict;
},
"Convert to dictionary with all fields")
.def(
"__repr__",
[](const HnswRabitqIndexParams &self) -> std::string {
return "{"
"\"type\":\"" +
index_type_to_string(self.type()) +
"\", \"metric_type\":\"" +
metric_type_to_string(self.metric_type()) +
"\", \"total_bits\":" + std::to_string(self.total_bits()) +
", \"num_clusters\":" + std::to_string(self.num_clusters()) +
", \"sample_count\":" + std::to_string(self.sample_count()) +
", \"m\":" + std::to_string(self.m()) +
", \"ef_construction\":" +
std::to_string(self.ef_construction()) +
", \"quantize_type\":\"" +
quantize_type_to_string(self.quantize_type()) + "\"}";
})
.def(py::pickle(
[](const HnswRabitqIndexParams &self) {
return py::make_tuple(self.metric_type(), self.total_bits(),
self.num_clusters(), self.m(),
self.ef_construction(), self.sample_count());
},
[](py::tuple t) {
if (t.size() != 6)
throw std::runtime_error(
"Invalid state for HnswRabitqIndexParams");
return std::make_shared<HnswRabitqIndexParams>(
t[0].cast<MetricType>(), t[1].cast<int>(), t[2].cast<int>(),
t[3].cast<int>(), t[4].cast<int>(), t[5].cast<int>());
}));
// FlatIndexParams
py::class_<FlatIndexParams, VectorIndexParams,
std::shared_ptr<FlatIndexParams>>
@@ -709,10 +812,81 @@ Args:
obj->set_is_linear(t[2].cast<bool>());
return obj;
}));
// binding hnsw rabitq query params
py::class_<HnswRabitqQueryParams, QueryParams,
std::shared_ptr<HnswRabitqQueryParams>>
hnsw_rabitq_query_params(m, "HnswRabitqQueryParam", R"pbdoc(
Query parameters for HNSW RaBitQ (Hierarchical Navigable Small World with RaBitQ quantization) index.
Controls the trade-off between search speed and accuracy via the `ef` parameter.
RaBitQ provides efficient quantization while maintaining high search quality.
Attributes:
type (IndexType): Always ``IndexType.HNSW_RABITQ``.
ef (int): Size of the dynamic candidate list during search.
Larger values improve recall but slow down search.
Default is 300.
radius (float): Search radius for range queries. Default is 0.0.
is_linear (bool): Force linear search. Default is False.
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
Examples:
>>> params = HnswRabitqQueryParam(ef=300)
>>> print(params.ef)
300
>>> print(params.to_dict() if hasattr(params, 'to_dict') else params)
{"type":"HNSW_RABITQ", "ef":300}
)pbdoc");
hnsw_rabitq_query_params
.def(py::init<int, float, bool, bool>(),
py::arg("ef") = core_interface::kDefaultHnswEfSearch,
py::arg("radius") = 0.0f, py::arg("is_linear") = false,
py::arg("is_using_refiner") = false,
R"pbdoc(
Constructs an HnswRabitqQueryParam instance.
Args:
ef (int, optional): Search-time candidate list size.
Higher values improve accuracy. Defaults to 300.
radius (float, optional): Search radius for range queries. Default is 0.0.
is_linear (bool, optional): Force linear search. Default is False.
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
)pbdoc")
.def_property_readonly(
"ef",
[](const HnswRabitqQueryParams &self) -> int { return self.ef(); },
"int: Size of the dynamic candidate list during HNSW RaBitQ search.")
.def("__repr__",
[](const HnswRabitqQueryParams &self) -> std::string {
return "{"
"\"type\":\"" +
index_type_to_string(self.type()) +
"\", \"ef\":" + std::to_string(self.ef()) +
", \"radius\":" + std::to_string(self.radius()) +
", \"is_linear\":" + std::to_string(self.is_linear()) +
", \"is_using_refiner\":" +
std::to_string(self.is_using_refiner()) + "}";
})
.def(py::pickle(
[](const HnswRabitqQueryParams &self) {
return py::make_tuple(self.ef(), self.radius(), self.is_linear(),
self.is_using_refiner());
},
[](py::tuple t) {
if (t.size() != 4)
throw std::runtime_error(
"Invalid state for HnswRabitqQueryParams");
auto obj =
std::make_shared<HnswRabitqQueryParams>(t[0].cast<int>());
obj->set_radius(t[1].cast<float>());
obj->set_is_linear(t[2].cast<bool>());
obj->set_is_using_refiner(t[3].cast<bool>());
return obj;
}));
}
void ZVecPyParams::bind_options(py::module_ &m) {
// binding collection options
void ZVecPyParams::bind_options(py::module_ &m) { // binding collection options
py::class_<CollectionOptions>(m, "CollectionOption", R"pbdoc(
Options for opening or creating a collection.
+3 -1
View File
@@ -96,6 +96,7 @@ Examples:
)pbdoc")
.value("UNDEFINED", IndexType::UNDEFINED)
.value("HNSW", IndexType::HNSW)
.value("HNSW_RABITQ", IndexType::HNSW_RABITQ)
.value("IVF", IndexType::IVF)
.value("FLAT", IndexType::FLAT)
.value("INVERT", IndexType::INVERT);
@@ -131,7 +132,8 @@ Examples:
.value("UNDEFINED", QuantizeType::UNDEFINED)
.value("FP16", QuantizeType::FP16)
.value("INT8", QuantizeType::INT8)
.value("INT4", QuantizeType::INT4);
.value("INT4", QuantizeType::INT4)
.value("RABITQ", QuantizeType::RABITQ);
}
void ZVecPyTyping::bind_status(py::module_ &m) {