feat: migrate multi-vector query and reranker logic to C++ (#405)

* feat: migrate multi-vector query and reranker logic to C++

- Add Reranker base class with RrfReRanker and WeightedReRanker implementations
- Add Collection::MultiQuery interface for multi-vector queries with reranking
- Add MultiVectorQuery struct in doc.h with forward declaration for Reranker
- Add C API bindings for reranker and MultiQuery (zvec_reranker_*, zvec_multi_vector_query_*, zvec_collection_multi_query)
- Add Python binding for reranker classes with py::function bridge for callback
- Validate duplicate field names in multi-vector queries (C++ and Python consistent)
- Remove TODO comment about concurrent execution (SQLEngine is not thread-safe)
- Update collection.h MultiQuery doc comment from concurrently to sequentially
- Add C++ collection tests (6 MultiQuery test cases)
- Add C API tests (reranker functions + multi_vector_query end-to-end)
- Implement Python test cases (11 previously skipped tests now active)
- Simplify Python query_executor validation for unified duplicate field check

* style: format Python files with ruff

* fix: adapt to main branch API changes (VectorQuery->Query rename, validate_and_sanitize)

* fix: multi_vector tests now use multiple same-type vector fields (dense2, sparse2)

* fix: suppress RET501 for intentional default return None in RerankFunction._get_object

* style: ruff format test_collection.py

* refact multi vector query

* format code

* fix(multi-vector): expose SubVectorQuery in Python binding, fix tests

- Register _SubVectorQuery in pybind11 with from_vector_query() factory
- Convert _VectorQuery to _SubVectorQuery in MultiVectorQueryExecutor
- Relax RRF/Weighted score assertion tolerance from 1e-10 to 1e-6
- Fix WeightedReRanker test metric to IP (matching HnswIndexParam default)

* style: ruff format query_executor.py

* fix: define _USE_MATH_DEFINES for M_PI on Windows (MSVC)

* refactor: include reranker.h directly in query.h instead of forward declaration

* refactor(reranker): move topn from member variable to rerank() parameter

* refact code

* style(python): fix ruff UP035/UP037 in multi_vector_reranker

- import Callable from collections.abc instead of typing (UP035)
- remove redundant quotes around MetricType annotations (UP037)

* chore: trigger PR sync

* refact code

* fix(examples): restore CMakeLists.txt formatting broken by clang-format

* refactor(reranker): remove redundant metrics_ map by querying schema directly, and use insert return value to avoid duplicate set lookup

* refactor(reranker): defer schema binding to query time and remove C API callback reranker
This commit is contained in:
lichen2015
2026-05-29 10:10:41 +08:00
committed by GitHub
parent e0ba23179b
commit f539580138
30 changed files with 2740 additions and 115 deletions
+1
View File
@@ -10,6 +10,7 @@ set(SRC_LISTS
binding.cc
model/python_collection.cc
model/python_doc.cc
model/python_reranker.cc
model/param/python_param.cc
model/schema/python_schema.cc
model/common/python_config.cc
+2
View File
@@ -16,6 +16,7 @@
#include "python_config.h"
#include "python_doc.h"
#include "python_param.h"
#include "python_reranker.h"
#include "python_schema.h"
#include "python_type.h"
@@ -26,6 +27,7 @@ PYBIND11_MODULE(_zvec, m) {
ZVecPyTyping::Initialize(m);
ZVecPyParams::Initialize(m);
ZVecPySchemas::Initialize(m);
ZVecPyReranker::Initialize(m);
ZVecPyConfig::Initialize(m);
ZVecPyDoc::Initialize(m);
ZVecPyCollection::Initialize(m);
@@ -0,0 +1,31 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <pybind11/pybind11.h>
#include <zvec/db/reranker.h>
namespace py = pybind11;
namespace zvec {
class ZVecPyReranker {
public:
ZVecPyReranker() = delete;
public:
static void Initialize(py::module_ &m);
};
} // namespace zvec
@@ -17,6 +17,7 @@
#include <pybind11/stl.h>
#include <zvec/core/interface/constants.h>
#include <zvec/db/index_params.h>
#include <zvec/db/query.h>
#include "python_doc.h"
namespace zvec {
@@ -1372,6 +1373,25 @@ Args:
}
void ZVecPyParams::bind_vector_query(py::module_ &m) {
// Bind SubQuery (used by MultiQuery)
py::class_<SubQuery>(m, "_SubQuery")
.def(py::init<>())
.def_readwrite("num_candidates", &SubQuery::num_candidates_)
.def_static(
"from_vector_query",
[](const VectorQuery &vq) {
SubQuery sub;
sub.num_candidates_ = vq.topk_;
sub.target_.field_name_ = vq.field_name_;
auto &clause = std::get<VectorClause>(sub.target_.clause_);
clause.query_vector_ = vq.query_vector_;
clause.sparse_indices_ = vq.query_sparse_indices_;
clause.sparse_values_ = vq.query_sparse_values_;
sub.target_.query_params_ = vq.query_params_;
return sub;
},
py::arg("vector_query"), "Create a SubQuery from a VectorQuery");
py::class_<VectorQuery>(m, "_VectorQuery")
.def(py::init<>())
// properties
@@ -261,6 +261,19 @@ void ZVecPyCollection::bind_dql_methods(
// return DocPtrList
return unwrap_expected(result);
})
// MultiQuery: multi query with reranker
.def(
"Query",
[](const Collection &self, const MultiQuery &query) {
Result<DocPtrList> result;
{
py::gil_scoped_release release;
result = self.Query(query);
}
// return DocPtrList
return unwrap_expected(result);
},
py::arg("query"), "Execute a multi query with re-ranking.")
.def("GroupByQuery",
[](const Collection &self, const GroupByVectorQuery &query) {
Result<GroupResults> result;
@@ -0,0 +1,59 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "python_reranker.h"
#include <pybind11/functional.h>
#include <pybind11/stl.h>
#include <zvec/db/collection.h>
#include <zvec/db/type.h>
namespace zvec {
void ZVecPyReranker::Initialize(py::module_ &m) {
// Bind Reranker base class (abstract, cannot be instantiated directly)
py::class_<Reranker, Reranker::Ptr>(m, "_Reranker");
// Bind ScoreBasedReranker intermediate class
py::class_<ScoreBasedReranker, Reranker, std::shared_ptr<ScoreBasedReranker>>(
m, "_ScoreBasedReranker");
// Bind RrfReranker
py::class_<RrfReranker, ScoreBasedReranker, std::shared_ptr<RrfReranker>>(
m, "_RrfReranker")
.def(py::init<int>(), py::arg("rank_constant") = 60)
.def_property_readonly("rank_constant", &RrfReranker::rank_constant);
// Bind WeightedReranker
py::class_<WeightedReranker, ScoreBasedReranker,
std::shared_ptr<WeightedReranker>>(m, "_WeightedReranker")
.def(py::init<std::map<std::string, double>>(), py::arg("weights"))
.def_property_readonly("weights", &WeightedReranker::weights);
// Bind CallbackReranker
py::class_<CallbackReranker, Reranker, std::shared_ptr<CallbackReranker>>(
m, "_CallbackReranker")
.def(py::init<CallbackReranker::Callback>(), py::arg("callback"));
// Bind MultiQuery struct
py::class_<MultiQuery>(m, "_MultiQuery")
.def(py::init<>())
.def_readwrite("queries", &MultiQuery::queries)
.def_readwrite("topk", &MultiQuery::topk)
.def_readwrite("filter", &MultiQuery::filter)
.def_readwrite("include_vector", &MultiQuery::include_vector)
.def_readwrite("output_fields", &MultiQuery::output_fields)
.def_readwrite("reranker", &MultiQuery::reranker);
}
} // namespace zvec