feat: migrate multi-vector query and reranker logic to C++ (#405)

* feat: migrate multi-vector query and reranker logic to C++

- Add Reranker base class with RrfReRanker and WeightedReRanker implementations
- Add Collection::MultiQuery interface for multi-vector queries with reranking
- Add MultiVectorQuery struct in doc.h with forward declaration for Reranker
- Add C API bindings for reranker and MultiQuery (zvec_reranker_*, zvec_multi_vector_query_*, zvec_collection_multi_query)
- Add Python binding for reranker classes with py::function bridge for callback
- Validate duplicate field names in multi-vector queries (C++ and Python consistent)
- Remove TODO comment about concurrent execution (SQLEngine is not thread-safe)
- Update collection.h MultiQuery doc comment from concurrently to sequentially
- Add C++ collection tests (6 MultiQuery test cases)
- Add C API tests (reranker functions + multi_vector_query end-to-end)
- Implement Python test cases (11 previously skipped tests now active)
- Simplify Python query_executor validation for unified duplicate field check

* style: format Python files with ruff

* fix: adapt to main branch API changes (VectorQuery->Query rename, validate_and_sanitize)

* fix: multi_vector tests now use multiple same-type vector fields (dense2, sparse2)

* fix: suppress RET501 for intentional default return None in RerankFunction._get_object

* style: ruff format test_collection.py

* refact multi vector query

* format code

* fix(multi-vector): expose SubVectorQuery in Python binding, fix tests

- Register _SubVectorQuery in pybind11 with from_vector_query() factory
- Convert _VectorQuery to _SubVectorQuery in MultiVectorQueryExecutor
- Relax RRF/Weighted score assertion tolerance from 1e-10 to 1e-6
- Fix WeightedReRanker test metric to IP (matching HnswIndexParam default)

* style: ruff format query_executor.py

* fix: define _USE_MATH_DEFINES for M_PI on Windows (MSVC)

* refactor: include reranker.h directly in query.h instead of forward declaration

* refactor(reranker): move topn from member variable to rerank() parameter

* refact code

* style(python): fix ruff UP035/UP037 in multi_vector_reranker

- import Callable from collections.abc instead of typing (UP035)
- remove redundant quotes around MetricType annotations (UP037)

* chore: trigger PR sync

* refact code

* fix(examples): restore CMakeLists.txt formatting broken by clang-format

* refactor(reranker): remove redundant metrics_ map by querying schema directly, and use insert return value to avoid duplicate set lookup

* refactor(reranker): defer schema binding to query time and remove C API callback reranker
This commit is contained in:
lichen2015
2026-05-29 10:10:41 +08:00
committed by GitHub
parent e0ba23179b
commit f539580138
30 changed files with 2740 additions and 115 deletions
+67 -3
View File
@@ -20,6 +20,7 @@ import os
from zvec import Doc, MetricType
from zvec.extension.multi_vector_reranker import (
CallbackReRanker,
RrfReRanker,
WeightedReRanker,
)
@@ -75,16 +76,17 @@ class TestRrfReRanker:
# ----------------------------
class TestWeightedReRanker:
def test_init(self):
metrics = {"vector1": MetricType.L2, "vector2": MetricType.COSINE}
weights = {"vector1": 0.7, "vector2": 0.3}
reranker = WeightedReRanker(
topn=5,
rerank_field="content",
metric=MetricType.L2,
metrics=metrics,
weights=weights,
)
assert reranker.topn == 5
assert reranker.rerank_field == "content"
assert reranker.metric == MetricType.L2
assert reranker.metrics == metrics
assert reranker.weights == weights
def test_normalize_score(self):
@@ -106,8 +108,9 @@ class TestWeightedReRanker:
reranker._normalize_score(1.0, "unsupported_metric")
def test_rerank(self):
metrics = {"vector1": MetricType.L2, "vector2": MetricType.L2}
weights = {"vector1": 0.7, "vector2": 0.3}
reranker = WeightedReRanker(topn=3, weights=weights, metric=MetricType.L2)
reranker = WeightedReRanker(topn=3, weights=weights, metrics=metrics)
doc1 = Doc(id="1", score=0.8)
doc2 = Doc(id="2", score=0.7)
@@ -122,9 +125,70 @@ class TestWeightedReRanker:
for doc in results:
assert hasattr(doc, "score")
def test_rerank_missing_metric_raises(self):
metrics = {"vector1": MetricType.L2}
reranker = WeightedReRanker(topn=3, metrics=metrics)
doc1 = Doc(id="1", score=0.8)
query_results = {"vector1": [doc1], "vector2": [doc1]}
with pytest.raises(ValueError, match="no metric type specified"):
reranker.rerank(query_results)
# ----------------------------
# CallbackReRanker Test Case
# ----------------------------
class TestCallbackReRanker:
def test_init(self):
def my_callback(query_results, topn):
return []
reranker = CallbackReRanker(callback=my_callback, topn=5)
assert reranker.topn == 5
def test_rerank(self):
def my_callback(query_results, topn):
all_docs = []
for docs in query_results.values():
all_docs.extend(docs)
all_docs.sort(key=lambda d: d.score, reverse=True)
return all_docs[:topn]
reranker = CallbackReRanker(callback=my_callback, topn=3)
doc1 = Doc(id="1", score=0.8)
doc2 = Doc(id="2", score=0.9)
doc3 = Doc(id="3", score=0.7)
doc4 = Doc(id="4", score=0.6)
query_results = {"vector1": [doc1, doc2], "vector2": [doc3, doc4]}
results = reranker.rerank(query_results)
assert len(results) == 3
scores = [doc.score for doc in results]
assert scores == sorted(scores, reverse=True)
def test_callback_with_topn(self):
received_topn = []
def my_callback(query_results, topn):
received_topn.append(topn)
return []
reranker = CallbackReRanker(callback=my_callback, topn=7)
reranker.rerank({"v1": [Doc(id="1", score=0.5)]})
assert received_topn == [7]
def test_get_object_returns_cpp_reranker(self):
def my_callback(query_results, topn):
return []
reranker = CallbackReRanker(callback=my_callback)
assert reranker._get_object() is not None
# ----------------------------
# QwenReRanker Test Case