feat(python): expose group-by search to Python API (#561)
Co-authored-by: jiliang.ljl <jiliang.ljl@alibaba-inc.com>
This commit is contained in:
co-authored by
jiliang.ljl
parent
2d00cb718b
commit
1afdea8dc5
@@ -5771,7 +5771,7 @@ zvec_group_by_vector_query_t *zvec_group_by_vector_query_create(void) {
|
||||
ZVEC_TRY_RETURN_NULL(
|
||||
"Failed to create GroupByVectorQuery",
|
||||
auto *query = new zvec::GroupByVectorQuery();
|
||||
query->group_count_ = 2; query->group_topk_ = 3;
|
||||
query->group_count_ = 2; query->topk_per_group_ = 3;
|
||||
return reinterpret_cast<zvec_group_by_vector_query_t *>(query);)
|
||||
return nullptr;
|
||||
}
|
||||
@@ -5841,23 +5841,23 @@ uint32_t zvec_group_by_vector_query_get_group_count(
|
||||
return ptr->group_count_;
|
||||
}
|
||||
|
||||
zvec_error_code_t zvec_group_by_vector_query_set_group_topk(
|
||||
zvec_group_by_vector_query_t *query, uint32_t topk) {
|
||||
zvec_error_code_t zvec_group_by_vector_query_set_topk_per_group(
|
||||
zvec_group_by_vector_query_t *query, uint32_t topk_per_group) {
|
||||
if (!query) {
|
||||
SET_LAST_ERROR(ZVEC_ERROR_INVALID_ARGUMENT,
|
||||
"GroupByVectorQuery pointer is null");
|
||||
return ZVEC_ERROR_INVALID_ARGUMENT;
|
||||
}
|
||||
auto *ptr = reinterpret_cast<zvec::GroupByVectorQuery *>(query);
|
||||
ptr->group_topk_ = topk;
|
||||
ptr->topk_per_group_ = topk_per_group;
|
||||
return ZVEC_OK;
|
||||
}
|
||||
|
||||
uint32_t zvec_group_by_vector_query_get_group_topk(
|
||||
uint32_t zvec_group_by_vector_query_get_topk_per_group(
|
||||
const zvec_group_by_vector_query_t *query) {
|
||||
if (!query) return 3;
|
||||
auto *ptr = reinterpret_cast<const zvec::GroupByVectorQuery *>(query);
|
||||
return ptr->group_topk_;
|
||||
return ptr->topk_per_group_;
|
||||
}
|
||||
|
||||
zvec_error_code_t zvec_group_by_vector_query_set_query_vector(
|
||||
|
||||
@@ -1952,6 +1952,71 @@ Args:
|
||||
}));
|
||||
}
|
||||
|
||||
void set_query_vector(QueryTarget &target, const FieldSchema &field_schema,
|
||||
const py::object &obj) {
|
||||
const DataType data_type = field_schema.data_type();
|
||||
|
||||
// Dense vector data is referenced by the query object. Callers
|
||||
// must not modify the source data until the query returns.
|
||||
if (FieldSchema::is_dense_vector_field(data_type)) {
|
||||
if (!py::isinstance<py::array>(obj)) {
|
||||
throw py::type_error("Dense vector[" + field_schema.name() +
|
||||
"] expects a ndarray, got " +
|
||||
std::string(py::str(py::type::of(obj))));
|
||||
}
|
||||
const auto arr = obj.cast<py::array>();
|
||||
if (arr.ndim() != 1) {
|
||||
throw py::type_error("Dense vector expects 1D array, got " +
|
||||
std::to_string(arr.ndim()) + "D");
|
||||
}
|
||||
const auto buf = arr.request();
|
||||
target.clause_ = VectorViewClause{
|
||||
std::string_view(static_cast<const char *>(buf.ptr),
|
||||
static_cast<size_t>(buf.size) * buf.itemsize),
|
||||
{},
|
||||
{}};
|
||||
return;
|
||||
}
|
||||
// sparse vector
|
||||
if (FieldSchema::is_sparse_vector_field(data_type)) {
|
||||
if (!py::isinstance<py::dict>(obj)) {
|
||||
throw py::type_error("Sparse vector[" + field_schema.name() +
|
||||
"] expects a Python dict, got " +
|
||||
std::string(py::str(py::type::of(obj))));
|
||||
}
|
||||
const auto sparse = obj.cast<py::dict>();
|
||||
|
||||
switch (data_type) {
|
||||
case DataType::SPARSE_VECTOR_FP16: {
|
||||
auto [indices, values] = serialize_sparse_vector<ailego::Float16>(
|
||||
sparse, [](const py::handle &h, size_t idx) {
|
||||
float f = checked_cast<float>(
|
||||
h, "Sparse value[" + std::to_string(idx) + "]", "FLOAT");
|
||||
return ailego::Float16(f);
|
||||
});
|
||||
target.set_sparse_vector(std::move(indices), std::move(values));
|
||||
break;
|
||||
}
|
||||
case DataType::SPARSE_VECTOR_FP32: {
|
||||
auto [indices, values] = serialize_sparse_vector<float>(
|
||||
sparse, [](const py::handle &h, size_t idx) {
|
||||
return checked_cast<float>(
|
||||
h, "Sparse value[" + std::to_string(idx) + "]", "FLOAT");
|
||||
});
|
||||
target.set_sparse_vector(std::move(indices), std::move(values));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw py::type_error("Unsupported sparse vector type: " +
|
||||
std::to_string(static_cast<int>(data_type)));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
throw py::type_error("Unsupported vector field type for field: " +
|
||||
field_schema.name());
|
||||
}
|
||||
|
||||
void ZVecPyParams::bind_vector_query(py::module_ &m) {
|
||||
// bind Fts
|
||||
py::class_<FtsClause>(m, "_Fts")
|
||||
@@ -2038,74 +2103,7 @@ void ZVecPyParams::bind_vector_query(py::module_ &m) {
|
||||
"set_vector",
|
||||
[](SearchQuery &self, const FieldSchema &field_schema,
|
||||
const py::object &obj) {
|
||||
const DataType data_type = field_schema.data_type();
|
||||
|
||||
// Dense vector data is referenced by the query object. Callers
|
||||
// must not modify the source data until the query returns.
|
||||
if (FieldSchema::is_dense_vector_field(data_type)) {
|
||||
if (!py::isinstance<py::array>(obj)) {
|
||||
throw py::type_error("Dense vector[" + field_schema.name() +
|
||||
"] expects a ndarray, got " +
|
||||
std::string(py::str(py::type::of(obj))));
|
||||
}
|
||||
const auto arr = obj.cast<py::array>();
|
||||
if (arr.ndim() != 1) {
|
||||
throw py::type_error("Dense vector expects 1D array, got " +
|
||||
std::to_string(arr.ndim()) + "D");
|
||||
}
|
||||
const auto buf = arr.request();
|
||||
self.target_.clause_ = VectorViewClause{
|
||||
std::string_view(
|
||||
static_cast<const char *>(buf.ptr),
|
||||
static_cast<size_t>(buf.size) * buf.itemsize),
|
||||
{},
|
||||
{}};
|
||||
return;
|
||||
}
|
||||
// sparse vector
|
||||
if (FieldSchema::is_sparse_vector_field(data_type)) {
|
||||
if (!py::isinstance<py::dict>(obj)) {
|
||||
throw py::type_error("Sparse vector[" + field_schema.name() +
|
||||
"] expects a Python dict, got " +
|
||||
std::string(py::str(py::type::of(obj))));
|
||||
}
|
||||
const auto sparse = obj.cast<py::dict>();
|
||||
|
||||
switch (data_type) {
|
||||
case DataType::SPARSE_VECTOR_FP16: {
|
||||
auto [indices, values] =
|
||||
serialize_sparse_vector<ailego::Float16>(
|
||||
sparse, [](const py::handle &h, size_t idx) {
|
||||
float f = checked_cast<float>(
|
||||
h, "Sparse value[" + std::to_string(idx) + "]",
|
||||
"FLOAT");
|
||||
return ailego::Float16(f);
|
||||
});
|
||||
self.target_.set_sparse_vector(std::move(indices),
|
||||
std::move(values));
|
||||
break;
|
||||
}
|
||||
case DataType::SPARSE_VECTOR_FP32: {
|
||||
auto [indices, values] = serialize_sparse_vector<float>(
|
||||
sparse, [](const py::handle &h, size_t idx) {
|
||||
return checked_cast<float>(
|
||||
h, "Sparse value[" + std::to_string(idx) + "]",
|
||||
"FLOAT");
|
||||
});
|
||||
self.target_.set_sparse_vector(std::move(indices),
|
||||
std::move(values));
|
||||
break;
|
||||
}
|
||||
default:
|
||||
throw py::type_error(
|
||||
"Unsupported sparse vector type: " +
|
||||
std::to_string(static_cast<int>(data_type)));
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
throw py::type_error("Unsupported vector field type for field: " +
|
||||
field_schema.name());
|
||||
set_query_vector(self.target_, field_schema, obj);
|
||||
},
|
||||
py::arg("field_schema"), py::arg("obj"), py::keep_alive<1, 3>(),
|
||||
"Set query vector. Dense vector source data must not be modified "
|
||||
@@ -2265,5 +2263,37 @@ void ZVecPyParams::bind_vector_query(py::module_ &m) {
|
||||
}
|
||||
return obj;
|
||||
}));
|
||||
|
||||
// _GroupByVectorQuery
|
||||
py::class_<GroupByVectorQuery>(m, "_GroupByVectorQuery")
|
||||
.def(py::init<>())
|
||||
.def_property(
|
||||
"field_name",
|
||||
[](const GroupByVectorQuery &q) { return q.target_.field_name_; },
|
||||
[](GroupByVectorQuery &q, std::string v) {
|
||||
q.target_.field_name_ = std::move(v);
|
||||
})
|
||||
.def_readwrite("filter", &GroupByVectorQuery::filter_)
|
||||
.def_readwrite("include_vector", &GroupByVectorQuery::include_vector_)
|
||||
.def_readwrite("output_fields", &GroupByVectorQuery::output_fields_)
|
||||
.def_readwrite("group_by_field_name",
|
||||
&GroupByVectorQuery::group_by_field_name_)
|
||||
.def_readwrite("group_count", &GroupByVectorQuery::group_count_)
|
||||
.def_readwrite("topk_per_group", &GroupByVectorQuery::topk_per_group_)
|
||||
.def_property(
|
||||
"query_params",
|
||||
[](const GroupByVectorQuery &q) { return q.target_.query_params_; },
|
||||
[](GroupByVectorQuery &q, QueryParams::Ptr p) {
|
||||
q.target_.query_params_ = std::move(p);
|
||||
})
|
||||
.def(
|
||||
"set_vector",
|
||||
[](GroupByVectorQuery &self, const FieldSchema &field_schema,
|
||||
const py::object &obj) {
|
||||
set_query_vector(self.target_, field_schema, obj);
|
||||
},
|
||||
py::arg("field_schema"), py::arg("obj"), py::keep_alive<1, 3>(),
|
||||
"Set query vector. Dense vector source data must not be modified "
|
||||
"until the query finishes.");
|
||||
}
|
||||
} // namespace zvec
|
||||
|
||||
@@ -48,6 +48,10 @@ T unwrap_expected(const tl::expected<T, Status> &exp) {
|
||||
}
|
||||
|
||||
void ZVecPyCollection::Initialize(pybind11::module_ &m) {
|
||||
py::class_<GroupResult>(m, "_GroupResult")
|
||||
.def_readonly("group_by_value", &GroupResult::group_by_value_)
|
||||
.def_readonly("docs", &GroupResult::docs_);
|
||||
|
||||
py::class_<Collection, Collection::Ptr> collection(m, "_Collection");
|
||||
bind_db_methods(collection);
|
||||
bind_ddl_methods(collection);
|
||||
@@ -281,7 +285,6 @@ void ZVecPyCollection::bind_dql_methods(
|
||||
py::gil_scoped_release release;
|
||||
result = self.GroupByQuery(query);
|
||||
}
|
||||
// return GroupResults
|
||||
return unwrap_expected(result);
|
||||
})
|
||||
.def(
|
||||
@@ -313,4 +316,4 @@ void ZVecPyCollection::bind_dql_methods(
|
||||
"for introspection and testing only; not part of the stable API.");
|
||||
}
|
||||
|
||||
} // namespace zvec
|
||||
} // namespace zvec
|
||||
|
||||
Reference in New Issue
Block a user