Initial commit
This commit is contained in:
@@ -0,0 +1,59 @@
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(CMAKE_CXX_STANDARD_REQUIRED ON)
|
||||
|
||||
find_package(pybind11 REQUIRED)
|
||||
|
||||
set(SRC_LISTS
|
||||
binding.cc
|
||||
model/python_collection.cc
|
||||
model/python_doc.cc
|
||||
model/param/python_param.cc
|
||||
model/schema/python_schema.cc
|
||||
model/common/python_config.cc
|
||||
typing/python_type.cc
|
||||
)
|
||||
|
||||
pybind11_add_module(_zvec ${SRC_LISTS})
|
||||
|
||||
if (CMAKE_SYSTEM_NAME STREQUAL "Linux")
|
||||
target_link_libraries(_zvec PRIVATE
|
||||
-Wl,--whole-archive
|
||||
$<TARGET_FILE:core_knn_flat_static>
|
||||
$<TARGET_FILE:core_knn_flat_sparse_static>
|
||||
$<TARGET_FILE:core_knn_hnsw_static>
|
||||
$<TARGET_FILE:core_knn_hnsw_sparse_static>
|
||||
$<TARGET_FILE:core_knn_ivf_static>
|
||||
$<TARGET_FILE:core_knn_cluster_static>
|
||||
$<TARGET_FILE:core_mix_reducer_static>
|
||||
$<TARGET_FILE:core_metric_static>
|
||||
$<TARGET_FILE:core_utility_static>
|
||||
$<TARGET_FILE:core_quantizer_static>
|
||||
-Wl,--no-whole-archive
|
||||
zvec
|
||||
)
|
||||
target_link_options(_zvec PRIVATE
|
||||
"LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/exports.map"
|
||||
)
|
||||
elseif (APPLE)
|
||||
target_link_libraries(_zvec PRIVATE
|
||||
-Wl,-force_load,$<TARGET_FILE:core_knn_flat_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_knn_flat_sparse_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_knn_hnsw_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_knn_hnsw_sparse_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_knn_ivf_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_knn_cluster_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_mix_reducer_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_metric_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_utility_static>
|
||||
-Wl,-force_load,$<TARGET_FILE:core_quantizer_static>
|
||||
zvec
|
||||
)
|
||||
target_link_libraries(_zvec PRIVATE
|
||||
-Wl,-exported_symbols_list,${CMAKE_CURRENT_SOURCE_DIR}/exports.mac
|
||||
)
|
||||
endif ()
|
||||
|
||||
target_include_directories(_zvec PRIVATE ${PYBIND11_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/src ${CMAKE_SOURCE_DIR}/src/binding/python/include)
|
||||
@@ -0,0 +1,33 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "python_collection.h"
|
||||
#include "python_config.h"
|
||||
#include "python_doc.h"
|
||||
#include "python_param.h"
|
||||
#include "python_schema.h"
|
||||
#include "python_type.h"
|
||||
|
||||
namespace zvec {
|
||||
PYBIND11_MODULE(_zvec, m) {
|
||||
m.doc() = "Zvec core module";
|
||||
|
||||
ZVecPyTyping::Initialize(m);
|
||||
ZVecPyParams::Initialize(m);
|
||||
ZVecPySchemas::Initialize(m);
|
||||
ZVecPyConfig::Initialize(m);
|
||||
ZVecPyDoc::Initialize(m);
|
||||
ZVecPyCollection::Initialize(m);
|
||||
}
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1 @@
|
||||
_PyInit__zvec
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
global:
|
||||
PyInit_*; # export only python related functions
|
||||
local:
|
||||
*;
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.#pragma once
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include "db/collection.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace zvec {
|
||||
|
||||
class ZVecPyCollection {
|
||||
public:
|
||||
ZVecPyCollection() = delete;
|
||||
|
||||
public:
|
||||
static void Initialize(py::module_ &m);
|
||||
|
||||
private:
|
||||
static void bind_db_methods(py::class_<Collection, Collection::Ptr> &col);
|
||||
static void bind_ddl_methods(py::class_<Collection, Collection::Ptr> &col);
|
||||
static void bind_dml_methods(py::class_<Collection, Collection::Ptr> &col);
|
||||
static void bind_dql_methods(py::class_<Collection, Collection::Ptr> &col);
|
||||
};
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,30 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.#pragma once
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include "db/common/config.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace zvec {
|
||||
|
||||
class ZVecPyConfig {
|
||||
public:
|
||||
ZVecPyConfig() = delete;
|
||||
|
||||
public:
|
||||
static void Initialize(py::module_ &m);
|
||||
};
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,34 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.#pragma once
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include "db/index/common/doc.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace zvec {
|
||||
|
||||
class ZVecPyDoc {
|
||||
public:
|
||||
ZVecPyDoc() = delete;
|
||||
|
||||
public:
|
||||
static void Initialize(py::module_ &m);
|
||||
|
||||
private:
|
||||
static void bind_doc_operator(py::module_ &m);
|
||||
static void bind_doc(py::module_ &m);
|
||||
};
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,37 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.#pragma once
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include "db/index/common/options.h"
|
||||
#include "db/index/common/type.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace zvec {
|
||||
|
||||
class ZVecPyParams {
|
||||
public:
|
||||
ZVecPyParams() = delete;
|
||||
|
||||
public:
|
||||
static void Initialize(py::module_ &m);
|
||||
|
||||
private:
|
||||
static void bind_index_params(py::module_ &m);
|
||||
static void bind_query_params(py::module_ &m);
|
||||
static void bind_options(py::module_ &m);
|
||||
static void bind_vector_query(py::module_ &m);
|
||||
};
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.#pragma once
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include "db/index/common/type.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace zvec {
|
||||
|
||||
class ZVecPySchemas {
|
||||
public:
|
||||
ZVecPySchemas() = delete;
|
||||
|
||||
public:
|
||||
static void Initialize(py::module_ &m);
|
||||
|
||||
private:
|
||||
static void bind_field_schema(py::module_ &m);
|
||||
static void bind_collection_schema(py::module_ &m);
|
||||
static void bind_collection_stats(py::module_ &m);
|
||||
};
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,38 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.#pragma once
|
||||
|
||||
#include <pybind11/pybind11.h>
|
||||
#include "db/common/status.h"
|
||||
#include "db/index/common/type.h"
|
||||
|
||||
namespace py = pybind11;
|
||||
|
||||
namespace zvec {
|
||||
|
||||
class ZVecPyTyping {
|
||||
public:
|
||||
ZVecPyTyping() = delete;
|
||||
|
||||
public:
|
||||
static void Initialize(py::module_ &m);
|
||||
|
||||
private:
|
||||
static void bind_datatypes(py::module_ &m);
|
||||
static void bind_index_types(py::module_ &m);
|
||||
static void bind_metric_types(py::module_ &m);
|
||||
static void bind_quantize_types(py::module_ &m);
|
||||
static void bind_status(py::module_ &m);
|
||||
};
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,190 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "python_config.h"
|
||||
#include <pybind11/stl.h>
|
||||
|
||||
namespace zvec {
|
||||
|
||||
inline bool has_key(py::dict d, const std::string &key) {
|
||||
return py::bool_(d.contains(key));
|
||||
}
|
||||
|
||||
template <typename T>
|
||||
std::optional<T> get_if(py::dict d, const std::string &key) {
|
||||
if (has_key(d, key)) {
|
||||
try {
|
||||
py::object obj = d[py::str(key)];
|
||||
return obj.cast<T>();
|
||||
} catch (const py::cast_error &) {
|
||||
throw py::type_error("Key '" + key + "' is not of expected type.");
|
||||
}
|
||||
}
|
||||
return std::nullopt;
|
||||
}
|
||||
|
||||
inline std::string to_lower(const std::string &s) {
|
||||
std::string lower;
|
||||
lower.reserve(s.size());
|
||||
std::transform(s.begin(), s.end(), std::back_inserter(lower), ::tolower);
|
||||
return lower;
|
||||
}
|
||||
|
||||
inline bool iequals(const std::string &a, const std::string &b) {
|
||||
return to_lower(a) == to_lower(b);
|
||||
}
|
||||
|
||||
GlobalConfig::LogLevel str_to_loglevel(const std::string &s) {
|
||||
if (iequals(s, "debug")) return GlobalConfig::LogLevel::DEBUG;
|
||||
if (iequals(s, "info")) return GlobalConfig::LogLevel::INFO;
|
||||
if (iequals(s, "warn") || iequals(s, "warning"))
|
||||
return GlobalConfig::LogLevel::WARN;
|
||||
if (iequals(s, "error")) return GlobalConfig::LogLevel::ERROR;
|
||||
if (iequals(s, "fatal")) return GlobalConfig::LogLevel::FATAL;
|
||||
throw py::value_error("Invalid log level: ");
|
||||
}
|
||||
|
||||
|
||||
void ZVecPyConfig::Initialize(pybind11::module_ &m) {
|
||||
m.def("Initialize", [](py::args args, py::kwargs kwargs) -> py::none {
|
||||
py::dict config_dict;
|
||||
// parse args
|
||||
for (auto &arg : args) {
|
||||
if (py::isinstance<py::dict>(arg)) {
|
||||
for (auto item : arg.cast<py::dict>()) {
|
||||
config_dict[item.first] = item.second;
|
||||
}
|
||||
} else {
|
||||
throw py::type_error("Positional argument must be a dict if provided");
|
||||
}
|
||||
}
|
||||
|
||||
// parser kwargs
|
||||
if (kwargs) {
|
||||
for (auto item : kwargs) {
|
||||
config_dict[item.first] = item.second;
|
||||
}
|
||||
}
|
||||
|
||||
if (config_dict.empty()) {
|
||||
return py::none();
|
||||
}
|
||||
|
||||
GlobalConfig::ConfigData data;
|
||||
// config memory_limit_mb
|
||||
if (has_key(config_dict, "memory_limit_mb")) {
|
||||
auto mb = get_if<int64_t>(config_dict, "memory_limit_mb").value();
|
||||
if (mb <= 0) throw py::value_error("memory_limit_mb must be positive");
|
||||
data.memory_limit_bytes = static_cast<uint64_t>(mb) * 1024 * 1024;
|
||||
}
|
||||
|
||||
// config log
|
||||
bool has_log_type = has_key(config_dict, "log_type");
|
||||
bool has_log_level = has_key(config_dict, "log_level");
|
||||
if (has_log_type || has_log_level) {
|
||||
std::string log_type = "console";
|
||||
std::string log_level_str = "warn";
|
||||
|
||||
if (has_log_type) {
|
||||
log_type = config_dict["log_type"].cast<std::string>();
|
||||
}
|
||||
if (has_log_level) {
|
||||
log_level_str = config_dict["log_level"].cast<std::string>();
|
||||
}
|
||||
auto log_level = str_to_loglevel(log_level_str);
|
||||
if (iequals(log_type, "file")) {
|
||||
std::string dir = DEFAULT_LOG_DIR;
|
||||
std::string basename = DEFAULT_LOG_BASENAME;
|
||||
uint32_t file_size = DEFAULT_LOG_FILE_SIZE;
|
||||
uint32_t overdue_days = DEFAULT_LOG_OVERDUE_DAYS;
|
||||
|
||||
if (has_key(config_dict, "log_dir")) {
|
||||
dir = get_if<std::string>(config_dict, "log_dir").value();
|
||||
}
|
||||
if (has_key(config_dict, "log_basename")) {
|
||||
basename = get_if<std::string>(config_dict, "log_basename").value();
|
||||
}
|
||||
if (has_key(config_dict, "log_file_size")) {
|
||||
auto s = get_if<int32_t>(config_dict, "log_file_size").value();
|
||||
if (s <= 0) {
|
||||
throw py::value_error("log_file_size must be positive");
|
||||
}
|
||||
file_size = static_cast<uint32_t>(s);
|
||||
}
|
||||
if (has_key(config_dict, "log_overdue_days")) {
|
||||
std::cout << " ** log_overdue_days: " << overdue_days << std::endl;
|
||||
auto d = get_if<int32_t>(config_dict, "log_overdue_days").value();
|
||||
if (d <= 0) {
|
||||
throw py::value_error("log_overdue_days must be positive");
|
||||
}
|
||||
overdue_days = static_cast<uint32_t>(d);
|
||||
}
|
||||
|
||||
data.log_config = std::make_shared<GlobalConfig::FileLogConfig>(
|
||||
log_level, dir, basename, file_size, overdue_days);
|
||||
|
||||
} else if (iequals(log_type, "console")) {
|
||||
data.log_config =
|
||||
std::make_shared<GlobalConfig::ConsoleLogConfig>(log_level);
|
||||
} else {
|
||||
throw py::value_error("log_type must be 'console' or 'file'");
|
||||
}
|
||||
}
|
||||
|
||||
// set query thread count
|
||||
if (has_key(config_dict, "query_threads")) {
|
||||
auto q = get_if<int32_t>(config_dict, "query_threads").value();
|
||||
if (q <= 0) throw py::value_error("query_threads must be positive");
|
||||
data.query_thread_count = static_cast<uint32_t>(q);
|
||||
}
|
||||
|
||||
// set optimize thread count
|
||||
if (has_key(config_dict, "optimize_threads")) {
|
||||
auto o = get_if<int32_t>(config_dict, "optimize_threads").value();
|
||||
if (o <= 0) throw py::value_error("optimize_threads must be positive");
|
||||
data.optimize_thread_count = static_cast<uint32_t>(o);
|
||||
}
|
||||
|
||||
// set invert_to_forward_scan_ratio
|
||||
if (has_key(config_dict, "invert_to_forward_scan_ratio")) {
|
||||
auto v =
|
||||
get_if<double>(config_dict, "invert_to_forward_scan_ratio").value();
|
||||
if (v < 0.0 || v > 1.0) {
|
||||
throw py::value_error(
|
||||
"invert_to_forward_scan_ratio must be in [0.0, 1.0]");
|
||||
}
|
||||
data.invert_to_forward_scan_ratio = static_cast<float>(v);
|
||||
}
|
||||
|
||||
// set brute_force_by_keys_ratio
|
||||
if (has_key(config_dict, "brute_force_by_keys_ratio")) {
|
||||
auto v = get_if<double>(config_dict, "brute_force_by_keys_ratio").value();
|
||||
if (v < 0.0 || v > 1.0) {
|
||||
throw py::value_error(
|
||||
"brute_force_by_keys_ratio must be in [0.0, 1.0]");
|
||||
}
|
||||
data.brute_force_by_keys_ratio = static_cast<float>(v);
|
||||
}
|
||||
|
||||
// initialize (contains validate)
|
||||
Status status = GlobalConfig::Instance().Initialize(data);
|
||||
if (!status.ok()) {
|
||||
throw std::runtime_error("Initialization failed: " + status.message());
|
||||
}
|
||||
return py::none();
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
} // namespace zvec
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,218 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "python_collection.h"
|
||||
#include <pybind11/stl.h>
|
||||
#include "db/collection.h"
|
||||
|
||||
namespace zvec {
|
||||
|
||||
inline void throw_if_error(const Status &status) {
|
||||
switch (status.code()) {
|
||||
case StatusCode::OK:
|
||||
return;
|
||||
case StatusCode::NOT_FOUND:
|
||||
throw py::key_error(status.message());
|
||||
case StatusCode::INVALID_ARGUMENT:
|
||||
throw py::value_error(status.message());
|
||||
case StatusCode::INTERNAL_ERROR:
|
||||
case StatusCode::ALREADY_EXISTS:
|
||||
case StatusCode::NOT_SUPPORTED:
|
||||
case StatusCode::PERMISSION_DENIED:
|
||||
case StatusCode::FAILED_PRECONDITION:
|
||||
case StatusCode::UNKNOWN:
|
||||
default:
|
||||
throw std::runtime_error(status.message());
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
template <typename T>
|
||||
T unwrap_expected(const tl::expected<T, Status> &exp) {
|
||||
if (exp.has_value()) {
|
||||
return exp.value();
|
||||
}
|
||||
throw_if_error(exp.error());
|
||||
return T{};
|
||||
}
|
||||
|
||||
void ZVecPyCollection::Initialize(pybind11::module_ &m) {
|
||||
py::class_<Collection, Collection::Ptr> collection(m, "_Collection");
|
||||
bind_db_methods(collection);
|
||||
bind_ddl_methods(collection);
|
||||
bind_dml_methods(collection);
|
||||
bind_dql_methods(collection);
|
||||
collection.def(py::pickle(
|
||||
[](const Collection &c) {
|
||||
return py::make_tuple(c.Path(), c.Schema(), c.Options());
|
||||
},
|
||||
[](py::tuple t) {
|
||||
if (t.size() != 3) {
|
||||
throw std::runtime_error("Invalid tuple size for Collection pickle");
|
||||
}
|
||||
std::string path = t[0].cast<std::string>();
|
||||
auto schema = t[1].cast<CollectionSchema>();
|
||||
CollectionOptions options = t[2].cast<CollectionOptions>();
|
||||
auto result = Collection::Open(path, options);
|
||||
// auto result = Collection::CreateAndOpen(path, schema, options);
|
||||
return unwrap_expected(result);
|
||||
}));
|
||||
}
|
||||
|
||||
void ZVecPyCollection::bind_db_methods(
|
||||
py::class_<Collection, Collection::Ptr> &col) {
|
||||
col.def_static("CreateAndOpen",
|
||||
[](const std::string &path, const CollectionSchema &schema,
|
||||
const CollectionOptions &options) {
|
||||
auto result =
|
||||
Collection::CreateAndOpen(path, schema, options);
|
||||
return unwrap_expected(result);
|
||||
})
|
||||
.def_static("Open", [](const std::string &path,
|
||||
const CollectionOptions &options) {
|
||||
auto result = Collection::Open(path, options);
|
||||
return unwrap_expected(result);
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
void ZVecPyCollection::bind_ddl_methods(
|
||||
py::class_<Collection, Collection::Ptr> &col) {
|
||||
// bind collection properties
|
||||
col.def("Path",
|
||||
[](const Collection &self) {
|
||||
auto ret = self.Path();
|
||||
return unwrap_expected(ret);
|
||||
})
|
||||
.def("Options",
|
||||
[](const Collection &self) {
|
||||
auto ret = self.Options();
|
||||
return unwrap_expected(ret);
|
||||
})
|
||||
.def("Schema",
|
||||
[](const Collection &self) {
|
||||
auto ret = self.Schema();
|
||||
return unwrap_expected(ret);
|
||||
})
|
||||
.def("Stats", [](const Collection &self) {
|
||||
auto ret = self.Stats();
|
||||
return unwrap_expected(ret);
|
||||
});
|
||||
|
||||
// bind collection ddl methods
|
||||
col.def("Destroy",
|
||||
[](Collection &self) {
|
||||
const auto status = self.Destroy();
|
||||
throw_if_error(status);
|
||||
})
|
||||
.def("Flush", [](Collection &self) {
|
||||
auto status = self.Flush();
|
||||
throw_if_error(status);
|
||||
});
|
||||
|
||||
// binding index ddl methods
|
||||
col.def("CreateIndex",
|
||||
[](Collection &self, const std::string &column_name,
|
||||
const IndexParams::Ptr &index_options,
|
||||
const CreateIndexOptions &options) {
|
||||
const auto status =
|
||||
self.CreateIndex(column_name, index_options, options);
|
||||
throw_if_error(status);
|
||||
})
|
||||
.def("DropIndex",
|
||||
[](Collection &self, const std::string &column_name) {
|
||||
const auto status = self.DropIndex(column_name);
|
||||
throw_if_error(status);
|
||||
})
|
||||
.def("Optimize", [](Collection &self, const OptimizeOptions &options) {
|
||||
const auto status = self.Optimize(options);
|
||||
throw_if_error(status);
|
||||
});
|
||||
|
||||
// binding column ddl methods
|
||||
col.def("AddColumn",
|
||||
[](Collection &self, const std::string &column_name,
|
||||
const FieldSchema::Ptr &column_schema,
|
||||
const std::string &expression, const AddColumnOptions &options) {
|
||||
const auto status =
|
||||
self.AddColumn(column_name, column_schema, expression, options);
|
||||
throw_if_error(status);
|
||||
})
|
||||
.def("DropColumn",
|
||||
[](Collection &self, std::string &column_name) {
|
||||
auto status = self.DropColumn(column_name);
|
||||
throw_if_error(status);
|
||||
})
|
||||
.def("AlterColumn", [](Collection &self, std::string &column_name,
|
||||
const std::string &rename,
|
||||
const FieldSchema::Ptr &new_column_schema,
|
||||
const AlterColumnOptions &options) {
|
||||
const auto status =
|
||||
self.AlterColumn(column_name, rename, new_column_schema, options);
|
||||
throw_if_error(status);
|
||||
});
|
||||
}
|
||||
|
||||
void ZVecPyCollection::bind_dml_methods(
|
||||
py::class_<Collection, Collection::Ptr> &col) {
|
||||
// bind collection upsert/insert/update/delete methods
|
||||
col.def("Insert",
|
||||
[](Collection &self, std::vector<Doc> &docs) {
|
||||
const auto result = self.Insert(docs);
|
||||
return unwrap_expected(result);
|
||||
})
|
||||
.def("Update",
|
||||
[](Collection &self, std::vector<Doc> &docs) {
|
||||
const auto result = self.Update(docs);
|
||||
return unwrap_expected(result);
|
||||
})
|
||||
.def("Upsert",
|
||||
[](Collection &self, std::vector<Doc> &docs) {
|
||||
const auto result = self.Upsert(docs);
|
||||
return unwrap_expected(result);
|
||||
})
|
||||
.def("Delete",
|
||||
[](Collection &self, const std::vector<std::string> &pks) {
|
||||
const auto result = self.Delete(pks);
|
||||
return unwrap_expected(result);
|
||||
})
|
||||
.def("DeleteByFilter", [](Collection &self, const std::string &filter) {
|
||||
const auto status = self.DeleteByFilter(filter);
|
||||
throw_if_error(status);
|
||||
});
|
||||
}
|
||||
|
||||
void ZVecPyCollection::bind_dql_methods(
|
||||
py::class_<Collection, Collection::Ptr> &col) {
|
||||
col.def("Query",
|
||||
[](const Collection &self, const VectorQuery &query) {
|
||||
const auto result = self.Query(query);
|
||||
// return DocPtrList
|
||||
return unwrap_expected(result);
|
||||
})
|
||||
.def("GroupByQuery",
|
||||
[](const Collection &self, const GroupByVectorQuery &query) {
|
||||
const auto result = self.GroupByQuery(query);
|
||||
// return GroupResults
|
||||
return unwrap_expected(result);
|
||||
})
|
||||
.def("Fetch",
|
||||
[](const Collection &self, const std::vector<std::string> &pks) {
|
||||
const auto result = self.Fetch(pks);
|
||||
// return DocPtrMap
|
||||
return unwrap_expected(result);
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,452 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "python_doc.h"
|
||||
#include <pybind11/numpy.h>
|
||||
#include <pybind11/stl.h>
|
||||
|
||||
namespace zvec {
|
||||
|
||||
template <typename T>
|
||||
T checked_cast(const py::object &obj, const std::string &field,
|
||||
const std::string &expected_type) {
|
||||
try {
|
||||
return obj.cast<T>();
|
||||
} catch (const py::cast_error &e) {
|
||||
std::string actual_type = std::string(py::str(py::type::of(obj)));
|
||||
std::string msg = "Field '" + field + "': expected " + expected_type +
|
||||
", got " + actual_type;
|
||||
throw py::type_error(msg);
|
||||
}
|
||||
}
|
||||
|
||||
void ZVecPyDoc::Initialize(pybind11::module_ &m) {
|
||||
bind_doc_operator(m);
|
||||
bind_doc(m);
|
||||
}
|
||||
|
||||
|
||||
void ZVecPyDoc::bind_doc_operator(py::module_ &m) {
|
||||
py::enum_<Operator>(m, "_DocOp")
|
||||
.value("INSERT", Operator::INSERT)
|
||||
.value("UPDATE", Operator::UPDATE)
|
||||
.value("DELETE", Operator::DELETE)
|
||||
.value("UPSERT", Operator::UPSERT);
|
||||
}
|
||||
|
||||
|
||||
void ZVecPyDoc::bind_doc(py::module_ &m) {
|
||||
// binding doc
|
||||
py::class_<Doc, Doc::Ptr> doc(m, "_Doc");
|
||||
|
||||
doc.def(py::init([]() { return std::make_shared<Doc>(); }))
|
||||
.def("set_pk", &Doc::set_pk)
|
||||
.def("pk", &Doc::pk)
|
||||
.def("set_score", &Doc::set_score)
|
||||
.def("score", &Doc::score)
|
||||
.def("has_field", &Doc::has)
|
||||
.def("field_names", &Doc::field_names)
|
||||
.def(py::pickle(
|
||||
[](const Doc &d) {
|
||||
std::vector<uint8_t> data = d.serialize();
|
||||
return py::bytes(reinterpret_cast<const char *>(data.data()),
|
||||
data.size());
|
||||
},
|
||||
[](py::bytes b) {
|
||||
py::buffer_info info(py::buffer(b).request());
|
||||
const uint8_t *buf = reinterpret_cast<const uint8_t *>(info.ptr);
|
||||
size_t size = static_cast<size_t>(info.size);
|
||||
Doc::Ptr d = Doc::deserialize(buf, size);
|
||||
if (!d) throw std::runtime_error("Failed to unpickle Doc");
|
||||
return d;
|
||||
}));
|
||||
|
||||
|
||||
// binding doc set field
|
||||
doc.def(
|
||||
"set_any",
|
||||
[](Doc &self, const std::string &field, const FieldSchema &field_schema,
|
||||
const py::object &obj) -> bool {
|
||||
if (obj.is_none()) {
|
||||
if (field_schema.nullable()) {
|
||||
self.set_null(field);
|
||||
return true;
|
||||
}
|
||||
throw py::value_error("Field '" + field +
|
||||
"': expected non-nullable type");
|
||||
}
|
||||
switch (field_schema.data_type()) {
|
||||
// base datatypes
|
||||
case DataType::STRING:
|
||||
return self.set(field,
|
||||
checked_cast<std::string>(obj, field, "STRING"));
|
||||
case DataType::BOOL:
|
||||
return self.set(field, checked_cast<bool>(obj, field, "BOOL"));
|
||||
case DataType::INT32:
|
||||
return self.set(field, checked_cast<int32_t>(obj, field, "INT32"));
|
||||
case DataType::INT64:
|
||||
return self.set(field, checked_cast<int64_t>(obj, field, "INT64"));
|
||||
case DataType::UINT32:
|
||||
return self.set(field,
|
||||
checked_cast<uint32_t>(obj, field, "UINT32"));
|
||||
case DataType::UINT64:
|
||||
return self.set(field,
|
||||
checked_cast<uint64_t>(obj, field, "UINT64"));
|
||||
case DataType::FLOAT:
|
||||
return self.set(field, checked_cast<float>(obj, field, "FLOAT"));
|
||||
case DataType::DOUBLE:
|
||||
return self.set(field, checked_cast<double>(obj, field, "DOUBLE"));
|
||||
|
||||
// array datatypes
|
||||
case DataType::ARRAY_STRING:
|
||||
return self.set(field, checked_cast<std::vector<std::string>>(
|
||||
obj, field, "ARRAY_STRING"));
|
||||
case DataType::ARRAY_BOOL:
|
||||
return self.set(field, checked_cast<std::vector<bool>>(
|
||||
obj, field, "ARRAY_BOOL"));
|
||||
case DataType::ARRAY_INT32:
|
||||
return self.set(field, checked_cast<std::vector<int32_t>>(
|
||||
obj, field, "ARRAY_INT32"));
|
||||
case DataType::ARRAY_UINT32:
|
||||
return self.set(field, checked_cast<std::vector<uint32_t>>(
|
||||
obj, field, "ARRAY_UINT32"));
|
||||
case DataType::ARRAY_INT64:
|
||||
return self.set(field, checked_cast<std::vector<int64_t>>(
|
||||
obj, field, "ARRAY_INT64"));
|
||||
case DataType::ARRAY_UINT64:
|
||||
return self.set(field, checked_cast<std::vector<uint64_t>>(
|
||||
obj, field, "ARRAY_UINT64"));
|
||||
case DataType::ARRAY_FLOAT:
|
||||
return self.set(field, checked_cast<std::vector<float>>(
|
||||
obj, field, "ARRAY_FLOAT"));
|
||||
case DataType::ARRAY_DOUBLE:
|
||||
return self.set(field, checked_cast<std::vector<double>>(
|
||||
obj, field, "ARRAY_DOUBLE"));
|
||||
|
||||
// dense vector datatypes
|
||||
case DataType::VECTOR_FP16: {
|
||||
const auto value = checked_cast<py::list>(
|
||||
obj, field, "VECTOR_FP16 (list of numbers)");
|
||||
std::vector<ailego::Float16> new_value;
|
||||
new_value.reserve(value.size());
|
||||
for (const auto &item : value) {
|
||||
try {
|
||||
new_value.emplace_back(item.cast<float>());
|
||||
} catch (const py::cast_error &e) {
|
||||
throw py::type_error("Vector '" + field +
|
||||
"': expected VECTOR_FP16, got " +
|
||||
std::string(py::str(py::type::of(obj))));
|
||||
}
|
||||
}
|
||||
return self.set(field, new_value);
|
||||
}
|
||||
case DataType::VECTOR_FP32:
|
||||
return self.set(field, checked_cast<std::vector<float>>(
|
||||
obj, field, "VECTOR_FP32"));
|
||||
case DataType::VECTOR_FP64:
|
||||
return self.set(field, checked_cast<std::vector<double>>(
|
||||
obj, field, "VECTOR_FP64"));
|
||||
case DataType::VECTOR_INT8:
|
||||
return self.set(field, checked_cast<std::vector<int8_t>>(
|
||||
obj, field, "VECTOR_INT8"));
|
||||
|
||||
// sparse vector datatypes
|
||||
case DataType::SPARSE_VECTOR_FP32: {
|
||||
const auto sparse_dict =
|
||||
checked_cast<py::dict>(obj, field, "SPARSE_VECTOR_FP32 (dict)");
|
||||
std::vector<uint32_t> indices;
|
||||
std::vector<float> values;
|
||||
for (const auto &item : sparse_dict) {
|
||||
try {
|
||||
indices.push_back(item.first.cast<uint32_t>());
|
||||
values.push_back(item.second.cast<float>());
|
||||
} catch (const py::cast_error &e) {
|
||||
throw py::type_error(
|
||||
"Vector '" + field +
|
||||
"': sparse vector key/value must be (uint32, float), "
|
||||
"got key=" +
|
||||
std::string(py::str(py::type::of(item.first))) +
|
||||
", value=" +
|
||||
std::string(py::str(py::type::of(item.second))));
|
||||
}
|
||||
}
|
||||
const std::pair<std::vector<uint32_t>, std::vector<float>>
|
||||
sparse_vector{std::move(indices), std::move(values)};
|
||||
return self.set(field, sparse_vector);
|
||||
}
|
||||
case DataType::SPARSE_VECTOR_FP16: {
|
||||
const auto sparse_dict =
|
||||
checked_cast<py::dict>(obj, field, "SPARSE_VECTOR_FP16 (dict)");
|
||||
std::vector<uint32_t> indices;
|
||||
std::vector<ailego::Float16> values;
|
||||
for (const auto &item : sparse_dict) {
|
||||
try {
|
||||
indices.push_back(item.first.cast<uint32_t>());
|
||||
values.push_back(ailego::Float16(item.second.cast<float>()));
|
||||
} catch (const py::cast_error &e) {
|
||||
throw py::type_error(
|
||||
"Field '" + field +
|
||||
"': sparse vector key/value must be (uint32, float), "
|
||||
"got key=" +
|
||||
std::string(py::str(py::type::of(item.first))) +
|
||||
", value=" +
|
||||
std::string(py::str(py::type::of(item.second))));
|
||||
}
|
||||
}
|
||||
const std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>>
|
||||
sparse_vector{std::move(indices), std::move(values)};
|
||||
return self.set(field, sparse_vector);
|
||||
}
|
||||
default:
|
||||
throw py::type_error("Unsupported type for field: " + field);
|
||||
}
|
||||
});
|
||||
|
||||
// binding doc get field
|
||||
doc.def(
|
||||
"get_any",
|
||||
[](Doc &self, const std::string &field,
|
||||
const DataType &type) -> py::object {
|
||||
switch (type) {
|
||||
// base datatypes
|
||||
case DataType::STRING:
|
||||
return py::cast(self.get<std::string>(field));
|
||||
case DataType::BOOL:
|
||||
return py::cast(self.get<bool>(field));
|
||||
case DataType::INT32:
|
||||
return py::cast(self.get<int32_t>(field));
|
||||
case DataType::UINT32:
|
||||
return py::cast(self.get<uint32_t>(field));
|
||||
case DataType::INT64:
|
||||
return py::cast(self.get<int64_t>(field));
|
||||
case DataType::UINT64:
|
||||
return py::cast(self.get<uint64_t>(field));
|
||||
case DataType::FLOAT:
|
||||
return py::cast(self.get<float>(field));
|
||||
case DataType::DOUBLE:
|
||||
return py::cast(self.get<double>(field));
|
||||
|
||||
// array datatypes
|
||||
case DataType::ARRAY_STRING:
|
||||
return py::cast(self.get<std::vector<std::string>>(field));
|
||||
case DataType::ARRAY_INT32:
|
||||
return py::cast(self.get<std::vector<int32_t>>(field));
|
||||
case DataType::ARRAY_INT64:
|
||||
return py::cast(self.get<std::vector<int64_t>>(field));
|
||||
case DataType::ARRAY_UINT32:
|
||||
return py::cast(self.get<std::vector<uint32_t>>(field));
|
||||
case DataType::ARRAY_UINT64:
|
||||
return py::cast(self.get<std::vector<uint64_t>>(field));
|
||||
case DataType::ARRAY_FLOAT:
|
||||
return py::cast(self.get<std::vector<float>>(field));
|
||||
case DataType::ARRAY_DOUBLE:
|
||||
return py::cast(self.get<std::vector<double>>(field));
|
||||
case DataType::ARRAY_BOOL:
|
||||
return py::cast(self.get<std::vector<bool>>(field));
|
||||
|
||||
// vector datatypes
|
||||
case DataType::VECTOR_INT8:
|
||||
return py::cast(self.get<std::vector<int8_t>>(field));
|
||||
case DataType::VECTOR_FP16: {
|
||||
auto value = self.get<std::vector<ailego::Float16>>(field);
|
||||
if (value.has_value()) {
|
||||
std::vector<float> new_value;
|
||||
new_value.reserve(value.value().size());
|
||||
for (auto &item : value.value()) {
|
||||
new_value.push_back(static_cast<float>(item));
|
||||
}
|
||||
return py::cast(new_value);
|
||||
}
|
||||
return py::none();
|
||||
}
|
||||
case DataType::VECTOR_FP32:
|
||||
return py::cast(self.get<std::vector<float>>(field));
|
||||
case DataType::VECTOR_FP64:
|
||||
return py::cast(self.get<std::vector<double>>(field));
|
||||
case DataType::SPARSE_VECTOR_FP16: {
|
||||
auto vector = self.get<
|
||||
std::pair<std::vector<uint32_t>, std::vector<ailego::Float16>>>(
|
||||
field);
|
||||
const auto &indices = vector->first;
|
||||
const auto &values = vector->second;
|
||||
py::dict d;
|
||||
for (size_t i = 0; i < indices.size(); ++i) {
|
||||
d[py::int_(indices[i])] =
|
||||
py::float_(static_cast<float>(values[i]));
|
||||
}
|
||||
return std::move(d);
|
||||
}
|
||||
case DataType::SPARSE_VECTOR_FP32: {
|
||||
auto vector =
|
||||
self.get<std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
||||
field);
|
||||
const auto &indices = vector->first;
|
||||
const auto &values = vector->second;
|
||||
py::dict d;
|
||||
for (size_t i = 0; i < indices.size(); ++i) {
|
||||
d[py::int_(indices[i])] = py::float_(values[i]);
|
||||
}
|
||||
return std::move(d);
|
||||
}
|
||||
default:
|
||||
throw py::type_error("Unsupported type for field: " + field);
|
||||
}
|
||||
});
|
||||
doc.def(
|
||||
"get_all",
|
||||
[](Doc &self, const CollectionSchema &schema) -> py::tuple {
|
||||
py::tuple result(4);
|
||||
// 1. set doc id and score
|
||||
result[0] = py::str(self.pk());
|
||||
result[1] = py::float_(self.score());
|
||||
|
||||
if (self.is_empty()) {
|
||||
result[2] = py::none();
|
||||
result[3] = py::none();
|
||||
return result;
|
||||
}
|
||||
// 2. set scalar fields
|
||||
py::dict fields;
|
||||
for (const auto &field_meta : schema.forward_fields()) {
|
||||
const std::string &field = field_meta->name();
|
||||
if (!self.has_value(field)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
try {
|
||||
auto val = [&]() -> py::object {
|
||||
switch (field_meta->data_type()) {
|
||||
// base datatypes
|
||||
case DataType::STRING:
|
||||
return py::str(self.get<std::string>(field).value());
|
||||
case DataType::BOOL:
|
||||
return py::cast(self.get<bool>(field));
|
||||
case DataType::INT32:
|
||||
return py::cast(self.get<int32_t>(field));
|
||||
case DataType::UINT32:
|
||||
return py::cast(self.get<uint32_t>(field));
|
||||
case DataType::INT64:
|
||||
return py::cast(self.get<int64_t>(field));
|
||||
case DataType::UINT64:
|
||||
return py::cast(self.get<uint64_t>(field));
|
||||
case DataType::FLOAT:
|
||||
return py::cast(self.get<float>(field));
|
||||
case DataType::DOUBLE:
|
||||
return py::cast(self.get<double>(field));
|
||||
|
||||
// array datatypes
|
||||
case DataType::ARRAY_STRING:
|
||||
return py::cast(self.get<std::vector<std::string>>(field));
|
||||
case DataType::ARRAY_INT32:
|
||||
return py::cast(self.get<std::vector<int32_t>>(field));
|
||||
case DataType::ARRAY_INT64:
|
||||
return py::cast(self.get<std::vector<int64_t>>(field));
|
||||
case DataType::ARRAY_UINT32:
|
||||
return py::cast(self.get<std::vector<uint32_t>>(field));
|
||||
case DataType::ARRAY_UINT64:
|
||||
return py::cast(self.get<std::vector<uint64_t>>(field));
|
||||
case DataType::ARRAY_FLOAT:
|
||||
return py::cast(self.get<std::vector<float>>(field));
|
||||
case DataType::ARRAY_DOUBLE:
|
||||
return py::cast(self.get<std::vector<double>>(field));
|
||||
case DataType::ARRAY_BOOL:
|
||||
return py::cast(self.get<std::vector<bool>>(field));
|
||||
default:
|
||||
throw py::type_error("Unsupported type for field: " + field);
|
||||
}
|
||||
}();
|
||||
fields[py::str(field)] = val;
|
||||
} catch (const std::exception &e) {
|
||||
fields[py::str(field)] = py::none();
|
||||
}
|
||||
}
|
||||
if (!fields.empty()) {
|
||||
result[2] = fields;
|
||||
} else {
|
||||
result[2] = py::none();
|
||||
}
|
||||
// 3. set vector fields
|
||||
py::dict vectors;
|
||||
for (const auto &vec_meta : schema.vector_fields()) {
|
||||
const std::string &vec = vec_meta->name();
|
||||
if (!self.has_value(vec)) continue;
|
||||
|
||||
try {
|
||||
auto array = [&]() -> py::object {
|
||||
switch (vec_meta->data_type()) {
|
||||
case DataType::VECTOR_INT8:
|
||||
return py::cast(self.get<std::vector<int8_t>>(vec));
|
||||
case DataType::VECTOR_FP16: {
|
||||
auto value = self.get<std::vector<ailego::Float16>>(vec);
|
||||
if (value.has_value()) {
|
||||
std::vector<float> new_value;
|
||||
new_value.reserve(value.value().size());
|
||||
for (auto &item : value.value()) {
|
||||
new_value.push_back(static_cast<float>(item));
|
||||
}
|
||||
return py::cast(new_value);
|
||||
}
|
||||
return py::none();
|
||||
}
|
||||
case DataType::VECTOR_FP32:
|
||||
return py::cast(self.get<std::vector<float>>(vec));
|
||||
case DataType::VECTOR_FP64:
|
||||
return py::cast(self.get<std::vector<double>>(vec));
|
||||
case DataType::SPARSE_VECTOR_FP16: {
|
||||
auto vector =
|
||||
self.get<std::pair<std::vector<uint32_t>,
|
||||
std::vector<ailego::Float16>>>(vec);
|
||||
const auto &indices = vector->first;
|
||||
const auto &values = vector->second;
|
||||
py::dict d;
|
||||
for (size_t i = 0; i < indices.size(); ++i) {
|
||||
d[py::int_(indices[i])] =
|
||||
py::float_(static_cast<float>(values[i]));
|
||||
}
|
||||
return std::move(d);
|
||||
}
|
||||
case DataType::SPARSE_VECTOR_FP32: {
|
||||
auto vector = self.get<
|
||||
std::pair<std::vector<uint32_t>, std::vector<float>>>(
|
||||
vec);
|
||||
const auto &indices = vector->first;
|
||||
const auto &values = vector->second;
|
||||
py::dict d;
|
||||
for (size_t i = 0; i < indices.size(); ++i) {
|
||||
d[py::int_(indices[i])] = py::float_(values[i]);
|
||||
}
|
||||
return std::move(d);
|
||||
}
|
||||
default:
|
||||
throw py::type_error("Unsupported type for field: " + vec);
|
||||
}
|
||||
}();
|
||||
vectors[py::str(vec)] = array;
|
||||
} catch (const std::exception &e) {
|
||||
vectors[py::str(vec)] = py::none();
|
||||
}
|
||||
}
|
||||
if (!vectors.empty()) {
|
||||
result[3] = vectors;
|
||||
} else {
|
||||
result[3] = py::none();
|
||||
}
|
||||
return result;
|
||||
},
|
||||
py::arg("schema"),
|
||||
"Get all fields and vectors as a tuple: (id, score, fields, vectors). "
|
||||
"Vectors are zero-copy numpy arrays (dense: ndarray, sparse: (indices, "
|
||||
"values) tuple).");
|
||||
}
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,162 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "python_schema.h"
|
||||
#include <pybind11/stl.h>
|
||||
#include "db/index/common/schema.h"
|
||||
#include "db/index/common/stats.h"
|
||||
|
||||
namespace zvec {
|
||||
|
||||
void ZVecPySchemas::Initialize(pybind11::module_ &parent) {
|
||||
auto m =
|
||||
parent.def_submodule("schema", "This module contains the schema of Zvec");
|
||||
|
||||
bind_field_schema(m);
|
||||
bind_collection_schema(m);
|
||||
bind_collection_stats(m);
|
||||
}
|
||||
|
||||
void ZVecPySchemas::bind_field_schema(py::module_ &m) {
|
||||
py::class_<FieldSchema, FieldSchema::Ptr>(m, "_FieldSchema")
|
||||
.def(py::init<const std::string &, DataType, uint32_t, bool,
|
||||
const IndexParams::Ptr &>(),
|
||||
py::arg("name"), py::arg("data_type"), py::arg("dimension") = 0,
|
||||
py::arg("nullable") = false, py::arg("index_param") = nullptr)
|
||||
.def_property_readonly("name", &FieldSchema::name)
|
||||
.def_property_readonly("data_type", &FieldSchema::data_type)
|
||||
.def_property_readonly("nullable", &FieldSchema::nullable)
|
||||
.def_property_readonly("dimension", &FieldSchema::dimension)
|
||||
.def_property_readonly("is_dense_vector", &FieldSchema::is_dense_vector)
|
||||
.def_property_readonly("is_sparse_vector", &FieldSchema::is_sparse_vector)
|
||||
.def_property_readonly("index_type",
|
||||
[](const FieldSchema &self) {
|
||||
return self.index_params()
|
||||
? self.index_type()
|
||||
: IndexType::UNDEFINED;
|
||||
})
|
||||
.def_property_readonly("index_param",
|
||||
[](const FieldSchema &self) -> py::object {
|
||||
if (self.index_params()) {
|
||||
return py::cast(self.index_params());
|
||||
}
|
||||
return py::none();
|
||||
})
|
||||
.def("__eq__", &FieldSchema::operator==)
|
||||
.def("__ne__", &FieldSchema::operator!=)
|
||||
.def(py::pickle(
|
||||
[](const FieldSchema &self) {
|
||||
return py::make_tuple(self.name(), self.data_type(),
|
||||
self.dimension(), self.nullable(),
|
||||
self.index_params()
|
||||
? py::cast(self.index_params())
|
||||
: py::none());
|
||||
},
|
||||
[](py::tuple t) {
|
||||
if (t.size() != 5) {
|
||||
throw std::runtime_error(
|
||||
"Invalid tuple size for FieldSchema pickle");
|
||||
}
|
||||
std::string name = t[0].cast<std::string>();
|
||||
DataType dtype = t[1].cast<DataType>();
|
||||
uint32_t dim = t[2].cast<uint32_t>();
|
||||
bool nullable = t[3].cast<bool>();
|
||||
|
||||
IndexParams::Ptr idx_params = nullptr;
|
||||
if (!t[4].is_none()) {
|
||||
idx_params = t[4].cast<IndexParams::Ptr>();
|
||||
}
|
||||
|
||||
return std::make_shared<FieldSchema>(name, dtype, dim, nullable,
|
||||
idx_params);
|
||||
}));
|
||||
}
|
||||
|
||||
void ZVecPySchemas::bind_collection_schema(py::module_ &m) {
|
||||
py::class_<CollectionSchema, CollectionSchema::Ptr>(m, "_CollectionSchema")
|
||||
.def(py::init<const std::string &, const FieldSchemaPtrList &>(),
|
||||
py::arg("name"), py::arg("fields"),
|
||||
"Construct with name and list of fields")
|
||||
.def_property_readonly("name", &CollectionSchema::name)
|
||||
.def("has_field", &CollectionSchema::has_field, py::arg("field_name"),
|
||||
"Check if a field exists.")
|
||||
.def(
|
||||
"get_field",
|
||||
[](const CollectionSchema &self, const std::string &name)
|
||||
-> const FieldSchema * { return self.get_field(name); },
|
||||
py::arg("field_name"), py::return_value_policy::reference_internal,
|
||||
"Get field by name (const pointer), returns None if not found.")
|
||||
.def(
|
||||
"get_forward_field",
|
||||
[](const CollectionSchema &self, const std::string &name)
|
||||
-> const FieldSchema * { return self.get_forward_field(name); },
|
||||
py::arg("field_name"), py::return_value_policy::reference_internal,
|
||||
"Get forward field (used for filtering).")
|
||||
.def(
|
||||
"get_vector_field",
|
||||
[](const CollectionSchema &self, const std::string &name)
|
||||
-> const FieldSchema * { return self.get_vector_field(name); },
|
||||
py::arg("field_name"), py::return_value_policy::reference_internal,
|
||||
"Get vector field by name.")
|
||||
.def("fields", &CollectionSchema::fields,
|
||||
"Return list of all field schemas.", py::return_value_policy::copy)
|
||||
.def("forward_fields", &CollectionSchema::forward_fields,
|
||||
"Return list of forward-indexed fields.",
|
||||
py::return_value_policy::copy)
|
||||
.def("vector_fields", &CollectionSchema::vector_fields,
|
||||
"Return list of vector fields.", py::return_value_policy::copy)
|
||||
.def("__eq__", &CollectionSchema::operator==)
|
||||
.def("__ne__", &CollectionSchema::operator!=)
|
||||
.def(py::pickle(
|
||||
[](const CollectionSchema &cs) {
|
||||
return py::make_tuple(cs.name(), cs.fields(),
|
||||
cs.max_doc_count_per_segment());
|
||||
},
|
||||
[](py::tuple t) {
|
||||
if (t.size() != 3)
|
||||
throw std::runtime_error("Invalid state for CollectionSchema!");
|
||||
|
||||
auto name = t[0].cast<std::string>();
|
||||
auto fields = t[1].cast<FieldSchemaPtrList>();
|
||||
auto max_docs = t[2].cast<uint64_t>();
|
||||
|
||||
auto cs = std::make_shared<CollectionSchema>(name, fields);
|
||||
cs->set_max_doc_count_per_segment(max_docs);
|
||||
return cs;
|
||||
}));
|
||||
}
|
||||
|
||||
void ZVecPySchemas::bind_collection_stats(py::module_ &m) {
|
||||
pybind11::class_<CollectionStats>(m, "CollectionStats")
|
||||
.def(pybind11::init<>())
|
||||
.def_property_readonly(
|
||||
"doc_count", [](const CollectionStats &c) { return c.doc_count; })
|
||||
.def_property_readonly(
|
||||
"index_completeness",
|
||||
[](const CollectionStats &c) { return c.index_completeness; })
|
||||
.def("__repr__", [](const CollectionStats &c) {
|
||||
std::string map_str = "{";
|
||||
bool first = true;
|
||||
for (const auto &[k, v] : c.index_completeness) {
|
||||
if (!first) map_str += ", ";
|
||||
map_str += "\"" + k + "\":" + std::to_string(v);
|
||||
first = false;
|
||||
}
|
||||
map_str += "}";
|
||||
return "{\"doc_count\":" + std::to_string(c.doc_count) +
|
||||
", \"index_completeness\":" + map_str + "}";
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,226 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "python_type.h"
|
||||
|
||||
namespace zvec {
|
||||
|
||||
void ZVecPyTyping::Initialize(pybind11::module_ &parent) {
|
||||
auto m = parent.def_submodule(
|
||||
"typing", "This module contains the basic data types of Zvec");
|
||||
// binding base types
|
||||
bind_datatypes(m);
|
||||
bind_index_types(m);
|
||||
bind_metric_types(m);
|
||||
bind_quantize_types(m);
|
||||
bind_status(m);
|
||||
}
|
||||
|
||||
void ZVecPyTyping::bind_datatypes(pybind11::module_ &m) {
|
||||
py::enum_<DataType>(m, "DataType", R"pbdoc(
|
||||
Enumeration of supported data types in Zvec.
|
||||
|
||||
Includes scalar types, dense/sparse vector types, and array types.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import DataType
|
||||
>>> print(DataType.FLOAT)
|
||||
DataType.FLOAT
|
||||
>>> print(DataType.VECTOR_FP32)
|
||||
DataType.VECTOR_FP32
|
||||
)pbdoc")
|
||||
// field type
|
||||
.value("STRING", DataType::STRING)
|
||||
.value("BOOL", DataType::BOOL)
|
||||
.value("INT32", DataType::INT32)
|
||||
.value("INT64", DataType::INT64)
|
||||
.value("FLOAT", DataType::FLOAT)
|
||||
.value("DOUBLE", DataType::DOUBLE)
|
||||
.value("UINT32", DataType::UINT32)
|
||||
.value("UINT64", DataType::UINT64)
|
||||
|
||||
|
||||
// dense vector type
|
||||
.value("VECTOR_FP16", DataType::VECTOR_FP16)
|
||||
.value("VECTOR_FP32", DataType::VECTOR_FP32)
|
||||
.value("VECTOR_FP64", DataType::VECTOR_FP64)
|
||||
.value("VECTOR_INT8", DataType::VECTOR_INT8)
|
||||
|
||||
|
||||
// sparse vector type
|
||||
.value("SPARSE_VECTOR_FP32", DataType::SPARSE_VECTOR_FP32)
|
||||
.value("SPARSE_VECTOR_FP16", DataType::SPARSE_VECTOR_FP16)
|
||||
|
||||
|
||||
// array type [not support bool/bytes]
|
||||
.value("ARRAY_STRING", DataType::ARRAY_STRING)
|
||||
.value("ARRAY_INT32", DataType::ARRAY_INT32)
|
||||
.value("ARRAY_INT64", DataType::ARRAY_INT64)
|
||||
.value("ARRAY_FLOAT", DataType::ARRAY_FLOAT)
|
||||
.value("ARRAY_DOUBLE", DataType::ARRAY_DOUBLE)
|
||||
.value("ARRAY_BOOL", DataType::ARRAY_BOOL)
|
||||
.value("ARRAY_UINT32", DataType::ARRAY_UINT32)
|
||||
.value("ARRAY_UINT64", DataType::ARRAY_UINT64)
|
||||
|
||||
|
||||
// non support
|
||||
// .value("BINARY", DataType::BINARY)
|
||||
// .value("ARRAY_BINARY", DataType::ARRAY_BINARY)
|
||||
// .value("VECTOR_INT4", DataType::VECTOR_INT4)
|
||||
// .value("VECTOR_INT16", DataType::VECTOR_INT16)
|
||||
// .value("VECTOR_BINARY32", DataType::VECTOR_BINARY32)
|
||||
// .value("VECTOR_BINARY64", DataType::VECTOR_BINARY64)
|
||||
// .value("UNDEFINED", DataType::UNDEFINED)
|
||||
;
|
||||
}
|
||||
|
||||
void ZVecPyTyping::bind_index_types(pybind11::module_ &m) {
|
||||
py::enum_<IndexType>(m, "IndexType", R"pbdoc(
|
||||
Enumeration of supported index types in Zvec.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import IndexType
|
||||
>>> print(IndexType.HNSW)
|
||||
IndexType.HNSW
|
||||
)pbdoc")
|
||||
.value("UNDEFINED", IndexType::UNDEFINED)
|
||||
.value("HNSW", IndexType::HNSW)
|
||||
.value("IVF", IndexType::IVF)
|
||||
.value("FLAT", IndexType::FLAT)
|
||||
.value("INVERT", IndexType::INVERT);
|
||||
}
|
||||
|
||||
void ZVecPyTyping::bind_metric_types(pybind11::module_ &m) {
|
||||
py::enum_<MetricType>(m, "MetricType", R"pbdoc(
|
||||
Enumeration of supported distance/similarity metrics.
|
||||
|
||||
- COSINE: Cosine similarity.
|
||||
- IP: Inner product (dot product).
|
||||
- L2: Euclidean distance (L2 norm).
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import MetricType
|
||||
>>> print(MetricType.COSINE)
|
||||
MetricType.COSINE
|
||||
)pbdoc")
|
||||
.value("COSINE", MetricType::COSINE)
|
||||
.value("IP", MetricType::IP)
|
||||
.value("L2", MetricType::L2);
|
||||
}
|
||||
|
||||
void ZVecPyTyping::bind_quantize_types(py::module_ &m) {
|
||||
py::enum_<QuantizeType>(m, "QuantizeType", R"pbdoc(
|
||||
Enumeration of supported quantization types for vector compression.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import QuantizeType
|
||||
>>> print(QuantizeType.INT8)
|
||||
QuantizeType.INT8
|
||||
)pbdoc")
|
||||
.value("UNDEFINED", QuantizeType::UNDEFINED)
|
||||
.value("FP16", QuantizeType::FP16)
|
||||
.value("INT8", QuantizeType::INT8)
|
||||
.value("INT4", QuantizeType::INT4);
|
||||
}
|
||||
|
||||
void ZVecPyTyping::bind_status(py::module_ &m) {
|
||||
// bind status code
|
||||
py::enum_<StatusCode>(m, "StatusCode", R"pbdoc(
|
||||
Enumeration of possible status codes for Zvec operations.
|
||||
|
||||
Used by the `Status` class to indicate success or failure reason.
|
||||
)pbdoc")
|
||||
.value("OK", StatusCode::OK)
|
||||
.value("NOT_FOUND", StatusCode::NOT_FOUND)
|
||||
.value("ALREADY_EXISTS", StatusCode::ALREADY_EXISTS)
|
||||
.value("INVALID_ARGUMENT", StatusCode::INVALID_ARGUMENT)
|
||||
.value("PERMISSION_DENIED", StatusCode::PERMISSION_DENIED)
|
||||
.value("FAILED_PRECONDITION", StatusCode::FAILED_PRECONDITION)
|
||||
.value("RESOURCE_EXHAUSTED", StatusCode::RESOURCE_EXHAUSTED)
|
||||
.value("UNAVAILABLE", StatusCode::UNAVAILABLE)
|
||||
.value("INTERNAL_ERROR", StatusCode::INTERNAL_ERROR)
|
||||
.value("NOT_SUPPORTED", StatusCode::NOT_SUPPORTED)
|
||||
.value("UNKNOWN", StatusCode::UNKNOWN);
|
||||
|
||||
// bind status
|
||||
py::class_<Status>(m, "Status", R"pbdoc(
|
||||
Represents the outcome of a Zvec operation.
|
||||
|
||||
A `Status` object is either OK (success) or carries an error code and message.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import Status, StatusCode
|
||||
>>> s = Status()
|
||||
>>> print(s.ok())
|
||||
True
|
||||
>>> s = Status(StatusCode.INVALID_ARGUMENT, "Field not found")
|
||||
>>> print(s.code() == StatusCode.INVALID_ARGUMENT)
|
||||
True
|
||||
>>> print(s.message())
|
||||
Field not found
|
||||
)pbdoc")
|
||||
.def(py::init<>())
|
||||
.def(py::init<StatusCode, const std::string &>(), py::arg("code"),
|
||||
py::arg("message") = "", R"pbdoc(
|
||||
Construct a status with the given code and optional message.
|
||||
|
||||
Args:
|
||||
code (StatusCode): The status code.
|
||||
message (str, optional): Error message. Defaults to empty string.
|
||||
)pbdoc")
|
||||
.def("ok", &Status::ok, "bool: Returns True if the status is OK.")
|
||||
.def("code", &Status::code, "StatusCode: Returns the status code.")
|
||||
.def("message", &Status::message,
|
||||
"str: Returns the error message (may be empty).")
|
||||
.def_static("OK", &Status::OK, "Create an OK status.")
|
||||
.def_static(
|
||||
"InvalidArgument",
|
||||
[](const std::string &msg) { return Status::InvalidArgument(msg); },
|
||||
py::arg("message"))
|
||||
.def_static(
|
||||
"NotFound",
|
||||
[](const std::string &msg) { return Status::NotFound(msg); },
|
||||
py::arg("message"))
|
||||
.def_static(
|
||||
"AlreadyExists",
|
||||
[](const std::string &msg) { return Status::AlreadyExists(msg); },
|
||||
py::arg("message"))
|
||||
.def_static(
|
||||
"InternalError",
|
||||
[](const std::string &msg) { return Status::InternalError(msg); },
|
||||
py::arg("message"))
|
||||
.def_static(
|
||||
"PermissionDenied",
|
||||
[](const std::string &msg) { return Status::PermissionDenied(msg); },
|
||||
py::arg("message"))
|
||||
.def("__eq__", [](const Status &self,
|
||||
const Status &other) { return self == other; })
|
||||
.def("__ne__", [](const Status &self,
|
||||
const Status &other) { return self != other; })
|
||||
.def("__repr__", [](const Status &self) {
|
||||
std::string result =
|
||||
"{"
|
||||
"\"code\":" +
|
||||
std::to_string(static_cast<int>(self.code()));
|
||||
|
||||
if (!self.message().empty()) {
|
||||
result += ", \"message\":\"" + self.message() + "\"";
|
||||
}
|
||||
|
||||
result += "}";
|
||||
return result;
|
||||
});
|
||||
}
|
||||
|
||||
} // namespace zvec
|
||||
Reference in New Issue
Block a user