From e2ea49d4acd4a7efe01f51159b474558f297c584 Mon Sep 17 00:00:00 2001 From: feihongxu0824 Date: Tue, 28 Jul 2026 11:51:44 +0800 Subject: [PATCH] fix: limit Windows all-in-one DLL exports (#611) --- cmake/bazel.cmake | 52 +++++- src/CMakeLists.txt | 73 +++++++- src/ailego/CMakeLists.txt | 11 +- src/ailego/utility/time_helper.cc | 41 ++++- src/binding/c/CMakeLists.txt | 1 + src/core/CMakeLists.txt | 23 ++- src/core/interface/index.cc | 31 ++++ src/core/interface/index_param.cc | 131 +++++++++++++- src/core/interface/indexes/diskann_index.cc | 69 +++++++- src/core/interface/vector_source.cc | 32 ++++ src/db/CMakeLists.txt | 10 +- .../zvec/ailego/buffer/block_eviction_queue.h | 3 +- .../zvec/ailego/buffer/vector_page_table.h | 7 +- src/include/zvec/ailego/container/params.h | 5 +- src/include/zvec/ailego/io/file.h | 3 +- src/include/zvec/ailego/io/mmap_file.h | 3 +- src/include/zvec/ailego/logger/logger.h | 5 +- .../zvec/ailego/parallel/thread_pool.h | 3 +- src/include/zvec/ailego/utility/file_helper.h | 3 +- .../zvec/ailego/utility/float_helper.h | 3 +- .../zvec/ailego/utility/string_helper.h | 3 +- src/include/zvec/ailego/utility/time_helper.h | 11 +- src/include/zvec/core/interface/index.h | 52 ++---- .../zvec/core/interface/index_factory.h | 5 +- src/include/zvec/core/interface/index_param.h | 160 ++++++++++-------- .../zvec/core/interface/vector_source.h | 12 +- src/include/zvec/db/collection.h | 5 +- src/include/zvec/db/config.h | 3 +- src/include/zvec/db/doc.h | 6 +- src/include/zvec/db/index_params.h | 21 +-- src/include/zvec/db/query.h | 15 +- src/include/zvec/db/query_params.h | 19 ++- src/include/zvec/db/reranker.h | 9 +- src/include/zvec/db/schema.h | 7 +- src/include/zvec/db/stats.h | 5 +- src/include/zvec/db/status.h | 11 +- src/include/zvec/export.h | 60 +++++++ src/include/zvec/turbo/turbo.h | 16 +- src/turbo/CMakeLists.txt | 11 ++ tests/CMakeLists.txt | 3 + tests/shared/CMakeLists.txt | 7 + tests/shared/shared_api_test.cc | 40 +++++ tools/db/CMakeLists.txt | 2 +- 43 files changed, 781 insertions(+), 211 deletions(-) create mode 100644 src/core/interface/vector_source.cc create mode 100644 src/include/zvec/export.h create mode 100644 tests/shared/CMakeLists.txt create mode 100644 tests/shared/shared_api_test.cc diff --git a/cmake/bazel.cmake b/cmake/bazel.cmake index b097340..01d950b 100644 --- a/cmake/bazel.cmake +++ b/cmake/bazel.cmake @@ -13,7 +13,8 @@ ## 1.3. Build a C/C++ static or shared library ## cc_library( ## NAME -## [STATIC] [SHARED] [STRICT] [ALWAYS_LINK] [EXCLUDE] [PACKED] [SRCS_NO_GLOB] +## [STATIC] [SHARED] [OBJECTS] [STRICT] [ALWAYS_LINK] [EXCLUDE] [PACKED] +## [SRCS_NO_GLOB] ## SRCS [file2 ...] ## [INCS dir1 ...] ## [PUBINCS public_dir1 ...] @@ -25,7 +26,10 @@ ## [DEPS target1 ...] ## [PACKED_EXCLUDES pattern1 ...] ## [VERSION ] +## [EXPORT_DEF ] ## ) +## OBJECTS creates _objects for a static library. EXPORT_DEF also +## creates _export_objects from the same sources and build settings. ## ## 1.4. Build a C/C++ executable program ## cc_binary( @@ -634,6 +638,14 @@ macro(_add_library _NAME _OPTION) endif() endmacro() +## Add a static library backed by an object library. +macro(_add_static_library_with_objects _NAME _OPTION) + add_library(${_NAME}_objects OBJECT ${_OPTION} ${ARGN}) + add_library( + ${_NAME} STATIC ${_OPTION} $ + ) +endmacro() + ## Link dependencies function(_targets_link_dependencies _NAME) foreach(LIB ${ARGN}) @@ -877,7 +889,9 @@ function(_cc_target_properties) endif() if(CC_ARGS_LIBS) - if(NOT TARGET_LINKABLE) + if("${TARGET_TYPE}" STREQUAL "OBJECT_LIBRARY") + target_link_libraries(${CC_ARGS_NAME} PRIVATE ${CC_ARGS_LIBS}) + elseif(NOT TARGET_LINKABLE) _targets_link_dependencies(${CC_ARGS_NAME} ${CC_ARGS_LIBS}) else() if ("${TARGET_TYPE}" STREQUAL "EXECUTABLE") @@ -918,9 +932,9 @@ endfunction() ## Build a C/C++ static or shared library function(cc_library) cmake_parse_arguments( - CC_ARGS - "STATIC;SHARED;EXCLUDE;PACKED;SRCS_NO_GLOB" - "NAME;VERSION" + CC_ARGS + "STATIC;SHARED;OBJECTS;EXCLUDE;PACKED;SRCS_NO_GLOB" + "NAME;VERSION;EXPORT_DEF" "SRCS;INCS;PUBINCS;DEFS;LIBS;CFLAGS;CXXFLAGS;LDFLAGS;DEPS;PACKED_EXCLUDES" ${ARGN} ) @@ -958,8 +972,16 @@ function(cc_library) set(EXCLUDE_OPTION EXCLUDE_FROM_ALL) endif() + if(CC_ARGS_OBJECTS AND (NOT CC_ARGS_STATIC OR CC_ARGS_SHARED)) + message(FATAL_ERROR "OBJECTS requires a static-only cc_library target") + endif() + if(CC_ARGS_SHARED AND CC_ARGS_STATIC) _add_library(${CC_ARGS_NAME} "${EXCLUDE_OPTION}" ${SOURCE_FILES}) + elseif(CC_ARGS_OBJECTS) + _add_static_library_with_objects( + ${CC_ARGS_NAME} "${EXCLUDE_OPTION}" ${SOURCE_FILES} + ) elseif(CC_ARGS_SHARED) add_library(${CC_ARGS_NAME} SHARED ${EXCLUDE_OPTION} ${SOURCE_FILES}) elseif(CC_ARGS_STATIC) @@ -982,6 +1004,26 @@ function(cc_library) ) endif() + if(CC_ARGS_EXPORT_DEF) + if(NOT CC_ARGS_OBJECTS) + message(FATAL_ERROR "EXPORT_DEF requires OBJECTS for ${CC_ARGS_NAME}") + endif() + add_library( + ${CC_ARGS_NAME}_export_objects OBJECT ${EXCLUDE_OPTION} ${SOURCE_FILES} + ) + _cc_target_properties( + NAME "${CC_ARGS_NAME}_export_objects" + INCS "${CC_ARGS_INCS};${CC_ARGS_PUBINCS}" + DEFS "${CC_ARGS_DEFS};${CC_ARGS_EXPORT_DEF}" + LIBS "${CC_ARGS_LIBS}" + CFLAGS "${CC_ARGS_CFLAGS}" + CXXFLAGS "${CC_ARGS_CXXFLAGS}" + LDFLAGS "${CC_ARGS_LDFLAGS}" + DEPS "${CC_ARGS_DEPS}" + "${CC_ARGS_UNPARSED_ARGUMENTS}" + ) + endif() + if(TARGET ${CC_ARGS_NAME}_static) _cc_target_properties( NAME "${CC_ARGS_NAME}_static" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 5fc4e34..2520d90 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,13 +15,14 @@ cc_directory(binding) # Build ALL-IN-ONE C++ Shared Libraries # ============================================================================= # Merges zvec internal static libraries into shared libraries while preserving -# C++ symbols for direct C++ linking. +# C++ symbols for direct C++ linking. Windows uses component-owned object +# variants so DLL export definitions never leak into static or plugin targets. include(GNUInstallDirs) find_package(Threads REQUIRED) function(zvec_add_all_in_one_shared TARGET_NAME OUTPUT_NAME) - cmake_parse_arguments(ZVEC_ALLIN "" "" "LIBS" ${ARGN}) + cmake_parse_arguments(ZVEC_ALLIN "" "" "LIBS;EXPORT_COMPONENTS" ${ARGN}) if(NOT ZVEC_ALLIN_LIBS) message(FATAL_ERROR "zvec_add_all_in_one_shared requires LIBS") endif() @@ -31,6 +32,12 @@ function(zvec_add_all_in_one_shared TARGET_NAME OUTPUT_NAME) message(FATAL_ERROR "Target ${ZVEC_ALLIN_LIB} is required by ${TARGET_NAME}") endif() endforeach() + foreach(ZVEC_ALLIN_EXPORT_COMPONENT ${ZVEC_ALLIN_EXPORT_COMPONENTS}) + if(NOT ZVEC_ALLIN_EXPORT_COMPONENT IN_LIST ZVEC_ALLIN_LIBS) + message(FATAL_ERROR + "Export component ${ZVEC_ALLIN_EXPORT_COMPONENT} is not part of ${TARGET_NAME}") + endif() + endforeach() file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${TARGET_NAME}_stub.cc "// Auto-generated stub for ${TARGET_NAME}\n" @@ -43,8 +50,12 @@ function(zvec_add_all_in_one_shared TARGET_NAME OUTPUT_NAME) set_target_properties(${TARGET_NAME} PROPERTIES OUTPUT_NAME "${OUTPUT_NAME}" POSITION_INDEPENDENT_CODE ON - WINDOWS_EXPORT_ALL_SYMBOLS ON ) + if(WIN32) + set_target_properties(${TARGET_NAME} PROPERTIES + WINDOWS_EXPORT_ALL_SYMBOLS OFF + ) + endif() if(WIN32) set_target_properties(${TARGET_NAME} PROPERTIES ARCHIVE_OUTPUT_NAME "${TARGET_NAME}" @@ -52,17 +63,39 @@ function(zvec_add_all_in_one_shared TARGET_NAME OUTPUT_NAME) endif() target_compile_features(${TARGET_NAME} PUBLIC cxx_std_17) - if(MSVC) + if(WIN32) foreach(ZVEC_ALLIN_LIB ${ZVEC_ALLIN_LIBS}) + if(ZVEC_ALLIN_LIB IN_LIST ZVEC_ALLIN_EXPORT_COMPONENTS) + set(ZVEC_ALLIN_OBJECT_TARGET ${ZVEC_ALLIN_LIB}_export_objects) + else() + set(ZVEC_ALLIN_OBJECT_TARGET ${ZVEC_ALLIN_LIB}_objects) + endif() + if(NOT TARGET ${ZVEC_ALLIN_OBJECT_TARGET}) + message(FATAL_ERROR + "Object target ${ZVEC_ALLIN_OBJECT_TARGET} is required by ${TARGET_NAME}") + endif() target_sources(${TARGET_NAME} PRIVATE - $ + $ ) endforeach() + set(ZVEC_ALLIN_LINK_LIBS) + foreach(ZVEC_ALLIN_LIB ${ZVEC_ALLIN_LIBS}) + get_target_property(ZVEC_ALLIN_LIB_DEPS ${ZVEC_ALLIN_LIB} LINK_LIBRARIES) + if(NOT ZVEC_ALLIN_LIB_DEPS) + continue() + endif() + foreach(ZVEC_ALLIN_LIB_DEP ${ZVEC_ALLIN_LIB_DEPS}) + if(NOT ZVEC_ALLIN_LIB_DEP IN_LIST ZVEC_ALLIN_LIBS) + list(APPEND ZVEC_ALLIN_LINK_LIBS ${ZVEC_ALLIN_LIB_DEP}) + endif() + endforeach() + endforeach() + list(APPEND ZVEC_ALLIN_LINK_LIBS Threads::Threads) + list(REMOVE_DUPLICATES ZVEC_ALLIN_LINK_LIBS) target_link_libraries(${TARGET_NAME} PRIVATE - ${ZVEC_ALLIN_LIBS} - Threads::Threads + ${ZVEC_ALLIN_LINK_LIBS} ) elseif(APPLE) foreach(ZVEC_ALLIN_LIB ${ZVEC_ALLIN_LIBS}) @@ -122,6 +155,23 @@ function(zvec_add_all_in_one_shared TARGET_NAME OUTPUT_NAME) ${PROJECT_SOURCE_DIR}/src ) + set(ZVEC_ALLIN_USE_DEFS) + foreach(ZVEC_ALLIN_LIB ${ZVEC_ALLIN_EXPORT_COMPONENTS}) + if(ZVEC_ALLIN_LIB STREQUAL zvec) + list(APPEND ZVEC_ALLIN_USE_DEFS ZVEC_DB_USE_SHARED) + elseif(ZVEC_ALLIN_LIB STREQUAL zvec_core) + list(APPEND ZVEC_ALLIN_USE_DEFS ZVEC_CORE_USE_SHARED) + elseif(ZVEC_ALLIN_LIB STREQUAL zvec_ailego) + list(APPEND ZVEC_ALLIN_USE_DEFS ZVEC_AILEGO_USE_SHARED) + elseif(ZVEC_ALLIN_LIB STREQUAL zvec_turbo) + list(APPEND ZVEC_ALLIN_USE_DEFS ZVEC_TURBO_USE_SHARED) + endif() + endforeach() + if(ZVEC_ALLIN_USE_DEFS) + list(REMOVE_DUPLICATES ZVEC_ALLIN_USE_DEFS) + target_compile_definitions(${TARGET_NAME} INTERFACE ${ZVEC_ALLIN_USE_DEFS}) + endif() + # Strip symbols in release builds to reduce library size. if(NOT CMAKE_BUILD_TYPE STREQUAL "Debug") if(UNIX AND NOT APPLE) @@ -148,6 +198,8 @@ if(BUILD_ZVEC_AILEGO_SHARED) zvec_add_all_in_one_shared(zvec_ailego_shared zvec_ailego LIBS zvec_ailego + EXPORT_COMPONENTS + zvec_ailego ) endif() @@ -157,6 +209,10 @@ if(BUILD_ZVEC_CORE_SHARED) zvec_core zvec_ailego zvec_turbo + EXPORT_COMPONENTS + zvec_core + zvec_ailego + zvec_turbo ) endif() @@ -167,5 +223,8 @@ if(BUILD_ZVEC_SHARED) zvec_core zvec_ailego zvec_turbo + EXPORT_COMPONENTS + zvec + zvec_ailego ) endif() diff --git a/src/ailego/CMakeLists.txt b/src/ailego/CMakeLists.txt index 29cf22c..1018ca0 100644 --- a/src/ailego/CMakeLists.txt +++ b/src/ailego/CMakeLists.txt @@ -117,8 +117,17 @@ if(NOT ANDROID AND AUTO_DETECT_ARCH) endif() endif() +set(ZVEC_AILEGO_LIBRARY_OPTIONS) +if(WIN32 AND + (BUILD_ZVEC_AILEGO_SHARED OR BUILD_ZVEC_CORE_SHARED OR BUILD_ZVEC_SHARED)) + list(APPEND ZVEC_AILEGO_LIBRARY_OPTIONS + OBJECTS + EXPORT_DEF ZVEC_AILEGO_BUILD_SHARED + ) +endif() + cc_library( - NAME zvec_ailego STATIC STRICT PACKED + NAME zvec_ailego STATIC STRICT PACKED ${ZVEC_AILEGO_LIBRARY_OPTIONS} SRCS ${ALL_SRCS} LIBS ${EXTRA_LIBS} VERSION "${GIT_SRCS_VER}" diff --git a/src/ailego/utility/time_helper.cc b/src/ailego/utility/time_helper.cc index 7f0231e..83b6954 100644 --- a/src/ailego/utility/time_helper.cc +++ b/src/ailego/utility/time_helper.cc @@ -111,6 +111,45 @@ size_t Realtime::Gmtime(const char *format, char *buf, size_t len) { time_t now = time(0); return strftime(buf, len, format, gmtime(&now)); } + +namespace { + +uint64_t FileTimeToTicks(FILETIME file_time) { + ULARGE_INTEGER value; + value.LowPart = file_time.dwLowDateTime; + value.HighPart = file_time.dwHighDateTime; + return value.QuadPart; +} + +uint64_t CurrentThreadCpuTime100NanoSeconds(void) { + FILETIME creation_time; + FILETIME exit_time; + FILETIME kernel_time; + FILETIME user_time; + if (!GetThreadTimes(GetCurrentThread(), &creation_time, &exit_time, + &kernel_time, &user_time)) { + return 0; + } + return FileTimeToTicks(kernel_time) + FileTimeToTicks(user_time); +} + +} // namespace + +uint64_t CPUtime::NanoSeconds(void) { + return CurrentThreadCpuTime100NanoSeconds() * 100u; +} + +uint64_t CPUtime::MicroSeconds(void) { + return CurrentThreadCpuTime100NanoSeconds() / 10u; +} + +uint64_t CPUtime::MilliSeconds(void) { + return CurrentThreadCpuTime100NanoSeconds() / 10000u; +} + +uint64_t CPUtime::Seconds(void) { + return CurrentThreadCpuTime100NanoSeconds() / 10000000u; +} #else uint64_t Monotime::NanoSeconds(void) { struct timespec tspec; @@ -212,4 +251,4 @@ uint64_t CPUtime::Seconds(void) { #endif // _WIN64 || _WIN32 } // namespace ailego -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/binding/c/CMakeLists.txt b/src/binding/c/CMakeLists.txt index 13dfdc0..a23a643 100644 --- a/src/binding/c/CMakeLists.txt +++ b/src/binding/c/CMakeLists.txt @@ -78,6 +78,7 @@ add_library(zvec_c_api SHARED set_target_properties(zvec_c_api PROPERTIES OUTPUT_NAME "zvec_c_api" POSITION_INDEPENDENT_CODE ON + WINDOWS_EXPORT_ALL_SYMBOLS OFF # Hide all symbols by default, only export C API CXX_VISIBILITY_PRESET hidden VISIBILITY_INLINES_HIDDEN ON diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt index 1441b1d..d875176 100644 --- a/src/core/CMakeLists.txt +++ b/src/core/CMakeLists.txt @@ -61,16 +61,19 @@ if(NOT RABITQ_SUPPORTED) endif() # Exclude algorithm/diskann implementation files from zvec_core when not -# supported (matching the hnsw_rabitq pattern above). When DISKANN_SUPPORTED, -# the diskann sources are packed into zvec_core directly, so libzvec_core.so -# includes diskann symbols without needing a separate whole-archive step. +# supported. Keep interface/indexes/diskann_index.cc in the core library so +# unsupported platforms still provide DiskAnnIndex's vtable and return +# IndexError_Unsupported from its guarded stubs. +# +# When DISKANN_SUPPORTED, the diskann sources are packed into zvec_core +# directly, so libzvec_core.so includes diskann symbols without needing a +# separate whole-archive step. # The standalone core_knn_diskann library (built by algorithm/diskann) is still # whole-archived into _zvec.so; since _zvec.so links zvec_core *normally* # (not --whole-archive), the linker skips zvec_core's diskann objects there # (symbols already resolved by core_knn_diskann_static) — no duplicates. if(NOT DISKANN_SUPPORTED) list(FILTER ALL_CORE_SRCS EXCLUDE REGEX ".*/algorithm/diskann/.*") - list(FILTER ALL_CORE_SRCS EXCLUDE REGEX ".*/interface/indexes/diskann_index\\.cc") endif() set(ZVEC_CORE_LIBS zvec_ailego zvec_turbo sparsehash magic_enum rabitqlib) @@ -79,8 +82,18 @@ if(CMAKE_SYSTEM_NAME STREQUAL "Linux") list(APPEND ZVEC_CORE_LIBS ${CMAKE_DL_LIBS}) endif() +set(ZVEC_CORE_LIBRARY_OPTIONS) +if(WIN32 AND (BUILD_ZVEC_CORE_SHARED OR BUILD_ZVEC_SHARED)) + list(APPEND ZVEC_CORE_LIBRARY_OPTIONS OBJECTS) + if(BUILD_ZVEC_CORE_SHARED) + list(APPEND ZVEC_CORE_LIBRARY_OPTIONS + EXPORT_DEF ZVEC_CORE_BUILD_SHARED + ) + endif() +endif() + cc_library( - NAME zvec_core STATIC STRICT PACKED + NAME zvec_core STATIC STRICT PACKED ${ZVEC_CORE_LIBRARY_OPTIONS} SRCS ${ALL_CORE_SRCS} LIBS ${ZVEC_CORE_LIBS} INCS . ${PROJECT_ROOT_DIR}/src/core diff --git a/src/core/interface/index.cc b/src/core/interface/index.cc index 755f673..56bf9f7 100644 --- a/src/core/interface/index.cc +++ b/src/core/interface/index.cc @@ -53,6 +53,37 @@ core::IndexContext::Pointer &Index::acquire_context() { return _context_list[context_index_]; } +int Index::Train() { + is_trained_ = true; + return 0; +} + +BaseIndexParam::Pointer Index::GetParam() const { + return std::make_shared(param_); +} + +bool Index::IsTrained() const { + return is_trained_; +} + +uint32_t Index::GetDocCount() const { + if (streamer_ == nullptr) { + return -1; + } + if (is_sparse_) { + return streamer_->create_sparse_provider()->count(); + } + return streamer_->create_provider()->count(); +} + +core::IndexStreamer::Pointer Index::index_searcher() { + return streamer_; +} + +core::IndexProvider::Pointer Index::create_index_provider() const { + return streamer_->create_provider(); +} + int Index::ParseMetricName(const BaseIndexParam ¶m) { std::string metric_name; if (is_sparse_) { diff --git a/src/core/interface/index_param.cc b/src/core/interface/index_param.cc index 5d75276..29ce8f4 100644 --- a/src/core/interface/index_param.cc +++ b/src/core/interface/index_param.cc @@ -18,6 +18,135 @@ namespace zvec { namespace core_interface { + +BaseIndexQueryParam::BaseIndexQueryParam() = default; +BaseIndexQueryParam::BaseIndexQueryParam(const BaseIndexQueryParam &) = default; +BaseIndexQueryParam::BaseIndexQueryParam(BaseIndexQueryParam &&) noexcept = + default; +BaseIndexQueryParam &BaseIndexQueryParam::operator=( + const BaseIndexQueryParam &) = default; +BaseIndexQueryParam &BaseIndexQueryParam::operator=( + BaseIndexQueryParam &&) noexcept = default; +BaseIndexQueryParam::~BaseIndexQueryParam() = default; + +FlatQueryParam::FlatQueryParam() = default; +FlatQueryParam::FlatQueryParam(const FlatQueryParam &) = default; +FlatQueryParam::FlatQueryParam(FlatQueryParam &&) noexcept = default; +FlatQueryParam &FlatQueryParam::operator=(const FlatQueryParam &) = default; +FlatQueryParam &FlatQueryParam::operator=(FlatQueryParam &&) noexcept = default; +FlatQueryParam::~FlatQueryParam() = default; + +BaseIndexQueryParam::Pointer FlatQueryParam::Clone() const { + return std::make_shared(*this); +} + +HNSWQueryParam::HNSWQueryParam() = default; +HNSWQueryParam::HNSWQueryParam(const HNSWQueryParam &) = default; +HNSWQueryParam::HNSWQueryParam(HNSWQueryParam &&) noexcept = default; +HNSWQueryParam &HNSWQueryParam::operator=(const HNSWQueryParam &) = default; +HNSWQueryParam &HNSWQueryParam::operator=(HNSWQueryParam &&) noexcept = default; +HNSWQueryParam::~HNSWQueryParam() = default; + +BaseIndexQueryParam::Pointer HNSWQueryParam::Clone() const { + return std::make_shared(*this); +} + +HNSWRabitqQueryParam::HNSWRabitqQueryParam() = default; +HNSWRabitqQueryParam::HNSWRabitqQueryParam(const HNSWRabitqQueryParam &) = + default; +HNSWRabitqQueryParam::HNSWRabitqQueryParam(HNSWRabitqQueryParam &&) noexcept = + default; +HNSWRabitqQueryParam &HNSWRabitqQueryParam::operator=( + const HNSWRabitqQueryParam &) = default; +HNSWRabitqQueryParam &HNSWRabitqQueryParam::operator=( + HNSWRabitqQueryParam &&) noexcept = default; +HNSWRabitqQueryParam::~HNSWRabitqQueryParam() = default; + +BaseIndexQueryParam::Pointer HNSWRabitqQueryParam::Clone() const { + return std::make_shared(*this); +} + +IVFQueryParam::IVFQueryParam() = default; +IVFQueryParam::IVFQueryParam(const IVFQueryParam &) = default; +IVFQueryParam::IVFQueryParam(IVFQueryParam &&) noexcept = default; +IVFQueryParam &IVFQueryParam::operator=(const IVFQueryParam &) = default; +IVFQueryParam &IVFQueryParam::operator=(IVFQueryParam &&) noexcept = default; +IVFQueryParam::~IVFQueryParam() = default; + +BaseIndexQueryParam::Pointer IVFQueryParam::Clone() const { + auto cloned_this = std::make_shared(*this); + cloned_this->l1QueryParam = l1QueryParam ? l1QueryParam->Clone() : nullptr; + cloned_this->l2QueryParam = l2QueryParam ? l2QueryParam->Clone() : nullptr; + return cloned_this; +} + +DiskAnnQueryParam::DiskAnnQueryParam() = default; +DiskAnnQueryParam::DiskAnnQueryParam(const DiskAnnQueryParam &) = default; +DiskAnnQueryParam::DiskAnnQueryParam(DiskAnnQueryParam &&) noexcept = default; +DiskAnnQueryParam &DiskAnnQueryParam::operator=(const DiskAnnQueryParam &) = + default; +DiskAnnQueryParam &DiskAnnQueryParam::operator=(DiskAnnQueryParam &&) noexcept = + default; +DiskAnnQueryParam::~DiskAnnQueryParam() = default; + +BaseIndexQueryParam::Pointer DiskAnnQueryParam::Clone() const { + return std::make_shared(*this); +} + +BaseIndexParam::BaseIndexParam(IndexType type, MetricType metric, int dim, + int ver) + : index_type(type), metric_type(metric), dimension(dim), version(ver) {} +BaseIndexParam::BaseIndexParam(const BaseIndexParam &) = default; +BaseIndexParam &BaseIndexParam::operator=(const BaseIndexParam &) = default; +BaseIndexParam::~BaseIndexParam() = default; + +IVFIndexParam::IVFIndexParam() : BaseIndexParam(IndexType::kIVF) {} +IVFIndexParam::IVFIndexParam(int nlist, int niters, + std::shared_ptr l1Index, + std::shared_ptr l2Index) + : BaseIndexParam(IndexType::kIVF), + nlist(nlist), + niters(niters), + l1Index(std::move(l1Index)), + l2Index(std::move(l2Index)) {} +IVFIndexParam::IVFIndexParam(MetricType metric, int dim, int nlist, int niters, + std::shared_ptr l1Index, + std::shared_ptr l2Index) + : BaseIndexParam(IndexType::kIVF, metric, dim), + nlist(nlist), + niters(niters), + l1Index(std::move(l1Index)), + l2Index(std::move(l2Index)) {} +IVFIndexParam::IVFIndexParam(const IVFIndexParam &) = default; +IVFIndexParam::IVFIndexParam(IVFIndexParam &&) = default; +IVFIndexParam &IVFIndexParam::operator=(const IVFIndexParam &) = default; +IVFIndexParam &IVFIndexParam::operator=(IVFIndexParam &&) = default; +IVFIndexParam::~IVFIndexParam() = default; + +BaseIndexQueryParam::Pointer VamanaQueryParam::Clone() const { + return std::make_shared(*this); +} + +HNSWRabitqIndexParam::HNSWRabitqIndexParam() + : BaseIndexParam(IndexType::kHNSWRabitq) {} +HNSWRabitqIndexParam::HNSWRabitqIndexParam(int m, int ef_construction) + : BaseIndexParam(IndexType::kHNSWRabitq), + m(m), + ef_construction(ef_construction) {} +HNSWRabitqIndexParam::HNSWRabitqIndexParam(MetricType metric, int dim, int m, + int ef_construction) + : BaseIndexParam(IndexType::kHNSWRabitq, metric, dim), + m(m), + ef_construction(ef_construction) {} +HNSWRabitqIndexParam::HNSWRabitqIndexParam(const HNSWRabitqIndexParam &) = + default; +HNSWRabitqIndexParam::HNSWRabitqIndexParam(HNSWRabitqIndexParam &&) = default; +HNSWRabitqIndexParam &HNSWRabitqIndexParam::operator=( + const HNSWRabitqIndexParam &) = default; +HNSWRabitqIndexParam &HNSWRabitqIndexParam::operator=(HNSWRabitqIndexParam &&) = + default; +HNSWRabitqIndexParam::~HNSWRabitqIndexParam() = default; + ailego::JsonObject BaseIndexParam::SerializeToJsonObject( bool omit_empty_value) const { ailego::JsonObject json_obj; @@ -340,4 +469,4 @@ bool QuantizerParam::DeserializeFromJsonObject( } // namespace core_interface -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/core/interface/indexes/diskann_index.cc b/src/core/interface/indexes/diskann_index.cc index 5af4467..e377f53 100644 --- a/src/core/interface/indexes/diskann_index.cc +++ b/src/core/interface/indexes/diskann_index.cc @@ -16,11 +16,76 @@ #include #include #include +#if DISKANN_SUPPORTED #include "algorithm/diskann/diskann_params.h" #include "holder_builder.h" +#endif namespace zvec::core_interface { +#if !DISKANN_SUPPORTED + +int DiskAnnIndex::CreateAndInitStreamer(const BaseIndexParam ¶m) { + (void)param; + LOG_ERROR("DiskAnn is not supported on this platform (Linux x86_64 only)"); + return core::IndexError_Unsupported; +} + +int DiskAnnIndex::Open(const std::string &file_path, + StorageOptions storage_options) { + (void)file_path; + (void)storage_options; + LOG_ERROR("DiskAnn is not supported on this platform (Linux x86_64 only)"); + return core::IndexError_Unsupported; +} + +int DiskAnnIndex::GenerateHolder() { + LOG_ERROR("DiskAnn is not supported on this platform (Linux x86_64 only)"); + return core::IndexError_Unsupported; +} + +int DiskAnnIndex::Add(const VectorData &vector, uint32_t doc_id) { + (void)vector; + (void)doc_id; + LOG_ERROR("DiskAnn is not supported on this platform (Linux x86_64 only)"); + return core::IndexError_Unsupported; +} + +int DiskAnnIndex::Train() { + LOG_ERROR("DiskAnn is not supported on this platform (Linux x86_64 only)"); + return core::IndexError_Unsupported; +} + +int DiskAnnIndex::_dense_fetch(const uint32_t doc_id, + VectorDataBuffer *vector_data_buffer) { + (void)doc_id; + (void)vector_data_buffer; + LOG_ERROR("DiskAnn is not supported on this platform (Linux x86_64 only)"); + return core::IndexError_Unsupported; +} + +int DiskAnnIndex::_prepare_for_search( + const VectorData &query, const BaseIndexQueryParam::Pointer &search_param, + core::IndexContext::Pointer &context) { + (void)query; + (void)search_param; + (void)context; + LOG_ERROR("DiskAnn is not supported on this platform (Linux x86_64 only)"); + return core::IndexError_Unsupported; +} + +int DiskAnnIndex::Merge(const std::vector &indexes, + const IndexFilter &filter, + const MergeOptions &options) { + (void)indexes; + (void)filter; + (void)options; + LOG_ERROR("DiskAnn is not supported on this platform (Linux x86_64 only)"); + return core::IndexError_Unsupported; +} + +#else + int DiskAnnIndex::CreateAndInitStreamer(const BaseIndexParam ¶m) { if (is_sparse_) { LOG_ERROR("Failed to create streamer. Sparse is not Supported."); @@ -271,4 +336,6 @@ int DiskAnnIndex::Merge(const std::vector &indexes, return 0; } -} // namespace zvec::core_interface \ No newline at end of file +#endif // DISKANN_SUPPORTED + +} // namespace zvec::core_interface diff --git a/src/core/interface/vector_source.cc b/src/core/interface/vector_source.cc new file mode 100644 index 0000000..ea19016 --- /dev/null +++ b/src/core/interface/vector_source.cc @@ -0,0 +1,32 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +namespace zvec { +namespace core { + +VectorSource::VectorSource() = default; + +VectorSource::~VectorSource() = default; + +void VectorSource::get_vectors(const uint32_t *ids, uint32_t count, + const void **out) const { + for (uint32_t i = 0; i < count; ++i) { + out[i] = get_vector(ids[i]); + } +} + +} // namespace core +} // namespace zvec diff --git a/src/db/CMakeLists.txt b/src/db/CMakeLists.txt index 69426d1..b8bb1de 100644 --- a/src/db/CMakeLists.txt +++ b/src/db/CMakeLists.txt @@ -30,8 +30,16 @@ if(NOT ANDROID AND AUTO_DETECT_ARCH) endif() endif() +set(ZVEC_DB_LIBRARY_OPTIONS) +if(WIN32 AND BUILD_ZVEC_SHARED) + list(APPEND ZVEC_DB_LIBRARY_OPTIONS + OBJECTS + EXPORT_DEF ZVEC_DB_BUILD_SHARED + ) +endif() + cc_library( - NAME zvec STATIC STRICT SRCS_NO_GLOB PACKED + NAME zvec STATIC STRICT SRCS_NO_GLOB PACKED ${ZVEC_DB_LIBRARY_OPTIONS} SRCS ${ALL_DB_SRCS} ${CMAKE_CURRENT_BINARY_DIR}/proto/zvec.pb.cc INCS . ${CMAKE_CURRENT_BINARY_DIR} PUBINCS ${PROJECT_ROOT_DIR}/src/include diff --git a/src/include/zvec/ailego/buffer/block_eviction_queue.h b/src/include/zvec/ailego/buffer/block_eviction_queue.h index fa5aff2..b93a62b 100644 --- a/src/include/zvec/ailego/buffer/block_eviction_queue.h +++ b/src/include/zvec/ailego/buffer/block_eviction_queue.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "concurrentqueue.h" #if defined(_MSC_VER) @@ -46,7 +47,7 @@ using eviction_key_t = size_t; using block_id_t = size_t; using version_t = size_t; -class EvictableBlockOwner { +class ZVEC_AILEGO_API EvictableBlockOwner { public: virtual ~EvictableBlockOwner() = default; diff --git a/src/include/zvec/ailego/buffer/vector_page_table.h b/src/include/zvec/ailego/buffer/vector_page_table.h index 02d19bb..3e1372f 100644 --- a/src/include/zvec/ailego/buffer/vector_page_table.h +++ b/src/include/zvec/ailego/buffer/vector_page_table.h @@ -33,6 +33,7 @@ #include #include #include +#include #include "block_eviction_queue.h" #include "concurrentqueue.h" @@ -45,7 +46,7 @@ namespace ailego { extern const size_t kVectorPageSize; -class VectorPageTable : public EvictableBlockOwner { +class ZVEC_AILEGO_API VectorPageTable : public EvictableBlockOwner { struct Entry { std::atomic ref_count; std::atomic in_evict_queue; @@ -196,7 +197,7 @@ class VectorPageTable : public EvictableBlockOwner { class VecBufferPoolHandle; -class VecBufferPool { +class ZVEC_AILEGO_API VecBufferPool { public: typedef std::shared_ptr Pointer; @@ -266,7 +267,7 @@ class VecBufferPool { std::unique_ptr block_mutexes_{}; }; -class VecBufferPoolHandle { +class ZVEC_AILEGO_API VecBufferPoolHandle { public: VecBufferPoolHandle(VecBufferPool &pool) : pool_(pool) {} VecBufferPoolHandle(VecBufferPoolHandle &&other) : pool_(other.pool_) {} diff --git a/src/include/zvec/ailego/container/params.h b/src/include/zvec/ailego/container/params.h index 6e14a59..a23c089 100644 --- a/src/include/zvec/ailego/container/params.h +++ b/src/include/zvec/ailego/container/params.h @@ -15,6 +15,7 @@ #pragma once #include +#include namespace zvec { namespace ailego { @@ -45,7 +46,7 @@ namespace ailego { /*! Index Params */ -class Params { +class ZVEC_AILEGO_API Params { public: //! Constructor Params(void) : hypercube_() {} @@ -773,4 +774,4 @@ class Params { #undef _TRYING_CONVERT_STRING } // namespace ailego -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/include/zvec/ailego/io/file.h b/src/include/zvec/ailego/io/file.h index 18c5f49..614262b 100644 --- a/src/include/zvec/ailego/io/file.h +++ b/src/include/zvec/ailego/io/file.h @@ -16,13 +16,14 @@ #include #include +#include namespace zvec { namespace ailego { /*! File Utility */ -class File { +class ZVEC_AILEGO_API File { public: //! Native Handle in OS typedef FileHelper::NativeHandle NativeHandle; diff --git a/src/include/zvec/ailego/io/mmap_file.h b/src/include/zvec/ailego/io/mmap_file.h index f6404a6..f19ec06 100644 --- a/src/include/zvec/ailego/io/mmap_file.h +++ b/src/include/zvec/ailego/io/mmap_file.h @@ -16,13 +16,14 @@ #include #include +#include namespace zvec { namespace ailego { /*! Memory Mapping File */ -class MMapFile { +class ZVEC_AILEGO_API MMapFile { public: //! Constructor MMapFile(void) diff --git a/src/include/zvec/ailego/logger/logger.h b/src/include/zvec/ailego/logger/logger.h index a7c90a9..37e576e 100644 --- a/src/include/zvec/ailego/logger/logger.h +++ b/src/include/zvec/ailego/logger/logger.h @@ -18,6 +18,7 @@ #include #include #include +#include // Define printf format attribute for GCC/Clang, empty for MSVC #if defined(__GNUC__) || defined(__clang__) @@ -79,7 +80,7 @@ namespace ailego { /*! Index Logger */ -struct Logger { +struct ZVEC_AILEGO_API Logger { //! Index Logger Pointer typedef std::shared_ptr Pointer; @@ -123,7 +124,7 @@ struct Logger { /*! Index Logger Broker */ -class LoggerBroker { +class ZVEC_AILEGO_API LoggerBroker { public: //! Register Logger static Logger::Pointer Register(Logger::Pointer logger) { diff --git a/src/include/zvec/ailego/parallel/thread_pool.h b/src/include/zvec/ailego/parallel/thread_pool.h index 150f014..a7b63ed 100644 --- a/src/include/zvec/ailego/parallel/thread_pool.h +++ b/src/include/zvec/ailego/parallel/thread_pool.h @@ -22,13 +22,14 @@ #include #include #include +#include namespace zvec { namespace ailego { /*! Thread Pool */ -class ThreadPool { +class ZVEC_AILEGO_API ThreadPool { public: /*! Thread Pool Task Group */ diff --git a/src/include/zvec/ailego/utility/file_helper.h b/src/include/zvec/ailego/utility/file_helper.h index c607cb4..5be07b4 100644 --- a/src/include/zvec/ailego/utility/file_helper.h +++ b/src/include/zvec/ailego/utility/file_helper.h @@ -20,13 +20,14 @@ #include #include #include +#include namespace zvec { namespace ailego { /*! File Helper Module */ -struct FileHelper { +struct ZVEC_AILEGO_API FileHelper { #if defined(_WIN32) || defined(_WIN64) //! Native Handle in Windows typedef void *NativeHandle; diff --git a/src/include/zvec/ailego/utility/float_helper.h b/src/include/zvec/ailego/utility/float_helper.h index 5dc2fe6..575da1b 100644 --- a/src/include/zvec/ailego/utility/float_helper.h +++ b/src/include/zvec/ailego/utility/float_helper.h @@ -16,13 +16,14 @@ #include #include +#include namespace zvec { namespace ailego { /*! Float Helper */ -struct FloatHelper { +struct ZVEC_AILEGO_API FloatHelper { //! Convert FP16 to FP32 static float ToFP32(uint16_t val); diff --git a/src/include/zvec/ailego/utility/string_helper.h b/src/include/zvec/ailego/utility/string_helper.h index 2dab937..ed21652 100644 --- a/src/include/zvec/ailego/utility/string_helper.h +++ b/src/include/zvec/ailego/utility/string_helper.h @@ -19,13 +19,14 @@ #include #include #include +#include namespace zvec { namespace ailego { /*! String Helper */ -struct StringHelper { +struct ZVEC_AILEGO_API StringHelper { //! Return true if the `ref` starts with the given prefix static bool StartsWith(const std::string &ref, const std::string &prefix); diff --git a/src/include/zvec/ailego/utility/time_helper.h b/src/include/zvec/ailego/utility/time_helper.h index 40bc381..e0013d2 100644 --- a/src/include/zvec/ailego/utility/time_helper.h +++ b/src/include/zvec/ailego/utility/time_helper.h @@ -16,13 +16,14 @@ #include #include +#include namespace zvec { namespace ailego { /*! Monotime */ -struct Monotime { +struct ZVEC_AILEGO_API Monotime { //! Retrieve monotonic time in nanoseconds static uint64_t NanoSeconds(void); @@ -38,7 +39,7 @@ struct Monotime { /*! Realtime */ -struct Realtime { +struct ZVEC_AILEGO_API Realtime { //! Retrieve system time in nanoseconds static uint64_t NanoSeconds(void); @@ -116,7 +117,7 @@ struct Realtime { /*! Thread-specific CPU time */ -struct CPUtime { +struct ZVEC_AILEGO_API CPUtime { //! Retrieve CPU time in nanoseconds static uint64_t NanoSeconds(void); @@ -132,7 +133,7 @@ struct CPUtime { /*! Elapsed Time */ -class ElapsedTime { +class ZVEC_AILEGO_API ElapsedTime { public: //! Constructor ElapsedTime(void) : stamp_(Monotime::NanoSeconds()) {} @@ -168,7 +169,7 @@ class ElapsedTime { /*! Elapsed CPU Time */ -class ElapsedCPUTime { +class ZVEC_AILEGO_API ElapsedCPUTime { public: //! Constructor ElapsedCPUTime(void) : stamp_(CPUtime::NanoSeconds()) {} diff --git a/src/include/zvec/core/interface/index.h b/src/include/zvec/core/interface/index.h index 118d06d..3006718 100644 --- a/src/include/zvec/core/interface/index.h +++ b/src/include/zvec/core/interface/index.h @@ -32,11 +32,12 @@ #include #include #include +#include #include "zvec/core/framework/index_provider.h" namespace zvec::core_interface { -class IndexFactory; +class ZVEC_CORE_API IndexFactory; struct DenseVector { const void *data; @@ -104,7 +105,7 @@ struct SearchResult { std::vector> group_reverted_sparse_values_list_{}; }; -class Index { +class ZVEC_CORE_API Index { public: typedef std::shared_ptr Pointer; virtual ~Index() = default; @@ -119,10 +120,7 @@ class Index { // // TODO: use holder // virtual int Build() = 0; - virtual int Train() { - is_trained_ = true; - return 0; - } + virtual int Train(); // virtual int Dump(const std::string &file_path) = 0; virtual int Merge(const std::vector &indexes, @@ -145,34 +143,17 @@ class Index { const core::VectorSource &src, SearchResult *result); - virtual BaseIndexParam::Pointer GetParam() const { - return std::make_shared(param_); - } + virtual BaseIndexParam::Pointer GetParam() const; - virtual bool IsTrained() const { - return is_trained_; - } + virtual bool IsTrained() const; bool IsDirty() const; - uint32_t GetDocCount() const { - if (streamer_ == nullptr) { - return -1; - } - if (is_sparse_) { - return streamer_->create_sparse_provider()->count(); - } else { - return streamer_->create_provider()->count(); - } - } + uint32_t GetDocCount() const; - core::IndexStreamer::Pointer index_searcher() { - return streamer_; - } + core::IndexStreamer::Pointer index_searcher(); - core::IndexProvider::Pointer create_index_provider() const { - return streamer_->create_provider(); - } + core::IndexProvider::Pointer create_index_provider() const; static std::string get_metric_name(MetricType metric_type, bool is_sparse); @@ -227,9 +208,6 @@ class Index { protected: bool init_context(); core::IndexContext::Pointer &acquire_context(); - void release_context() { - // context_list_[get_context_index()]->reset(); - } protected: bool is_trained_{false}; @@ -256,7 +234,7 @@ class Index { }; -class FlatIndex : public Index { +class ZVEC_CORE_API FlatIndex : public Index { public: FlatIndex() = default; // FlatIndex(const FlatIndexParam ¶m) : param_(param) {} @@ -274,7 +252,7 @@ class FlatIndex : public Index { FlatIndexParam param_{}; }; -class IVFIndex : public Index { +class ZVEC_CORE_API IVFIndex : public Index { public: IVFIndex() = default; @@ -307,7 +285,7 @@ class IVFIndex : public Index { }; -class HNSWIndex : public Index { +class ZVEC_CORE_API HNSWIndex : public Index { public: HNSWIndex() = default; @@ -338,7 +316,7 @@ class HNSWIndex : public Index { HNSWIndexParam param_{}; }; -class VamanaIndex : public Index { +class ZVEC_CORE_API VamanaIndex : public Index { public: VamanaIndex() = default; @@ -355,7 +333,7 @@ class VamanaIndex : public Index { VamanaIndexParam param_{}; }; -class HNSWRabitqIndex : public Index { +class ZVEC_CORE_API HNSWRabitqIndex : public Index { public: HNSWRabitqIndex() = default; @@ -372,7 +350,7 @@ class HNSWRabitqIndex : public Index { HNSWRabitqIndexParam param_{}; }; -class DiskAnnIndex : public Index { +class ZVEC_CORE_API DiskAnnIndex : public Index { public: DiskAnnIndex() = default; diff --git a/src/include/zvec/core/interface/index_factory.h b/src/include/zvec/core/interface/index_factory.h index d0be6ee..6cf3cb3 100644 --- a/src/include/zvec/core/interface/index_factory.h +++ b/src/include/zvec/core/interface/index_factory.h @@ -17,11 +17,12 @@ #include #include #include +#include namespace zvec::core_interface { // 索引的工厂类 -class IndexFactory { +class ZVEC_CORE_API IndexFactory { public: static Index::Pointer CreateAndInitIndex(const BaseIndexParam ¶m); @@ -51,4 +52,4 @@ class IndexFactory { }; -} // namespace zvec::core_interface \ No newline at end of file +} // namespace zvec::core_interface diff --git a/src/include/zvec/core/interface/index_param.h b/src/include/zvec/core/interface/index_param.h index 78bebd5..877e6e3 100644 --- a/src/include/zvec/core/interface/index_param.h +++ b/src/include/zvec/core/interface/index_param.h @@ -24,6 +24,7 @@ #include #include #include +#include #include "zvec/core/framework/index_framework.h" namespace zvec::core_interface { @@ -31,10 +32,10 @@ namespace zvec::core_interface { // #define MAX_EF_CONSTRUCTION 65536 // #define MAX_EF_SEARCH 100 -class IndexFactory; -class Index; -class BaseIndexParam; -class BaseIndexQueryParam; +class ZVEC_CORE_API IndexFactory; +class ZVEC_CORE_API Index; +class ZVEC_CORE_API BaseIndexParam; +class ZVEC_CORE_API BaseIndexQueryParam; struct StorageOptions { enum class StorageType { kNone, kMMAP, kMemory, kBufferPool }; @@ -96,7 +97,7 @@ enum class QuantizerType { kUniformInt8, // Global uniform int8 quantization (shared scale/bias). }; -struct SerializableBase { +struct ZVEC_CORE_API SerializableBase { std::string SerializeToJson(bool omit_empty_value = false) const { return zvec::ailego::JsonValue(SerializeToJsonObject(omit_empty_value)) .as_json_string() @@ -119,7 +120,7 @@ struct SerializableBase { }; // TODO: maybe a base class for quantizer? -struct QuantizerParam : public SerializableBase { +struct ZVEC_CORE_API QuantizerParam : public SerializableBase { QuantizerType type = QuantizerType::kNone; int num_subquantizers = 8; // M int num_bits = 8; // bits per subquantizer @@ -175,11 +176,16 @@ struct GroupByParam { }; // --- Query Parameters (can be passed to search methods) --- -class BaseIndexQueryParam { +class ZVEC_CORE_API BaseIndexQueryParam { public: using Pointer = std::shared_ptr; - virtual ~BaseIndexQueryParam() = default; + BaseIndexQueryParam(); + BaseIndexQueryParam(const BaseIndexQueryParam &); + BaseIndexQueryParam(BaseIndexQueryParam &&) noexcept; + BaseIndexQueryParam &operator=(const BaseIndexQueryParam &); + BaseIndexQueryParam &operator=(BaseIndexQueryParam &&) noexcept; + virtual ~BaseIndexQueryParam(); uint32_t topk = 10; bool fetch_vector = false; @@ -193,75 +199,97 @@ class BaseIndexQueryParam { virtual Pointer Clone() const = 0; }; -struct FlatQueryParam : public BaseIndexQueryParam { +struct ZVEC_CORE_API FlatQueryParam : public BaseIndexQueryParam { using Pointer = std::shared_ptr; - BaseIndexQueryParam::Pointer Clone() const override { - return std::make_shared(*this); - } + FlatQueryParam(); + FlatQueryParam(const FlatQueryParam &); + FlatQueryParam(FlatQueryParam &&) noexcept; + FlatQueryParam &operator=(const FlatQueryParam &); + FlatQueryParam &operator=(FlatQueryParam &&) noexcept; + ~FlatQueryParam() override; + + BaseIndexQueryParam::Pointer Clone() const override; }; -struct HNSWQueryParam : public BaseIndexQueryParam { +struct ZVEC_CORE_API HNSWQueryParam : public BaseIndexQueryParam { using Pointer = std::shared_ptr; + HNSWQueryParam(); + HNSWQueryParam(const HNSWQueryParam &); + HNSWQueryParam(HNSWQueryParam &&) noexcept; + HNSWQueryParam &operator=(const HNSWQueryParam &); + HNSWQueryParam &operator=(HNSWQueryParam &&) noexcept; + ~HNSWQueryParam() override; + uint32_t ef_search = kDefaultHnswEfSearch; uint32_t prefetch_offset = kDefaultPrefetchOffset; uint32_t prefetch_lines = kDefaultPrefetchLines; - BaseIndexQueryParam::Pointer Clone() const override { - return std::make_shared(*this); - } + BaseIndexQueryParam::Pointer Clone() const override; }; -struct HNSWRabitqQueryParam : public BaseIndexQueryParam { +struct ZVEC_CORE_API HNSWRabitqQueryParam : public BaseIndexQueryParam { using Pointer = std::shared_ptr; + HNSWRabitqQueryParam(); + HNSWRabitqQueryParam(const HNSWRabitqQueryParam &); + HNSWRabitqQueryParam(HNSWRabitqQueryParam &&) noexcept; + HNSWRabitqQueryParam &operator=(const HNSWRabitqQueryParam &); + HNSWRabitqQueryParam &operator=(HNSWRabitqQueryParam &&) noexcept; + ~HNSWRabitqQueryParam() override; + uint32_t ef_search = kDefaultHnswEfSearch; - BaseIndexQueryParam::Pointer Clone() const override { - return std::make_shared(*this); - } + BaseIndexQueryParam::Pointer Clone() const override; }; -struct IVFQueryParam : public BaseIndexQueryParam { +struct ZVEC_CORE_API IVFQueryParam : public BaseIndexQueryParam { + IVFQueryParam(); + IVFQueryParam(const IVFQueryParam &); + IVFQueryParam(IVFQueryParam &&) noexcept; + IVFQueryParam &operator=(const IVFQueryParam &); + IVFQueryParam &operator=(IVFQueryParam &&) noexcept; + ~IVFQueryParam() override; + int nprobe = 10; std::shared_ptr l1QueryParam = nullptr; std::shared_ptr l2QueryParam = nullptr; using Pointer = std::shared_ptr; - BaseIndexQueryParam::Pointer Clone() const override { - auto cloned_this = std::make_shared(*this); - cloned_this->l1QueryParam = l1QueryParam ? l1QueryParam->Clone() : nullptr; - cloned_this->l2QueryParam = l2QueryParam ? l2QueryParam->Clone() : nullptr; - return cloned_this; - } + BaseIndexQueryParam::Pointer Clone() const override; }; -struct DiskAnnQueryParam : public BaseIndexQueryParam { +struct ZVEC_CORE_API DiskAnnQueryParam : public BaseIndexQueryParam { using Pointer = std::shared_ptr; + DiskAnnQueryParam(); + DiskAnnQueryParam(const DiskAnnQueryParam &); + DiskAnnQueryParam(DiskAnnQueryParam &&) noexcept; + DiskAnnQueryParam &operator=(const DiskAnnQueryParam &); + DiskAnnQueryParam &operator=(DiskAnnQueryParam &&) noexcept; + ~DiskAnnQueryParam() override; + // Beam-search candidate list size used at query time. Larger values improve // recall at the cost of latency. uint32_t list_size = kDefaultDiskAnnListSize; - BaseIndexQueryParam::Pointer Clone() const override { - return std::make_shared(*this); - } + BaseIndexQueryParam::Pointer Clone() const override; }; // --- Construction Parameters --- // template -class BaseIndexParam : public SerializableBase { +class ZVEC_CORE_API BaseIndexParam : public SerializableBase { public: using Pointer = std::shared_ptr; explicit BaseIndexParam(IndexType type = IndexType::kNone, MetricType metric = MetricType::kL2sq, int dim = 0, - int ver = 0) - : index_type(type), metric_type(metric), dimension(dim), version(ver) {} - - virtual ~BaseIndexParam() = default; + int ver = 0); + BaseIndexParam(const BaseIndexParam &); + BaseIndexParam &operator=(const BaseIndexParam &); + virtual ~BaseIndexParam(); IndexType index_type = IndexType::kNone; MetricType metric_type = MetricType::kL2sq; @@ -293,7 +321,7 @@ class BaseIndexParam : public SerializableBase { bool omit_empty_value = false) const override; }; -struct FlatIndexParam : public BaseIndexParam { +struct ZVEC_CORE_API FlatIndexParam : public BaseIndexParam { using Pointer = std::shared_ptr; FlatIndexParam() : BaseIndexParam(IndexType::kFlat) {} @@ -305,7 +333,7 @@ struct FlatIndexParam : public BaseIndexParam { bool omit_empty_value = false) const override; }; -struct IVFIndexParam : public BaseIndexParam { +struct ZVEC_CORE_API IVFIndexParam : public BaseIndexParam { using Pointer = std::shared_ptr; int nlist = 1024; int niters = 10; @@ -314,24 +342,17 @@ struct IVFIndexParam : public BaseIndexParam { bool use_soar = false; // Constructors with delegation - IVFIndexParam() : BaseIndexParam(IndexType::kIVF) {} - + IVFIndexParam(); IVFIndexParam(int nlist, int niters, std::shared_ptr l1Index, - std::shared_ptr l2Index) - : BaseIndexParam(IndexType::kIVF), - nlist(nlist), - niters(niters), - l1Index(std::move(l1Index)), - l2Index(std::move(l2Index)) {} - + std::shared_ptr l2Index); IVFIndexParam(MetricType metric, int dim, int nlist, int niters, std::shared_ptr l1Index, - std::shared_ptr l2Index) - : BaseIndexParam(IndexType::kIVF, metric, dim), - nlist(nlist), - niters(niters), - l1Index(std::move(l1Index)), - l2Index(std::move(l2Index)) {} + std::shared_ptr l2Index); + IVFIndexParam(const IVFIndexParam &); + IVFIndexParam(IVFIndexParam &&); + IVFIndexParam &operator=(const IVFIndexParam &); + IVFIndexParam &operator=(IVFIndexParam &&); + ~IVFIndexParam() override; // query param: // topk of l1Index's param ==== IVFIndexQueryParam.nprobe @@ -341,7 +362,7 @@ struct IVFIndexParam : public BaseIndexParam { // IVFIndexParam.quantization === l2Index's quantization }; -struct HNSWIndexParam : public BaseIndexParam { +struct ZVEC_CORE_API HNSWIndexParam : public BaseIndexParam { using Pointer = std::shared_ptr; int m = kDefaultHnswNeighborCnt; int ef_construction = kDefaultHnswEfConstruction; @@ -366,7 +387,7 @@ struct HNSWIndexParam : public BaseIndexParam { bool omit_empty_value = false) const override; }; -struct VamanaIndexParam : public BaseIndexParam { +struct ZVEC_CORE_API VamanaIndexParam : public BaseIndexParam { using Pointer = std::shared_ptr; int max_degree = kDefaultVamanaMaxDegree; int search_list_size = kDefaultVamanaSearchListSize; @@ -396,19 +417,17 @@ struct VamanaIndexParam : public BaseIndexParam { bool omit_empty_value = false) const override; }; -struct VamanaQueryParam : public BaseIndexQueryParam { +struct ZVEC_CORE_API VamanaQueryParam : public BaseIndexQueryParam { using Pointer = std::shared_ptr; uint32_t ef_search = kDefaultVamanaEfSearch; uint32_t prefetch_offset = kDefaultPrefetchOffset; uint32_t prefetch_lines = kDefaultPrefetchLines; - BaseIndexQueryParam::Pointer Clone() const override { - return std::make_shared(*this); - } + BaseIndexQueryParam::Pointer Clone() const override; }; -struct HNSWRabitqIndexParam : public BaseIndexParam { +struct ZVEC_CORE_API HNSWRabitqIndexParam : public BaseIndexParam { using Pointer = std::shared_ptr; // HNSW parameters @@ -423,17 +442,14 @@ struct HNSWRabitqIndexParam : public BaseIndexParam { core::IndexReformer::Pointer reformer = nullptr; // Constructors with delegation - HNSWRabitqIndexParam() : BaseIndexParam(IndexType::kHNSWRabitq) {} - - HNSWRabitqIndexParam(int m, int ef_construction) - : BaseIndexParam(IndexType::kHNSWRabitq), - m(m), - ef_construction(ef_construction) {} - - HNSWRabitqIndexParam(MetricType metric, int dim, int m, int ef_construction) - : BaseIndexParam(IndexType::kHNSWRabitq, metric, dim), - m(m), - ef_construction(ef_construction) {} + HNSWRabitqIndexParam(); + HNSWRabitqIndexParam(int m, int ef_construction); + HNSWRabitqIndexParam(MetricType metric, int dim, int m, int ef_construction); + HNSWRabitqIndexParam(const HNSWRabitqIndexParam &); + HNSWRabitqIndexParam(HNSWRabitqIndexParam &&); + HNSWRabitqIndexParam &operator=(const HNSWRabitqIndexParam &); + HNSWRabitqIndexParam &operator=(HNSWRabitqIndexParam &&); + ~HNSWRabitqIndexParam() override; protected: bool DeserializeFromJsonObject(const ailego::JsonObject &json_obj) override; @@ -441,7 +457,7 @@ struct HNSWRabitqIndexParam : public BaseIndexParam { bool omit_empty_value = false) const override; }; -struct DiskAnnIndexParam : public BaseIndexParam { +struct ZVEC_CORE_API DiskAnnIndexParam : public BaseIndexParam { using Pointer = std::shared_ptr; int max_degree = kDefaultDiskAnnMaxDegree; diff --git a/src/include/zvec/core/interface/vector_source.h b/src/include/zvec/core/interface/vector_source.h index 717ba51..66b4da5 100644 --- a/src/include/zvec/core/interface/vector_source.h +++ b/src/include/zvec/core/interface/vector_source.h @@ -15,22 +15,20 @@ #pragma once #include +#include namespace zvec { namespace core { -class VectorSource { +class ZVEC_CORE_API VectorSource { public: - virtual ~VectorSource() = default; + VectorSource(); + virtual ~VectorSource(); virtual const void *get_vector(uint32_t node_id) const = 0; virtual void get_vectors(const uint32_t *ids, uint32_t count, - const void **out) const { - for (uint32_t i = 0; i < count; ++i) { - out[i] = get_vector(ids[i]); - } - } + const void **out) const; }; } // namespace core diff --git a/src/include/zvec/db/collection.h b/src/include/zvec/db/collection.h index 83a289c..c4b7757 100644 --- a/src/include/zvec/db/collection.h +++ b/src/include/zvec/db/collection.h @@ -21,10 +21,11 @@ #include #include #include +#include namespace zvec { -class Collection { +class ZVEC_API Collection { public: using Ptr = std::shared_ptr; @@ -119,4 +120,4 @@ class Collection { const std::string &column_name) const = 0; }; -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/include/zvec/db/config.h b/src/include/zvec/db/config.h index 4403f35..4bc4f5c 100644 --- a/src/include/zvec/db/config.h +++ b/src/include/zvec/db/config.h @@ -20,6 +20,7 @@ #include #include #include +#include namespace zvec { @@ -31,7 +32,7 @@ const std::string FILE_LOG_TYPE_NAME = "AppendLogger"; const std::string DEFAULT_LOG_DIR = "./logs"; const std::string DEFAULT_LOG_BASENAME = "zvec.log"; -class GlobalConfig : public ailego::Singleton { +class ZVEC_API GlobalConfig : public ailego::Singleton { friend class ailego::Singleton; public: diff --git a/src/include/zvec/db/doc.h b/src/include/zvec/db/doc.h index 3dbe9a7..785d9c1 100644 --- a/src/include/zvec/db/doc.h +++ b/src/include/zvec/db/doc.h @@ -23,12 +23,13 @@ #include #include #include +#include namespace zvec { using float16_t = ailego::Float16; -class Doc { +class ZVEC_API Doc { public: using Value = std::variant< std::monostate, // 0 - represents null value @@ -355,7 +356,8 @@ class Doc { std::unordered_map fields_; }; -std::string get_value_type_name(const Doc::Value &value, bool is_vector); +ZVEC_API std::string get_value_type_name(const Doc::Value &value, + bool is_vector); using DocPtrList = std::vector; diff --git a/src/include/zvec/db/index_params.h b/src/include/zvec/db/index_params.h index 31ec5b6..5509e1e 100644 --- a/src/include/zvec/db/index_params.h +++ b/src/include/zvec/db/index_params.h @@ -20,6 +20,7 @@ #include #include #include +#include #include "zvec/core/framework/index_provider.h" #include "zvec/core/framework/index_reformer.h" @@ -33,7 +34,7 @@ struct FtsPipelineHelper; /* * Column index params */ -class IndexParams { +class ZVEC_API IndexParams { public: using Ptr = std::shared_ptr; @@ -68,7 +69,7 @@ class IndexParams { /* * Scalar: Invert index params */ -class InvertIndexParams : public IndexParams { +class ZVEC_API InvertIndexParams : public IndexParams { public: InvertIndexParams(bool enable_range_optimization = true, bool enable_extended_wildcard = false) @@ -153,7 +154,7 @@ class QuantizerParam { /* * Column index params */ -class VectorIndexParams : public IndexParams { +class ZVEC_API VectorIndexParams : public IndexParams { public: VectorIndexParams(IndexType type, MetricType metric_type, QuantizeType quantize_type = QuantizeType::UNDEFINED, @@ -207,7 +208,7 @@ class VectorIndexParams : public IndexParams { /* * Vector: Hnsw index params */ -class HnswIndexParams : public VectorIndexParams { +class ZVEC_API HnswIndexParams : public VectorIndexParams { public: HnswIndexParams( MetricType metric_type, int m = core_interface::kDefaultHnswNeighborCnt, @@ -285,7 +286,7 @@ class HnswIndexParams : public VectorIndexParams { bool use_contiguous_memory_{false}; }; -class HnswRabitqIndexParams : public VectorIndexParams { +class ZVEC_API HnswRabitqIndexParams : public VectorIndexParams { public: HnswRabitqIndexParams( MetricType metric_type, @@ -397,7 +398,7 @@ class HnswRabitqIndexParams : public VectorIndexParams { core::IndexReformer::Pointer rabitq_reformer_; }; -class FlatIndexParams : public VectorIndexParams { +class ZVEC_API FlatIndexParams : public VectorIndexParams { public: FlatIndexParams(MetricType metric_type, QuantizeType quantize_type = QuantizeType::UNDEFINED, @@ -446,7 +447,7 @@ inline FlatIndexParams MakeDefaultQuantVectorIndexParams( return FlatIndexParams(metric_type, quantize_type, quantizer_param); } -class IVFIndexParams : public VectorIndexParams { +class ZVEC_API IVFIndexParams : public VectorIndexParams { public: IVFIndexParams(MetricType metric_type, int n_list = 1024, int n_iters = 10, bool use_soar = false, @@ -520,7 +521,7 @@ class IVFIndexParams : public VectorIndexParams { bool use_soar_; }; -class DiskAnnIndexParams : public VectorIndexParams { +class ZVEC_API DiskAnnIndexParams : public VectorIndexParams { public: DiskAnnIndexParams(MetricType metric_type, int max_degree = 100, int list_size = 50, int pq_chunk_num = 0, @@ -601,7 +602,7 @@ class DiskAnnIndexParams : public VectorIndexParams { /* * Vector: Vamana index params */ -class VamanaIndexParams : public VectorIndexParams { +class ZVEC_API VamanaIndexParams : public VectorIndexParams { public: VamanaIndexParams( MetricType metric_type, @@ -743,7 +744,7 @@ class VamanaIndexParams : public VectorIndexParams { * * Not copyable. Use shared_ptr for shared ownership. */ -class FtsIndexParams : public IndexParams { +class ZVEC_API FtsIndexParams : public IndexParams { public: FtsIndexParams(std::string tokenizer_name = "standard", std::vector filters = {"lowercase"}, diff --git a/src/include/zvec/db/query.h b/src/include/zvec/db/query.h index 6d30ccc..c983ac4 100644 --- a/src/include/zvec/db/query.h +++ b/src/include/zvec/db/query.h @@ -23,12 +23,13 @@ #include #include #include +#include namespace zvec { struct VectorViewClause; -struct VectorClause { +struct ZVEC_API VectorClause { std::string query_vector_; std::string sparse_indices_; std::string sparse_values_; @@ -55,7 +56,7 @@ struct FtsClause { std::string match_string_; }; -struct QueryTarget { +struct ZVEC_API QueryTarget { std::string field_name_; std::variant clause_; QueryParams::Ptr query_params_; @@ -131,7 +132,7 @@ inline void QueryTarget::set_sparse_vector(std::string indices, vc.sparse_values_ = std::move(values); } -struct SearchQuery { +struct ZVEC_API SearchQuery { QueryTarget target_; int topk_{0}; std::string filter_; @@ -149,16 +150,18 @@ struct SearchQuery { }; // Validate topk and output_fields bounds. -Status validate_topk_and_output_fields( +ZVEC_API Status validate_topk_and_output_fields( int topk, const std::optional> &output_fields); // Sort sparse indices in-place and check for duplicates. // Returns error if duplicates are found after sorting. -Status sanitize_sparse_vector(VectorClause &vc, const FieldSchema *schema); +ZVEC_API Status sanitize_sparse_vector(VectorClause &vc, + const FieldSchema *schema); // Materializes VectorViewClause into VectorClause if needed, then sorts // sparse indices in place. Operates on the QueryTarget's clause_ variant. -Status sanitize_sparse_vector(QueryTarget &target, const FieldSchema *schema); +ZVEC_API Status sanitize_sparse_vector(QueryTarget &target, + const FieldSchema *schema); struct GroupByVectorQuery { QueryTarget target_; diff --git a/src/include/zvec/db/query_params.h b/src/include/zvec/db/query_params.h index da90b7b..9ef417b 100644 --- a/src/include/zvec/db/query_params.h +++ b/src/include/zvec/db/query_params.h @@ -17,13 +17,14 @@ #include #include #include +#include namespace zvec { /* * Query Index params */ -class QueryParams { +class ZVEC_API QueryParams { public: using Ptr = std::shared_ptr; @@ -65,7 +66,7 @@ class QueryParams { bool is_using_refiner_{false}; }; -class HnswQueryParams : public QueryParams { +class ZVEC_API HnswQueryParams : public QueryParams { public: HnswQueryParams( int ef = core_interface::kDefaultHnswEfSearch, float radius = 0.0f, @@ -113,7 +114,7 @@ class HnswQueryParams : public QueryParams { uint32_t prefetch_lines_{core_interface::kDefaultPrefetchLines}; }; -class IVFQueryParams : public QueryParams { +class ZVEC_API IVFQueryParams : public QueryParams { public: IVFQueryParams(int nprobe = 10, bool is_using_refiner = false, float scale_factor = 10) @@ -145,7 +146,7 @@ class IVFQueryParams : public QueryParams { float scale_factor_{10}; }; -class HnswRabitqQueryParams : public QueryParams { +class ZVEC_API HnswRabitqQueryParams : public QueryParams { public: HnswRabitqQueryParams(int ef = core_interface::kDefaultHnswEfSearch, float radius = 0.0f, bool is_linear = false, @@ -170,7 +171,7 @@ class HnswRabitqQueryParams : public QueryParams { int ef_; }; -class FlatQueryParams : public QueryParams { +class ZVEC_API FlatQueryParams : public QueryParams { public: FlatQueryParams(bool is_using_refiner = false, float scale_factor = 10) : QueryParams(IndexType::FLAT) { @@ -192,7 +193,7 @@ class FlatQueryParams : public QueryParams { float scale_factor_{10}; }; -class DiskAnnQueryParams : public QueryParams { +class ZVEC_API DiskAnnQueryParams : public QueryParams { public: DiskAnnQueryParams(int list_size = 300) : QueryParams(IndexType::DISKANN) { set_list_size(list_size); @@ -214,7 +215,7 @@ class DiskAnnQueryParams : public QueryParams { int list_size_; }; -class VamanaQueryParams : public QueryParams { +class ZVEC_API VamanaQueryParams : public QueryParams { public: VamanaQueryParams( int ef_search = core_interface::kDefaultVamanaEfSearch, @@ -263,7 +264,7 @@ class VamanaQueryParams : public QueryParams { uint32_t prefetch_lines_{core_interface::kDefaultPrefetchLines}; }; -class FtsQueryParams : public QueryParams { +class ZVEC_API FtsQueryParams : public QueryParams { public: using Ptr = std::shared_ptr; @@ -284,4 +285,4 @@ class FtsQueryParams : public QueryParams { std::string default_operator_; }; -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/include/zvec/db/reranker.h b/src/include/zvec/db/reranker.h index e6e9256..a65d192 100644 --- a/src/include/zvec/db/reranker.h +++ b/src/include/zvec/db/reranker.h @@ -19,6 +19,7 @@ #include #include #include +#include namespace zvec { namespace reranker { @@ -66,10 +67,10 @@ using RerankParams = std::variant; /// normalization) /// @param topn Maximum number of results to return /// @return Re-ranked document list (length <= topn) -Result rerank(const RerankParams ¶ms, - const std::vector &results, - const std::vector &fields, - int topn); +ZVEC_API Result rerank(const RerankParams ¶ms, + const std::vector &results, + const std::vector &fields, + int topn); } // namespace reranker } // namespace zvec diff --git a/src/include/zvec/db/schema.h b/src/include/zvec/db/schema.h index c899f47..1b22193 100644 --- a/src/include/zvec/db/schema.h +++ b/src/include/zvec/db/schema.h @@ -19,6 +19,7 @@ #include #include #include +#include namespace zvec { @@ -28,7 +29,7 @@ const uint64_t MAX_DOC_COUNT_PER_SEGMENT_MIN_THRESHOLD = 1000; /* * Field schema */ -class FieldSchema { +class ZVEC_API FieldSchema { public: using Ptr = std::shared_ptr; @@ -284,7 +285,7 @@ using FieldSchemaPtrMap = std::unordered_map; /* * Collection schema */ -class CollectionSchema { +class ZVEC_API CollectionSchema { public: using Ptr = std::shared_ptr; @@ -421,4 +422,4 @@ class CollectionSchema { uint64_t max_doc_count_per_segment_{MAX_DOC_COUNT_PER_SEGMENT}; }; -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/include/zvec/db/stats.h b/src/include/zvec/db/stats.h index ea9bf9a..dbe138d 100644 --- a/src/include/zvec/db/stats.h +++ b/src/include/zvec/db/stats.h @@ -16,13 +16,14 @@ #include #include #include +#include namespace zvec { /* * Collection stats */ -struct CollectionStats { +struct ZVEC_API CollectionStats { uint64_t doc_count{0}; // column -> completeness std::unordered_map index_completeness; @@ -32,4 +33,4 @@ struct CollectionStats { std::string to_string_formatted(int indent_level = 0) const; }; -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/include/zvec/db/status.h b/src/include/zvec/db/status.h index dac3f8f..3a4404d 100644 --- a/src/include/zvec/db/status.h +++ b/src/include/zvec/db/status.h @@ -16,14 +16,15 @@ #include #include #include +#include namespace zvec { -class Status; +class ZVEC_API Status; template using Result = tl::expected; -std::ostream &operator<<(std::ostream &os, const Status &s); +ZVEC_API std::ostream &operator<<(std::ostream &os, const Status &s); /** * @brief Enumeration of common error codes. @@ -43,7 +44,7 @@ enum class StatusCode { }; // Helper: get default message for code -const char *GetDefaultMessage(StatusCode code); +ZVEC_API const char *GetDefaultMessage(StatusCode code); /** * @class Status @@ -56,7 +57,7 @@ const char *GetDefaultMessage(StatusCode code); * @note This class is thread-compatible: const methods can be called from * multiple threads. */ -class Status { +class ZVEC_API Status { public: /// @brief Default constructor: OK status Status() noexcept : code_(StatusCode::OK) {} @@ -178,4 +179,4 @@ class Status { std::string msg_; }; -} // namespace zvec \ No newline at end of file +} // namespace zvec diff --git a/src/include/zvec/export.h b/src/include/zvec/export.h new file mode 100644 index 0000000..a836194 --- /dev/null +++ b/src/include/zvec/export.h @@ -0,0 +1,60 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#if defined(_WIN32) || defined(__CYGWIN__) +#define ZVEC_HELPER_DLL_EXPORT __declspec(dllexport) +#define ZVEC_HELPER_DLL_IMPORT __declspec(dllimport) +#else +#if defined(__GNUC__) && __GNUC__ >= 4 +#define ZVEC_HELPER_DLL_EXPORT __attribute__((visibility("default"))) +#define ZVEC_HELPER_DLL_IMPORT __attribute__((visibility("default"))) +#else +#define ZVEC_HELPER_DLL_EXPORT +#define ZVEC_HELPER_DLL_IMPORT +#endif +#endif + +#if defined(ZVEC_DB_BUILD_SHARED) +#define ZVEC_API ZVEC_HELPER_DLL_EXPORT +#elif defined(ZVEC_DB_USE_SHARED) +#define ZVEC_API ZVEC_HELPER_DLL_IMPORT +#else +#define ZVEC_API +#endif + +#if defined(ZVEC_CORE_BUILD_SHARED) +#define ZVEC_CORE_API ZVEC_HELPER_DLL_EXPORT +#elif defined(ZVEC_CORE_USE_SHARED) +#define ZVEC_CORE_API ZVEC_HELPER_DLL_IMPORT +#else +#define ZVEC_CORE_API +#endif + +#if defined(ZVEC_AILEGO_BUILD_SHARED) +#define ZVEC_AILEGO_API ZVEC_HELPER_DLL_EXPORT +#elif defined(ZVEC_AILEGO_USE_SHARED) +#define ZVEC_AILEGO_API ZVEC_HELPER_DLL_IMPORT +#else +#define ZVEC_AILEGO_API +#endif + +#if defined(ZVEC_TURBO_BUILD_SHARED) +#define ZVEC_TURBO_API ZVEC_HELPER_DLL_EXPORT +#elif defined(ZVEC_TURBO_USE_SHARED) +#define ZVEC_TURBO_API ZVEC_HELPER_DLL_IMPORT +#else +#define ZVEC_TURBO_API +#endif diff --git a/src/include/zvec/turbo/turbo.h b/src/include/zvec/turbo/turbo.h index 04d0e3a..5baa8c4 100644 --- a/src/include/zvec/turbo/turbo.h +++ b/src/include/zvec/turbo/turbo.h @@ -17,6 +17,7 @@ #include #include #include +#include namespace zvec::turbo { @@ -116,15 +117,15 @@ enum class CpuArchType { kSVE2 }; -DistanceFunc get_distance_func(MetricType metric_type, DataType data_type, - QuantizeType quantize_type, - CpuArchType cpu_arch_type = CpuArchType::kAuto); - -BatchDistanceFunc get_batch_distance_func( +ZVEC_TURBO_API DistanceFunc get_distance_func( MetricType metric_type, DataType data_type, QuantizeType quantize_type, CpuArchType cpu_arch_type = CpuArchType::kAuto); -QueryPreprocessFunc get_query_preprocess_func( +ZVEC_TURBO_API BatchDistanceFunc get_batch_distance_func( + MetricType metric_type, DataType data_type, QuantizeType quantize_type, + CpuArchType cpu_arch_type = CpuArchType::kAuto); + +ZVEC_TURBO_API QueryPreprocessFunc get_query_preprocess_func( MetricType metric_type, DataType data_type, QuantizeType quantize_type, CpuArchType cpu_arch_type = CpuArchType::kAuto); @@ -134,7 +135,8 @@ QueryPreprocessFunc get_query_preprocess_func( // uniform-specific accessor intentionally kept outside of the generic // (metric/data/quantize) dispatch above; data_type is retained so the // interface can grow to cover other output types (e.g. fp16) in the future. -UniformQuantizeFunc get_uniform_quantize_func(DataType data_type); +ZVEC_TURBO_API UniformQuantizeFunc +get_uniform_quantize_func(DataType data_type); // Returns rotator kernels dispatched for the current CPU. RotatorKernels get_rotator_kernels( diff --git a/src/turbo/CMakeLists.txt b/src/turbo/CMakeLists.txt index 2ce56d4..7b2e117 100644 --- a/src/turbo/CMakeLists.txt +++ b/src/turbo/CMakeLists.txt @@ -45,8 +45,19 @@ if(NOT ANDROID AND AUTO_DETECT_ARCH) endif() endif() +set(ZVEC_TURBO_LIBRARY_OPTIONS) +if(WIN32 AND (BUILD_ZVEC_CORE_SHARED OR BUILD_ZVEC_SHARED)) + list(APPEND ZVEC_TURBO_LIBRARY_OPTIONS OBJECTS) + if(BUILD_ZVEC_CORE_SHARED) + list(APPEND ZVEC_TURBO_LIBRARY_OPTIONS + EXPORT_DEF ZVEC_TURBO_BUILD_SHARED + ) + endif() +endif() + cc_library( NAME zvec_turbo STATIC STRICT ALWAYS_LINK PACKED + ${ZVEC_TURBO_LIBRARY_OPTIONS} SRCS ${ALL_SRCS} LIBS zvec_ailego INCS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/distance ${PROJECT_ROOT_DIR}/src/include diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index e3b54ee..8a89cd3 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,6 +8,9 @@ cc_directories(ailego) cc_directories(db) cc_directories(core) cc_directories(turbo) +if(BUILD_ZVEC_SHARED AND NOT IOS AND NOT ANDROID) + cc_directories(shared) +endif() if(BUILD_C_BINDINGS) cc_directories(c) endif() diff --git a/tests/shared/CMakeLists.txt b/tests/shared/CMakeLists.txt new file mode 100644 index 0000000..8de4198 --- /dev/null +++ b/tests/shared/CMakeLists.txt @@ -0,0 +1,7 @@ +include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake) + +cc_test( + NAME shared_api_test STRICT + SRCS shared_api_test.cc + LIBS zvec_shared +) diff --git a/tests/shared/shared_api_test.cc b/tests/shared/shared_api_test.cc new file mode 100644 index 0000000..cc157da --- /dev/null +++ b/tests/shared/shared_api_test.cc @@ -0,0 +1,40 @@ +// Copyright 2025-present the zvec project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include +#include +#include + +int main() { + zvec::float16_t half_value(1.5F); + if (std::fabs(static_cast(half_value) - 1.5F) > 0.001F) { + return 1; + } + + zvec::CollectionSchema schema("shared-api-test"); + auto field = std::make_shared("id", zvec::DataType::INT64); + if (!schema.add_field(std::move(field)).ok() || !schema.has_field("id")) { + return 2; + } + + zvec::Doc doc; + doc.set_pk("1"); + if (!doc.set("id", 1)) { + return 3; + } + + return 0; +} diff --git a/tools/db/CMakeLists.txt b/tools/db/CMakeLists.txt index fc224e3..8265e42 100644 --- a/tools/db/CMakeLists.txt +++ b/tools/db/CMakeLists.txt @@ -4,7 +4,7 @@ cc_binary( NAME fts_bench PACKED SRCS fts_bench_main.cc LIBS - zvec_shared + zvec gflags roaring rocksdb