Initial commit

This commit is contained in:
sanyi
2025-12-30 11:02:17 +08:00
commit 6524fabd80
841 changed files with 293919 additions and 0 deletions
+31
View File
@@ -0,0 +1,31 @@
# http://clang.llvm.org/docs/ClangFormatStyleOptions.html
# Defines the Google C++ style for automatic reformatting.
BasedOnStyle: Google
MaxEmptyLinesToKeep: 2
DerivePointerAlignment: false
PointerAlignment: Right
AllowShortFunctionsOnASingleLine: Empty
IncludeBlocks: Merge
IncludeCategories:
- Regex: '^<linux/[0-9A-Za-z]+'
Priority: 100
- Regex: '^<mach/[0-9A-Za-z]+'
Priority: 101
- Regex: '^<mach-o/[0-9A-Za-z]+'
Priority: 102
- Regex: '^<sys/[0-9A-Za-z]+'
Priority: 103
- Regex: '^<[0-9A-Za-z]+\.h>$'
Priority: 200
- Regex: '^<[0-9A-Za-z_]+>$'
Priority: 201
- Regex: '^<[0-9A-Za-z_]+\.[0-9A-Za-z]+>$'
Priority: 202
- Regex: '^<[0-9A-Za-z_]+/[0-9A-Za-z]+'
Priority: 203
- Regex: '^\"[0-9A-Za-z_]+/[0-9A-Za-z]+'
Priority: 300
- Regex: '^\"[0-9A-Za-z_]+\.[0-9A-Za-z]+\"$'
Priority: 301
- Regex: '.*'
Priority: 1000
+73
View File
@@ -0,0 +1,73 @@
name: Bug Report
description: Report a bug or unexpected behavior (e.g., crash, incorrect vector query, memory leak)
title: "[Bug]: "
labels: ["bug", "triage"]
body:
- type: markdown
attributes:
value: |
Thank you for reporting! Please provide detailed info so we can reproduce and fix it quickly.
- type: textarea
id: description
attributes:
label: Description
description: What happened? What did you expect?
placeholder: |
e.g. "Query with vector field crashes when using Zvec Python API"
validations:
required: true
- type: textarea
id: steps_to_reproduce
attributes:
label: Steps to Reproduce
description: Exact steps to trigger the issue (code snippets welcome)
placeholder: |
1. Build Zvec with CMake (Debug/Release)
2. Run Python script: `python test.py`
3. Call `collection.query(VectorQuery())`
4. Process segfaults / hangs / returns wrong results
render: python
validations:
required: true
- type: textarea
id: logs_or_trace
attributes:
label: Logs / Stack Trace
description: Paste relevant logs, LLDB/GDB backtrace, or CI failures
placeholder: |
Thread 1 "python" received signal SIGSEGV, Segmentation fault.
0x0000000104a2c3f0 in std::__1::shared_ptr<...>::...
render: shell
validations:
required: false
- type: input
id: os
attributes:
label: Operating System
placeholder: macOS 14 (M1), Ubuntu 22.04, Windows 11 (WSL2)
validations:
required: true
- type: input
id: build_env
attributes:
label: Build & Runtime Environment
description: Compiler, CMake, Python, key dependencies
placeholder: |
clang 15.0.0, CMake 4.1.2, Python 3.11.9, magic_enum v0.9.7 (via git submodule)
validations:
required: true
- type: checkboxes
id: additional_context
attributes:
label: Additional Context
options:
- label: I've checked `git status` — no uncommitted submodule changes
- label: I built with `CMAKE_BUILD_TYPE=Debug`
- label: This occurs with or without `COVERAGE=ON`
- label: The issue involves Python ↔ C++ integration (pybind11)
+9
View File
@@ -0,0 +1,9 @@
blank_issues_enabled: false
contact_links:
- name: Documentation
url: https://github.com/your-org/zvec#readme
about: Check the README, build guide, and API docs first.
- name: Python API Examples
url: https://github.com/your-org/zvec/tree/main/examples
about: See working usage examples.
@@ -0,0 +1,55 @@
name: Feature Request
description: Suggest a new feature or improvement (e.g., better memory control, new query option)
title: "[Feature]: "
labels: ["enhancement"]
body:
- type: markdown
attributes:
value: |
Thanks for your idea! Help us understand the motivation and scope.
- type: textarea
id: problem_or_motivation
attributes:
label: Problem / Motivation
description: What problem does this solve? Why is it needed?
placeholder: |
e.g. "Current vector queries don't allow filtering by metadata + distance threshold at once"
validations:
required: true
- type: textarea
id: proposed_solution
attributes:
label: Proposed Solution
description: How should it work? API sketch or pseudocode welcome.
placeholder: |
Add `filter=` and `max_distance=` args to `Zvec.query()`:
```python
results = db.query(vector, filter="category == 'A'", max_distance=0.5)
```
render: python
validations:
required: false
- type: textarea
id: alternatives
attributes:
label: Alternatives Considered
description: Are there workarounds? Why not use them?
validations:
required: false
- type: dropdown
id: impact_area
attributes:
label: Affected Area
multiple: true
options:
- label: C++ Core (storage, indexing)
- label: Python API / Bindings
- label: Build System (CMake, Homebrew pkg)
- label: Testing / CI / Coverage
- label: Documentation
validations:
required: false
+32
View File
@@ -0,0 +1,32 @@
codecov:
require_ci_to_pass: true
coverage:
precision: 2
round: down
range: "60...75"
status:
project:
default: false
patch:
default: false
parsers:
gcov:
branch_detection:
conditional: true
loop: true
method: false
macro: false
comment:
require_changes: false
layout: "reach,diff,flags,tree"
behavior: default
ignore:
- "thirdparty/"
- "tests/"
+57
View File
@@ -0,0 +1,57 @@
name: Build Wheels
on:
workflow_dispatch:
jobs:
build_wheels:
name: Build wheels on self-hosted manylinux_2_28
runs-on: linux_x64
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
submodules: recursive
- name: Set up Python (for cibuildwheel controller)
uses: actions/setup-python@v5
with:
python-version: '3.11'
- name: Install cibuildwheel
run: |
pip install --upgrade pip
pip install cibuildwheel==2.17.0
- name: Build wheels using cibuildwheel
run: |
python -m cibuildwheel --output-dir wheelhouse
# Save list of built wheels for publishing
ls wheelhouse/*.whl | tee $GITHUB_STEP_SUMMARY
echo "wheels=$(ls wheelhouse/*.whl | tr '\n' ' ')" >> $GITHUB_ENV
- name: Publish to TestPyPI
if: success() && github.event_name == 'workflow_dispatch'
env:
TWINE_USERNAME: __token__
TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
run: |
pip install twine
twine upload --skip-existing --verbose wheelhouse/*.whl
- name: (Optional) Install and test from TestPyPI
if: success() && github.event_name == 'workflow_dispatch'
run: |
# Create a clean venv
python -m venv test_env
source test_env/bin/activate
pip install --upgrade pip
# Install from TestPyPI (must allow pre-releases if version has dev/alpha)
pip install --index-url https://test.pypi.org/simple/ \
--extra-index-url https://pypi.org/simple/ \
zvec
# Run a simple smoke test
python -c "import zvec; print('Import OK:', zvec.__version__)"
shell: bash
@@ -0,0 +1,83 @@
# =============================================================================
# Dockerfile.linux_x64_glibc228
# Purpose: Ubuntu 18.10 gcc-9 + glibc 2.28 + CMake 3.30.0 + PyBind11 build environment
# Warning: ubuntu:18.10 is EOL; use only for glibc 2.28 compatibility testing.
# =============================================================================
# Use official Ubuntu 18.10 (Cosmic Cuttlefish)
# glibc version: 2.28 (confirmed via `ldd --version`)
FROM ubuntu:18.10
# Replace Ubuntu mirror with old-releases.ubuntu.com for older glibc compatibility
RUN sed -i 's|http://\(.*\)/ubuntu|http://old-releases.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
sed -i 's|http://security.ubuntu.com/ubuntu|http://old-releases.ubuntu.com/ubuntu|g' /etc/apt/sources.list
# Add Ubuntu 20.04 (focal) repo for GCC 9 ONLY
RUN echo "deb http://archive.ubuntu.com/ubuntu/ focal main universe" >> /etc/apt/sources.list && \
echo "deb http://security.ubuntu.com/ubuntu/ focal-security main universe" >> /etc/apt/sources.list
# Prevent interactive prompts & set non-root user
ENV DEBIAN_FRONTEND=noninteractive \
TZ=Etc/UTC
# Create non-root user for safety (optional but recommended)
RUN useradd -m -u 1000 builder && \
mkdir -p /workspace && chown builder:builder /workspace
# Install base system dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends \
build-essential \
gcc-9 g++-9 \
ninja-build git curl ca-certificates vim wget lcov gnupg clang-format-18\
rsync lsb-release \
uuid-dev zlib1g-dev libssl-dev libffi-dev \
pybind11-dev && \
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90 \
--slave /usr/bin/g++ g++ /usr/bin/g++-9 && \
rm -rf /var/lib/apt/lists/*
# Install Miniforge (Conda) as root, then assign to builder
ENV MINIFORGE_VERSION="latest"
ENV MINIFORGE_HOME="/opt/miniforge3"
RUN curl -sSL "https://github.com/conda-forge/miniforge/releases/${MINIFORGE_VERSION}/download/Miniforge3-Linux-x86_64.sh" -o miniforge.sh && \
bash miniforge.sh -b -p ${MINIFORGE_HOME} && \
rm miniforge.sh && \
chown -R builder:builder ${MINIFORGE_HOME}
# Switch to non-root user
USER builder
ENV PATH="${MINIFORGE_HOME}/bin:${PATH}"
WORKDIR /workspace
# Create conda envs for supported Python versions
RUN conda create -n py310 python=3.10 -y && \
conda create -n py311 python=3.11 -y && \
conda create -n py312 python=3.12 -y
RUN conda clean --all -f -y
# Install CMake 3.30.0 from Kitware official binary
# Ref: https://github.com/Kitware/CMake/releases/tag/v3.30.0
RUN mkdir -p /tmp/cmake && cd /tmp/cmake && \
curl -sSL -o cmake.tar.gz \
"https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-x86_64.tar.gz" && \
tar -xzf cmake.tar.gz --strip-components=1 -C /tmp/cmake && \
mkdir -p /home/builder/.local && \
mv * /home/builder/.local/ && \
chown -R builder:builder /home/builder/.local && \
rm -rf /tmp/cmake
# Add CMake to PATH
ENV PATH="/home/builder/.local/bin:${PATH}"
# Verify installations
RUN cmake --version && \
conda info && \
conda env list && \
python --version && \
gcc --version && \
ldd --version | head -n1
# Final setup
WORKDIR /workspace
+124
View File
@@ -0,0 +1,124 @@
name: Zvec LinuxX64 CI
on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
workflow_dispatch:
jobs:
build:
runs-on: linux_x64
strategy:
matrix:
python-version: ['3.10']
fail-fast: false
container:
image: zvec-registry.cn-hongkong.cr.aliyuncs.com/zvec/zvec:0.0.2
options: --user root
steps:
- name: Activate Conda environment
run: |
# Map version to env name
if [[ "${{ matrix.python-version }}" == "3.10" ]]; then
ENV_NAME="py310"
elif [[ "${{ matrix.python-version }}" == "3.11" ]]; then
ENV_NAME="py311"
elif [[ "${{ matrix.python-version }}" == "3.12" ]]; then
ENV_NAME="py312"
else
echo "Unsupported Python version"
exit 1
fi
echo "CONDA_ENV_NAME=$ENV_NAME" >> $GITHUB_ENV
source /opt/miniforge3/bin/activate $ENV_NAME
pip install --upgrade pip pytest pytest-cov ruff
python --version
shell: bash
- name: Prepare clean build directory
run: |
export CLEAN_WORKSPACE="/tmp/zvec"
mkdir -p "$CLEAN_WORKSPACE"
cd "$CLEAN_WORKSPACE"
git config --global --add safe.directory "$CLEAN_WORKSPACE"
git clone --recursive "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" .
if [ -n "${{ github.event.pull_request.head.sha }}" ]; then
git checkout "${{ github.event.pull_request.head.sha }}"
else
git checkout "${{ github.sha }}"
fi
echo "CLEAN_WORKSPACE=$CLEAN_WORKSPACE" >> $GITHUB_ENV
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: bash
- name: Run Ruff Linter
run: |
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
cd "$CLEAN_WORKSPACE"
ruff check .
shell: bash
- name: Run Ruff Formatter Check (ensure code is formatted)
run: |
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
cd "$CLEAN_WORKSPACE"
ruff format --check .
shell: bash
- name: Run clang-format Check
run: |
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
cd "$CLEAN_WORKSPACE"
CPP_FILES=$(find . -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.hpp" -o -name "*.cc" -o -name "*.cxx" \) \
! -path "./build/*" \
! -path "./tests/*" \
! -path "./scripts/*" \
! -path "./python/*" \
! -path "./thirdparty/*" \
! -path "./.git/*")
if [ -z "$CPP_FILES" ]; then
echo "No C++ files found to check."
exit 0
fi
clang-format-18 --dry-run --Werror $CPP_FILES
shell: bash
- name: Install Python dependencies and build package
run: |
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
cd "$CLEAN_WORKSPACE"
NPROC=$(nproc 2>/dev/null || echo $(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2))
echo "ParallelGroup: Using $NPROC parallel jobs for build"
CMAKE_GENERATOR="Unix Makefiles" \
CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \
pip install -v .
shell: bash
- name: Run Python Tests with Coverage
run: |
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
cd "$CLEAN_WORKSPACE"
python -m pytest python/tests/ --cov=zvec --cov-report=xml --no-cov-on-fail
shell: bash
- name: Run Cpp Tests with Coverage
run: |
cd "$CLEAN_WORKSPACE/build"
make unittest -j$(nproc)
shell: bash
+85
View File
@@ -0,0 +1,85 @@
name: Zvec MacArm64 CI
on:
push:
branches: [ "main" ]
pull_request:
branches: [ "main" ]
workflow_dispatch:
jobs:
build:
runs-on: mac_m1_arm
steps:
- name: Run Ruff Linter
run: |
cd "$CLEAN_WORKSPACE"
ruff check .
shell: bash
- name: Run Ruff Formatter Check (ensure code is formatted)
run: |
cd "$CLEAN_WORKSPACE"
ruff format --check .
shell: bash
- name: Run clang-format Check
run: |
cd "$CLEAN_WORKSPACE"
CPP_FILES=$(find . -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.hpp" -o -name "*.cc" -o -name "*.cxx" \) \
! -path "./build/*" \
! -path "./tests/*" \
! -path "./scripts/*" \
! -path "./python/*" \
! -path "./thirdparty/*" \
! -path "./.git/*")
if [ -z "$CPP_FILES" ]; then
echo "No C++ files found to check."
exit 0
fi
clang-format --dry-run --Werror $CPP_FILES
shell: bash
- name: Prepare clean build directory
run: |
export CLEAN_WORKSPACE="/tmp/zvec"
rm -rf "$CLEAN_WORKSPACE"
mkdir -p "$CLEAN_WORKSPACE"
cd "$CLEAN_WORKSPACE"
git config --global --add safe.directory "$CLEAN_WORKSPACE"
git clone --recursive "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" .
if [ -n "${{ github.event.pull_request.head.sha }}" ]; then
git checkout "${{ github.event.pull_request.head.sha }}"
else
git checkout "${{ github.sha }}"
fi
echo "CLEAN_WORKSPACE=$CLEAN_WORKSPACE" >> $GITHUB_ENV
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: bash
- name: Install Python dependencies and build package
run: |
cd "$CLEAN_WORKSPACE"
pip install --upgrade pip pytest pytest-cov ruff
pip install -v .
shell: bash
- name: Run Python Tests with Coverage
run: |
cd "$CLEAN_WORKSPACE"
python -m pytest python/tests/ --cov=zvec --cov-report=xml --no-cov-on-fail
shell: bash
- name: Run Cpp Tests with Coverage
run: |
cd "$CLEAN_WORKSPACE/build"
make unittest -j 16
shell: bash
+95
View File
@@ -0,0 +1,95 @@
name: Nightly Coverage Report
on:
schedule:
# Runs daily at 00:00 CST (China Standard Time) = 16:00 UTC
- cron: '0 16 * * *'
workflow_dispatch:
jobs:
coverage:
runs-on: linux_x64
strategy:
matrix:
python-version: ['3.10']
fail-fast: false
container:
image: zvec-registry.cn-hongkong.cr.aliyuncs.com/zvec/zvec:0.0.2
options: --user root
steps:
- name: Activate Conda environment
run: |
if [[ "${{ matrix.python-version }}" == "3.10" ]]; then
ENV_NAME="py310"
elif [[ "${{ matrix.python-version }}" == "3.11" ]]; then
ENV_NAME="py311"
elif [[ "${{ matrix.python-version }}" == "3.12" ]]; then
ENV_NAME="py312"
else
echo "Unsupported Python version"
exit 1
fi
echo "CONDA_ENV_NAME=$ENV_NAME" >> $GITHUB_ENV
source /opt/miniforge3/bin/activate "$ENV_NAME"
python --version
shell: bash
- name: Prepare clean build directory
run: |
export CLEAN_WORKSPACE="/tmp/zvec"
mkdir -p "$CLEAN_WORKSPACE"
cd "$CLEAN_WORKSPACE"
git config --global --add safe.directory "$CLEAN_WORKSPACE"
git clone --recursive "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" .
git checkout main # Always use main for nightly
echo "CLEAN_WORKSPACE=$CLEAN_WORKSPACE" >> $GITHUB_ENV
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
shell: bash
- name: Build with COVERAGE config
run: |
source /opt/miniforge3/bin/activate "${{ env.CONDA_ENV_NAME }}"
cd "$CLEAN_WORKSPACE"
pip install --upgrade pip pytest pytest-cov
NPROC=$(nproc 2>/dev/null || echo 2)
echo "Using $NPROC parallel jobs"
CMAKE_GENERATOR="Unix Makefiles" \
CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \
pip install -v . \
--config-settings="cmake.build-type=COVERAGE"
shell: bash
- name: Run Python Tests with Coverage
run: |
source /opt/miniforge3/bin/activate "${{ env.CONDA_ENV_NAME }}"
cd "$CLEAN_WORKSPACE"
python -m pytest python/tests/ --cov=zvec --cov-report=xml
shell: bash
- name: Run C++ Tests and Generate Coverage
run: |
cd "$CLEAN_WORKSPACE/build"
make unittest -j$(nproc) # Run all (nightly can afford it)
cd "$CLEAN_WORKSPACE"
# Ensure gcov.sh is executable
chmod +x scripts/gcov.sh
bash scripts/gcov.sh -k
shell: bash
- name: Upload Coverage to Codecov
uses: codecov/codecov-action@v4
with:
files: ${{ env.CLEAN_WORKSPACE }}/proxima-zvec-filtered.lcov.info,${{ env.CLEAN_WORKSPACE }}/coverage.xml
flags: python,cpp,nightly
name: nightly-linux-py${{ matrix.python-version }}
token: ${{ secrets.CODECOV_TOKEN }}
+50
View File
@@ -0,0 +1,50 @@
.*
*~
bazel-*
/*build*
bin/*
lib/*
var/*
thirdparty
venv*
tests/integration/conf/*
tests/de_integration/conf/*
**/__pycache__/*
tests/bench/log/*
tests/integration/integration
tests/integration/log
tests/integration/*.log
tests/de_integration/log
tests/de_integration/*.log
!.git*
!.clang-format
!.circleci
!.drone.yml
sdk/python/dist/
compile_commands.json
dist
html
*.lcov.info
# Dependencies
/node_modules
# Production
/build
# Generated files
.docusaurus
.cache-loader
# Misc
.DS_Store
.env.local
.env.development.local
.env.test.local
.env.production.local
npm-debug.log*
yarn-debug.log*
yarn-error.log*
allure-*
+41
View File
@@ -0,0 +1,41 @@
[submodule "thirdparty/googletest/googletest-1.10.0"]
path = thirdparty/googletest/googletest-1.10.0
url = https://github.com/google/googletest.git
[submodule "thirdparty/sparsehash/sparsehash-2.0.4"]
path = thirdparty/sparsehash/sparsehash-2.0.4
url = https://github.com/sparsehash/sparsehash.git
ignore = untracked
[submodule "thirdparty/gflags/gflags-2.2.2"]
path = thirdparty/gflags/gflags-2.2.2
url = https://github.com/gflags/gflags.git
[submodule "thirdparty/rocksdb/rocksdb-8.1.1"]
path = thirdparty/rocksdb/rocksdb-8.1.1
url = https://github.com/facebook/rocksdb.git
[submodule "thirdparty/yaml-cpp/yaml-cpp-0.6.3"]
path = thirdparty/yaml-cpp/yaml-cpp-0.6.3
url = https://github.com/jbeder/yaml-cpp.git
[submodule "thirdparty/arrow/apache-arrow-21.0.0"]
path = thirdparty/arrow/apache-arrow-21.0.0
url = https://github.com/apache/arrow.git
ignore = all
[submodule "thirdparty/CRoaring/CRoaring-2.0.4"]
path = thirdparty/CRoaring/CRoaring-2.0.4
url = https://github.com/RoaringBitmap/CRoaring.git
[submodule "thirdparty/glog/glog-0.5.0"]
path = thirdparty/glog/glog-0.5.0
url = https://github.com/google/glog.git
ignore = all
[submodule "thirdparty/protobuf/protobuf-3.21.12"]
path = thirdparty/protobuf/protobuf-3.21.12
url = https://github.com/protocolbuffers/protobuf.git
[submodule "thirdparty/lz4/lz4-1.9.4"]
path = thirdparty/lz4/lz4-1.9.4
url = https://github.com/lz4/lz4.git
[submodule "thirdparty/antlr/antlr4"]
path = thirdparty/antlr/antlr4
url = https://github.com/antlr/antlr4.git
ignore = all
[submodule "thirdparty/magic_enum/magic_enum-0.9.7"]
path = thirdparty/magic_enum/magic_enum-0.9.7
url = https://github.com/Neargye/magic_enum.git
ignore = all
+30
View File
@@ -0,0 +1,30 @@
repos:
- repo: https://github.com/astral-sh/ruff-pre-commit
rev: v0.14.4
hooks:
- id: ruff-check
name: Ruff (lint + fix)
args: [--config=pyproject.toml, --fix]
types_or: [python, pyi]
- id: ruff-format
name: Ruff (format)
args: [--config=pyproject.toml]
types_or: [python, pyi]
- repo: https://github.com/pre-commit/mirrors-clang-format
rev: v18.1.8
hooks:
- id: clang-format
name: clang-format (C++)
args: [--style=file, --fallback-style=none]
types_or: [c++, c]
exclude: ^thirdparty/.*$
- repo: https://github.com/gitleaks/gitleaks
rev: v8.21.4
hooks:
- id: gitleaks
name: Detect hardcoded secrets (gitleaks)
description: Scans staged files for secrets using gitleaks rules
+60
View File
@@ -0,0 +1,60 @@
cmake_minimum_required(VERSION 3.13)
cmake_policy(SET CMP0077 NEW)
project(zvec)
set(CC_CXX_STANDARD 17)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type")
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type")
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-as-needed")
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-as-needed")
endif()
if(NOT DEFINED PROJECT_ROOT_DIR OR NOT PROJECT_ROOT_DIR)
set(PROJECT_ROOT_DIR ${CMAKE_SOURCE_DIR} CACHE PATH "Root directory of the project" FORCE)
endif()
message(STATUS "PROJECT_ROOT_DIR = ${PROJECT_ROOT_DIR}")
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include_directories(${PROJECT_ROOT_DIR}/src)
option(BUILD_PYTHON_BINDINGS "Build Python bindings using pybind11" OFF)
message(STATUS "BUILD_PYTHON_BINDINGS:${BUILD_PYTHON_BINDINGS}")
option(BUILD_TOOLS "Build tools" ON)
message(STATUS "BUILD_TOOLS:${BUILD_TOOLS}")
cc_directory(thirdparty)
cc_directories(src)
cc_directories(tests)
if(BUILD_TOOLS)
cc_directories(tools)
endif ()
git_version(GIT_SRCS_VER ${PROJECT_ROOT_DIR})
set(CPACK_PACKAGE_VERSION ${GIT_SRCS_VER})
set(CPACK_PACKAGE_NAME zvec)
include(CPack)
if(BUILD_PYTHON_BINDINGS)
if(APPLE)
set(CMAKE_STRIP "")
message(STATUS "Disabled strip on macOS to preserve code signature")
endif()
include(GNUInstallDirs)
if(DEFINED SKBUILD_PLATLIB_DIR)
set(ZVEC_PY_INSTALL_DIR "${SKBUILD_PLATLIB_DIR}")
elseif(DEFINED Python_SITEARCH)
set(ZVEC_PY_INSTALL_DIR "${Python_SITEARCH}")
else()
set(ZVEC_PY_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}")
endif()
MESSAGE(STATUS "Zvec install path: ${ZVEC_PY_INSTALL_DIR}")
install(TARGETS _zvec LIBRARY DESTINATION ${ZVEC_PY_INSTALL_DIR})
endif()
+41
View File
@@ -0,0 +1,41 @@
# Zvec Code of Conduct
## Our Pledge
We pledge to foster an open, respectful, and harassment-free environment for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
## Expected Behavior
- Use welcoming and inclusive language
- Respect differing viewpoints and experiences
- Gracefully accept constructive criticism
- Focus on what is best for the community
- Show empathy and kindness toward others
## Unacceptable Behavior
- Harassment, intimidation, or discriminatory conduct
- Trolling, insulting, or derogatory comments
- Public or private harassment
- Publishing others’ private information without consent
- Any conduct that would reasonably be considered inappropriate in a professional setting
## Enforcement
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team
at **zvec@alibaba-inc.com** (replace with your contact). All complaints will be reviewed and investigated promptly
and fairly.
The project team is obligated to respect the privacy and security of the reporter.
Consequences may include:
- A formal warning
- Temporary or permanent ban from project spaces
- Removal of contributions (e.g. comments, PRs)
## Attribution
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at
https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
[homepage]: https://www.contributor-covenant.org
+104
View File
@@ -0,0 +1,104 @@
# Contributing to Zvec
First off, thank you for considering contributing to Zvec! 🙌
Whether you're reporting a bug, proposing a feature, improving documentation, or submitting code — every contribution helps make Zvec better.
## Code of Conduct
By participating, you agree to abide by our [Code of Conduct](CODE_OF_CONDUCT.md). Please be respectful, collaborative, and inclusive.
---
## Development Setup
### Prerequisites
- Python ≥ 3.9
- CMake ≥ 3.15 (`cmake --version`)
- A C++17-compatible compiler (e.g., `g++-11+`, `clang++`, Apple Clang on macOS)
### Clone & Initialize
```bash
git clone --recursive https://github.com/your-org/zvec.git
cd zvec
```
> 💡 **Submodule tip**: If you forgot `--recursive`, run:
> `git submodule update --init --recursive`
### Build from Source (Editable Install)
```bash
pip install -e ".[dev]"
# This installs dev dependencies (pytest, ruff, etc.) and builds the C++ extension in-place
```
> ✅ Verify:
> ```bash
> python -c "import zvec; print('Success!')"
> ```
---
## Testing
### Run All Tests
```bash
pytest python/tests/ -v
```
### Run with Coverage (Debug/CI)
```bash
pytest python/tests/ --cov=zvec --cov-report=term-missing
```
> 🔎 See full rules in `[tool.ruff]` section of `pyproject.toml`.
---
## Build Customization
You can control build behavior via environment variables or `pyproject.toml`:
| Option | How to Set | Description |
|--------|------------|-------------|
| **Build Type** | `CMAKE_BUILD_TYPE=Debug` | `Debug`, `Release`, or `Coverage` (for gcov/lcov) |
| **Generator** | `CMAKE_GENERATOR="Unix Makefiles"` | Default: `Ninja`; use Make if preferred |
| **AVX-512** | `ENABLE_SKYLAKE_AVX512=ON` | Enable AVX-512 optimizations (x86_64 only) |
Example (Debug + Make):
```bash
CMAKE_BUILD_TYPE=Debug CMAKE_GENERATOR="Unix Makefiles" pip install -v .
```
---
## Submitting Changes
1. Fork the repo and create a feature branch (`feat/...`, `fix/...`, `docs/...`)
2. Write clear commit messages (e.g., `fix(query): handle null vector in dense_fp32`)
3. Ensure tests pass & linter is clean
4. Open a Pull Request to `main`
5. Link related issue (e.g., `Closes #123`)
✅ **PRs should include**:
- Test coverage for new behavior
- Updates to documentation (if applicable)
- Reasoning behind non-obvious design choices
---
## Documentation
- User guides: `docs/` (built with MkDocs)
- API reference: generated from docstrings (follow [Google style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings))
- Build & deploy: `mkdocs serve` / `mkdocs build`
---
## Need Help
- Browse [existing issues](https://github.com/graphscope/zvec/issues)
- For sensitive/security issues: email `zvec@alibaba-inc.com`
---
✨ Thanks again for being part of Zvec!
+201
View File
@@ -0,0 +1,201 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction,
and distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by
the copyright owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all
other entities that control, are controlled by, or are under common
control with that entity. For the purposes of this definition,
"control" means (i) the power, direct or indirect, to cause the
direction or management of such entity, whether by contract or
otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity
exercising permissions granted by this License.
"Source" form shall mean the preferred form for making modifications,
including but not limited to software source code, documentation
source, and configuration files.
"Object" form shall mean any form resulting from mechanical
transformation or translation of a Source form, including but
not limited to compiled object code, generated documentation,
and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or
Object form, made available under the License, as indicated by a
copyright notice that is included in or attached to the work
(an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object
form, that is based on (or derived from) the Work and for which the
editorial revisions, annotations, elaborations, or other modifications
represent, as a whole, an original work of authorship. For the purposes
of this License, Derivative Works shall not include works that remain
separable from, or merely link (or bind by name) to the interfaces of,
the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including
the original version of the Work and any modifications or additions
to that Work or Derivative Works thereof, that is intentionally
submitted to Licensor for inclusion in the Work by the copyright owner
or by an individual or Legal Entity authorized to submit on behalf of
the copyright owner. For the purposes of this definition, "submitted"
means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems,
and issue tracking systems that are managed by, or on behalf of, the
Licensor for the purpose of discussing and improving the Work, but
excluding communication that is conspicuously marked or otherwise
designated in writing by the copyright owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity
on behalf of whom a Contribution has been received by Licensor and
subsequently incorporated within the Work.
2. Grant of Copyright License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the
Work and such Derivative Works in Source or Object form.
3. Grant of Patent License. Subject to the terms and conditions of
this License, each Contributor hereby grants to You a perpetual,
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
(except as stated in this section) patent license to make, have made,
use, offer to sell, sell, import, and otherwise transfer the Work,
where such license applies only to those patent claims licensable
by such Contributor that are necessarily infringed by their
Contribution(s) alone or by combination of their Contribution(s)
with the Work to which such Contribution(s) was submitted. If You
institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work
or a Contribution incorporated within the Work constitutes direct
or contributory patent infringement, then any patent licenses
granted to You under this License for that Work shall terminate
as of the date such litigation is filed.
4. Redistribution. You may reproduce and distribute copies of the
Work or Derivative Works thereof in any medium, with or without
modifications, and in Source or Object form, provided that You
meet the following conditions:
(a) You must give any other recipients of the Work or
Derivative Works a copy of this License; and
(b) You must cause any modified files to carry prominent notices
stating that You changed the files; and
(c) You must retain, in the Source form of any Derivative Works
that You distribute, all copyright, patent, trademark, and
attribution notices from the Source form of the Work,
excluding those notices that do not pertain to any part of
the Derivative Works; and
(d) If the Work includes a "NOTICE" text file as part of its
distribution, then any Derivative Works that You distribute must
include a readable copy of the attribution notices contained
within such NOTICE file, excluding those notices that do not
pertain to any part of the Derivative Works, in at least one
of the following places: within a NOTICE text file distributed
as part of the Derivative Works; within the Source form or
documentation, if provided along with the Derivative Works; or,
within a display generated by the Derivative Works, if and
wherever such third-party notices normally appear. The contents
of the NOTICE file are for informational purposes only and
do not modify the License. You may add Your own attribution
notices within Derivative Works that You distribute, alongside
or as an addendum to the NOTICE text from the Work, provided
that such additional attribution notices cannot be construed
as modifying the License.
You may add Your own copyright statement to Your modifications and
may provide additional or different license terms and conditions
for use, reproduction, or distribution of Your modifications, or
for any such Derivative Works as a whole, provided Your use,
reproduction, and distribution of the Work otherwise complies with
the conditions stated in this License.
5. Submission of Contributions. Unless You explicitly state otherwise,
any Contribution intentionally submitted for inclusion in the Work
by You to the Licensor shall be under the terms and conditions of
this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify
the terms of any separate license agreement you may have executed
with Licensor regarding such Contributions.
6. Trademarks. This License does not grant permission to use the trade
names, trademarks, service marks, or product names of the Licensor,
except as required for reasonable and customary use in describing the
origin of the Work and reproducing the content of the NOTICE file.
7. Disclaimer of Warranty. Unless required by applicable law or
agreed to in writing, Licensor provides the Work (and each
Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied, including, without limitation, any warranties or conditions
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
PARTICULAR PURPOSE. You are solely responsible for determining the
appropriateness of using or redistributing the Work and assume any
risks associated with Your exercise of permissions under this License.
8. Limitation of Liability. In no event and under no legal theory,
whether in tort (including negligence), contract, or otherwise,
unless required by applicable law (such as deliberate and grossly
negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special,
incidental, or consequential damages of any character arising as a
result of this License or out of the use or inability to use the
Work (including but not limited to damages for loss of goodwill,
work stoppage, computer failure or malfunction, or any and all
other commercial damages or losses), even if such Contributor
has been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability. While redistributing
the Work or Derivative Works thereof, You may choose to offer,
and charge a fee for, acceptance of support, warranty, indemnity,
or other liability obligations and/or rights consistent with this
License. However, in accepting such obligations, You may act only
on Your own behalf and on Your sole responsibility, not on behalf
of any other Contributor, and only if You agree to indemnify,
defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason
of your accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work.
To apply the Apache License to your work, attach the following
boilerplate notice, with the fields enclosed by brackets "[]"
replaced with your own identifying information. (Don't include
the brackets!) The text should be enclosed in the appropriate
comment syntax for the file format. We also recommend that a
file or class name and description of purpose be included on the
same "printed page" as the copyright notice for easier
identification within third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
+99
View File
@@ -0,0 +1,99 @@
<h1 align=" center">
<picture>
<source media="(prefers-color-scheme: dark)" srcset="https://zvec.oss-cn-hongkong.aliyuncs.com/logo/github_log_2.svg" />
<img src="https://zvec.oss-cn-hongkong.aliyuncs.com/logo/github_logo_1.svg" width="400" alt="zvec logo" />
</picture>
</h1>
<p align="center">
<a href="https://codecov.io/github/GraphScope/zvec" style="text-decoration: none;"><img src="https://codecov.io/github/GraphScope/zvec/graph/badge.svg?token=PN16Q9PSZX" alt="Code Coverage"/></a>
<a href="https://github.com/GraphScope/zvec/actions" style="text-decoration:none;"><img src="https://github.com/GraphScope/zvec/actions/workflows/linux_x64_docker_ci.yml/badge.svg" alt="Linux x64 CI"/></a>
<a href="https://github.com/GraphScope/zvec/releases" style="text-decoration: none;"><img src="https://img.shields.io/github/v/release/GraphScope/zvec" alt="Latest Release"/></a>
<a href="https://github.com/GraphScope/zvec/blob/main/LICENSE" style="text-decoration: none;"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License"/></a>
</p>
# Zvec
**Zvec** is an open-source, in-process vector database — lightweight, lightning-fast, and designed to embed directly into applications. Built on **Proxima** (Alibaba's battle-tested vector search engine), it delivers production-grade, low-latency, scalable similarity search with minimal setup.
[📚 Quick Start](https://zvec.org/en/docs/quickstart/) | 🌐 [Website](https://zvec.org/en/) | [📖 Documentation](https://zvec.org/en/docs/) | [📊 Benchmarks](https://zvec.org/en/docs/benchmarks/)
## 💫 Features
- **Blazing Fast**: Searches billions of vectors in milliseconds.
- **Simple, Just Works**: Install with `pip install zvec` and start searching in seconds. No servers, no config, no fuss.
- **Dense + Sparse Vectors**: Work with both dense and sparse embeddings, with native support for multi-vector queries in a single call.
- **Hybrid Search**: Combine semantic similarity with structured filters for precise results.
- **Runs Anywhere**: As an in-process library, Zvec runs wherever your code runs — notebooks, servers, CLI tools, or even edge devices.
## 📦 Installation
Install Zvec from PyPI with a single command:
```bash
pip install zvec
```
**Requirements**:
- Python 3.9
- **Supported platforms**: Linux (x86_64) or macOS (ARM64/x86_64)
### Building from Source
If you prefer to build Zvec from source, you can clone the repository and install it locally.
**Note**: *The repository includes Git submodules, so cloning may take a few minutes depending on your network*.
```bash
# Clone the repository
git clone --recurse-submodules https://github.com/GraphScope/zvec.git
cd zvec
# Install from source
pip install .
```
**Build Requirements**:
- Python 3.9 or higher
- **Supported platforms**: Linux (x86_64) or macOS (ARM64/x86_64)
- C++17 compatible compiler
For detailed build instructions, check the [Building from Source](https://zvec.org/en/docs/build/) guide.
## ⚡ One-Minute Example
```python
import zvec
# Define collection schema
schema = zvec.CollectionSchema(
name="example",
vectors=zvec.VectorSchema("embedding", zvec.DataType.VECTOR_FP32, 4),
)
# Create collection
collection = zvec.create_and_open(path="./zvec_example", schema=schema,)
# Insert documents
collection.insert([
zvec.Doc(id="doc_1", vectors={"embedding": [0.1, 0.2, 0.3, 0.4]}),
zvec.Doc(id="doc_2", vectors={"embedding": [0.2, 0.3, 0.4, 0.1]}),
])
# Search by vector similarity
results = collection.query(
zvec.VectorQuery("embedding", vector=[0.4, 0.3, 0.3, 0.1]),
topk=10
)
# Results: list of {'id': str, 'score': float, ...}, sorted by relevance
print(results)
```
## ❤️ Contributing
We welcome and appreciate contributions from the community!
Whether you're fixing a bug, adding a feature, or improving documentation, your help makes Zvec better for everyone.
Check out our [Contributing Guide](./CONTRIBUTING.md) to get started!
+2338
View File
File diff suppressed because it is too large Load Diff
+585
View File
@@ -0,0 +1,585 @@
## https://en.wikipedia.org/wiki/List_of_Intel_CPU_microarchitectures
## https://en.wikipedia.org/wiki/List_of_AMD_CPU_microarchitectures
## https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html
## Intel Microarchitectures
option(ENABLE_NEHALEM "Enable Intel Nehalem CPU microarchitecture" OFF)
option(ENABLE_SANDYBRIDGE "Enable Intel Sandy Bridge CPU microarchitecture" OFF)
option(ENABLE_HASWELL "Enable Intel Haswell CPU microarchitecture" OFF)
option(ENABLE_BROADWELL "Enable Intel Broadwell CPU microarchitecture" OFF)
option(ENABLE_SKYLAKE "Enable Intel Skylake CPU microarchitecture" OFF)
option(ENABLE_SKYLAKE_AVX512 "Enable Intel Skylake Server CPU microarchitecture" OFF)
option(ENABLE_SAPPHIRERAPIDS "Enable Intel Sapphire Rapids Server CPU microarchitecture" OFF)
option(ENABLE_EMERALDRAPIDS "Enable Intel Emerald Rapids Server CPU microarchitecture" OFF)
option(ENABLE_GRANITERAPIDS "Enable Intel Granite Rapids Server CPU microarchitecture" OFF)
## AMD Microarchitectures
option(ENABLE_ZEN1 "Enable AMD Zen+ Family 17h CPU microarchitecture" OFF)
option(ENABLE_ZEN2 "Enable AMD Zen 2 Family 17h CPU microarchitecture" OFF)
option(ENABLE_ZEN3 "Enable AMD Zen 3 Family 19h CPU microarchitecture" OFF)
## Intel Instructions
option(ENABLE_SSE "Enable Intel SSE instructions" OFF)
option(ENABLE_SSE2 "Enable Intel SSE2 instructions" OFF)
option(ENABLE_SSE3 "Enable Intel SSE3 instructions" OFF)
option(ENABLE_SSSE3 "Enable Intel SSSE3 instructions" OFF)
option(ENABLE_SSE4.1 "Enable Intel SSE4.1 instructions" OFF)
option(ENABLE_SSE4.2 "Enable Intel SSE4.2 instructions" OFF)
option(ENABLE_AVX "Enable Intel AVX instructions" OFF)
option(ENABLE_AVX2 "Enable Intel AVX2 instructions" OFF)
option(ENABLE_AVX512F "Enable Intel AVX512F instructions" OFF)
option(ENABLE_AVX512BW "Enable Intel AVX512BW instructions" OFF)
option(ENABLE_AVX512CD "Enable Intel AVX512CD instructions" OFF)
option(ENABLE_AVX512DQ "Enable Intel AVX512DQ instructions" OFF)
option(ENABLE_AVX512ER "Enable Intel AVX512ER instructions" OFF)
option(ENABLE_AVX512IFMA52 "Enable Intel AVX512IFMA52 instructions" OFF)
option(ENABLE_AVX512PF "Enable Intel AVX512PF instructions" OFF)
option(ENABLE_AVX512VL "Enable Intel AVX512VL instructions" OFF)
option(ENABLE_AVX512VPOPCNTDQ "Enable Intel AVX512VPOPCNTDQ instructions" OFF)
option(ENABLE_AVX5124FMAPS "Enable Intel AVX5124FMAPS instructions" OFF)
option(ENABLE_AVX5124VNNIW "Enable Intel AVX5124VNNIW instructions" OFF)
option(ENABLE_AVX512BF16 "Enable Intel AVX512BF16 instructions" OFF)
option(ENABLE_AVX512BITALG "Enable Intel AVX512BITALG instructions" OFF)
option(ENABLE_AVX512VBMI "Enable Intel AVX512VBMI instructions" OFF)
option(ENABLE_AVX512VBMI2 "Enable Intel AVX512VBMI2 instructions" OFF)
option(ENABLE_AVX512VNNI "Enable Intel AVX512VNNI instructions" OFF)
option(ENABLE_AVX512VP2INTERSECT "Enable Intel AVX512VP2INTERSECT instructions" OFF)
option(ENABLE_AVX512FP16 "Enable Intel AVX512FP16 instructions" OFF)
option(ENABLE_AVXVNNI "Enable Intel AVXVNNI instructions" OFF)
option(ENABLE_FMA "Enable Intel FMA instructions" OFF)
option(ENABLE_F16C "Enable Intel F16C instructions" OFF)
## ARM architectures
option(ENABLE_ARMV8A "Enable ARMv8-a architecture" OFF)
option(ENABLE_ARMV8.1A "Enable ARMv8.1-a architecture" OFF)
option(ENABLE_ARMV8.2A "Enable ARMv8.2-a architecture" OFF)
option(ENABLE_ARMV8.3A "Enable ARMv8.3-a architecture" OFF)
option(ENABLE_ARMV8.4A "Enable ARMv8.4-a architecture" OFF)
option(ENABLE_ARMV8.5A "Enable ARMv8.5-a architecture" OFF)
option(ENABLE_ARMV8.6A "Enable ARMv8.6-a architecture" OFF)
## OpenMP option
option(ENABLE_OPENMP "Enable OpenMP support" OFF)
function(_AppendFlags _RESULT _FLAG)
if(${_RESULT} AND NOT "${${_RESULT}}" MATCHES "${_FLAG}")
set(${_RESULT} "${${_RESULT}} ${_FLAG}" PARENT_SCOPE)
else()
set(${_RESULT} "${_FLAG}" PARENT_SCOPE)
endif()
endfunction()
include(CheckCCompilerFlag)
if(NOT MSVC)
if(ENABLE_ARMV8.6A)
CHECK_C_COMPILER_FLAG("-march=armv8.6-a" COMPILER_SUPPORT_ARMV8.6A)
if(COMPILER_SUPPORT_ARMV8.6A)
_AppendFlags(CMAKE_C_FLAGS -march=armv8.6-a)
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.6-a)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.6-a\'.")
endif()
endif()
if(ENABLE_ARMV8.5A)
CHECK_C_COMPILER_FLAG("-march=armv8.5-a" COMPILER_SUPPORT_ARMV8.5A)
if(COMPILER_SUPPORT_ARMV8.5A)
_AppendFlags(CMAKE_C_FLAGS -march=armv8.5-a)
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.5-a)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.5-a\'.")
endif()
endif()
if(ENABLE_ARMV8.4A)
CHECK_C_COMPILER_FLAG("-march=armv8.4-a" COMPILER_SUPPORT_ARMV8.4A)
if(COMPILER_SUPPORT_ARMV8.4A)
_AppendFlags(CMAKE_C_FLAGS -march=armv8.4-a)
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.4-a)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.4-a\'.")
endif()
endif()
if(ENABLE_ARMV8.3A)
CHECK_C_COMPILER_FLAG("-march=armv8.3-a" COMPILER_SUPPORT_ARMV8.3A)
if(COMPILER_SUPPORT_ARMV8.3A)
_AppendFlags(CMAKE_C_FLAGS -march=armv8.3-a)
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.3-a)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.3-a\'.")
endif()
endif()
if(ENABLE_ARMV8.2A)
CHECK_C_COMPILER_FLAG("-march=armv8.2-a" COMPILER_SUPPORT_ARMV8.2A)
if(COMPILER_SUPPORT_ARMV8.2A)
_AppendFlags(CMAKE_C_FLAGS -march=armv8.2-a)
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.2-a)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.2-a\'.")
endif()
endif()
if(ENABLE_ARMV8.1A)
CHECK_C_COMPILER_FLAG("-march=armv8.1-a" COMPILER_SUPPORT_ARMV8.1A)
if(COMPILER_SUPPORT_ARMV8.1A)
_AppendFlags(CMAKE_C_FLAGS -march=armv8.1-a)
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.1-a)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.1-a\'.")
endif()
endif()
if(ENABLE_ARMV8A)
CHECK_C_COMPILER_FLAG("-march=armv8-a" COMPILER_SUPPORT_ARMV8A)
if(COMPILER_SUPPORT_ARMV8A)
_AppendFlags(CMAKE_C_FLAGS -march=armv8-a)
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8-a)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8-a\'.")
endif()
endif()
if(ENABLE_ZEN1)
CHECK_C_COMPILER_FLAG("-march=znver1" COMPILER_SUPPORT_ZEN1)
if(COMPILER_SUPPORT_ZEN1)
_AppendFlags(CMAKE_C_FLAGS -march=znver1)
_AppendFlags(CMAKE_CXX_FLAGS -march=znver1)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=znver1\'.")
endif()
endif()
if(ENABLE_ZEN2)
CHECK_C_COMPILER_FLAG("-march=znver2" COMPILER_SUPPORT_ZEN2)
if(COMPILER_SUPPORT_ZEN2)
_AppendFlags(CMAKE_C_FLAGS -march=znver2)
_AppendFlags(CMAKE_CXX_FLAGS -march=znver2)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=znver2\'.")
endif()
endif()
if(ENABLE_ZEN3)
CHECK_C_COMPILER_FLAG("-march=znver3" COMPILER_SUPPORT_ZEN3)
if(COMPILER_SUPPORT_ZEN3)
_AppendFlags(CMAKE_C_FLAGS -march=znver3)
_AppendFlags(CMAKE_CXX_FLAGS -march=znver3)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=znver3\'.")
endif()
endif()
if(ENABLE_NEHALEM)
CHECK_C_COMPILER_FLAG("-march=nehalem" COMPILER_SUPPORT_NEHALEM)
if(COMPILER_SUPPORT_NEHALEM)
_AppendFlags(CMAKE_C_FLAGS -march=nehalem)
_AppendFlags(CMAKE_CXX_FLAGS -march=nehalem)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=nehalem\'.")
endif()
endif()
if(ENABLE_SANDYBRIDGE)
CHECK_C_COMPILER_FLAG("-march=sandybridge" COMPILER_SUPPORT_SANDYBRIDGE)
if(COMPILER_SUPPORT_SANDYBRIDGE)
_AppendFlags(CMAKE_C_FLAGS -march=sandybridge)
_AppendFlags(CMAKE_CXX_FLAGS -march=sandybridge)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=sandybridge\'.")
endif()
endif()
if(ENABLE_HASWELL)
CHECK_C_COMPILER_FLAG("-march=haswell" COMPILER_SUPPORT_HASWELL)
if(COMPILER_SUPPORT_HASWELL)
_AppendFlags(CMAKE_C_FLAGS -march=haswell)
_AppendFlags(CMAKE_CXX_FLAGS -march=haswell)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=haswell\'.")
endif()
endif()
if(ENABLE_BROADWELL)
CHECK_C_COMPILER_FLAG("-march=broadwell" COMPILER_SUPPORT_BROADWELL)
if(COMPILER_SUPPORT_BROADWELL)
_AppendFlags(CMAKE_C_FLAGS -march=broadwell)
_AppendFlags(CMAKE_CXX_FLAGS -march=broadwell)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=broadwell\'.")
endif()
endif()
if(ENABLE_SKYLAKE)
CHECK_C_COMPILER_FLAG("-march=skylake" COMPILER_SUPPORT_SKYLAKE)
if(COMPILER_SUPPORT_SKYLAKE)
_AppendFlags(CMAKE_C_FLAGS -march=skylake)
_AppendFlags(CMAKE_CXX_FLAGS -march=skylake)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=skylake\'.")
endif()
endif()
if(ENABLE_SKYLAKE_AVX512)
CHECK_C_COMPILER_FLAG("-march=skylake-avx512" COMPILER_SUPPORT_SKYLAKE_AVX512)
if(COMPILER_SUPPORT_SKYLAKE_AVX512)
_AppendFlags(CMAKE_C_FLAGS -march=skylake-avx512)
_AppendFlags(CMAKE_CXX_FLAGS -march=skylake-avx512)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=skylake-avx512\'.")
endif()
endif()
if(ENABLE_SAPPHIRERAPIDS)
CHECK_C_COMPILER_FLAG("-march=sapphirerapids" COMPILER_SUPPORT_SAPPHIRERAPIDS)
if(COMPILER_SUPPORT_SAPPHIRERAPIDS)
_AppendFlags(CMAKE_C_FLAGS -march=sapphirerapids)
_AppendFlags(CMAKE_CXX_FLAGS -march=sapphirerapids)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=sapphirerapids\'.")
endif()
endif()
if(ENABLE_EMERALDRAPIDS)
CHECK_C_COMPILER_FLAG("-march=emeraldrapids" COMPILER_SUPPORT_EMERALDRAPID)
if(COMPILER_SUPPORT_EMERALDRAPID)
_AppendFlags(CMAKE_C_FLAGS -march=emeraldrapids)
_AppendFlags(CMAKE_CXX_FLAGS -march=emeraldrapids)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=emeraldrapids\'.")
endif()
endif()
if(ENABLE_GRANITERAPIDS)
CHECK_C_COMPILER_FLAG("-march=graniterapids" COMPILER_SUPPORT_GRANITERAPIDS)
if(COMPILER_SUPPORT_GRANITERAPIDS)
_AppendFlags(CMAKE_C_FLAGS -march=graniterapids)
_AppendFlags(CMAKE_CXX_FLAGS -march=graniterapids)
else()
message(SEND_ERROR "Compiler cannot support flag \'-march=graniterapids\'.")
endif()
endif()
if(ENABLE_AVX512BW)
CHECK_C_COMPILER_FLAG("-mavx512bw" COMPILER_SUPPORT_AVX512BW)
if(COMPILER_SUPPORT_AVX512BW)
_AppendFlags(CMAKE_C_FLAGS -mavx512bw)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512bw)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512bw\'.")
endif()
endif()
if(ENABLE_AVX512CD)
CHECK_C_COMPILER_FLAG("-mavx512cd" COMPILER_SUPPORT_AVX512CD)
if(COMPILER_SUPPORT_AVX512CD)
_AppendFlags(CMAKE_C_FLAGS -mavx512cd)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512cd)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512cd\'.")
endif()
endif()
if(ENABLE_AVX512DQ)
CHECK_C_COMPILER_FLAG("-mavx512dq" COMPILER_SUPPORT_AVX512DQ)
if(COMPILER_SUPPORT_AVX512DQ)
_AppendFlags(CMAKE_C_FLAGS -mavx512dq)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512dq)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512dq\'.")
endif()
endif()
if(ENABLE_AVX512ER)
CHECK_C_COMPILER_FLAG("-mavx512er" COMPILER_SUPPORT_AVX512ER)
if(COMPILER_SUPPORT_AVX512ER)
_AppendFlags(CMAKE_C_FLAGS -mavx512er)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512er)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512er\'.")
endif()
endif()
if(ENABLE_AVX512IFMA52)
CHECK_C_COMPILER_FLAG("-mavx512ifma52" COMPILER_SUPPORT_AVX512IFMA52)
if(COMPILER_SUPPORT_AVX512IFMA52)
_AppendFlags(CMAKE_C_FLAGS -mavx512ifma52)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512ifma52)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512ifma52\'.")
endif()
endif()
if(ENABLE_AVX512PF)
CHECK_C_COMPILER_FLAG("-mavx512pf" COMPILER_SUPPORT_AVX512PF)
if(COMPILER_SUPPORT_AVX512PF)
_AppendFlags(CMAKE_C_FLAGS -mavx512pf)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512pf)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512pf\'.")
endif()
endif()
if(ENABLE_AVX512VL)
CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL)
if(COMPILER_SUPPORT_AVX512VL)
_AppendFlags(CMAKE_C_FLAGS -mavx512vl)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vl)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vl\'.")
endif()
endif()
if(ENABLE_AVX512VPOPCNTDQ)
CHECK_C_COMPILER_FLAG("-mavx512vpopcntdq" COMPILER_SUPPORT_AVX512VPOPCNTDQ)
if(COMPILER_SUPPORT_AVX512VPOPCNTDQ)
_AppendFlags(CMAKE_C_FLAGS -mavx512vpopcntdq)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vpopcntdq)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vpopcntdq\'.")
endif()
endif()
if(ENABLE_AVX5124FMAPS)
CHECK_C_COMPILER_FLAG("-mavx5124fmaps" COMPILER_SUPPORT_AVX5124FMAPS)
if(COMPILER_SUPPORT_AVX5124FMAPS)
_AppendFlags(CMAKE_C_FLAGS -mavx5124fmaps)
_AppendFlags(CMAKE_CXX_FLAGS -mavx5124fmaps)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx5124fmaps\'.")
endif()
endif()
if(ENABLE_AVX5124VNNIW)
CHECK_C_COMPILER_FLAG("-mavx5124vnniw" COMPILER_SUPPORT_AVX5124VNNIW)
if(COMPILER_SUPPORT_AVX5124VNNIW)
_AppendFlags(CMAKE_C_FLAGS -mavx5124vnniw)
_AppendFlags(CMAKE_CXX_FLAGS -mavx5124vnniw)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx5124vnniw\'.")
endif()
endif()
if(ENABLE_AVX512BF16)
CHECK_C_COMPILER_FLAG("-mavx512bf16" COMPILER_SUPPORT_AVX512BF16)
if(COMPILER_SUPPORT_AVX512BF16)
_AppendFlags(CMAKE_C_FLAGS -mavx512bf16)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512bf16)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512bf16\'.")
endif()
endif()
if(ENABLE_AVX512BITALG)
CHECK_C_COMPILER_FLAG("-mavx512bitalg" COMPILER_SUPPORT_AVX512BITALG)
if(COMPILER_SUPPORT_AVX512BITALG)
_AppendFlags(CMAKE_C_FLAGS -mavx512bitalg)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512bitalg)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512bitalg\'.")
endif()
endif()
if(ENABLE_AVX512VBMI)
CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512VBMI)
if(COMPILER_SUPPORT_AVX512VBMI)
_AppendFlags(CMAKE_C_FLAGS -mavx512vbmi)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vbmi)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vbmi\'.")
endif()
endif()
if(ENABLE_AVX512VBMI2)
CHECK_C_COMPILER_FLAG("-mavx512vbmi2" COMPILER_SUPPORT_AVX512VBMI2)
if(COMPILER_SUPPORT_AVX512VBMI2)
_AppendFlags(CMAKE_C_FLAGS -mavx512vbmi2)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vbmi2)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vbmi2\'.")
endif()
endif()
if(ENABLE_AVX512VNNI)
CHECK_C_COMPILER_FLAG("-mavx512vnni" COMPILER_SUPPORT_AVX512VNNI)
if(COMPILER_SUPPORT_AVX512VNNI)
_AppendFlags(CMAKE_C_FLAGS -mavx512vnni)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vnni)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vnni\'.")
endif()
endif()
if(ENABLE_AVX512VP2INTERSECT)
CHECK_C_COMPILER_FLAG("-mavx512vp2intersect" COMPILER_SUPPORT_AVX512VP2INTERSECT)
if(COMPILER_SUPPORT_AVX512VP2INTERSECT)
_AppendFlags(CMAKE_C_FLAGS -mavx512vp2intersect)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vp2intersect)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vp2intersect\'.")
endif()
endif()
if(ENABLE_AVX512FP16)
CHECK_C_COMPILER_FLAG("-mavx512fp16" COMPILER_SUPPORT_AVX512FP16)
if(COMPILER_SUPPORT_AVX512FP16)
_AppendFlags(CMAKE_C_FLAGS -mavx512fp16)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512fp16)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512fp16\'.")
endif()
endif()
if(ENABLE_AVXVNNI)
CHECK_C_COMPILER_FLAG("-mavxvnni" COMPILER_SUPPORT_AVXVNNI)
if(COMPILER_SUPPORT_AVXVNNI)
_AppendFlags(CMAKE_C_FLAGS -mavxvnni)
_AppendFlags(CMAKE_CXX_FLAGS -mavxvnni)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavxvnni\'.")
endif()
endif()
if(ENABLE_AVX512F)
CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F)
if(COMPILER_SUPPORT_AVX512F)
_AppendFlags(CMAKE_C_FLAGS -mavx512f)
_AppendFlags(CMAKE_CXX_FLAGS -mavx512f)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx512f\'.")
endif()
elseif(ENABLE_AVX2)
CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2)
if(COMPILER_SUPPORT_AVX2)
_AppendFlags(CMAKE_C_FLAGS -mavx2)
_AppendFlags(CMAKE_CXX_FLAGS -mavx2)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx2\'.")
endif()
elseif(ENABLE_AVX)
CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX)
if(COMPILER_SUPPORT_AVX)
_AppendFlags(CMAKE_C_FLAGS -mavx)
_AppendFlags(CMAKE_CXX_FLAGS -mavx)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mavx\'.")
endif()
elseif(ENABLE_SSE4.2)
CHECK_C_COMPILER_FLAG("-msse4.2" COMPILER_SUPPORT_SSE4.2)
if(COMPILER_SUPPORT_SSE4.2)
_AppendFlags(CMAKE_C_FLAGS -msse4.2)
_AppendFlags(CMAKE_CXX_FLAGS -msse4.2)
else()
message(SEND_ERROR "Compiler cannot support flag \'-msse4.2\'.")
endif()
elseif(ENABLE_SSE4.1)
CHECK_C_COMPILER_FLAG("-msse4.1" COMPILER_SUPPORT_SSE4.1)
if(COMPILER_SUPPORT_SSE4.1)
_AppendFlags(CMAKE_C_FLAGS -msse4.1)
_AppendFlags(CMAKE_CXX_FLAGS -msse4.1)
else()
message(SEND_ERROR "Compiler cannot support flag \'-msse4.1\'.")
endif()
elseif(ENABLE_SSSE3)
CHECK_C_COMPILER_FLAG("-mssse3" COMPILER_SUPPORT_SSSE3)
if(COMPILER_SUPPORT_SSSE3)
_AppendFlags(CMAKE_C_FLAGS -mssse3)
_AppendFlags(CMAKE_CXX_FLAGS -mssse3)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mssse3\'.")
endif()
elseif(ENABLE_SSE3)
CHECK_C_COMPILER_FLAG("-msse3" COMPILER_SUPPORT_SSE3)
if(COMPILER_SUPPORT_SSE3)
_AppendFlags(CMAKE_C_FLAGS -msse3)
_AppendFlags(CMAKE_CXX_FLAGS -msse3)
else()
message(SEND_ERROR "Compiler cannot support flag \'-msse3\'.")
endif()
elseif(ENABLE_SSE2)
CHECK_C_COMPILER_FLAG("-msse2" COMPILER_SUPPORT_SSE2)
if(COMPILER_SUPPORT_SSE2)
_AppendFlags(CMAKE_C_FLAGS -msse2)
_AppendFlags(CMAKE_CXX_FLAGS -msse2)
else()
message(SEND_ERROR "Compiler cannot support flag \'-msse2\'.")
endif()
elseif(ENABLE_SSE)
CHECK_C_COMPILER_FLAG("-msse" COMPILER_SUPPORT_SSE)
if(COMPILER_SUPPORT_SSE)
_AppendFlags(CMAKE_C_FLAGS -msse)
_AppendFlags(CMAKE_CXX_FLAGS -msse)
else()
message(SEND_ERROR "Compiler cannot support flag \'-msse\'.")
endif()
endif()
if(ENABLE_FMA)
CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA)
if(COMPILER_SUPPORT_FMA)
_AppendFlags(CMAKE_C_FLAGS -mfma)
_AppendFlags(CMAKE_CXX_FLAGS -mfma)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mfma\'.")
endif()
endif()
if(ENABLE_F16C)
CHECK_C_COMPILER_FLAG("-mf16c" COMPILER_SUPPORT_F16C)
if(COMPILER_SUPPORT_F16C)
_AppendFlags(CMAKE_C_FLAGS -mf16c)
_AppendFlags(CMAKE_CXX_FLAGS -mf16c)
else()
message(SEND_ERROR "Compiler cannot support flag \'-mf16c\'.")
endif()
endif()
else()
if(ENABLE_AVX512F OR ENABLE_AVX512CD OR ENABLE_AVX512BW OR ENABLE_AVX512DQ OR ENABLE_AVX512VL)
CHECK_C_COMPILER_FLAG("/arch:AVX512" COMPILER_SUPPORT_AVX512F)
if(COMPILER_SUPPORT_AVX512F)
_AppendFlags(CMAKE_C_FLAGS /arch:AVX512)
_AppendFlags(CMAKE_CXX_FLAGS /arch:AVX512)
else()
message(SEND_ERROR "Compiler cannot support flag \'/arch:AVX512\'.")
endif()
elseif(ENABLE_AVX2)
CHECK_C_COMPILER_FLAG("/arch:AVX2" COMPILER_SUPPORT_AVX2)
if(COMPILER_SUPPORT_AVX2)
_AppendFlags(CMAKE_C_FLAGS /arch:AVX2)
_AppendFlags(CMAKE_CXX_FLAGS /arch:AVX2)
else()
message(SEND_ERROR "Compiler cannot support flag \'/arch:AVX2\'.")
endif()
elseif(ENABLE_AVX)
CHECK_C_COMPILER_FLAG("/arch:AVX" COMPILER_SUPPORT_AVX)
if(COMPILER_SUPPORT_AVX)
_AppendFlags(CMAKE_C_FLAGS /arch:AVX)
_AppendFlags(CMAKE_CXX_FLAGS /arch:AVX)
else()
message(SEND_ERROR "Compiler cannot support flag \'/arch:AVX\'.")
endif()
elseif(ENABLE_SSE4.2 OR ENABLE_SSE4.1 OR ENABLE_SSE3 OR ENABLE_SSSE3 OR ENABLE_SSE2)
CHECK_C_COMPILER_FLAG("/arch:SSE2" COMPILER_SUPPORT_SSE2)
if(COMPILER_SUPPORT_SSE2)
_AppendFlags(CMAKE_C_FLAGS /arch:SSE2)
_AppendFlags(CMAKE_CXX_FLAGS /arch:SSE2)
endif()
elseif(ENABLE_SSE)
CHECK_C_COMPILER_FLAG("/arch:SSE" COMPILER_SUPPORT_SSE)
if(COMPILER_SUPPORT_SSE)
_AppendFlags(CMAKE_C_FLAGS /arch:SSE)
_AppendFlags(CMAKE_CXX_FLAGS /arch:SSE)
endif()
endif()
endif()
# OpenMP support
if(ENABLE_OPENMP)
find_package(OpenMP REQUIRED)
if(OPENMP_FOUND)
_AppendFlags(CMAKE_C_FLAGS ${OpenMP_C_FLAGS})
_AppendFlags(CMAKE_CXX_FLAGS ${OpenMP_CXX_FLAGS})
endif()
endif()
+28
View File
@@ -0,0 +1,28 @@
function(apply_patch_once patch_name target_dir patch_file)
set(mark_file "${target_dir}/.${patch_name}_patched")
if(EXISTS "${mark_file}")
#message(STATUS "Patch '${patch_name}' already applied to ${target_dir}, skipping.")
return()
endif()
if(NOT EXISTS "${patch_file}")
message(FATAL_ERROR "Patch file '${patch_file}' not found!")
endif()
#message(STATUS "Applying patch '${patch_name}' to ${target_dir} ...")
execute_process(
COMMAND patch -p1 -i "${patch_file}"
WORKING_DIRECTORY "${target_dir}"
RESULT_VARIABLE patch_result
OUTPUT_VARIABLE patch_stdout
ERROR_VARIABLE patch_stderr
)
if(NOT patch_result EQUAL 0)
message(FATAL_ERROR "Failed to apply patch '${patch_name}' to ${target_dir}:\n${patch_stderr}")
else()
#message(STATUS "Patch '${patch_name}' applied successfully:\n${patch_stdout}")
file(WRITE "${mark_file}" "patched")
endif()
endfunction()
+242
View File
@@ -0,0 +1,242 @@
######################################################################################################
# Zvec: High-Performance Vector Database with PyBind11 & C++ Backend
######################################################################################################
[project]
name = "zvec"
dynamic = ["version"]
description = "A high-performance vector database engine with native C++ backend and Python bindings"
readme = "README.md"
license = { text = "Apache-2.0" }
authors = [
{ name = "zvec", email = "zvec@alibaba-inc.com" },
]
maintainers = [
{ name = "Zvec Core Team", email = "zvec@alibaba-inc.com" },
]
requires-python = ">=3.9"
classifiers = [
"Development Status :: 3 - Alpha",
"Intended Audience :: Developers",
"Intended Audience :: Education",
"Intended Audience :: Science/Research",
"License :: OSI Approved :: Apache Software License",
"Operating System :: POSIX :: Linux",
"Operating System :: MacOS",
"Programming Language :: C++",
"Programming Language :: Python :: 3 :: Only",
"Programming Language :: Python :: 3.10",
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Topic :: Database",
"Topic :: Scientific/Engineering :: Artificial Intelligence",
"Topic :: Software Development :: Libraries :: Python Modules",
]
keywords = [
"vector-database", "ann", "nearest-neighbor"
]
dependencies = [
"numpy >=1.23",
]
[project.urls]
Homepage = "https://github.com/graphscope/zvec"
Repository = "https://github.com/graphscope/zvec"
"Bug Tracker" = "https://github.com/graphscope/zvec/issues"
"Documentation" = "https://zvec.org"
[project.optional-dependencies]
test = [
"pytest >=8.0",
"pytest-cov >=4.1",
"pytest-mock >=3.12",
"cibuildwheel == 2.17.0",
]
docs = [
"mkdocs >=1.5",
"mkdocs-material >=9.5",
"mkdocstrings[python] >=0.24",
]
dev = [
"ruff >=0.4",
"black >=24.0",
"mypy >=1.8",
"pre-commit >=3.6",
"build >=1.0",
"twine >=4.0",
"numpy >=1.23",
# Inherit test deps
"pytest >=8.0",
"pytest-cov >=4.1",
"pytest-mock >=3.12",
"cibuildwheel == 2.17.0",
# Inherit docs deps
"mkdocs >=1.5",
"mkdocs-material >=9.5",
"mkdocstrings[python] >=0.24",
"pybind11-stubgen>=2.5.5",
"pybind11 >=3.0",
]
######################################################################################################
# BUILD SYSTEM CONFIGURATION (scikit-build-core)
######################################################################################################
[build-system]
requires = ["scikit-build-core >=0.11", "pybind11 >=3.0", "setuptools_scm>=8.0"]
build-backend = "scikit_build_core.build"
[tool.scikit-build]
# Core settings
minimum-version = "0.11"
metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
# CMake configuration
cmake.version = ">=3.15,<4.0"
ninja.version = ">=1.11"
cmake.build-type = "Release"
install.strip = true # Strip symbols in release builds to reduce wheel size
# Build directory
build-dir = "build"
# Platform support
wheel.expand-macos-universal-tags = true
wheel.packages = ["python/zvec"]
# Source distribution
sdist.include = [
"README.md",
"LICENSE",
"pyproject.toml",
"CMakeLists.txt",
"src/**/*",
"stub/zvec/**/*",
"python/zvec/py.typed",
]
# CMake defines (env-overridable)
[tool.scikit-build.cmake.define]
BUILD_TOOLS = "OFF"
BUILD_PYTHON_BINDINGS = "ON"
ENABLE_SKYLAKE_AVX512 = { env = "ENABLE_SKYLAKE_AVX512", default = "OFF" }
ENABLE_SAPPHIRERAPIDS = { env = "ENABLE_SAPPHIRERAPIDS", default = "OFF" }
######################################################################################################
# TESTING & QUALITY
######################################################################################################
[tool.pytest.ini_options]
minversion = "8.0"
addopts = [
"-ra",
"--showlocals",
"--strict-markers",
"--strict-config",
"--tb=short",
]
xfail_strict = true
log_cli_level = "INFO"
filterwarnings = [
"error",
"ignore::pytest.PytestCacheWarning",
# Ignore numpy deprecation warnings in tests (if any)
"ignore:.*numpy.*:DeprecationWarning",
]
testpaths = ["python/tests"]
markers = [
"title: Custom marker for test title/description",
# "slow: marks tests as slow",
]
######################################################################################################
# BUILD WHEEL
######################################################################################################
[tool.cibuildwheel]
build = [
"cp310-*",
"cp311-*",
"cp312-*",
]
build-frontend = "build"
test-requires = ["pytest", "numpy"]
build-verbosity = 1
[tool.cibuildwheel.linux]
archs = ["x86_64"]
test-command = "cd {project} && pytest python/tests -v --tb=short"
manylinux-x86_64-image = "manylinux_2_28"
skip = "*musllinux*"
[tool.cibuildwheel.macos]
archs = ["arm64"]
environment = { MACOSX_DEPLOYMENT_TARGET = "11.0" }
######################################################################################################
# CODE QUALITY & FORMATTING (Ruff)
######################################################################################################
[tool.ruff]
target-version = "py310"
line-length = 88
exclude = [
"build/",
"dist/",
".git/",
".venv/",
"venv/",
]
[tool.ruff.lint]
extend-select = [
"B", # flake8-bugbear
"I", # isort
"ARG", # flake8-unused-arguments
"C4", # flake8-comprehensions
"EM", # flake8-errmsg
"ICN", # flake8-import-conventions
"G", # flake8-logging-format
"PGH", # pygrep-hooks
"PIE", # flake8-pie
"PL", # pylint
"PT", # flake8-pytest-style
"PTH", # flake8-use-pathlib
"RET", # flake8-return
"RUF", # Ruff-specific
"SIM", # flake8-simplify
"T20", # flake8-print
"UP", # pyupgrade
"YTT", # flake8-2020
"EXE", # flake8-executable
"NPY", # NumPy-specific
"PD", # pandas-vet
]
ignore = [
"PLR0913", # Too many arguments (common in bindings)
"PLR2004", # Magic value used in comparison
"UP045", "UP007", # Use list() instead of [] (breaks C++ init)
"EM101", "EM102", # Exception messages as literals (ok in tests/utils)
"B008", # Mutable default args (cautiously allowed in config)
"E731", # Lambda assignment (used in callbacks)
"B019", # `functools.lru_cache` on methods (handled manually)
"PLR0912", # Too many branches
]
fixable = ["ALL"]
unfixable = []
[tool.ruff.lint.isort]
required-imports = ["from __future__ import annotations"]
known-first-party = ["zvec"]
[tool.ruff.lint.per-file-ignores]
"python/tests/**" = ["ALL"]
"bench/core/**" = ["ALL"]
"python/zvec/__init__.py" = [
"F401", # Unused import (for __all__)
"E402", # Module level import not at top (C++ module init order)
"PLE0605", # Invalid format for __all__
"RUF022", # __all__ is not sorted
]
"python/zvec/model/doc.py" = [
"RUF023", # Unused sort (for __slot__)
]
[tool.ruff.format]
indent-style = "space"
quote-style = "double"
line-ending = "lf"
skip-magic-trailing-comma = false
+302
View File
@@ -0,0 +1,302 @@
import logging
import math
import numpy as np
from zvec import (
MetricType,
DataType,
QuantizeType,
Doc,
CollectionSchema,
FieldSchema,
VectorSchema,
)
from typing import Dict
def is_float_equal(actual, expected, rel_tol=1e-5, abs_tol=1e-8):
if actual is None and expected is None:
return True
return math.isclose(actual, expected, rel_tol=rel_tol, abs_tol=abs_tol)
def is_dense_vector_equal(vec1, vec2, rtol=1e-5, atol=1e-8):
"""Compare two dense vectors with tolerance."""
return np.allclose(vec1, vec2, rtol=rtol, atol=atol)
def is_sparse_vector_equal(vec1, vec2, rtol=1e-5, atol=1e-8):
"""Compare two sparse vectors with tolerance."""
# Check if they have the same keys
if set(vec1.keys()) != set(vec2.keys()):
return False
# Check if all values are close
for key in vec1:
if not math.isclose(vec1[key], vec2[key], rel_tol=rtol, abs_tol=atol):
return False
return True
def is_float_array_equal(arr1, arr2, rtol=1e-5, atol=1e-8):
"""Compare two float arrays with tolerance."""
return np.allclose(arr1, arr2, rtol=rtol, atol=atol)
def is_double_array_equal(arr1, arr2, rtol=1e-9, atol=1e-12):
"""Compare two double arrays with tolerance."""
return np.allclose(arr1, arr2, rtol=rtol, atol=atol)
def is_int_array_equal(arr1, arr2):
"""Compare two integer arrays with exact equality."""
return np.array_equal(arr1, arr2)
def cosine_distance_dense(
vec1,
vec2,
dtype: DataType = DataType.VECTOR_FP32,
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
):
if dtype == DataType.VECTOR_FP16 or quantize_type == QuantizeType.FP16:
vec1 = [np.float16(a) for a in vec1]
vec2 = [np.float16(b) for b in vec2]
dot_product = sum(a * b for a, b in zip(vec1, vec2))
magnitude1 = math.sqrt(sum(a * a for a in vec1))
magnitude2 = math.sqrt(sum(b * b for b in vec2))
if magnitude1 == 0 or magnitude2 == 0:
return 0.0
return 1 - dot_product / (magnitude1 * magnitude2)
def dp_distance_dense(
vec1,
vec2,
dtype: DataType = DataType.VECTOR_FP32,
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
):
if dtype == DataType.VECTOR_FP16 or quantize_type == QuantizeType.FP16:
return sum(np.float16(a) * np.float16(b) for a, b in zip(vec1, vec2))
return sum(a * b for a, b in zip(vec1, vec2))
def euclidean_distance_dense(
vec1,
vec2,
dtype: DataType = DataType.VECTOR_FP32,
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
):
if dtype == DataType.VECTOR_FP16 or quantize_type == QuantizeType.FP16:
return sum((np.float16(a) - np.float16(b)) ** 2 for a, b in zip(vec1, vec2))
return sum((a - b) ** 2 for a, b in zip(vec1, vec2))
def distance_dense(
vec1,
vec2,
metric: MetricType,
data_type: DataType = DataType.VECTOR_FP32,
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
):
if metric == MetricType.COSINE:
return cosine_distance_dense(vec1, vec2, data_type, quantize_type)
elif metric == MetricType.L2:
return euclidean_distance_dense(vec1, vec2, data_type, quantize_type)
elif metric == MetricType.IP:
return dp_distance_dense(vec1, vec2, data_type, quantize_type)
else:
raise ValueError("Unsupported metric type")
def dp_distance_sparse(
vec1,
vec2,
data_type: DataType = DataType.SPARSE_VECTOR_FP32,
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
):
dot_product = 0.0
for dim in set(vec1.keys()) & set(vec2.keys()):
if (
data_type == DataType.SPARSE_VECTOR_FP16
or quantize_type == QuantizeType.FP16
):
vec1[dim] = np.float16(vec1[dim])
vec2[dim] = np.float16(vec2[dim])
dot_product += vec1[dim] * vec2[dim]
return dot_product
def distance(
vec1,
vec2,
metric: MetricType,
data_type: DataType,
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
):
is_sparse = (
data_type == DataType.SPARSE_VECTOR_FP32
or data_type == DataType.SPARSE_VECTOR_FP16
)
if is_sparse:
if metric != MetricType.IP:
raise ValueError("Unsupported metric type for sparse vectors")
if is_sparse:
return dp_distance_sparse(vec1, vec2, data_type, quantize_type)
else:
return distance_dense(vec1, vec2, metric, data_type, quantize_type)
def calculate_rrf_score(rank, k=60):
return 1.0 / (k + rank + 1)
def calculate_multi_vector_rrf_scores(query_results: Dict[str, Doc], k=60):
rrf_scores = {}
for vector_name, docs in query_results.items():
for rank, doc in enumerate(docs):
doc_id = doc.id
rrf_score = calculate_rrf_score(rank, k)
if doc_id in rrf_scores:
rrf_scores[doc_id] += rrf_score
else:
rrf_scores[doc_id] = rrf_score
return rrf_scores
def calculate_multi_vector_weighted_scores(
query_results: Dict[str, Doc], weights: Dict[str, float], metric: MetricType
):
def _normalize_score(score: float, metric: MetricType) -> float:
if metric == MetricType.L2:
return 1.0 - 2 * math.atan(score) / math.pi
if metric == MetricType.IP:
return 0.5 + math.atan(score) / math.pi
if metric == MetricType.COSINE:
return 1.0 - score / 2.0
raise ValueError("Unsupported metric type")
weighted_scores = {}
for vector_name, docs in query_results.items():
weight = weights.get(vector_name, 1.0)
for doc in docs:
doc_id = doc.id
weighted_score = (_normalize_score(doc.score, metric)) * weight
if doc_id in weighted_scores:
weighted_scores[doc_id] += weighted_score
else:
weighted_scores[doc_id] = weighted_score
return weighted_scores
def is_field_equal(field1, field2, schema: FieldSchema) -> bool:
if field1 is None and field2 is None:
return True
if field1 is None or field2 is None:
return False
if schema.data_type == DataType.ARRAY_FLOAT:
return is_float_array_equal(field1, field2)
elif schema.data_type == DataType.ARRAY_DOUBLE:
return is_double_array_equal(field1, field2)
elif schema.data_type in [
DataType.ARRAY_INT32,
DataType.ARRAY_INT64,
DataType.ARRAY_BOOL,
DataType.ARRAY_STRING,
DataType.ARRAY_UINT32,
DataType.ARRAY_UINT64,
DataType.ARRAY_INT64,
]:
return is_int_array_equal(field1, field2)
elif schema.data_type in [DataType.FLOAT, DataType.DOUBLE]:
return is_float_equal(field1, field2)
return field1 == field2
def is_vector_equal(vec1, vec2, schema: VectorSchema) -> bool:
if (
schema.data_type == DataType.SPARSE_VECTOR_FP16
or schema.data_type == DataType.VECTOR_FP16
):
# skip fp16 vector equal
return True
is_sparse = (
schema.data_type == DataType.SPARSE_VECTOR_FP32
or schema.data_type == DataType.SPARSE_VECTOR_FP16
)
if is_sparse:
return is_sparse_vector_equal(vec1, vec2)
else:
return is_dense_vector_equal(vec1, vec2)
def is_doc_equal(
doc1: Doc,
doc2: Doc,
schema: CollectionSchema,
except_score: bool = True,
include_vector: bool = True,
):
if doc1.id != doc2.id:
logging.error("doc ids are not equal")
return False
reduce_field_names = set(doc1.field_names() + doc2.field_names())
reduce_vector_names = set(doc1.vector_names() + doc2.vector_names())
is_doc1_fields_empty = doc1.fields is None or doc1.fields == {}
is_doc2_fields_empty = doc2.fields is None or doc2.fields == {}
if is_doc1_fields_empty or is_doc2_fields_empty:
if is_doc1_fields_empty != is_doc2_fields_empty:
return False
else:
for field_name in reduce_field_names:
field_schema = schema.field(field_name)
if field_schema is None:
return False
if is_field_equal(
doc1.field(field_name), doc2.field(field_name), field_schema
):
continue
else:
logging.error(f"{field_name} are not equal")
return False
if include_vector:
is_doc1_vectors_empty = doc1.vectors is None or doc1.vectors == {}
is_doc2_vectors_empty = doc2.vectors is None or doc2.vectors == {}
if is_doc1_vectors_empty or is_doc2_vectors_empty:
if is_doc1_fields_empty != is_doc2_vectors_empty:
return False
else:
for vector_name in reduce_vector_names:
vector_schema = schema.vector(vector_name)
if vector_schema is None:
return False
if is_vector_equal(
doc1.vector(vector_name), doc2.vector(vector_name), vector_schema
):
continue
else:
return False
return True
+377
View File
@@ -0,0 +1,377 @@
from zvec import CollectionSchema, Doc
from support_helper import *
import numpy as np
from typing import Literal, Optional, Union, Tuple
import random
import string
def generate_constant_vector(
i: int, dimension: int, dtype: Literal["int8", "float16", "float32"] = "float32"
):
if dtype == "int8":
vec = [i % 128] * dimension
vec[i % dimension] = (i + 1) % 128
else:
vec = [i / 256.0] * dimension
vec[i % dimension] = (i + 1) / 256.0
return vec
def generate_sparse_vector(i: int):
return {i: i + 0.1}
def generate_vectordict(i: int, schema: CollectionSchema) -> Doc:
doc_fields = {}
doc_vectors = {}
doc_fields = {}
doc_vectors = {}
for field in schema.fields:
if field.data_type == DataType.BOOL:
doc_fields[field.name] = i % 2 == 0
elif field.data_type == DataType.INT32:
doc_fields[field.name] = i
elif field.data_type == DataType.UINT32:
doc_fields[field.name] = i
elif field.data_type == DataType.INT64:
doc_fields[field.name] = i
elif field.data_type == DataType.UINT64:
doc_fields[field.name] = i
elif field.data_type == DataType.FLOAT:
doc_fields[field.name] = float(i) + 0.1
elif field.data_type == DataType.DOUBLE:
doc_fields[field.name] = float(i) + 0.11
elif field.data_type == DataType.STRING:
doc_fields[field.name] = f"test_{i}"
elif field.data_type == DataType.ARRAY_BOOL:
doc_fields[field.name] = [i % 2 == 0, i % 3 == 0]
elif field.data_type == DataType.ARRAY_INT32:
doc_fields[field.name] = [i, i + 1, i + 2]
elif field.data_type == DataType.ARRAY_UINT32:
doc_fields[field.name] = [i, i + 1, i + 2]
elif field.data_type == DataType.ARRAY_INT64:
doc_fields[field.name] = [i, i + 1, i + 2]
elif field.data_type == DataType.ARRAY_UINT64:
doc_fields[field.name] = [i, i + 1, i + 2]
elif field.data_type == DataType.ARRAY_FLOAT:
doc_fields[field.name] = [float(i + 0.1), float(i + 1.1), float(i + 2.1)]
elif field.data_type == DataType.ARRAY_DOUBLE:
doc_fields[field.name] = [float(i + 0.11), float(i + 1.11), float(i + 2.11)]
elif field.data_type == DataType.ARRAY_STRING:
doc_fields[field.name] = [f"test_{i}", f"test_{i + 1}", f"test_{i + 2}"]
else:
raise ValueError(f"Unsupported field type: {field.data_type}")
for vector in schema.vectors:
if vector.data_type == DataType.VECTOR_FP16:
doc_vectors[vector.name] = generate_constant_vector(
i, vector.dimension, "float16"
)
elif vector.data_type == DataType.VECTOR_FP32:
doc_vectors[vector.name] = generate_constant_vector(
i, vector.dimension, "float32"
)
elif vector.data_type == DataType.VECTOR_INT8:
doc_vectors[vector.name] = generate_constant_vector(
i,
vector.dimension,
"int8",
)
elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
doc_vectors[vector.name] = generate_sparse_vector(i)
elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
doc_vectors[vector.name] = generate_sparse_vector(i)
else:
raise ValueError(f"Unsupported vector type: {vector.data_type}")
return doc_fields, doc_vectors
def generate_doc(i: int, schema: CollectionSchema) -> Doc:
doc_fields = {}
doc_vectors = {}
doc_fields, doc_vectors = generate_vectordict(i, schema)
doc = Doc(id=str(i), fields=doc_fields, vectors=doc_vectors)
return doc
def generate_update_doc(i: int, schema: CollectionSchema) -> Doc:
doc_fields = {}
doc_vectors = {}
for field in schema.fields:
if field.data_type == DataType.BOOL:
doc_fields[field.name] = (i + 1) % 2 == 0
elif field.data_type == DataType.INT32:
doc_fields[field.name] = i + 1
elif field.data_type == DataType.UINT32:
doc_fields[field.name] = i + 1
elif field.data_type == DataType.INT64:
doc_fields[field.name] = i + 1
elif field.data_type == DataType.UINT64:
doc_fields[field.name] = i + 1
elif field.data_type == DataType.FLOAT:
doc_fields[field.name] = float(i + 1) + 0.1
elif field.data_type == DataType.DOUBLE:
doc_fields[field.name] = float(i) + 0.11
elif field.data_type == DataType.STRING:
doc_fields[field.name] = f"test_{i + 1}"
elif field.data_type == DataType.ARRAY_BOOL:
doc_fields[field.name] = [(i + 1) % 2 == 0, (i + 1) % 3 == 0]
elif field.data_type == DataType.ARRAY_INT32:
doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
elif field.data_type == DataType.ARRAY_UINT32:
doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
elif field.data_type == DataType.ARRAY_INT64:
doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
elif field.data_type == DataType.ARRAY_UINT64:
doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
elif field.data_type == DataType.ARRAY_FLOAT:
doc_fields[field.name] = [
float((i + 1) + 0.1),
float((i + 1) + 1.1),
float((i + 1) + 2.1),
]
elif field.data_type == DataType.ARRAY_DOUBLE:
doc_fields[field.name] = [
float((i + 1) + 0.11),
float((i + 1) + 1.11),
float((i + 1) + 2.11),
]
elif field.data_type == DataType.ARRAY_STRING:
doc_fields[field.name] = [
f"test_{i + 1}",
f"test_{(i + 1) + 1}",
f"test_{(i + 1) + 2}",
]
else:
raise ValueError(f"Unsupported field type: {field.data_type}")
for vector in schema.vectors:
if vector.data_type == DataType.VECTOR_FP16:
doc_vectors[vector.name] = generate_constant_vector(
i + 1, DEFAULT_VECTOR_DIMENSION, "float16"
)
elif vector.data_type == DataType.VECTOR_FP32:
doc_vectors[vector.name] = generate_constant_vector(
i + 1, DEFAULT_VECTOR_DIMENSION, "float32"
)
elif vector.data_type == DataType.VECTOR_INT8:
doc_vectors[vector.name] = generate_constant_vector(
i + 1,
DEFAULT_VECTOR_DIMENSION,
"int8",
)
elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
doc_vectors[vector.name] = generate_sparse_vector(i)
elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
doc_vectors[vector.name] = generate_sparse_vector(i)
else:
raise ValueError(f"Unsupported vector type: {vector.data_type}")
doc = Doc(id=str(i), fields=doc_fields, vectors=doc_vectors)
return doc
def generate_doc_random(i, schema: CollectionSchema) -> Doc:
doc_fields = {}
doc_vectors = {}
random.seed(i)
for field in schema.fields:
if field.data_type == DataType.BOOL:
doc_fields[field.name] = random.choice([True, False])
elif field.data_type == DataType.INT32:
doc_fields[field.name] = random.randint(-2147483648, 2147483647)
elif field.data_type == DataType.UINT32:
doc_fields[field.name] = random.randint(0, 4294967295)
elif field.data_type == DataType.INT64:
doc_fields[field.name] = random.randint(
-9223372036854775808, 9223372036854775807
)
elif field.data_type == DataType.UINT64:
doc_fields[field.name] = random.randint(0, 18446744073709551615)
elif field.data_type == DataType.FLOAT:
doc_fields[field.name] = random.uniform(-3.4028235e38, 3.4028235e38)
elif field.data_type == DataType.DOUBLE:
doc_fields[field.name] = random.uniform(
-1.7976931348623157e308, 1.7976931348623157e308
)
elif field.data_type == DataType.STRING:
length = random.randint(1, 999)
doc_fields[field.name] = "".join(
random.choices(string.ascii_letters + string.digits, k=length)
)
elif field.data_type == DataType.ARRAY_BOOL:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.choice([True, False]) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_INT32:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.randint(-2147483648, 2147483647) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_UINT32:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.randint(0, 4294967295) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_INT64:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.randint(-9223372036854775808, 9223372036854775807)
for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_UINT64:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.randint(0, 18446744073709551615) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_FLOAT:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.uniform(-3.4028235e38, 3.4028235e38) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_DOUBLE:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.uniform(-1.7976931348623157e308, 1.7976931348623157e308)
for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_STRING:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
"".join(
random.choices(
string.ascii_letters + string.digits, k=random.randint(1, 100)
)
)
for _ in range(array_length)
]
else:
raise ValueError(f"Unsupported field type: {field.data_type}")
for vector in schema.vectors:
if vector.data_type == DataType.VECTOR_FP16:
doc_vectors[vector.name] = generate_constant_vector(
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float16"
)
elif vector.data_type == DataType.VECTOR_FP32:
doc_vectors[vector.name] = generate_constant_vector(
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float32"
)
elif vector.data_type == DataType.VECTOR_INT8:
doc_vectors[vector.name] = generate_constant_vector(
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "int8"
)
elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
else:
raise ValueError(f"Unsupported vector type: {vector.data_type}")
doc = Doc(id=i, fields=doc_fields, vectors=doc_vectors)
return doc
def generate_vectordict_random(schema: CollectionSchema):
doc_fields = {}
doc_vectors = {}
for field in schema.fields:
if field.data_type == DataType.BOOL:
doc_fields[field.name] = random.choice([True, False])
elif field.data_type == DataType.INT32:
doc_fields[field.name] = random.randint(-2147483648, 2147483647)
elif field.data_type == DataType.UINT32:
doc_fields[field.name] = random.randint(0, 4294967295)
elif field.data_type == DataType.INT64:
doc_fields[field.name] = random.randint(
-9223372036854775808, 9223372036854775807
)
elif field.data_type == DataType.UINT64:
doc_fields[field.name] = random.randint(0, 18446744073709551615)
elif field.data_type == DataType.FLOAT:
doc_fields[field.name] = random.uniform(-3.4028235e38, 3.4028235e38)
elif field.data_type == DataType.DOUBLE:
doc_fields[field.name] = random.uniform(
-1.7976931348623157e308, 1.7976931348623157e308
)
elif field.data_type == DataType.STRING:
length = random.randint(1, 999)
doc_fields[field.name] = "".join(
random.choices(string.ascii_letters + string.digits, k=length)
)
elif field.data_type == DataType.ARRAY_BOOL:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.choice([True, False]) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_INT32:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.randint(-2147483648, 2147483647) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_UINT32:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.randint(0, 4294967295) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_INT64:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.randint(-9223372036854775808, 9223372036854775807)
for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_UINT64:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.randint(0, 18446744073709551615) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_FLOAT:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.uniform(-3.4028235e38, 3.4028235e38) for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_DOUBLE:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
random.uniform(-1.7976931348623157e308, 1.7976931348623157e308)
for _ in range(array_length)
]
elif field.data_type == DataType.ARRAY_STRING:
array_length = random.randint(0, 10)
doc_fields[field.name] = [
"".join(
random.choices(
string.ascii_letters + string.digits, k=random.randint(1, 100)
)
)
for _ in range(array_length)
]
else:
raise ValueError(f"Unsupported field type: {field.data_type}")
for vector in schema.vectors:
if vector.data_type == DataType.VECTOR_FP16:
doc_vectors[vector.name] = generate_constant_vector(
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float16"
)
elif vector.data_type == DataType.VECTOR_FP32:
doc_vectors[vector.name] = generate_constant_vector(
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float32"
)
elif vector.data_type == DataType.VECTOR_INT8:
doc_vectors[vector.name] = generate_constant_vector(
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "int8"
)
elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
else:
raise ValueError(f"Unsupported vector type: {vector.data_type}")
return doc_fields, doc_vectors
+319
View File
@@ -0,0 +1,319 @@
import pytest
import logging
from typing import Any, Generator
import zvec
from zvec import (
CollectionOption,
InvertIndexParam,
HnswIndexParam,
IVFIndexParam,
FieldSchema,
VectorSchema,
CollectionSchema,
Collection,
Doc,
VectorQuery,
)
from support_helper import *
@pytest.fixture(scope="session")
def basic_schema(collection_name="test_collection") -> CollectionSchema:
return CollectionSchema(
name=collection_name if len(collection_name) > 0 else "test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name", DataType.STRING, nullable=False, index_param=InvertIndexParam()
),
FieldSchema("weight", DataType.FLOAT, nullable=True),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
),
VectorSchema(
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
),
],
)
@pytest.fixture(scope="session")
def full_schema(
nullable: bool = False,
has_index: bool = False,
) -> CollectionSchema:
scalar_index_param = None
vector_index_param = None
if has_index:
scalar_index_param = InvertIndexParam(enable_range_optimization=True)
vector_index_param = HnswIndexParam()
fields = []
for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
fields.append(
FieldSchema(
v,
k,
nullable=nullable,
index_param=scalar_index_param,
)
)
vetors = []
for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
vetors.append(
VectorSchema(
v,
k,
dimension=DEFAULT_VECTOR_DIMENSION,
index_param=vector_index_param,
)
)
return CollectionSchema(
name="full_collection",
fields=fields,
vectors=vetors,
)
@pytest.fixture(scope="function")
def full_schema_new(request) -> CollectionSchema:
if hasattr(request, "param"):
nullable, has_index, vector_index = request.param
else:
nullable, has_index, vector_index = True, False, HnswIndexParam()
scalar_index_param = None
vector_index_param = None
if has_index:
scalar_index_param = InvertIndexParam(enable_range_optimization=True)
vector_index_param = vector_index
fields = []
for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
fields.append(
FieldSchema(
v,
k,
nullable=nullable,
index_param=scalar_index_param,
)
)
vectors = []
for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
vectors.append(
VectorSchema(
v,
k,
dimension=DEFAULT_VECTOR_DIMENSION,
index_param=vector_index_param,
)
)
return CollectionSchema(
name="full_collection_new",
fields=fields,
vectors=vectors,
)
@pytest.fixture(scope="function")
def full_schema_ivf(request) -> CollectionSchema:
if hasattr(request, "param"):
nullable, has_index, vector_index = request.param
else:
nullable, has_index, vector_index = True, False, IVFIndexParam()
scalar_index_param = None
vector_index_param = None
if has_index:
scalar_index_param = InvertIndexParam(enable_range_optimization=True)
vector_index_param = vector_index
fields = []
for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
fields.append(
FieldSchema(
v,
k,
nullable=nullable,
index_param=scalar_index_param,
)
)
vectors = []
for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
if v in ["vector_fp16_field", "vector_fp32_field"]:
vectors.append(
VectorSchema(
v,
k,
dimension=DEFAULT_VECTOR_DIMENSION,
index_param=vector_index_param,
)
)
return CollectionSchema(
name="full_collection_ivf",
fields=fields,
vectors=vectors,
)
@pytest.fixture(scope="function")
def single_vector_schema(
data_type: DataType,
) -> CollectionSchema:
vector_schema = [
VectorSchema(
DEFAULT_VECTOR_FIELD_NAME[data_type],
data_type,
DEFAULT_VECTOR_DIMENSION,
)
]
return CollectionSchema(
name="full_collection",
vectors=vector_schema,
)
@pytest.fixture(scope="function")
def single_vector_schema_with_index_param(
data_type: DataType, index_param
) -> CollectionSchema:
vector_schema = [
VectorSchema(
DEFAULT_VECTOR_FIELD_NAME[data_type],
data_type,
DEFAULT_VECTOR_DIMENSION,
index_param,
)
]
return CollectionSchema(
name="full_collection",
vectors=vector_schema,
)
def create_collection_fixture(
collection_temp_dir, schema: CollectionSchema, collection_option: CollectionOption
) -> Generator[Any, Any, Collection]:
"""Common helper function to create and manage collection fixtures."""
coll = zvec.create_and_open(
path=str(collection_temp_dir),
schema=schema,
option=collection_option,
)
assert coll is not None, "Failed to create and open collection"
assert coll.path == str(collection_temp_dir)
assert coll.schema.name == schema.name
assert list(coll.schema.fields) == list(schema.fields)
assert list(coll.schema.vectors) == list(schema.vectors)
assert coll.option.read_only == collection_option.read_only
assert coll.option.enable_mmap == collection_option.enable_mmap
try:
yield coll
finally:
if hasattr(coll, "destroy") and coll is not None:
try:
coll.destroy()
except Exception as e:
logging.warning(f"Warning: failed to destroy collection: {e}")
@pytest.fixture(scope="function")
def basic_collection(
collection_temp_dir, basic_schema, collection_option
) -> Generator[Any, Any, Collection]:
yield from create_collection_fixture(
collection_temp_dir, basic_schema, collection_option
)
@pytest.fixture(scope="function")
def collection_option():
return CollectionOption(read_only=False, enable_mmap=True)
@pytest.fixture(scope="function")
def collection_temp_dir(tmp_path_factory):
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection_path"
return str(collection_path)
@pytest.fixture(scope="function")
def full_collection(
collection_temp_dir,
full_schema,
collection_option,
nullable: bool = True,
has_index: bool = False,
) -> Generator[Any, Any, Collection]:
yield from create_collection_fixture(
collection_temp_dir, full_schema, collection_option
)
@pytest.fixture(scope="function")
def full_collection_new(
collection_temp_dir, full_schema_new, collection_option
) -> Generator[Any, Any, Collection]:
yield from create_collection_fixture(
collection_temp_dir, full_schema_new, collection_option
)
@pytest.fixture(scope="function")
def full_collection_ivf(
collection_temp_dir, full_schema_ivf, collection_option
) -> Generator[Any, Any, Collection]:
yield from create_collection_fixture(
collection_temp_dir, full_schema_ivf, collection_option
)
@pytest.fixture
def sample_field_list(nullable: bool = True, scalar_index_param=None, name_prefix=""):
field_list = []
for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
field_list.append(
FieldSchema(
f"{name_prefix}_{v}" if len(name_prefix) > 0 else v,
k,
nullable=nullable,
index_param=scalar_index_param,
)
)
return field_list
@pytest.fixture
def sample_vector_list(vector_index_param=None, name_prefix=""):
vector_list = []
for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
vector_list.append(
VectorSchema(
f"{name_prefix}_{v}" if len(name_prefix) > 0 else v,
k,
dimension=DEFAULT_VECTOR_DIMENSION,
index_param=vector_index_param,
)
)
return vector_list
+209
View File
@@ -0,0 +1,209 @@
from zvec import (
CollectionOption,
IndexOption,
OptimizeOption,
InvertIndexParam,
HnswIndexParam,
IVFIndexParam,
FlatIndexParam,
AlterColumnOption,
AddColumnOption,
DataType,
MetricType,
QuantizeType,
)
VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP = {
DataType.VECTOR_FP32: [
HnswIndexParam(),
HnswIndexParam(
metric_type=MetricType.IP,
m=16,
ef_construction=100,
quantize_type=QuantizeType.INT8,
),
HnswIndexParam(
metric_type=MetricType.COSINE,
m=24,
ef_construction=150,
quantize_type=QuantizeType.INT4,
),
HnswIndexParam(
metric_type=MetricType.L2,
m=32,
ef_construction=200,
quantize_type=QuantizeType.FP16,
),
FlatIndexParam(),
FlatIndexParam(metric_type=MetricType.IP, quantize_type=QuantizeType.INT4),
FlatIndexParam(metric_type=MetricType.L2, quantize_type=QuantizeType.INT8),
FlatIndexParam(metric_type=MetricType.COSINE, quantize_type=QuantizeType.FP16),
IVFIndexParam(),
IVFIndexParam(
metric_type=MetricType.IP,
quantize_type=QuantizeType.INT4,
n_list=100,
n_iters=10,
use_soar=False,
),
IVFIndexParam(
metric_type=MetricType.L2,
quantize_type=QuantizeType.INT8,
n_list=200,
n_iters=20,
use_soar=True,
),
IVFIndexParam(
metric_type=MetricType.COSINE,
quantize_type=QuantizeType.FP16,
n_list=150,
n_iters=15,
use_soar=False,
),
],
DataType.VECTOR_FP16: [
HnswIndexParam(),
FlatIndexParam(),
# IVFIndexParam(),
],
DataType.VECTOR_INT8: [
HnswIndexParam(),
FlatIndexParam(),
# IVFIndexParam(),
],
DataType.SPARSE_VECTOR_FP32: [
HnswIndexParam(),
FlatIndexParam(),
HnswIndexParam(
metric_type=MetricType.IP,
m=16,
ef_construction=100,
quantize_type=QuantizeType.FP16,
),
],
DataType.SPARSE_VECTOR_FP16: [
HnswIndexParam(),
FlatIndexParam(),
HnswIndexParam(
metric_type=MetricType.IP,
m=16,
ef_construction=100,
),
],
}
VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS = [
(data_type, param)
for data_type, params in VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP.items()
for param in params
]
INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP = {
DataType.VECTOR_FP32: [
InvertIndexParam(),
],
DataType.VECTOR_FP16: [
InvertIndexParam(),
],
DataType.VECTOR_INT8: [
InvertIndexParam(),
],
DataType.SPARSE_VECTOR_FP32: [
HnswIndexParam(metric_type=MetricType.L2),
FlatIndexParam(metric_type=MetricType.COSINE),
IVFIndexParam(),
InvertIndexParam(),
],
DataType.SPARSE_VECTOR_FP16: [
HnswIndexParam(metric_type=MetricType.L2),
FlatIndexParam(metric_type=MetricType.COSINE),
IVFIndexParam(),
InvertIndexParam(),
],
}
INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS = [
(data_type, param)
for data_type, params in INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP.items()
for param in params
]
COLLECTION_NAME_MAX_LENGTH = 64
COLLECTION_NAME_VALID_LIST = [
"col",
"C0llECTION",
"Collection1",
"collection_2",
"123collection-",
"a" * COLLECTION_NAME_MAX_LENGTH,
]
COLLECTION_NAME_INVALID_LIST = [
"l",
"1C",
"",
" ",
None,
"abcdefghijklmnopqrstuvwxzy123456abcdefghijklmnopqrstuvwxzy1234561",
"test/",
"!@#$%^&*()test",
]
FIELD_NAME_VALID_LIST = [
"1",
"12",
"col",
"ID",
"name1",
"Weigt_12-",
"123age",
"name_with_underscores",
"123numeric_start",
"name-with-dashes",
]
FIELD_NAME_INVALID_LIST = [
"",
" ",
None,
"abcdefghijklmnopqrstuvwxzy1234561",
"test/",
"!@#$%^&*()test",
"name@with#special$chars",
"name with spaces",
]
FIELD_LIST_MAX_LENGTH = 1024
VECTOR_LIST_MAX_LENGTH = 5
DENSE_VECTOR_MAX_DIMENSION = 20000
SPARSE_VECTOR_MAX_DIMENSION = 4096
FIELD_VECTOR_LIST_DIMENSION_VALID_LIST = [
# field_list_len, vector_list_len, dimension
(1, 1, 1),
(2, 2, 512),
(512, 3, 1024),
(1024, 4, 20000),
]
FIELD_VECTOR_LIST_DIMENSION_INVALID_LIST = [
# field_list_len, vector_list_len, dimension
(1, 1, 0),
(1, 1, -1),
(1, 1, "1"),
(1, 1, 20001),
]
INCOMPATIBLE_CONSTRUCTOR_ERROR_MSG = "incompatible constructor arguments"
SCHEMA_VALIDATE_ERROR_MSG = "schema validate failed"
CREATE_READ_ONLY_ERROR_MSG = "Unable to create collection with read-only mode"
INCOMPATIBLE_FUNCTION_ERROR_MSG = "incompatible function arguments"
INVALID_PATH_ERROR_MSG = "path validate failed"
INDEX_NON_EXISTENT_COLUMN_ERROR_MSG = "not found in schema"
ACCESS_DESTROYED_COLLECTION_ERROR_MSG = "is already destroyed"
COLLECTION_PATH_NOT_EXIST_ERROR_MSG = "not exist"
NOT_SUPPORT_ADD_COLUMN_ERROR_MSG = "Only support basic numeric data type"
NOT_EXIST_COLUMN_TO_DROP_ERROR_MSG = "Column not exists"
+126
View File
@@ -0,0 +1,126 @@
from zvec import (
CollectionOption,
IndexOption,
OptimizeOption,
InvertIndexParam,
HnswIndexParam,
IVFIndexParam,
FlatIndexParam,
DataType,
IndexType,
QuantizeType,
)
SUPPORT_SCALAR_DATA_TYPES = [
DataType.BOOL,
DataType.FLOAT,
DataType.DOUBLE,
DataType.INT32,
DataType.INT64,
DataType.UINT32,
DataType.UINT64,
DataType.STRING,
DataType.ARRAY_BOOL,
DataType.ARRAY_FLOAT,
DataType.ARRAY_DOUBLE,
DataType.ARRAY_INT32,
DataType.ARRAY_INT64,
DataType.ARRAY_UINT32,
DataType.ARRAY_UINT64,
DataType.ARRAY_STRING,
]
DEFAULT_SCALAR_FIELD_NAME = {
DataType.BOOL: "bool_field",
DataType.FLOAT: "float_field",
DataType.DOUBLE: "double_field",
DataType.INT32: "int32_field",
DataType.INT64: "int64_field",
DataType.UINT32: "uint32_field",
DataType.UINT64: "uint64_field",
DataType.STRING: "string_field",
DataType.ARRAY_BOOL: "array_bool_field",
DataType.ARRAY_FLOAT: "array_float_field",
DataType.ARRAY_DOUBLE: "array_double_field",
DataType.ARRAY_INT32: "array_int32_field",
DataType.ARRAY_INT64: "array_int64_field",
DataType.ARRAY_UINT32: "array_uint32_field",
DataType.ARRAY_UINT64: "array_uint64_field",
DataType.ARRAY_STRING: "array_string_field",
}
SUPPORT_SCALAR_INDEX_TYPES = [
IndexType.INVERT,
]
SUPPORT_VECTOR_DATA_TYPES = [
DataType.VECTOR_FP16,
DataType.VECTOR_FP32,
DataType.VECTOR_INT8,
DataType.SPARSE_VECTOR_FP32,
DataType.SPARSE_VECTOR_FP16,
]
SUPPORT_VECTOR_INDEX_TYPES = [
IndexType.FLAT,
IndexType.HNSW,
IndexType.IVF,
]
DEFAULT_VECTOR_FIELD_NAME = {
DataType.VECTOR_FP16: "vector_fp16_field",
DataType.VECTOR_FP32: "vector_fp32_field",
DataType.VECTOR_INT8: "vector_int8_field",
DataType.SPARSE_VECTOR_FP32: "sparse_vector_fp32_field",
DataType.SPARSE_VECTOR_FP16: "sparse_vector_fp16_field",
}
DEFAULT_VECTOR_DIMENSION = 128
SUPPORT_VECTOR_DATA_TYPE_INDEX_MAP = {
DataType.VECTOR_FP16: [IndexType.FLAT, IndexType.HNSW, IndexType.IVF],
DataType.VECTOR_FP32: [IndexType.FLAT, IndexType.HNSW, IndexType.IVF],
DataType.VECTOR_INT8: [IndexType.FLAT, IndexType.HNSW],
DataType.SPARSE_VECTOR_FP32: [IndexType.FLAT, IndexType.HNSW],
DataType.SPARSE_VECTOR_FP16: [IndexType.FLAT, IndexType.HNSW],
}
SUPPORT_VECTOR_DATA_TYPE_INDEX_MAP_PARAMS = [
(data_type, index_type)
for data_type, index_types in SUPPORT_VECTOR_DATA_TYPE_INDEX_MAP.items()
for index_type in index_types
]
DEFAULT_INDEX_PARAMS = {
IndexType.FLAT: FlatIndexParam(),
IndexType.HNSW: HnswIndexParam(),
IndexType.IVF: IVFIndexParam(),
IndexType.INVERT: InvertIndexParam(),
}
SUPPORT_VECTOR_DATA_TYPE_QUANT_MAP = {
DataType.VECTOR_FP32: [QuantizeType.FP16, QuantizeType.INT8, QuantizeType.INT4],
DataType.SPARSE_VECTOR_FP32: [QuantizeType.FP16],
}
SUPPORT_ADD_COLUMN_DATA_TYPE = [
DataType.INT32,
DataType.UINT32,
DataType.INT64,
DataType.UINT64,
DataType.FLOAT,
DataType.DOUBLE,
]
NOT_SUPPORT_ADD_COLUMN_DATA_TYPE = [
DataType.BOOL,
DataType.STRING,
DataType.ARRAY_BOOL,
DataType.ARRAY_INT32,
DataType.ARRAY_INT64,
DataType.ARRAY_UINT32,
DataType.ARRAY_UINT64,
DataType.ARRAY_FLOAT,
DataType.ARRAY_DOUBLE,
DataType.ARRAY_STRING,
]
@@ -0,0 +1,429 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import pytest
import threading
import numpy as np
import zvec
from zvec import (
CollectionOption,
InvertIndexParam,
HnswIndexParam,
Collection,
Doc,
DataType,
FieldSchema,
VectorSchema,
)
class TestCollectionConcurrency:
@pytest.fixture(scope="function")
def test_collection(self, tmp_path_factory):
"""Fixture to create a test collection"""
collection_schema = zvec.CollectionSchema(
name="test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name",
DataType.STRING,
nullable=False,
index_param=InvertIndexParam(),
),
FieldSchema("weight", DataType.FLOAT, nullable=True),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
),
VectorSchema(
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
),
],
)
collection_option = CollectionOption(read_only=False, enable_mmap=True)
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
coll = zvec.create_and_open(
path=str(collection_path),
schema=collection_schema,
option=collection_option,
)
assert coll is not None, "Failed to create and open collection"
yield coll
# Clean up
if hasattr(coll, "destroy") and coll is not None:
try:
coll.destroy()
except Exception as e:
print(f"Warning: failed to destroy collection: {e}")
def test_concurrent_read_write(self, test_collection: Collection):
results = []
def insert_docs(thread_id):
try:
docs = [
Doc(
id=f"{thread_id}_{i}",
fields={
"id": int(f"{thread_id}{i}"),
"name": f"thread_{thread_id}_doc_{i}",
"weight": float(i),
},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: float(i), 2: float(i * 2)},
},
)
for i in range(5)
]
result = test_collection.insert(docs)
results.append((thread_id, "insert", len(result)))
except Exception as e:
results.append((thread_id, "insert_exception", str(e)))
def query_docs(thread_id):
try:
result = test_collection.query(filter="id > 0", topk=10)
results.append((thread_id, "query", len(result)))
except Exception as e:
results.append((thread_id, "query_exception", str(e)))
# Create threads for concurrent operations
threads = []
# Start insert threads
for i in range(3):
thread = threading.Thread(target=insert_docs, args=(i,))
threads.append(thread)
thread.start()
# Start query threads
for i in range(3):
thread = threading.Thread(target=query_docs, args=(i,))
threads.append(thread)
thread.start()
# Wait for all threads to complete
for thread in threads:
thread.join()
# Analyze results
insert_results = [r for r in results if r[1] == "insert"]
query_results = [r for r in results if r[1] == "query"]
logging.info(
f"Concurrent read/write results - Inserts: {len(insert_results)}, Queries: {len(query_results)}"
)
# At least some operations should succeed
assert len(insert_results) + len(query_results) > 0
def test_concurrent_query(self, test_collection: Collection):
# First insert some data
docs = [
Doc(
id=f"{i}",
fields={"id": i, "name": f"test_{i}", "weight": float(i)},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: float(i), 2: float(i * 2)},
},
)
for i in range(20)
]
insert_result = test_collection.insert(docs)
assert len(insert_result) == 20
results = []
def query_operation(thread_id):
"""Perform query operation from a thread"""
try:
result = test_collection.query(filter=f"id > {thread_id}", topk=5)
results.append((thread_id, "query", len(result)))
except Exception as e:
results.append((thread_id, "query_exception", str(e)))
# Create multiple threads for concurrent queries
threads = []
for i in range(5):
thread = threading.Thread(target=query_operation, args=(i,))
threads.append(thread)
thread.start()
# Wait for all threads to complete
for thread in threads:
thread.join()
# Analyze results
query_results = [r for r in results if r[1] == "query"]
logging.info(f"Concurrent query results - Queries: {len(query_results)}")
# All query operations should succeed
assert len(query_results) == 5
def test_concurrent_modifications(self, test_collection: Collection):
# First insert some data
docs = [
Doc(
id=f"{i}",
fields={"id": i, "name": f"test_{i}", "weight": float(i)},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: float(i), 2: float(i * 2)},
},
)
for i in range(10)
]
insert_result = test_collection.insert(docs)
assert len(insert_result) == 10
results = []
def update_operation(thread_id):
"""Perform update operation from a thread"""
try:
# Each thread updates different documents
update_docs = [
Doc(
id=f"{i}",
fields={
"id": i,
"name": f"updated_by_thread_{thread_id}",
"weight": float(i + thread_id),
},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: float(i) + 0.5, 2: float(i * 2) + 0.5},
},
)
for i in range(thread_id * 2, thread_id * 2 + 2)
]
result = test_collection.update(update_docs)
results.append((thread_id, "update", len(result)))
except Exception as e:
results.append((thread_id, "update_exception", str(e)))
def delete_operation(thread_id):
"""Perform delete operation from a thread"""
try:
# Each thread deletes different documents
delete_ids = [f"{thread_id * 2 + 2}", f"{thread_id * 2 + 3}"]
result = test_collection.delete(delete_ids)
results.append((thread_id, "delete", len(result)))
except Exception as e:
results.append((thread_id, "delete_exception", str(e)))
# Create threads for concurrent operations
threads = []
# Start update threads
for i in range(3):
thread = threading.Thread(target=update_operation, args=(i,))
threads.append(thread)
thread.start()
# Start delete threads
for i in range(2):
thread = threading.Thread(target=delete_operation, args=(i,))
threads.append(thread)
thread.start()
# Wait for all threads to complete
for thread in threads:
thread.join()
# Analyze results
update_results = [r for r in results if r[1] == "update"]
delete_results = [r for r in results if r[1] == "delete"]
logging.info(
f"Concurrent modification results - Updates: {len(update_results)}, Deletes: {len(delete_results)}"
)
# At least some operations should succeed
assert len(update_results) + len(delete_results) > 0
def test_read_write_locking(self, test_collection: Collection):
# Perform operations that should be thread-safe
docs = [
Doc(
id=f"{i}",
fields={"id": i, "name": f"test_{i}", "weight": float(i)},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: float(i), 2: float(i * 2)},
},
)
for i in range(5)
]
# Insert data
insert_result = test_collection.insert(docs)
assert len(insert_result) == 5
# Concurrent operations should not cause data corruption
results = []
def mixed_operation(thread_id):
"""Perform mixed operations from a thread"""
try:
# Mix of read and write operations
if thread_id % 2 == 0:
# Read operation
result = test_collection.fetch([f"{thread_id % 5}"])
results.append((thread_id, "read", len(result)))
else:
# Write operation
doc = Doc(
id=f"{thread_id % 5}",
fields={
"id": thread_id % 5,
"name": f"mixed_op_{thread_id}",
"weight": float(thread_id),
},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: float(thread_id), 2: float(thread_id * 2)},
},
)
result = test_collection.upsert(doc)
results.append((thread_id, "write", len(result)))
except Exception as e:
results.append((thread_id, "exception", str(e)))
# Create multiple threads
threads = []
for i in range(10):
thread = threading.Thread(target=mixed_operation, args=(i,))
threads.append(thread)
thread.start()
# Wait for all threads to complete
for thread in threads:
thread.join()
# Verify that the collection is still in a consistent state
final_result = test_collection.query()
assert len(final_result) >= 0 # Should not crash or return corrupted data
def test_race_condition_detection(self, test_collection: Collection):
# Insert initial data
docs = [
Doc(
id=f"{i}",
fields={"id": i, "name": f"initial_{i}", "weight": float(i)},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: float(i), 2: float(i * 2)},
},
)
for i in range(10)
]
insert_result = test_collection.insert(docs)
assert len(insert_result) == 10
# Perform many rapid concurrent operations
operation_count = 100
results = []
def rapid_operation(op_id):
"""Perform rapid operations"""
try:
# Alternate between different types of operations
if op_id % 4 == 0:
# Insert
doc = Doc(
id=f"rapid_{op_id}",
fields={
"id": op_id,
"name": f"rapid_{op_id}",
"weight": float(op_id),
},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: float(op_id), 2: float(op_id * 2)},
},
)
result = test_collection.insert(doc)
results.append(("insert", len(result)))
elif op_id % 4 == 1:
# Update
doc = Doc(
id=f"{op_id % 10}",
fields={
"id": op_id % 10,
"name": f"rapid_update_{op_id}",
"weight": float(op_id),
},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: float(op_id), 2: float(op_id * 2)},
},
)
result = test_collection.update(doc)
results.append(("update", len(result)))
elif op_id % 4 == 2:
# Query
result = test_collection.query(filter=f"id > {op_id % 5}", topk=3)
results.append(("query", len(result)))
else:
# Fetch
result = test_collection.fetch([f"{op_id % 10}"])
results.append(("fetch", len(result)))
except Exception as e:
results.append(("exception", str(e)))
# Create many threads for rapid concurrent operations
threads = []
for i in range(operation_count):
thread = threading.Thread(target=rapid_operation, args=(i,))
threads.append(thread)
thread.start()
# Wait for all threads to complete
for thread in threads:
thread.join()
# Verify collection is still functional
final_query = test_collection.query()
assert len(final_query) >= 0 # Should not be corrupted
logging.info(
f"Rapid concurrent operations completed - Total operations: {len(results)}"
)
@@ -0,0 +1,792 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import threading
import os
from distance_helper import *
from fixture_helper import *
from doc_helper import *
from params_helper import *
def check_collection_info(
coll: Collection, schema: CollectionSchema, option: CollectionOption, path: str
):
assert coll is not None, "Failed to create and open collection"
assert coll.path == path
assert coll.schema.name == schema.name
assert list(coll.schema.fields) == list(schema.fields)
assert list(coll.schema.vectors) == list(schema.vectors)
assert coll.option.read_only == option.read_only
assert coll.option.enable_mmap == option.enable_mmap
def check_collection_basic(coll: Collection, optimize: bool = False):
schema = coll.schema
docs = [generate_doc(i, schema) for i in range(10)]
results = coll.insert(docs=docs)
assert len(results) == len(docs)
for result in results:
assert result.ok()
assert coll.stats.doc_count == len(docs)
def check_fetch_query():
results = coll.fetch([str(i) for i in range(len(docs))])
assert len(results) == len(docs)
for i in range(len(docs)):
assert str(i) in results
results = coll.query()
assert len(results) == len(docs)
check_fetch_query()
if optimize:
coll.optimize()
check_fetch_query()
def check_collection_full(coll: Collection):
test_doc = generate_doc(1, coll.schema)
insert_result = coll.insert(test_doc)
assert insert_result.ok()
stats = coll.stats
assert stats.doc_count == 1
fetched_docs = coll.fetch(ids=["1"])
assert len(fetched_docs) == 1
assert "1" in fetched_docs
assert fetched_docs["1"] is not None
assert is_doc_equal(fetched_docs["1"], test_doc, coll.schema)
query_result = coll.query()
assert len(query_result) == 1
updated_doc = Doc(
id="1",
fields={"int32_field": 1},
vectors={"vector_fp32_field": [0.2] * 128},
)
update_result = coll.update(updated_doc)
assert update_result.ok()
upserted_doc = generate_doc(1, coll.schema)
upsert_result = coll.upsert(upserted_doc)
assert upsert_result.ok()
# 8. Delete document
delete_result = coll.delete("1")
assert delete_result.ok()
# Verify document was deleted
stats = coll.stats
assert stats.doc_count == 0
valid_collection_options = [
# (read_only, enable_mmap)
(False, True),
(False, False),
]
invalid_collection_options = [
# (read_only, enable_mmap)
(True, True),
(True, False),
]
duplicate_names_test = [
("field1", "field1", "vector1", "vector2"),
("field1", "field2", "vector1", "vector1"),
(
"shared_name1",
"shared_name2",
"shared_name1",
"shared_name2",
),
]
long_names = [
"a" * 100, # 100 characters
"b" * 200, # 200 characters
]
valid_path_list = [
"/tmp/nonexistent/directory/test_collection",
"test/collection/with/slashes",
]
invalid_path_list = [
"invalid:path",
"",
"test_collection_with_spaces ",
"test@#$%collection",
]
class TestCreateAndOpen:
@pytest.mark.parametrize("collection_name", COLLECTION_NAME_VALID_LIST)
def test_valid_collection_name(
self,
collection_temp_dir,
collection_name,
collection_option,
sample_field_list,
sample_vector_list,
):
collection_schema = zvec.CollectionSchema(
name=collection_name,
fields=sample_field_list,
vectors=sample_vector_list,
)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
check_collection_info(
coll, collection_schema, collection_option, collection_temp_dir
)
check_collection_basic(coll)
coll.destroy()
@pytest.mark.parametrize("collection_name", COLLECTION_NAME_INVALID_LIST)
def test_invalid_collection_name(
self,
collection_temp_dir,
collection_name,
collection_option,
sample_field_list,
sample_vector_list,
):
with pytest.raises(Exception) as exc_info:
collection_schema = zvec.CollectionSchema(
name=collection_name,
fields=sample_field_list,
vectors=sample_vector_list,
)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
@pytest.mark.parametrize("name_prefix", FIELD_NAME_VALID_LIST)
def test_valid_field_vector_name(
self,
collection_temp_dir,
collection_option,
name_prefix,
sample_field_list,
sample_vector_list,
):
collection_schema = zvec.CollectionSchema(
name="test_collection",
fields=sample_field_list,
vectors=sample_vector_list,
)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
check_collection_info(
coll, collection_schema, collection_option, collection_temp_dir
)
check_collection_basic(coll)
coll.destroy()
@pytest.mark.parametrize("field_name", FIELD_NAME_INVALID_LIST)
def test_invalid_field_name(
self, collection_temp_dir, collection_option, field_name
):
with pytest.raises(Exception) as exc_info:
field_list = [FieldSchema(field_name, DataType.STRING)]
vector_list = [
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
)
]
collection_schema = zvec.CollectionSchema(
name="collection_name", fields=field_list, vectors=vector_list
)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
@pytest.mark.parametrize("vector_name", FIELD_NAME_INVALID_LIST)
def test_invalid_vector_name(
self, collection_temp_dir, collection_option, vector_name
):
with pytest.raises(Exception) as exc_info:
field_list = [
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
)
]
vector_list = [
VectorSchema(vector_name, DataType.VECTOR_FP32, dimension=128)
]
collection_schema = zvec.CollectionSchema(
name="collection_name", fields=field_list, vectors=vector_list
)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
@pytest.mark.parametrize(
"field_list_len,vector_list_len,dimension",
FIELD_VECTOR_LIST_DIMENSION_VALID_LIST,
)
def test_valid_field_vector_size_dimension(
self,
collection_temp_dir,
collection_option,
field_list_len,
vector_list_len,
dimension,
):
field_list = []
vector_list = []
for i in range(0, field_list_len):
field_list.append(
FieldSchema("id_" + str(i), DataType.INT64, nullable=True)
)
for i in range(0, vector_list_len):
vector_list.append(
VectorSchema(
"dense_vector_" + str(i),
DataType.VECTOR_FP32,
dimension=dimension,
index_param=HnswIndexParam(),
)
)
collection_schema = zvec.CollectionSchema(
name="test_dense_vector_list", fields=field_list, vectors=vector_list
)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
check_collection_info(
coll, collection_schema, collection_option, collection_temp_dir
)
check_collection_basic(coll)
coll.destroy()
@pytest.mark.parametrize(
"field_list_len,vector_list_len,dimension",
FIELD_VECTOR_LIST_DIMENSION_INVALID_LIST,
)
def test_invalid_field_vector_size_dimension(
self,
collection_temp_dir,
collection_option,
vector_list_len,
field_list_len,
dimension,
):
with pytest.raises(Exception) as exc_info:
field_list = []
vector_list = []
for i in range(0, field_list_len):
field_list.append(
FieldSchema(
"id_" + str(i),
DataType.INT64,
nullable=False,
)
)
for i in range(0, vector_list_len):
vector_list.append(
VectorSchema(
"dense_vector_" + str(i),
DataType.VECTOR_FP32,
dimension=dimension,
index_param=HnswIndexParam(),
)
)
collection_schema = zvec.CollectionSchema(
name="test_dense_vector_list", fields=field_list, vectors=vector_list
)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
def test_valid_single_vector_field_construction(
self, collection_temp_dir, collection_option
):
field = FieldSchema(
"id",
DataType.INT64,
nullable=True,
index_param=InvertIndexParam(enable_range_optimization=True),
)
vector = VectorSchema(
"dense_vector",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
)
collection_schema = zvec.CollectionSchema(
name="test_single_dense_vector_non_list",
fields=field,
vectors=vector, # Non-list form
)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
check_collection_info(
coll, collection_schema, collection_option, collection_temp_dir
)
check_collection_basic(coll)
coll.destroy()
def test_collection_concurrent_create(
self, collection_temp_dir, basic_schema, collection_option
):
results = []
errors = []
lock = threading.Lock()
# Function to be executed by each thread
def create_collection_thread(thread_id):
try:
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=basic_schema,
option=collection_option,
)
with lock:
results.append((thread_id, coll))
except Exception as e:
with lock:
errors.append((thread_id, str(e)))
threads = []
for i in range(5):
thread = threading.Thread(target=create_collection_thread, args=(i,))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
assert len(results) == 1, (
f"Expected exactly one successful creation, but got {len(results)}"
)
assert len(errors) == 4, (
f"Expected exactly four failures, but got {len(errors)}"
)
successful_thread_id, successful_collection = results[0]
assert successful_collection is not None, (
"Successful creation should return a valid collection"
)
assert successful_collection.path == collection_temp_dir, (
"Collection path mismatch"
)
def test_create_open_loop(
self, collection_temp_dir, collection_option, full_schema
):
for cycle in range(10):
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=full_schema,
option=collection_option,
)
assert coll is not None, (
f"Failed to create and open collection in cycle {cycle}"
)
assert coll.path == collection_temp_dir, (
f"Collection path mismatch in cycle {cycle}"
)
del coll
reopened_coll = zvec.open(
path=collection_temp_dir, option=collection_option
)
assert reopened_coll is not None, (
f"Failed to reopen collection in cycle {cycle}"
)
assert reopened_coll.path == collection_temp_dir, (
f"Reopened collection path mismatch in cycle {cycle}"
)
check_collection_full(reopened_coll)
reopened_coll.destroy()
@pytest.mark.parametrize(
"data_type, index_param", VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS
)
def test_valid_vector_index_params(
self,
data_type,
index_param,
single_vector_schema_with_index_param,
collection_temp_dir,
collection_option,
):
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=single_vector_schema_with_index_param,
option=collection_option,
)
check_collection_info(
coll,
single_vector_schema_with_index_param,
collection_option,
collection_temp_dir,
)
check_collection_basic(coll, True)
@pytest.mark.parametrize(
"data_type, index_param", INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS
)
def test_invalid_vector_index_params(
self,
data_type,
index_param,
single_vector_schema_with_index_param,
collection_temp_dir,
collection_option,
):
with pytest.raises(Exception) as exc_info:
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=single_vector_schema_with_index_param,
option=collection_option,
)
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
def test_open_concurrent_same_path(self, tmp_path_factory, collection_option):
"""Test concurrent opening of the same collection path.
- Multi-threading concurrency: 5 threads simultaneously open the same collection
- Result verification: Verify that only one can open successfully, others must fail
"""
# Create a temporary directory and path for the collection
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "concurrent_open_test_collection"
# First, create a collection that we'll try to open concurrently
field_list = [
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name", DataType.STRING, nullable=False, index_param=InvertIndexParam()
),
]
vector_list = [
VectorSchema(
"dense_vector",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
)
]
collection_schema = zvec.CollectionSchema(
name="concurrent_open_test_collection",
fields=field_list,
vectors=vector_list,
)
# Create the collection first
coll = zvec.create_and_open(
path=str(collection_path),
schema=collection_schema,
option=collection_option,
)
# Close the collection so we can test opening it
if hasattr(coll, "close") and coll is not None:
coll.close()
# Shared variables to collect results from threads
results = []
errors = []
# Lock for thread-safe operations
lock = threading.Lock()
# Clean up the created collection reference
del coll
# Function to be executed by each thread
def open_collection_thread(thread_id):
try:
reopened_coll = zvec.open(
path=str(collection_path), option=collection_option
)
with lock:
results.append((thread_id, reopened_coll))
# Clean up the collection if opened successfully
if hasattr(reopened_coll, "close") and reopened_coll is not None:
reopened_coll.close()
except Exception as e:
with lock:
errors.append((thread_id, str(e)))
# Create and start 5 threads
threads = []
for i in range(5):
thread = threading.Thread(target=open_collection_thread, args=(i,))
threads.append(thread)
thread.start()
# Wait for all threads to complete
for thread in threads:
thread.join()
# Verify results:
# 1. Only one open should succeed (exactly one collection in results)
# 2. Others should fail (4 errors in errors)
assert len(results) == 1, (
f"Expected exactly one successful open, but got {len(results)}"
)
assert len(errors) == 4, (
f"Expected exactly four failures, but got {len(errors)}"
)
# Additional verification: check that the successful open has a valid collection
successful_thread_id, successful_collection = results[0]
assert successful_collection is not None, (
"Successful open should return a valid collection"
)
assert successful_collection.path == str(collection_path), (
"Collection path mismatch"
)
@pytest.mark.parametrize("read_only,enable_mmap", valid_collection_options)
def test_valid_option(
self, collection_temp_dir, basic_schema, read_only, enable_mmap
):
option = CollectionOption(read_only=read_only, enable_mmap=enable_mmap)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=basic_schema,
option=option,
)
check_collection_info(coll, basic_schema, option, collection_temp_dir)
check_collection_basic(coll)
coll.destroy()
def test_valid_none_option(self, collection_temp_dir, basic_schema):
zvec.create_and_open(
path=collection_temp_dir,
schema=basic_schema,
option=None,
)
@pytest.mark.parametrize("read_only,enable_mmap", invalid_collection_options)
def test_invalid_option(
self, collection_temp_dir, basic_schema, read_only, enable_mmap
):
with pytest.raises(Exception) as exc_info:
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=basic_schema,
option=CollectionOption(read_only=read_only, enable_mmap=enable_mmap),
)
assert CREATE_READ_ONLY_ERROR_MSG in str(exc_info.value), str(exc_info.value)
@pytest.mark.parametrize(
"field_name1,field_name2,vector_name1,vector_name2",
duplicate_names_test,
)
def test_duplicate_field_names(
self,
collection_temp_dir,
collection_option,
field_name1,
field_name2,
vector_name1,
vector_name2,
):
with pytest.raises(Exception) as exc_info:
collection_schema = zvec.CollectionSchema(
name="test_collection",
fields=[
FieldSchema(
field_name1,
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
field_name2,
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
],
vectors=[
VectorSchema(
vector_name1,
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
),
VectorSchema(
vector_name2,
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
),
],
)
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
@pytest.mark.parametrize("long_name", long_names)
def test_invalid_long_field_names(
self, collection_option, collection_temp_dir, long_name
):
collection_schema = zvec.CollectionSchema(
name=long_name,
fields=[
FieldSchema(
long_name + "_field",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
],
vectors=[
VectorSchema(
long_name + "_vector",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
)
],
)
with pytest.raises(Exception) as exc_info:
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
def test_invalid_empty_fields_and_vectors(
self, collection_temp_dir, collection_option
):
collection_schema = zvec.CollectionSchema(
name="test_collection",
fields=[], # Empty fields
vectors=[], # Empty vectors
)
with pytest.raises(Exception) as exc_info:
coll = zvec.create_and_open(
path=collection_temp_dir,
schema=collection_schema,
option=collection_option,
)
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
@pytest.mark.parametrize("valid_path", valid_path_list)
def test_valid_path(self, basic_schema, collection_option, valid_path):
if os.path.exists(valid_path):
import shutil
shutil.rmtree(valid_path)
coll = zvec.create_and_open(
path=valid_path, schema=basic_schema, option=collection_option
)
check_collection_info(coll, basic_schema, collection_option, valid_path)
coll.destroy()
@pytest.mark.parametrize("invalid_path", invalid_path_list)
def test_invalid_path(self, basic_schema, collection_option, invalid_path):
with pytest.raises(Exception) as exc_info:
coll = zvec.create_and_open(
path=invalid_path, schema=basic_schema, option=collection_option
)
assert INVALID_PATH_ERROR_MSG in str(exc_info.value), str(exc_info.value)
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,328 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import pytest
import numpy as np
import zvec
from zvec import (
CollectionOption,
InvertIndexParam,
HnswIndexParam,
DataType,
Collection,
Doc,
FieldSchema,
VectorSchema,
VectorQuery,
)
class TestCollectionExceptionHandling:
@pytest.fixture(scope="function")
def test_collection(self, tmp_path_factory):
"""Fixture to create a test collection"""
collection_schema = zvec.CollectionSchema(
name="test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name",
DataType.STRING,
nullable=False,
index_param=InvertIndexParam(),
),
FieldSchema("weight", DataType.FLOAT, nullable=True),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
),
VectorSchema(
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
),
],
)
collection_option = CollectionOption(read_only=False, enable_mmap=True)
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
coll = zvec.create_and_open(
path=str(collection_path),
schema=collection_schema,
option=collection_option,
)
assert coll is not None, "Failed to create and open collection"
yield coll
# Clean up
if hasattr(coll, "destroy") and coll is not None:
try:
coll.destroy()
except Exception as e:
print(f"Warning: failed to destroy collection: {e}")
def test_create_and_open_missing_path(self, tmp_path_factory):
collection_schema = zvec.CollectionSchema(
name="test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name",
DataType.STRING,
nullable=False,
index_param=InvertIndexParam(),
),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
)
],
)
collection_option = CollectionOption(read_only=False, enable_mmap=True)
with pytest.raises(Exception) as exc_info:
coll = zvec.create_and_open(
schema=collection_schema, option=collection_option
)
assert exc_info.value is not None, (
"Expected exception for missing path parameter"
)
def test_create_and_open_missing_schema(self, tmp_path_factory):
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
collection_option = CollectionOption(read_only=False, enable_mmap=True)
with pytest.raises(Exception) as exc_info:
coll = zvec.create_and_open(
path=str(collection_path), option=collection_option
)
assert exc_info.value is not None, (
"Expected exception for missing schema parameter"
)
def test_open_missing_path(self):
collection_option = CollectionOption(read_only=False, enable_mmap=True)
with pytest.raises(Exception) as exc_info:
coll = zvec.open(option=collection_option)
assert exc_info.value is not None, (
"Expected exception for missing path parameter"
)
def test_insert_missing_docs(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
result = test_collection.insert()
assert exc_info.value is not None, (
"Expected exception for missing docs parameter"
)
def test_update_missing_docs(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
result = test_collection.update()
assert exc_info.value is not None, (
"Expected exception for missing docs parameter"
)
def test_upsert_missing_docs(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
result = test_collection.upsert()
assert exc_info.value is not None, (
"Expected exception for missing docs parameter"
)
def test_delete_missing_ids(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
result = test_collection.delete()
assert exc_info.value is not None, (
"Expected exception for missing ids parameter"
)
def test_fetch_missing_ids(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
result = test_collection.fetch()
assert exc_info.value is not None, (
"Expected exception for missing ids parameter"
)
def test_query_missing_vectorquery_field_name(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
result = test_collection.query(vectors=[VectorQuery()])
assert exc_info.value is not None, (
"Expected exception for missing VectorQuery field_name parameter"
)
def test_add_column_missing_field_schema(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
test_collection.add_column()
assert exc_info.value is not None, (
"Expected exception for missing field_schema parameter"
)
def test_alter_column_missing_old_name(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
test_collection.alter_column(new_name="new_name")
assert exc_info.value is not None, (
"Expected exception for missing old_name parameter"
)
def test_alter_column_missing_new_name(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
test_collection.alter_column(old_name="old_name")
assert exc_info.value is not None, (
"Expected exception for missing new_name parameter"
)
def test_drop_column_missing_field_name(self, test_collection: Collection):
with pytest.raises(Exception) as exc_info:
test_collection.drop_column()
assert exc_info.value is not None, (
"Expected exception for missing field_name parameter"
)
def test_invalid_parameter_types(self, test_collection: Collection):
# This test depends on specific implementation details
# Generally, we would expect TypeErrors or similar exceptions
pass
def test_missing_required_parameters(self, test_collection: Collection):
# This test depends on specific implementation details
# Generally, we would expect TypeErrors or similar exceptions
pass
def test_empty_collection_operations(self, tmp_path_factory):
collection_schema = zvec.CollectionSchema(
name="empty_test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name",
DataType.STRING,
nullable=False,
index_param=InvertIndexParam(),
),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
)
],
)
collection_option = CollectionOption(read_only=False, enable_mmap=True)
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "empty_test_collection"
coll = zvec.create_and_open(
path=str(collection_path),
schema=collection_schema,
option=collection_option,
)
assert coll is not None, "Failed to create and open collection"
# Test fetch on empty collection
result = coll.fetch(["1"])
assert len(result) >= 0 # May be empty or have special handling
# Test query on empty collection
result = coll.query()
assert len(result) == 0
# Test update on empty collection
doc = Doc(
id="1",
fields={"id": 1, "name": "test"},
vectors={"dense": np.random.random(128).tolist()},
)
result = coll.update(doc)
# Should handle gracefully, possibly with NOT_FOUND status
# Clean up
if hasattr(coll, "destroy") and coll is not None:
try:
coll.destroy()
except Exception as e:
print(f"Warning: failed to destroy collection: {e}")
def test_resource_management(self, test_collection: Collection):
doc = Doc(
id="1",
fields={"id": 1, "name": "test", "weight": 80.5},
vectors={
"dense": np.random.random(128).tolist(),
"sparse": {1: 1.0, 2: 2.0},
},
)
# Insert
result = test_collection.insert(doc)
assert result.ok()
# Fetch
result = test_collection.fetch(["1"])
assert len(result) == 1
# Query
result = test_collection.query()
assert len(result) >= 0
# Update
result = test_collection.update(doc)
assert result.ok()
# Delete
result = test_collection.delete("1")
assert result.ok()
def test_exception_resource_cleanup(self, test_collection: Collection):
# This test would need to simulate exception conditions
# which is difficult without specific failure injection points
pass
+960
View File
@@ -0,0 +1,960 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import threading
import numpy as np
from fixture_helper import *
COLLECTION_OPTION_TEST_CASES_VALID = [
# (read_only, enable_mmap, description)
(False, True, "Read-write with mmap enabled"),
(False, False, "Read-write with mmap disabled"),
(True, True, "Read-only with mmap enabled"),
(True, False, "Read-only with mmap disabled"),
]
# Test data for invalid paths
INVALID_PATH_LIST = [
"/nonexistent/directory/test_collection",
"invalid:path",
"", # Empty path
]
@pytest.fixture(scope="session")
def collection_schema():
return zvec.CollectionSchema(
name="test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name", DataType.STRING, nullable=False, index_param=InvertIndexParam()
),
FieldSchema(
"weight", DataType.FLOAT, nullable=False, index_param=InvertIndexParam()
),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
),
VectorSchema(
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
),
],
)
@pytest.fixture
def single_doc():
id = 0
return Doc(
id=f"{id}",
fields={"id": id, "name": "test"},
vectors={
"dense": [id + 0.1] * 128,
},
)
@pytest.fixture(scope="function")
def test_collection(
tmp_path_factory, collection_schema, collection_option
) -> Generator[Any, Any, Collection]:
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
coll = zvec.create_and_open(
path=str(collection_path), schema=collection_schema, option=collection_option
)
assert coll is not None, "Failed to create and open collection"
assert coll.path == str(collection_path)
assert coll.schema.name == collection_schema.name
assert list(coll.schema.fields) == list(collection_schema.fields)
assert list(coll.schema.vectors) == list(collection_schema.vectors)
assert coll.option.read_only == collection_option.read_only
assert coll.option.enable_mmap == collection_option.enable_mmap
try:
yield coll
finally:
if hasattr(coll, "destroy") and coll is not None:
try:
coll.destroy()
except Exception as e:
print(f"Warning: failed to destroy collection: {e}")
class TestCollectionOpen:
def test_open_basic_functionality(
self, tmp_path_factory, collection_schema, collection_option
):
import sys
import time
import os
# Create unique temp directory
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
# Ensure the path exists
collection_path_str = str(collection_path)
print(f"DEBUG: Collection path: {collection_path_str}")
print(f"DEBUG: Temp directory exists: {temp_dir.exists()}")
# Create and open collection first
created_coll = zvec.create_and_open(
path=collection_path_str, schema=collection_schema, option=collection_option
)
assert created_coll is not None, (
f"Failed to create collection, returned None instead of valid Collection object. Path: {collection_path_str}"
)
assert created_coll.path == collection_path_str, (
f"Collection path mismatch. Expected: {collection_path_str}, Actual: {created_coll.path}"
)
assert created_coll.schema.name == "test_collection", (
f"Collection schema name mismatch. Expected: test_collection, Actual: {created_coll.schema.name}"
)
# Insert multiple documents to verify persistence
docs = []
for i in range(3):
doc = Doc(
id=f"{i}",
fields={"id": i, "name": f"test_{i}", "weight": float(i * 10)},
vectors={
"dense": [float(j + i) for j in range(128)],
"sparse": {j: float(j + i) for j in range(5)},
},
)
docs.append(doc)
result = created_coll.insert(docs)
assert len(result) == 3, f"Expected 3 insertion results, but got {len(result)}"
for i, res in enumerate(result):
assert res.ok(), (
f"Insertion result {i} is not OK. Status code: {res.code()}, Message: {res.message()}"
)
# Verify documents were inserted using fetch interface
fetched_docs_after_insert = created_coll.fetch(["0", "1", "2"])
assert len(fetched_docs_after_insert) == 3, (
f"Expected 3 fetched documents after insertion, but got {len(fetched_docs_after_insert)}"
)
assert "0" in fetched_docs_after_insert, (
"Document with ID '0' not found in fetched results after insertion"
)
assert "1" in fetched_docs_after_insert, (
"Document with ID '1' not found in fetched results after insertion"
)
assert "2" in fetched_docs_after_insert, (
"Document with ID '2' not found in fetched results after insertion"
)
# Verify fetched document content after insertion
for i in range(3):
doc = fetched_docs_after_insert[f"{i}"]
assert doc is not None, (
f"Fetched document with ID '{i}' is None after insertion"
)
assert doc.id == f"{i}", (
f"Document ID mismatch for document '{i}' after insertion. Expected: {i}, Actual: {doc.id}"
)
assert doc.field("id") == i, (
f"Document id field mismatch for document '{i}' after insertion. Expected: {i}, Actual: {doc.field('id')}"
)
assert doc.field("name") == f"test_{i}", (
f"Document name field mismatch for document '{i}' after insertion. Expected: test_{i}, Actual: {doc.field('name')}"
)
assert doc.field("weight") == float(i * 10), (
f"Document weight field mismatch for document '{i}' after insertion. Expected: {float(i * 10)}, Actual: {doc.field('weight')}"
)
# Verify vector access after insertion
assert doc.vector("dense") is not None, (
f"Document {i} should have dense vector after insertion"
)
assert doc.vector("sparse") is not None, (
f"Document {i} should have sparse vector after insertion"
)
# Verify vector types after insertion
assert isinstance(doc.vector("dense"), list), (
f"Document {i} dense vector should be dict after insertion, got {type(doc.vector('dense'))}"
)
assert isinstance(doc.vector("sparse"), dict), (
f"Document {i} sparse vector should be dict after insertion, got {type(doc.vector('sparse'))}"
)
# Verify documents were inserted using stats
stats = created_coll.stats
assert stats is not None, "Collection stats should not be None"
assert stats.doc_count == 3, (
f"Document count mismatch after insertion. Expected: 3, Actual: {stats.doc_count}"
)
# Store the collection path before cleanup
collection_path = created_coll.path
# Clean up the created collection reference
del created_coll
# Wait and verify the path still exists
print(f"DEBUG: Collection path after destroy: {collection_path}")
print(f"DEBUG: Path exists after destroy: {os.path.exists(collection_path)}")
# Now open the existing collection
try:
print(f"DEBUG: Path exists before open: {os.path.exists(collection_path)}")
# List contents of parent directory for debugging
parent_dir = os.path.dirname(collection_path)
if os.path.exists(parent_dir):
print(f"DEBUG: Parent directory contents: {os.listdir(parent_dir)}")
opened_coll = zvec.open(path=collection_path, option=collection_option)
assert opened_coll is not None, (
f"Failed to open existing collection at path: {collection_path}. Returned None instead of valid Collection object"
)
assert opened_coll.path == collection_path, (
f"Opened collection path mismatch. Expected: {collection_path}, Actual: {opened_coll.path}"
)
assert opened_coll.schema.name == "test_collection", (
f"Opened collection schema name mismatch. Expected: test_collection, Actual: {opened_coll.schema.name}"
)
# Check reference count of opened collection
opened_ref_count = sys.getrefcount(opened_coll)
print(f"DEBUG: Reference count of opened collection: {opened_ref_count}")
# Verify data persistence
# Verify data persistence using fetch interface
fetched_docs = opened_coll.fetch(["0", "1", "2"])
assert len(fetched_docs) == 3, (
f"Expected 3 fetched documents after reopening, but got {len(fetched_docs)}"
)
assert "0" in fetched_docs, (
"Document with ID '0' not found in fetched results after reopening"
)
assert "1" in fetched_docs, (
"Document with ID '1' not found in fetched results after reopening"
)
assert "2" in fetched_docs, (
"Document with ID '2' not found in fetched results after reopening"
)
# Verify fetched document content after reopening collection
for i in range(3):
doc = fetched_docs[f"{i}"]
assert doc is not None, (
f"Fetched document with ID '{i}' is None after reopening collection"
)
assert doc.id == f"{i}", (
f"Document ID mismatch for document '{i}' after reopening. Expected: {i}, Actual: {doc.id}"
)
assert doc.field("id") == i, (
f"Document id field mismatch for document '{i}' after reopening. Expected: {i}, Actual: {doc.field('id')}"
)
assert doc.field("name") == f"test_{i}", (
f"Document name field mismatch for document '{i}' after reopening. Expected: test_{i}, Actual: {doc.field('name')}"
)
assert doc.field("weight") == float(i * 10), (
f"Document weight field mismatch for document '{i}' after reopening. Expected: {float(i * 10)}, Actual: {doc.field('weight')}"
)
# Verify vector access after reopening
assert doc.vector("dense") is not None, (
f"Document {i} should have dense vector after reopening"
)
assert doc.vector("sparse") is not None, (
f"Document {i} should have sparse vector after reopening"
)
# Verify vector types after reopening
assert isinstance(doc.vector("dense"), list), (
f"Document {i} dense vector should be dict after reopening, got {type(doc.vector('dense'))}"
)
assert isinstance(doc.vector("sparse"), dict), (
f"Document {i} sparse vector should be dict after reopening, got {type(doc.vector('sparse'))}"
)
# Verify score attribute exists
assert hasattr(doc, "score"), (
f"Document {i} should have a score attribute after reopening"
)
assert isinstance(doc.score, (int, float)), (
f"Document {i} score should be numeric after reopening, got {type(doc.score)}"
)
# For fetch operations, score is typically 0.0
assert doc.score == 0.0, (
f"Document {i} score should be 0.0 for fetch operation after reopening, but got {doc.score}"
)
# Test query functionality
query_result = opened_coll.query(include_vector=True)
assert len(query_result) == 3, (
f"Expected 3 query results, but got {len(query_result)}"
)
# Verify query results have proper structure and content with detailed validation
returned_doc_ids = set()
for doc in query_result:
# Verify basic document structure
assert doc.id is not None, f"Query result document should have an ID"
assert doc.id in ["0", "1", "2"], (
f"Query result document ID should be one of ['0', '1', '2'], but got {doc.id}"
)
returned_doc_ids.add(doc.id)
# Verify field access
assert doc.field("id") is not None, (
f"Document {doc.id} should have id field"
)
assert doc.field("name") is not None, (
f"Document {doc.id} should have name field"
)
assert doc.field("weight") is not None, (
f"Document {doc.id} should have weight field"
)
# Verify field values
expected_id = int(doc.id)
assert doc.field("id") == expected_id, (
f"Document {doc.id} id field mismatch. Expected: {expected_id}, Actual: {doc.field('id')}"
)
assert doc.field("name") == f"test_{expected_id}", (
f"Document {doc.id} name field mismatch. Expected: test_{expected_id}, Actual: {doc.field('name')}"
)
assert doc.field("weight") == float(expected_id * 10), (
f"Document {doc.id} weight field mismatch. Expected: {float(expected_id * 10)}, Actual: {doc.field('weight')}"
)
# Verify vector access
assert doc.vector("dense") is not None, (
f"Document {doc.id} should have dense vector"
)
assert doc.vector("sparse") is not None, (
f"Document {doc.id} should have sparse vector"
)
# Verify vector types
assert isinstance(doc.vector("dense"), list), (
f"Document {doc.id} dense vector should be list, got {type(doc.vector('dense'))}"
)
assert isinstance(doc.vector("sparse"), dict), (
f"Document {doc.id} sparse vector should be dict, got {type(doc.vector('sparse'))}"
)
# Verify score attribute exists
assert hasattr(doc, "score"), (
f"Document {doc.id} should have a score attribute"
)
assert isinstance(doc.score, (int, float)), (
f"Document {doc.id} score should be numeric, got {type(doc.score)}"
)
# Verify all expected documents are returned
expected_doc_ids = {"0", "1", "2"}
assert returned_doc_ids == expected_doc_ids, (
f"Query should return all expected documents. Expected: {expected_doc_ids}, Actual: {returned_doc_ids}"
)
# === Enhanced validation based on test_collection_dql_operations.py ===
# Verify vector field names accessibility for all documents
for doc in query_result:
vector_names = doc.vector_names()
expected_vector_names = {"dense", "sparse"}
assert set(vector_names) == expected_vector_names, (
f"Document {doc.id} vector names mismatch. Expected: {expected_vector_names}, Actual: {set(vector_names)}"
)
# Verify all vector fields can be accessed
for vector_name in expected_vector_names:
vector_data = doc.vector(vector_name)
assert vector_data is not None, (
f"Document {doc.id} should have accessible vector '{vector_name}'"
)
if vector_name == "dense":
assert isinstance(vector_data, list), (
f"Document {doc.id} vector '{vector_name}' should be list, got {type(vector_data)}"
)
else:
assert isinstance(vector_data, dict), (
f"Document {doc.id} vector '{vector_name}' should be dict, got {type(vector_data)}"
)
# Test query with filter
filtered_result = opened_coll.query(filter="id >= 1", include_vector=True)
assert len(filtered_result) == 2, (
f"Expected 2 filtered query results (id >= 1), but got {len(filtered_result)}"
)
# Verify filtered query results
filtered_doc_ids = set()
for doc in filtered_result:
assert doc.id is not None, (
f"Filtered query result document should have an ID"
)
assert doc.id in ["1", "2"], (
f"Filtered query result document ID should be one of ['1', '2'], but got {doc.id}"
)
filtered_doc_ids.add(doc.id)
# Verify filter condition is satisfied
doc_id = int(doc.id)
assert doc_id >= 1, (
f"Document {doc.id} should satisfy filter condition id >= 1"
)
# Verify document structure
assert doc.field("id") is not None, (
f"Document {doc.id} should have id field"
)
assert doc.field("name") is not None, (
f"Document {doc.id} should have name field"
)
assert doc.field("weight") is not None, (
f"Document {doc.id} should have weight field"
)
# Verify field values
assert doc.field("id") == doc_id, (
f"Document {doc.id} id field mismatch. Expected: {doc_id}, Actual: {doc.field('id')}"
)
assert doc.field("name") == f"test_{doc_id}", (
f"Document {doc.id} name field mismatch. Expected: test_{doc_id}, Actual: {doc.field('name')}"
)
assert doc.field("weight") == float(doc_id * 10), (
f"Document {doc.id} weight field mismatch. Expected: {float(doc_id * 10)}, Actual: {doc.field('weight')}"
)
# Verify vector access
assert doc.vector("dense") is not None, (
f"Document {doc.id} should have dense vector"
)
assert doc.vector("sparse") is not None, (
f"Document {doc.id} should have sparse vector"
)
# Verify score attribute exists
assert hasattr(doc, "score"), (
f"Document {doc.id} should have a score attribute"
)
assert isinstance(doc.score, (int, float)), (
f"Document {doc.id} score should be numeric, got {type(doc.score)}"
)
# Verify filtered documents
expected_filtered_ids = {"1", "2"}
assert filtered_doc_ids == expected_filtered_ids, (
f"Filtered query should return expected documents. Expected: {expected_filtered_ids}, Actual: {filtered_doc_ids}"
)
# Test vector query functionality for dense vectors
query_vector_dense = [0.1] * 128
vector_query_result = opened_coll.query(
VectorQuery(field_name="dense", vector=query_vector_dense)
)
assert len(vector_query_result) > 0, (
f"Expected at least 1 vector query result, but got {len(vector_query_result)}"
)
# Verify vector query results structure
for doc in vector_query_result[:3]: # Check first 3 results
assert doc.id is not None, (
f"Vector query result document should have an ID"
)
assert doc.id in ["0", "1", "2"], (
f"Vector query result document ID should be one of ['0', '1', '2'], but got {doc.id}"
)
# Verify document structure
assert doc.field("id") is not None, (
f"Document {doc.id} should have id field"
)
assert doc.field("name") is not None, (
f"Document {doc.id} should have name field"
)
assert doc.field("weight") is not None, (
f"Document {doc.id} should have weight field"
)
# Verify vector access
assert doc.vector("dense") is not None, (
f"Document {doc.id} should have dense vector"
)
assert doc.vector("sparse") is not None, (
f"Document {doc.id} should have sparse vector"
)
# Verify score attribute exists and is numeric
assert hasattr(doc, "score"), (
f"Document {doc.id} should have a score attribute"
)
assert isinstance(doc.score, (int, float)), (
f"Document {doc.id} score should be numeric, got {type(doc.score)}"
)
# For dense vector queries, score should typically be non-negative (depending on metric)
# Note: This may vary based on the metric type used
assert doc.score >= 0 or doc.score < 0, (
f"Document {doc.id} score should be a valid number"
)
# Test vector query functionality for sparse vectors
query_vector_sparse = {1: 1.0, 2: 2.0, 3: 3.0}
sparse_vector_query_result = opened_coll.query(
VectorQuery(field_name="sparse", vector=query_vector_sparse)
)
assert len(sparse_vector_query_result) > 0, (
f"Expected at least 1 sparse vector query result, but got {len(sparse_vector_query_result)}"
)
# Verify sparse vector query results structure
for doc in sparse_vector_query_result[:3]: # Check first 3 results
assert doc.id is not None, (
f"Sparse vector query result document should have an ID"
)
assert doc.id in ["0", "1", "2"], (
f"Sparse vector query result document ID should be one of ['0', '1', '2'], but got {doc.id}"
)
# Verify document structure
assert doc.field("id") is not None, (
f"Document {doc.id} should have id field"
)
assert doc.field("name") is not None, (
f"Document {doc.id} should have name field"
)
assert doc.field("weight") is not None, (
f"Document {doc.id} should have weight field"
)
# Verify vector access
assert doc.vector("dense") is not None, (
f"Document {doc.id} should have dense vector"
)
assert doc.vector("sparse") is not None, (
f"Document {doc.id} should have sparse vector"
)
# Verify score attribute exists and is numeric
assert hasattr(doc, "score"), (
f"Document {doc.id} should have a score attribute"
)
assert isinstance(doc.score, (int, float)), (
f"Document {doc.id} score should be numeric, got {type(doc.score)}"
)
# Clean up
if hasattr(opened_coll, "destroy") and opened_coll is not None:
opened_coll.destroy()
print("DEBUG: Opened collection destroyed successfully")
except Exception as e:
logging.error("Exception occurred: [{}]".format(e))
raise e
@pytest.mark.parametrize(
"read_only,enable_mmap,description", COLLECTION_OPTION_TEST_CASES_VALID
)
@pytest.mark.parametrize("createAndopen_enable_mmap", [True, False])
def test_open_with_different_collection_options_valid(
self,
tmp_path_factory,
createAndopen_enable_mmap,
read_only,
enable_mmap,
description,
collection_schema,
):
# Create collection with initial option
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
initial_option = CollectionOption(
read_only=False, enable_mmap=createAndopen_enable_mmap
)
# Create and open collection first
created_coll = zvec.create_and_open(
path=str(collection_path), schema=collection_schema, option=initial_option
)
assert created_coll is not None, "Failed to create collection"
# Clean up the created collection reference
del created_coll
# Now open with different options
collection_option = CollectionOption(
read_only=read_only, enable_mmap=enable_mmap
)
try:
opened_coll = zvec.open(path=str(collection_path), option=collection_option)
assert opened_coll is not None, (
f"Failed to open collection with option: {description}. Returned None instead of valid Collection object. Path: {collection_path}"
)
assert opened_coll.path == str(collection_path), (
f"Opened collection path mismatch. Expected: {collection_path}, Actual: {opened_coll.path}"
)
assert opened_coll.schema.name == collection_schema.name, (
f"Opened collection schema name mismatch. Expected: {collection_schema.name}, Actual: {opened_coll.schema.name}"
)
assert opened_coll.option.read_only == read_only, (
f"Opened collection read_only option mismatch. Expected: {read_only}, Actual: {opened_coll.option.read_only}"
)
assert opened_coll.option.enable_mmap == createAndopen_enable_mmap, (
f"Opened collection mmap option mismatch. Expected: {createAndopen_enable_mmap}, Actual: {opened_coll.option.enable_mmap}"
)
# Clean up
if (
hasattr(opened_coll, "destroy")
and opened_coll is not None
and read_only == False
):
opened_coll.destroy()
except Exception as e:
logging.error("Exception occurred: [{}]".format(e))
pytest.fail(f"Failed to open collection with different options: {e}")
def test_open_with_none_option(self, tmp_path_factory, collection_schema):
# Create collection
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
initial_option = CollectionOption(read_only=False, enable_mmap=True)
# Create and open collection first
created_coll = zvec.create_and_open(
path=str(collection_path), schema=collection_schema, option=initial_option
)
assert created_coll is not None, (
f"Failed to create collection. Returned None instead of valid Collection object. Path: {collection_path}"
)
# Clean up the created collection reference
del created_coll
# Now open with None option
with pytest.raises(Exception) as exc_info:
zvec.open(path=str(collection_path), option=None)
assert "incompatible function arguments" in str(exc_info.value), (
f"Expected 'incompatible function arguments' error, but got: {exc_info.value}"
)
def test_reopen_collection(self, tmp_path_factory):
# Prepare schema
collection_schema = zvec.CollectionSchema(
name="test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name",
DataType.STRING,
nullable=False,
index_param=InvertIndexParam(),
),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
)
],
)
collection_option = CollectionOption(read_only=False, enable_mmap=True)
# Create collection
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
# Create and open collection
coll1 = zvec.create_and_open(
path=str(collection_path),
schema=collection_schema,
option=collection_option,
)
assert coll1 is not None, "Failed to create and open collection"
# Insert some data
doc = Doc(
id="1",
fields={"id": 1, "name": "test"},
vectors={"dense": np.random.random(128).tolist()},
)
result = coll1.insert(doc)
assert result.ok()
# Close the first collection (delete reference)
del coll1
# Reopen the collection
coll2 = zvec.open(path=str(collection_path), option=collection_option)
assert coll2 is not None, "Failed to reopen collection"
assert coll2.path == str(collection_path)
assert coll2.schema.name == collection_schema.name
# Verify data is still there
fetched_docs = coll2.fetch(["1"])
assert "1" in fetched_docs
fetched_doc = fetched_docs["1"]
assert fetched_doc.id == "1"
assert fetched_doc.field("name") == "test"
# Clean up
if hasattr(coll2, "destroy") and coll2 is not None:
try:
coll2.destroy()
except Exception as e:
print(f"Warning: failed to destroy collection: {e}")
def test_open_concurrent_same_path(self, tmp_path_factory):
# First create a collection
collection_schema = zvec.CollectionSchema(
name="test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name",
DataType.STRING,
nullable=False,
index_param=InvertIndexParam(),
),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
)
],
)
collection_option = CollectionOption(read_only=False, enable_mmap=True)
# Create collection path
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
# First create the collection
created_coll = zvec.create_and_open(
path=str(collection_path),
schema=collection_schema,
option=collection_option,
)
assert created_coll is not None, "Failed to create collection"
# Close the collection so we can test concurrent opening
if hasattr(created_coll, "close") and created_coll is not None:
created_coll.close()
# Shared variables to collect results from threads
results = []
errors = []
# Lock for thread-safe operations
lock = threading.Lock()
# Clean up the created collection reference
del created_coll
# Function to be executed by each thread
def open_collection_thread(thread_id):
try:
coll = zvec.open(path=str(collection_path), option=collection_option)
with lock:
results.append((thread_id, coll))
# Close the collection if opened successfully
if hasattr(coll, "close") and coll is not None:
coll.close()
except Exception as e:
with lock:
errors.append((thread_id, str(e)))
# Create 5 threads to call open concurrently
threads = []
for i in range(5):
thread = threading.Thread(target=open_collection_thread, args=(i,))
threads.append(thread)
thread.start()
# Wait for all threads to complete
for thread in threads:
thread.join()
# Verify concurrency safety: only one should succeed, others should fail
assert len(results) == 1, (
f"Expected exactly one successful open, but got {len(results)}"
)
assert len(errors) == 4, (
f"Expected exactly four failures, but got {len(errors)}"
)
# Additional verification: check that the successful open has a valid collection
successful_thread_id, successful_collection = results[0]
assert successful_collection is not None, (
"Successful open should return a valid collection"
)
assert successful_collection.path == str(collection_path), (
"Collection path mismatch"
)
# Clean up the successfully opened collection
if (
hasattr(successful_collection, "destroy")
and successful_collection is not None
):
try:
successful_collection.destroy()
except Exception as e:
print(f"Warning: failed to destroy collection: {e}")
def test_open_with_corrupted_files(self, tmp_path_factory):
# First create a collection
collection_schema = zvec.CollectionSchema(
name="test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name",
DataType.STRING,
nullable=False,
index_param=InvertIndexParam(),
),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
)
],
)
collection_option = CollectionOption(read_only=False, enable_mmap=True)
# Create collection path
temp_dir = tmp_path_factory.mktemp("zvec")
collection_path = temp_dir / "test_collection"
# First create the collection
created_coll = zvec.create_and_open(
path=str(collection_path),
schema=collection_schema,
option=collection_option,
)
assert created_coll is not None, "Failed to create collection"
# Close the collection so we can manipulate its files
if hasattr(created_coll, "close") and created_coll is not None:
created_coll.close()
# Test case 1: Delete some files in the collection directory (simulate partial corruption)
import os
import shutil
import random
# Get the collection directory path
collection_dir = str(collection_path)
# List all files in the collection directory
files_in_dir = []
for root, dirs, files in os.walk(collection_dir):
for file in files:
files_in_dir.append(os.path.join(root, file))
# Randomly delete approximately half of the files to simulate partial corruption
if files_in_dir:
# Shuffle the list to randomly select files
random.shuffle(files_in_dir)
files_to_delete = files_in_dir[: len(files_in_dir) // 2]
for file_path in files_to_delete:
try:
os.remove(file_path)
except Exception as e:
pass # Ignore errors during deletion
# Try to open the collection with missing files - should raise an exception
with pytest.raises(Exception):
zvec.open(path=str(collection_path), option=collection_option)
# Test case 2: Delete all files in the collection directory (simulate complete corruption)
# Recreate the collection
recreated_coll = zvec.create_and_open(
path=str(collection_path) + "_all",
schema=collection_schema,
option=collection_option,
)
assert recreated_coll is not None, "Failed to recreate collection"
# Close the collection so we can manipulate its files
if hasattr(recreated_coll, "close") and recreated_coll is not None:
recreated_coll.close()
# Delete all files in the collection directory
try:
shutil.rmtree(collection_dir)
os.makedirs(collection_dir) # Recreate empty directory
except Exception as e:
pass # Ignore errors during deletion
# Try to open the collection with missing files - should raise an exception
with pytest.raises(Exception):
zvec.open(path=str(collection_path), option=collection_option)
+307
View File
@@ -0,0 +1,307 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import pytest
import tempfile
import os
import sys
import subprocess
import zvec
import zvec
from zvec import LogType, LogLevel
# Error messages
INITIALIZATION_ERROR_MSG = "initialization failed"
RUNTIME_ERROR_MSG = "RuntimeError"
VALUE_ERROR_MSG = "ValueError"
TYPE_ERROR_MSG = "TypeError"
# ==================== helper ====================
def run_in_subprocess(func):
def wrapper(*args, **kwargs):
if os.getenv("RUNNING_IN_SUBPROCESS"):
return func(*args, **kwargs)
env = os.environ.copy()
env["RUNNING_IN_SUBPROCESS"] = "1"
env["PYTEST_CURRENT_TEST"] = func.__name__
import inspect
filepath = inspect.getfile(func)
qualname = func.__qualname__.replace(".", "::")
test_id = f"{filepath}::{qualname}"
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
env["PYTHONPATH"] = project_root + ":" + env.get("PYTHONPATH", "")
cmd = [sys.executable, "-m", "pytest", "-v", "-s", test_id]
result = subprocess.run(cmd, env=env, capture_output=True, text=True)
if result.returncode != 0:
pytest.fail(
f"Subprocess test {func.__name__} failed with code {result.returncode}\n"
f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
)
return wrapper
# ==================== Fixtures ====================
@pytest.fixture(scope="function")
def temp_log_dir(tmp_path_factory):
return tmp_path_factory.mktemp("logs")
# ==================== Tests ====================
class TestDbConfigInitialization:
@run_in_subprocess
def test_init_default(self):
# default config
# log_type: Optional[LogType] = LogType.CONSOLE,
# log_level: Optional[LogLevel] = LogLevel.WARN,
# log_dir: Optional[str] = "./logs",
# log_basename: Optional[str] = "zvec.log",
# log_file_size: Optional[int] = 2048,
# log_overdue_days: Optional[int] = 7,
zvec.init()
@run_in_subprocess
def test_init_file_logger(self):
from pathlib import Path
import shutil
zvec.init(
log_level=LogLevel.DEBUG,
log_type=LogType.FILE,
)
# assert logdir exist
log_dir = Path("./logs")
assert log_dir.exists()
# validate write log
col = zvec.create_and_open(
"/tmp/test/1",
zvec.CollectionSchema(
name="test",
vectors=zvec.VectorSchema(
dimension=4,
data_type=zvec.DataType.VECTOR_FP32,
name="image",
),
),
)
col.insert(docs=[zvec.Doc(id="1", vectors={"image": [1.0, 2.0, 3.0, 4.0]})])
assert any(log_dir.glob("zvec.log.*"))
# clear
col.destroy()
shutil.rmtree(log_dir, ignore_errors=True)
@run_in_subprocess
def test_init_with_mixed_config(self):
zvec.init(
memory_limit_mb=128,
log_type=LogType.FILE,
query_threads=1,
log_level=LogLevel.WARN,
)
@run_in_subprocess
def test_repeated_initialization_runtime_error(self):
zvec.init()
with pytest.raises(RuntimeError):
zvec.init()
class TestDbConfigMemoryLimitValidation:
@run_in_subprocess
def test_memory_limit_min_valid(self):
# MIN_MEMORY_LIMIT_BYTES is 100M
with pytest.raises(RuntimeError):
zvec.init(memory_limit_mb=99)
@run_in_subprocess
def test_memory_limit_invalid_value(self):
# memory_limit_mb must >= 0 and must be int and if None, set default value
with pytest.raises(ValueError):
zvec.init(memory_limit_mb=0)
with pytest.raises(ValueError):
zvec.init(memory_limit_mb=-1)
with pytest.raises(TypeError):
zvec.init(memory_limit_mb="512")
with pytest.raises(TypeError):
zvec.init(memory_limit_mb=512.5)
class TestDbConfigThreadValidation:
@run_in_subprocess
def test_query_threads(self):
zvec.init(query_threads=1)
@run_in_subprocess
def test_query_threads_invalid(self):
# query_threads must >= 0 and must be int and if None, set default value
with pytest.raises(ValueError):
zvec.init(query_threads=0)
with pytest.raises(ValueError):
zvec.init(query_threads=-1)
with pytest.raises(TypeError):
zvec.init(query_threads="value")
with pytest.raises(TypeError):
zvec.init(query_threads=512.5)
with pytest.raises(TypeError):
zvec.init(query_threads="512")
@run_in_subprocess
def test_optimize_threads(self):
zvec.init(optimize_threads=1)
@run_in_subprocess
def test_optimize_threads_invalid(self):
# optimize_threads must >= 0 and must be int and if None, set default value
with pytest.raises(ValueError):
zvec.init(optimize_threads=0)
with pytest.raises(ValueError):
zvec.init(optimize_threads=-1)
with pytest.raises(TypeError):
zvec.init(optimize_threads="value")
with pytest.raises(TypeError):
zvec.init(optimize_threads=512.5)
with pytest.raises(TypeError):
zvec.init(optimize_threads="512")
class TestDbConfigRatioValidation:
@run_in_subprocess
def test_init_invert_to_forward_scan_ratio(self):
# must be in [0,1]
zvec.init(invert_to_forward_scan_ratio=0.8)
@run_in_subprocess
def test_init_invert_to_forward_scan_ratio_invalid(self):
with pytest.raises(ValueError):
zvec.init(invert_to_forward_scan_ratio=1.1)
with pytest.raises(ValueError):
zvec.init(invert_to_forward_scan_ratio=-0.1)
with pytest.raises(TypeError):
zvec.init(invert_to_forward_scan_ratio="0.8")
@run_in_subprocess
def test_init_brute_force_by_keys_ratio(self):
zvec.init(brute_force_by_keys_ratio=0.8)
@run_in_subprocess
def test_init_brute_force_by_keys_ratio_invalid(self):
with pytest.raises(ValueError):
zvec.init(brute_force_by_keys_ratio=1.1)
with pytest.raises(ValueError):
zvec.init(brute_force_by_keys_ratio=-0.1)
with pytest.raises(TypeError):
zvec.init(brute_force_by_keys_ratio="0.8")
class TestDbConfigLogValidation:
@run_in_subprocess
def test_log_type_valid(self):
zvec.init(log_type=LogType.CONSOLE)
@run_in_subprocess
def test_log_type_invalid(self):
with pytest.raises(TypeError):
zvec.init(log_type="FILE")
with pytest.raises(TypeError):
zvec.init(log_type="")
with pytest.raises(TypeError):
zvec.init(log_type="invalid")
with pytest.raises(TypeError):
zvec.init(log_type=123)
@run_in_subprocess
def test_log_level_valid(self):
zvec.init(log_level=LogLevel.ERROR)
@run_in_subprocess
def test_log_level_invalid(self):
with pytest.raises(TypeError):
zvec.init(log_level="WARN")
with pytest.raises(TypeError):
zvec.init(log_level="")
with pytest.raises(TypeError):
zvec.init(log_level="invalid")
with pytest.raises(TypeError):
zvec.init(log_level=123)
@run_in_subprocess
def test_init_file_logger(self):
from pathlib import Path
import shutil
temp_dir = tempfile.mkdtemp(prefix="log_test_")
abs_temp_dir = os.path.abspath(temp_dir)
zvec.init(
log_level=LogLevel.DEBUG,
log_type=LogType.FILE,
log_dir=abs_temp_dir,
log_basename="test",
)
# assert logdir exist
log_dir = Path(abs_temp_dir)
assert log_dir.exists()
# validate write log
col = zvec.create_and_open(
"/tmp/test/1",
zvec.CollectionSchema(
name="test",
vectors=zvec.VectorSchema(
dimension=4,
data_type=zvec.DataType.VECTOR_FP32,
name="image",
),
),
)
col.insert(docs=[zvec.Doc(id="1", vectors={"image": [1.0, 2.0, 3.0, 4.0]})])
assert any(log_dir.glob("test.*"))
# clear
col.destroy()
shutil.rmtree(log_dir, ignore_errors=True)
@run_in_subprocess
def test_log_file_size_invalid(self):
with pytest.raises(TypeError):
zvec.init(log_type=LogType.FILE, log_file_size="df")
with pytest.raises(ValueError):
zvec.init(log_type=LogType.FILE, log_file_size=0)
with pytest.raises(ValueError):
zvec.init(log_type=LogType.FILE, log_file_size=-1)
@run_in_subprocess
def test_log_overdue_days_invalid(self):
with pytest.raises(TypeError):
zvec.init(log_type=LogType.FILE, log_overdue_days="df")
with pytest.raises(ValueError):
zvec.init(log_type=LogType.FILE, log_overdue_days=0)
with pytest.raises(ValueError):
zvec.init(log_type=LogType.FILE, log_overdue_days=-1)
File diff suppressed because it is too large Load Diff
+584
View File
@@ -0,0 +1,584 @@
from __future__ import annotations
import math
import pytest
from _zvec import _Doc
from zvec.model.convert import convert_to_py_doc, convert_to_cpp_doc
from zvec import Doc, CollectionSchema, DataType, FieldSchema, VectorSchema
# ----------------------------
# Convert Cpp Doc Test Case
# ----------------------------
class TestConvertCppDoc:
def test_default(self):
doc = Doc(id="1")
schema = CollectionSchema(
name="test_collection",
fields=FieldSchema("name", DataType.STRING),
)
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
assert cpp_doc is not None
assert cpp_doc.pk() == doc.id
def test_with_field_notin_schema(self):
doc = Doc(id="1", fields={"name": "Tom"})
schema = CollectionSchema(
name="test_collection",
fields=[
FieldSchema("id", DataType.UINT64),
FieldSchema("salary", DataType.UINT32),
FieldSchema("age", DataType.INT32),
FieldSchema("create_at", DataType.INT64),
FieldSchema("author", DataType.STRING),
FieldSchema("weight", DataType.FLOAT),
],
)
with pytest.raises(ValueError):
convert_to_cpp_doc(doc, collection_schema=schema)
def test_with_scalar_fields(self):
schema = CollectionSchema(
name="test_collection",
fields=[
FieldSchema("id", DataType.UINT64),
FieldSchema("salary", DataType.UINT32),
FieldSchema("age", DataType.INT32),
FieldSchema("create_at", DataType.INT64),
FieldSchema("author", DataType.STRING),
FieldSchema("weight", DataType.FLOAT),
FieldSchema("bmi", DataType.DOUBLE),
FieldSchema("is_male", DataType.BOOL),
],
)
doc = Doc(
id="1",
fields={
"id": 1,
"salary": 1000,
"age": 18,
"create_at": 1640995200,
"bmi": 80.0 / 200.0,
"author": "Tom",
"weight": 80.0,
"is_male": True,
},
)
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
assert cpp_doc is not None
assert cpp_doc.pk() == doc.id
assert cpp_doc.get_any("id", DataType.UINT64) == 1
assert cpp_doc.get_any("salary", DataType.UINT32) == 1000
assert cpp_doc.get_any("age", DataType.INT32) == 18
assert cpp_doc.get_any("create_at", DataType.INT64) == 1640995200
assert cpp_doc.get_any("author", DataType.STRING) == "Tom"
assert math.isclose(
cpp_doc.get_any("weight", DataType.FLOAT), 80.0, rel_tol=1e-6
)
assert math.isclose(
cpp_doc.get_any("bmi", DataType.DOUBLE), 80.0 / 200.0, rel_tol=1e-6
)
assert cpp_doc.get_any("is_male", DataType.BOOL) == True
def test_with_array_fields(self):
schema = CollectionSchema(
name="test_collection",
fields=[
FieldSchema("tags", DataType.ARRAY_STRING),
FieldSchema("ids", DataType.ARRAY_UINT64),
FieldSchema("marks", DataType.ARRAY_UINT32),
FieldSchema("x", DataType.ARRAY_INT32),
FieldSchema("y", DataType.ARRAY_INT64),
FieldSchema("scores", DataType.ARRAY_FLOAT),
FieldSchema("ratios", DataType.ARRAY_DOUBLE),
FieldSchema("results", DataType.ARRAY_BOOL),
],
)
doc = Doc(
id="1",
fields={
"tags": ["tag1", "tag2", "tag3"],
"ids": [111111111111, 222222222222, 333333333333],
"marks": [100, 200, 300],
"x": [1, 2, 3],
"y": [100, 200, 300],
"scores": [1.1, 2.2, 3.3],
"ratios": [0.1, 0.2, 0.3],
"results": [True, False, True],
},
)
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
assert cpp_doc is not None
assert cpp_doc.pk() == doc.id
assert cpp_doc.get_any("tags", DataType.ARRAY_STRING) == doc.field("tags")
assert cpp_doc.get_any("ids", DataType.ARRAY_UINT64) == doc.field("ids")
assert cpp_doc.get_any("marks", DataType.ARRAY_UINT32) == doc.field("marks")
assert cpp_doc.get_any("x", DataType.ARRAY_INT32) == doc.field("x")
assert cpp_doc.get_any("y", DataType.ARRAY_INT64) == doc.field("y")
scores = cpp_doc.get_any("scores", DataType.ARRAY_FLOAT)
for i in range(len(doc.field("scores"))):
assert math.isclose(scores[i], doc.field("scores")[i], rel_tol=1e-1)
ratios = cpp_doc.get_any("ratios", DataType.ARRAY_DOUBLE)
for i in range(len(doc.field("ratios"))):
assert math.isclose(ratios[i], doc.field("ratios")[i], rel_tol=1e-1)
results = cpp_doc.get_any("results", DataType.ARRAY_BOOL)
for i in range(len(doc.field("results"))):
assert results[i] == doc.field("results")[i]
def test_with_dense_vector_fields(self):
schema = CollectionSchema(
name="test_collection",
vectors=[
VectorSchema(
name="embedding",
data_type=DataType.VECTOR_FP16,
dimension=4,
),
VectorSchema(
name="image",
data_type=DataType.VECTOR_FP32,
dimension=8,
),
VectorSchema(
name="text",
data_type=DataType.VECTOR_INT8,
dimension=32,
),
],
)
doc = Doc(
id="1",
vectors={
"embedding": [1.1] * 4,
"image": [2.2] * 8,
"text": [4] * 32,
},
)
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
assert cpp_doc is not None
assert cpp_doc.pk() == doc.id
embedding_vector = cpp_doc.get_any("embedding", DataType.VECTOR_FP16)
assert len(embedding_vector) == 4
for i in range(4):
assert math.isclose(
embedding_vector[i], doc.vector("embedding")[i], rel_tol=1e-1
)
image_vector = cpp_doc.get_any("image", DataType.VECTOR_FP32)
assert len(image_vector) == 8
for i in range(8):
assert math.isclose(image_vector[i], doc.vector("image")[i], rel_tol=1e-1)
text_vector = cpp_doc.get_any("text", DataType.VECTOR_INT8)
assert len(text_vector) == 32
for i in range(32):
assert text_vector[i] == doc.vectors["text"][i]
def test_with_sparse_vector_fields(self):
schema = CollectionSchema(
name="test_collection",
vectors=[
VectorSchema(
name="author",
data_type=DataType.SPARSE_VECTOR_FP32,
),
VectorSchema(
name="content",
data_type=DataType.SPARSE_VECTOR_FP16,
),
],
)
doc = Doc(
id="1",
vectors={
"author": {1: 1.1, 2: 2.2, 3: 3.3},
"content": {4: 4.4, 5: 5.5, 6: 6.6},
},
)
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
assert cpp_doc is not None
assert cpp_doc.pk() == doc.id
author_vector = cpp_doc.get_any("author", DataType.SPARSE_VECTOR_FP32)
assert isinstance(author_vector, dict)
for key, value in doc.vector("author").items():
assert math.isclose(author_vector[key], value, rel_tol=1e-1)
content_vector = cpp_doc.get_any("content", DataType.SPARSE_VECTOR_FP16)
assert isinstance(content_vector, dict)
for key, value in doc.vector("content").items():
assert math.isclose(content_vector[key], value, rel_tol=1e-1)
def test_with_scalar_fields_error_datatype(self):
schema = CollectionSchema(
name="test_collection",
fields=[
FieldSchema("id", DataType.UINT64),
FieldSchema("salary", DataType.UINT32),
FieldSchema("age", DataType.INT32),
FieldSchema("create_at", DataType.INT64),
FieldSchema("author", DataType.STRING),
FieldSchema("weight", DataType.FLOAT),
FieldSchema("bmi", DataType.DOUBLE),
FieldSchema("is_male", DataType.BOOL),
],
)
doc = Doc(
id="1",
fields={
"id": "1",
},
)
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"salary": "1000"})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"age": "18"})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"create_at": "2021-01-01"})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"author": 1})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"weight": "80.5"})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"bmi": "25.0"})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"is_male": "true"})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
def test_with_array_fields_error_datatype(self):
schema = CollectionSchema(
name="test_collection",
fields=[
FieldSchema("tags", DataType.ARRAY_STRING),
FieldSchema("ids", DataType.ARRAY_UINT64),
FieldSchema("marks", DataType.ARRAY_UINT32),
FieldSchema("x", DataType.ARRAY_INT32),
FieldSchema("y", DataType.ARRAY_INT64),
FieldSchema("scores", DataType.ARRAY_FLOAT),
FieldSchema("ratios", DataType.ARRAY_DOUBLE),
FieldSchema("results", DataType.ARRAY_BOOL),
],
)
doc = Doc(id="1", fields={"tags": [1, 2, 3]})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"ids": ["1", "2", "3"]})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"marks": [1.1, 2.2, 3.3]})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"x": [1.1, 2.2, 3.3]})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"y": [1.1, 2.2, 3.3]})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"scores": ["1", "2", "3"]})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"ratios": ["1", "2", "3"]})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", fields={"results": ["1", "2", "3"]})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
def test_with_vector_fields_error_datatype(self):
schema = CollectionSchema(
name="test_collection",
vectors=[
VectorSchema(
name="embedding",
data_type=DataType.VECTOR_FP16,
dimension=4,
),
VectorSchema(
name="image",
data_type=DataType.VECTOR_FP32,
dimension=8,
),
VectorSchema(
name="text",
data_type=DataType.VECTOR_INT8,
dimension=32,
),
],
)
doc = Doc(id="1", vectors={"image": ["1.1"] * 4})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", vectors={"text": ["1"] * 4})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(id="1", vectors={"embedding": ["1"] * 4})
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
def test_with_sparse_vector_error_datatype(self):
schema = CollectionSchema(
name="test_collection",
vectors=[
VectorSchema(
name="author",
data_type=DataType.SPARSE_VECTOR_FP32,
),
VectorSchema(
name="content",
data_type=DataType.SPARSE_VECTOR_FP16,
),
],
)
doc = Doc(
id="1",
vectors={
"author": {"1": 1.1, "2": 2.2, "3": 3.3},
},
)
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(
id="1",
vectors={
"content": {"1": 1.1, "2": 2.2, "3": 3.3},
},
)
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
doc = Doc(
id="1",
vectors={
"author": {1: "1", 2: "2", 3: "3"},
},
)
with pytest.raises(TypeError):
convert_to_cpp_doc(doc, collection_schema=schema)
# ----------------------------
# Convert Py Doc Test Case
# ----------------------------
class TestConvertPyDoc:
def test_default(self):
doc = _Doc()
doc.set_pk("1")
doc.set_score(1.0)
schema = CollectionSchema(
name="test_collection",
fields=FieldSchema("name", DataType.STRING),
)
py_doc = convert_to_py_doc(doc, schema)
assert py_doc.id == "1"
assert py_doc.score == 1.0
def test_with_scalar_fields(self):
schema = CollectionSchema(
name="test_collection",
fields=[
FieldSchema("id", DataType.UINT64),
FieldSchema("salary", DataType.UINT32),
FieldSchema("age", DataType.INT32),
FieldSchema("create_at", DataType.INT64),
FieldSchema("author", DataType.STRING),
FieldSchema("weight", DataType.FLOAT),
FieldSchema("bmi", DataType.DOUBLE),
FieldSchema("is_male", DataType.BOOL),
],
)
doc = _Doc()
doc.set_pk("1")
doc.set_any("id", schema.field("id")._get_object(), 1111111111111111)
doc.set_any("salary", schema.field("salary")._get_object(), 1000)
doc.set_any("age", schema.field("age")._get_object(), 18)
doc.set_any("create_at", schema.field("create_at")._get_object(), 1640995200)
doc.set_any("author", schema.field("author")._get_object(), "Tom")
doc.set_any("weight", schema.field("weight")._get_object(), 80.0)
doc.set_any("bmi", schema.field("bmi")._get_object(), 80.0 / 200.0)
doc.set_any("is_male", schema.field("is_male")._get_object(), True)
py_doc = convert_to_py_doc(doc, schema)
assert py_doc.id == "1"
assert py_doc.field("id") == 1111111111111111
assert py_doc.field("salary") == 1000
assert py_doc.field("age") == 18
assert py_doc.field("create_at") == 1640995200
assert py_doc.field("author") == "Tom"
assert py_doc.field("weight") == 80.0
assert py_doc.field("bmi") == 80.0 / 200.0
assert py_doc.field("is_male") == True
def test_with_array_fields(self):
schema = CollectionSchema(
name="test_collection",
fields=[
FieldSchema("tags", DataType.ARRAY_STRING),
FieldSchema("ids", DataType.ARRAY_UINT64),
FieldSchema("marks", DataType.ARRAY_UINT32),
FieldSchema("x", DataType.ARRAY_INT32),
FieldSchema("y", DataType.ARRAY_INT64),
FieldSchema("scores", DataType.ARRAY_FLOAT),
FieldSchema("ratios", DataType.ARRAY_DOUBLE),
FieldSchema("results", DataType.ARRAY_BOOL),
],
)
doc = _Doc()
doc.set_pk("1")
doc.set_any(
"tags", schema.field("tags")._get_object(), ["tag1", "tag2", "tag3"]
)
doc.set_any(
"ids",
schema.field("ids")._get_object(),
[111111111111, 222222222222, 3333333333333],
)
doc.set_any("marks", schema.field("marks")._get_object(), [1000, 2000, 3000])
doc.set_any("x", schema.field("x")._get_object(), [1, 2, 3])
doc.set_any("y", schema.field("y")._get_object(), [100, 200, 300])
doc.set_any("scores", schema.field("scores")._get_object(), [0.1, 0.2, 0.3])
doc.set_any("ratios", schema.field("ratios")._get_object(), [0.1, 0.2, 0.3])
doc.set_any(
"results", schema.field("results")._get_object(), [True, False, True]
)
py_doc = convert_to_py_doc(doc, schema)
assert py_doc.field("tags") == ["tag1", "tag2", "tag3"]
assert py_doc.field("ids") == [111111111111, 222222222222, 3333333333333]
assert py_doc.field("marks") == [1000, 2000, 3000]
assert py_doc.field("x") == [1, 2, 3]
assert py_doc.field("y") == [100, 200, 300]
scores = doc.get_any("scores", DataType.ARRAY_FLOAT)
for i in range(len(scores)):
assert math.isclose(scores[i], py_doc.field("scores")[i], rel_tol=1e-1)
ratios = doc.get_any("ratios", DataType.ARRAY_DOUBLE)
for i in range(len(ratios)):
assert math.isclose(ratios[i], py_doc.field("ratios")[i], rel_tol=1e-1)
results = doc.get_any("results", DataType.ARRAY_BOOL)
for i in range(len(results)):
assert results[i] == py_doc.field("results")[i]
def test_with_dense_vector_fields(self):
schema = CollectionSchema(
name="test_collection",
vectors=[
VectorSchema(
name="embedding",
data_type=DataType.VECTOR_FP16,
dimension=4,
),
VectorSchema(
name="image",
data_type=DataType.VECTOR_FP32,
dimension=8,
),
VectorSchema(
name="text",
data_type=DataType.VECTOR_INT8,
dimension=32,
),
],
)
doc = _Doc()
doc.set_pk("1")
doc.set_any("embedding", schema.vector("embedding")._get_object(), [1.1] * 4)
doc.set_any("image", schema.vector("image")._get_object(), [2.2] * 8)
doc.set_any("text", schema.vector("text")._get_object(), [4] * 32)
py_doc = convert_to_py_doc(doc, schema)
assert py_doc.id == "1"
embedding_vector = py_doc.vector("embedding")
assert len(embedding_vector) == 4
for i in range(4):
assert math.isclose(
py_doc.vector("embedding")[i], embedding_vector[i], rel_tol=1e-1
)
image_vector = py_doc.vector("image")
assert len(image_vector) == 8
for i in range(8):
assert math.isclose(
py_doc.vector("image")[i], image_vector[i], rel_tol=1e-1
)
text_vector = py_doc.vector("text")
assert len(text_vector) == 32
for i in range(32):
assert py_doc.vector("text")[i] == text_vector[i]
def test_with_sparse_vector_fields(self):
schema = CollectionSchema(
name="test_collection",
vectors=[
VectorSchema(
name="author",
data_type=DataType.SPARSE_VECTOR_FP32,
),
VectorSchema(
name="content",
data_type=DataType.SPARSE_VECTOR_FP16,
),
],
)
doc = _Doc()
doc.set_pk("1")
doc.set_any(
"author", schema.vector("author")._get_object(), {1: 1.1, 2: 2.2, 3: 3.3}
)
doc.set_any(
"content", schema.vector("content")._get_object(), {4: 4.4, 5: 5.5, 6: 6.6}
)
py_doc = convert_to_py_doc(doc, schema)
assert py_doc.id == "1"
author_vector = py_doc.vector("author")
assert isinstance(author_vector, dict)
for key, value in doc.get_any("author", DataType.SPARSE_VECTOR_FP32).items():
assert math.isclose(author_vector[key], value, rel_tol=1e-1)
content_vector = py_doc.vector("content")
assert isinstance(content_vector, dict)
for key, value in doc.get_any("content", DataType.SPARSE_VECTOR_FP16).items():
assert math.isclose(content_vector[key], value, rel_tol=1e-1)
+269
View File
@@ -0,0 +1,269 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import math
import pytest
from _zvec import _Doc
from zvec import FieldSchema, VectorSchema, Doc, DataType
# ----------------------------
# PyDoc Test Case
# ----------------------------
class TestPyDoc:
def test_default(self):
Doc(id="1")
def test_with_single_vector(self):
doc = Doc(id="1", vectors={"dense": [1, 2, 3]})
assert doc is not None
assert doc.id == "1"
assert doc.vector("dense") == [1, 2, 3]
def test_with_hybrid_vectors(self):
doc = Doc(
id="1", vectors={"dense": [1, 2, 3], "sparse": {1: 1.0, 2: 2.0, 3: 3.0}}
)
assert doc is not None
assert doc.id == "1"
assert doc.vector("dense") == [1, 2, 3]
assert doc.vector("sparse") == {1: 1.0, 2: 2.0, 3: 3.0}
def test_with_multi_vectors(self):
doc = Doc(
id="1",
vectors={
"image": [1, 2, 3],
"description": [4, 5, 6],
"keys": {1: 1.0, 2: 2.0, 3: 3.0},
},
fields={"author": "Tom", "age": 19, "is_male": True, "weight": 60.5},
)
assert doc is not None
assert doc.id == "1"
assert doc.vector("image") == [1, 2, 3]
assert doc.vector("description") == [4, 5, 6]
assert doc.vector("keys") == {1: 1.0, 2: 2.0, 3: 3.0}
assert doc.field("author") == "Tom"
assert doc.field("age") == 19
assert doc.field("is_male") == True
assert doc.field("weight") == 60.5
def test_with_numpy_array(self):
import numpy as np
doc = Doc._from_tuple(
(
"1",
0.0,
None,
{
"image": np.array([1, 2, 3]),
"description": np.random.random(512),
"keys": {1: 1.0, 2: 2.0, 3: 3.0},
},
)
)
assert doc is not None
assert doc.id == "1"
assert doc.vector("image") == [1, 2, 3]
assert doc.vector("keys") == {1: 1.0, 2: 2.0, 3: 3.0}
# ----------------------------
# CppDoc Test Case
# ----------------------------
class TestCppDoc:
def test_default(self):
doc = _Doc()
assert doc is not None
def test_doc_set_pk(self):
doc = _Doc()
doc.set_pk("1")
assert doc.pk() == "1"
def test_doc_set_score(self):
doc = _Doc()
doc.set_score(0.9)
assert math.isclose(doc.score(), 0.9, rel_tol=1e-6)
def test_doc_get_null_field(self):
doc = _Doc()
schema = FieldSchema("author", DataType.STRING, nullable=True)
doc.set_any("author", schema._get_object(), None)
assert doc.has_field("author")
assert doc.get_any("author", schema.data_type) is None
def test_doc_get_set_has_null_field(self):
doc = _Doc()
schema = FieldSchema("author", DataType.STRING, nullable=False)
with pytest.raises(ValueError):
doc.set_any("author", schema._get_object(), None)
def test_doc_get_set_has_string_field(self):
doc = _Doc()
schema = FieldSchema("author", DataType.STRING)
doc.set_any("author", schema._get_object(), "Tom")
assert doc.has_field("author")
assert doc.get_any("author", DataType.STRING) == "Tom"
def test_doc_get_set_has_bool_field(self):
doc = _Doc()
schema = FieldSchema("is_male", DataType.BOOL)
doc.set_any("is_male", schema._get_object(), True)
assert doc.has_field("is_male")
assert doc.get_any("is_male", DataType.BOOL) == True
def test_doc_get_set_has_int32_field(self):
doc = _Doc()
schema = FieldSchema("age", DataType.INT32)
doc.set_any("age", schema._get_object(), 19)
assert doc.has_field("age")
assert doc.get_any("age", DataType.INT32) == 19
def test_doc_get_set_has_int64_field(self):
doc = _Doc()
schema = FieldSchema("id", DataType.INT64)
doc.set_any("id", schema._get_object(), 1111111111111111111)
assert doc.has_field("id")
assert doc.get_any("id", DataType.INT64) == 1111111111111111111
def test_doc_get_set_has_float_field(self):
doc = _Doc()
schema = FieldSchema("weight", DataType.FLOAT)
doc.set_any("weight", schema._get_object(), 60.5)
assert doc.has_field("weight")
assert math.isclose(doc.get_any("weight", DataType.FLOAT), 60.5, rel_tol=1e-6)
def test_doc_get_set_has_double_field(self):
doc = _Doc()
schema = FieldSchema("height", DataType.DOUBLE)
doc.set_any("height", schema._get_object(), 1.77777777777)
assert doc.has_field("height")
assert math.isclose(
doc.get_any("height", DataType.DOUBLE), 1.7777777777, rel_tol=1e-9
)
def test_doc_get_set_has_uint32_field(self):
doc = _Doc()
schema = FieldSchema("id", DataType.UINT32)
doc.set_any("id", schema._get_object(), 4294967295)
assert doc.has_field("id")
assert doc.get_any("id", DataType.UINT32) == 4294967295
def test_doc_get_set_has_uint64_field(self):
doc = _Doc()
schema = FieldSchema("id", DataType.UINT64)
doc.set_any("id", schema._get_object(), 18446744073709551615)
assert doc.has_field("id")
assert doc.get_any("id", DataType.UINT64) == 18446744073709551615
def test_doc_get_set_has_array_string_field(self):
doc = _Doc()
schema = FieldSchema("tags", DataType.ARRAY_STRING)
doc.set_any("tags", schema._get_object(), ["tag1", "tag2", "tag3"])
assert doc.has_field("tags")
assert doc.get_any("tags", DataType.ARRAY_STRING) == ["tag1", "tag2", "tag3"]
def test_doc_get_set_has_array_int32_field(self):
doc = _Doc()
schema = FieldSchema("ids", DataType.ARRAY_INT32)
doc.set_any("ids", schema._get_object(), [1, 2, 3])
assert doc.has_field("ids")
assert doc.get_any("ids", DataType.ARRAY_INT32) == [1, 2, 3]
def test_doc_get_set_has_array_int64_field(self):
doc = _Doc()
schema = FieldSchema("ids", DataType.ARRAY_INT64)
doc.set_any("ids", schema._get_object(), [1, 2, 3])
assert doc.has_field("ids")
assert doc.get_any("ids", DataType.ARRAY_INT64) == [1, 2, 3]
def test_doc_get_set_has_array_float_field(self):
doc = _Doc()
schema = FieldSchema("weights", DataType.ARRAY_FLOAT)
doc.set_any("weights", schema._get_object(), [1.0, 2.0, 3.0])
assert doc.has_field("weights")
assert doc.get_any("weights", DataType.ARRAY_FLOAT) == [1.0, 2.0, 3.0]
def test_doc_get_set_has_array_double_field(self):
doc = _Doc()
schema = FieldSchema("heights", DataType.ARRAY_DOUBLE)
doc.set_any("heights", schema._get_object(), [1.0, 2.0, 3.0])
assert doc.has_field("heights")
assert doc.get_any("heights", DataType.ARRAY_DOUBLE) == [1.0, 2.0, 3.0]
def test_doc_get_set_has_array_bool_field(self):
doc = _Doc()
schema = FieldSchema("bools", DataType.ARRAY_BOOL)
doc.set_any("bools", schema._get_object(), [True, False, True])
assert doc.has_field("bools")
assert doc.get_any("bools", DataType.ARRAY_BOOL) == [True, False, True]
def test_doc_get_set_has_vector_fp16(self):
doc = _Doc()
schema = VectorSchema("image", DataType.VECTOR_FP16)
doc.set_any("image", schema._get_object(), [1.0, 2.0, 3.0])
assert doc.has_field("image")
image_vector = doc.get_any("image", DataType.VECTOR_FP16)
assert image_vector is not None
for i in range(len(image_vector)):
assert math.isclose(image_vector[i], [1.0, 2.0, 3.0][i], rel_tol=1e-6)
def test_doc_get_set_has_vector_fp32(self):
doc = _Doc()
schema = VectorSchema("image", DataType.VECTOR_FP32)
doc.set_any("image", schema._get_object(), [1.111111, 2.222222, 3.333333])
assert doc.has_field("image")
vector = doc.get_any("image", DataType.VECTOR_FP32)
assert vector is not None
for i in range(len(vector)):
assert math.isclose(
vector[i], [1.111111, 2.222222, 3.333333][i], rel_tol=1e-6
)
def test_doc_get_set_has_vector_int8(self):
doc = _Doc()
schema = VectorSchema("image", DataType.VECTOR_INT8)
doc.set_any("image", schema._get_object(), [1, 2, 3])
assert doc.has_field("image")
assert doc.get_any("image", DataType.VECTOR_INT8) == [1, 2, 3]
def test_doc_get_set_has_sparse_vector_fp32(self):
doc = _Doc()
sparse = {1: 1.111111, 2: 2.222222, 3: 3.333333}
schema = VectorSchema("key", DataType.SPARSE_VECTOR_FP32)
doc.set_any("key", schema._get_object(), sparse)
assert doc.has_field("key")
vector = doc.get_any("key", DataType.SPARSE_VECTOR_FP32)
assert vector is not None
assert isinstance(vector, dict)
for key, value in sparse.items():
assert math.isclose(vector[key], value, rel_tol=1e-6)
def test_doc_get_set_has_sparse_vector_fp16(self):
doc = _Doc()
sparse = {1: 1.1, 2: 2.2, 3: 3.3}
schema = VectorSchema("key", DataType.SPARSE_VECTOR_FP16)
doc.set_any("key", schema._get_object(), sparse)
assert doc.has_field("key")
vector = doc.get_any("key", DataType.SPARSE_VECTOR_FP16)
assert vector is not None
assert isinstance(vector, dict)
for key, value in sparse.items():
assert math.isclose(vector[key], value, rel_tol=1e-1)
+131
View File
@@ -0,0 +1,131 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import os
from http import HTTPStatus
from unittest.mock import MagicMock, patch
import pytest
from zvec.extension import QwenEmbeddingFunction
# ----------------------------
# QwenEmbeddingFunction Test Case
# ----------------------------
class TestQwenEmbeddingFunction:
def test_init_with_api_key(self):
# Test initialization with explicit API key
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
assert embedding_func.dimension == 128
assert embedding_func.model == "text-embedding-v4"
assert embedding_func._api_key == "test_key"
@patch.dict(os.environ, {"DASHSCOPE_API_KEY": "env_key"})
def test_init_with_env_api_key(self):
# Test initialization with API key from environment
embedding_func = QwenEmbeddingFunction(dimension=128)
assert embedding_func._api_key == "env_key"
def test_init_without_api_key(self):
# Test initialization without API key raises ValueError
with pytest.raises(ValueError, match="DashScope API key is required"):
QwenEmbeddingFunction(dimension=128)
@patch.dict(os.environ, {"DASHSCOPE_API_KEY": ""})
def test_init_with_empty_env_api_key(self):
# Test initialization with empty API key from environment
with pytest.raises(ValueError, match="DashScope API key is required"):
QwenEmbeddingFunction(dimension=128)
def test_model_property(self):
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
assert embedding_func.model == "text-embedding-v4"
embedding_func = QwenEmbeddingFunction(
dimension=128, model="custom-model", api_key="test_key"
)
assert embedding_func.model == "custom-model"
@patch("zvec.extension.embedding.require_module")
def test_embed_with_empty_text(self, mock_require_module):
# Test embed method with empty text raises ValueError
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
with pytest.raises(
ValueError, match="Input text cannot be empty or whitespace only"
):
embedding_func.embed("")
with pytest.raises(TypeError):
embedding_func.embed(None)
@patch("zvec.extension.embedding.require_module")
def test_embed_success(self, mock_require_module):
# Test successful embedding
mock_dashscope = MagicMock()
mock_response = MagicMock()
mock_response.status_code = HTTPStatus.OK
mock_response.output = {"embeddings": [{"embedding": [0.1, 0.2, 0.3]}]}
mock_dashscope.TextEmbedding.call.return_value = mock_response
mock_require_module.return_value = mock_dashscope
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
result = embedding_func.embed("test text")
assert result == [0.1, 0.2, 0.3]
mock_dashscope.TextEmbedding.call.assert_called_once_with(
model="text-embedding-v4",
input="test text",
dimension=128,
output_type="dense",
)
@patch("zvec.extension.embedding.require_module")
def test_embed_http_error(self, mock_require_module):
# Test embedding with HTTP error
mock_dashscope = MagicMock()
mock_response = MagicMock()
mock_response.status_code = HTTPStatus.BAD_REQUEST
mock_response.message = "Bad Request"
mock_dashscope.TextEmbedding.call.return_value = mock_response
mock_require_module.return_value = mock_dashscope
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
with pytest.raises(ValueError):
embedding_func.embed("test text")
@patch("zvec.extension.embedding.require_module")
def test_embed_invalid_response(self, mock_require_module):
# Test embedding with invalid response (wrong number of embeddings)
mock_dashscope = MagicMock()
mock_response = MagicMock()
mock_response.status_code = HTTPStatus.OK
mock_response.output.embeddings = []
mock_dashscope.TextEmbedding.call.return_value = mock_response
mock_require_module.return_value = mock_dashscope
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
with pytest.raises(ValueError):
embedding_func.embed("test text")
@pytest.mark.skip(reason="Qwen Embedding is not available in CI")
def test_embed(self):
# Test embedding with invalid dimension
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="xxx")
dense = embedding_func("test text")
assert len(dense) == 128
+415
View File
@@ -0,0 +1,415 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import sys
import time
import numpy as np
import pytest
from zvec import (
AddColumnOption,
AlterColumnOption,
CollectionOption,
FlatIndexParam,
HnswIndexParam,
IndexOption,
InvertIndexParam,
IVFIndexParam,
OptimizeOption,
HnswQueryParam,
IVFQueryParam,
VectorQuery,
IndexType,
MetricType,
QuantizeType,
DataType,
VectorSchema,
)
from _zvec.param import _VectorQuery
# ----------------------------
# Invert Index Param Test Case
# ----------------------------
class TestInvertIndexParam:
def test_default(self):
param = InvertIndexParam()
assert param.enable_range_optimization is False
assert param.enable_extended_wildcard is False
assert param.type == IndexType.INVERT
def test_custom(self):
param = InvertIndexParam(
enable_range_optimization=True, enable_extended_wildcard=True
)
assert param.enable_range_optimization is True
assert param.enable_extended_wildcard is True
def test_readonly(self):
param = InvertIndexParam()
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
param.enable_range_optimization = False
param.enable_extended_wildcard = False
# ----------------------------
# Hnsw Index Param Test Case
# ----------------------------
class TestHnswIndexParam:
def test_default(self):
param = HnswIndexParam()
assert param.metric_type == MetricType.IP
assert param.m == 100
assert param.ef_construction == 500
assert param.quantize_type == QuantizeType.UNDEFINED
assert param.type == IndexType.HNSW
def test_custom(self):
param = HnswIndexParam(
metric_type=MetricType.L2,
m=10,
ef_construction=1000,
quantize_type=QuantizeType.FP16,
)
assert param.metric_type == MetricType.L2
assert param.m == 10
assert param.ef_construction == 1000
assert param.quantize_type == QuantizeType.FP16
@pytest.mark.parametrize(
"attr", ["metric_type", "m", "ef_construction", "quantize_type"]
)
def test_readonly_attributes(self, attr):
param = HnswIndexParam()
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
setattr(param, attr, getattr(param, attr))
# ----------------------------
# Flat Index Param Test Case
# ----------------------------
class TestFlatIndexParam:
def test_default(self):
param = FlatIndexParam()
assert param.type == IndexType.FLAT
assert param.quantize_type == QuantizeType.UNDEFINED
assert param.metric_type == MetricType.IP
def test_custom(self):
param = FlatIndexParam(
metric_type=MetricType.L2, quantize_type=QuantizeType.INT8
)
assert param.metric_type == MetricType.L2
assert param.quantize_type == QuantizeType.INT8
@pytest.mark.parametrize("attr", ["metric_type", "quantize_type"])
def test_readonly_attributes(self, attr):
param = FlatIndexParam()
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
setattr(param, attr, getattr(param, attr))
# ----------------------------
# Ivf Index Param Test Case
# ----------------------------
class TestIVFIndexParam:
def test_default(self):
param = IVFIndexParam()
assert param.metric_type == MetricType.IP
assert param.n_list == 0
assert param.quantize_type == QuantizeType.UNDEFINED
assert param.type == IndexType.IVF
def test_custom(self):
param = IVFIndexParam(
metric_type=MetricType.L2, n_list=1000, quantize_type=QuantizeType.FP16
)
assert param.metric_type == MetricType.L2
assert param.n_list == 1000
assert param.quantize_type == QuantizeType.FP16
assert param.type == IndexType.IVF
@pytest.mark.parametrize("attr", ["metric_type", "n_list", "quantize_type"])
def test_readonly_attributes(self, attr):
param = IVFIndexParam()
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
setattr(param, attr, getattr(param, attr))
# ----------------------------
# CollectionOption Test Case
# ----------------------------
class TestCollectionOption:
def test_default(self):
option = CollectionOption()
assert option is not None
assert option.read_only == False
assert option.enable_mmap == True
def test_custom(self):
option = CollectionOption(read_only=True, enable_mmap=False)
assert option.read_only == True
assert option.enable_mmap == False
option = CollectionOption(read_only=False, enable_mmap=True)
assert option.read_only == False
assert option.enable_mmap == True
@pytest.mark.parametrize("attr", ["read_only", "enable_mmap"])
def test_readonly_attributes(self, attr):
param = CollectionOption()
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
setattr(param, attr, getattr(param, attr))
# ----------------------------
# IndexOption Test Case
# ----------------------------
class TestIndexOption:
def test_default(self):
option = IndexOption()
assert option is not None
assert option.concurrency == 0
def test_custom(self):
option = IndexOption(concurrency=10)
assert option.concurrency == 10
@pytest.mark.parametrize("attr", ["concurrency"])
def test_readonly_attributes(self, attr):
param = IndexOption()
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
setattr(param, attr, getattr(param, attr))
# ----------------------------
# AddColumnOption Test Case
# ----------------------------
class TestAddColumnOption:
def test_default(self):
option = AddColumnOption()
assert option is not None
assert option.concurrency == 0
def test_custom(self):
option = AddColumnOption(concurrency=10)
assert option.concurrency == 10
@pytest.mark.parametrize("attr", ["concurrency"])
def test_readonly_attributes(self, attr):
param = AddColumnOption()
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
setattr(param, attr, getattr(param, attr))
# ----------------------------
# AlterColumnOption Test Case
# ----------------------------
class TestAlterColumnOption:
def test_default(self):
option = AlterColumnOption()
assert option is not None
assert option.concurrency == 0
def test_custom(self):
option = AlterColumnOption(concurrency=10)
assert option.concurrency == 10
@pytest.mark.parametrize("attr", ["concurrency"])
def test_readonly_attributes(self, attr):
param = AlterColumnOption()
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
setattr(param, attr, getattr(param, attr))
# ----------------------------
# OptimizeOption Test Case
# ----------------------------
class TestOptimizeOption:
def test_default(self):
option = OptimizeOption()
assert option is not None
assert option.concurrency == 0
def test_custom(self):
option = OptimizeOption(concurrency=10)
assert option.concurrency == 10
@pytest.mark.parametrize("attr", ["concurrency"])
def test_readonly_attributes(self, attr):
param = OptimizeOption()
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
setattr(param, attr, getattr(param, attr))
# ----------------------------
# HnswQueryParam Test Case
# ----------------------------
class TestHnswQueryParam:
def test_default(self):
param = HnswQueryParam()
assert param is not None
assert param.ef == 300
assert param.is_using_refiner == False
assert param.radius == 0
assert param.is_linear == False
def test_custom(self):
param = HnswQueryParam(ef=10, is_using_refiner=True, radius=30, is_linear=True)
assert param.ef == 10
assert param.is_using_refiner == True
assert param.radius == 30
assert param.is_linear == True
def test_readonly_attributes(self):
param = HnswQueryParam()
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
param.ef = 10
param.is_using_refiner = True
param.radius = 30
param.is_linear = True
# # ----------------------------
# # IVFQueryParam Test Case
# # ----------------------------
# class TestIVFQueryParam:
# def test_default(self):
# param = IVFQueryParam()
# assert param is not None
# assert param.nprobe == 10
# assert param.is_using_refiner == False
# assert param.radius == 0
# assert param.is_linear == False
# assert param.scale_factor == 10
#
# def test_custom(self):
# param = IVFQueryParam(
# nprobe=20,
# is_using_refiner=True,
# radius=30,
# is_linear=True,
# scale_factor=40
# )
# assert param.nprobe == 20
# assert param.is_using_refiner == True
# assert param.radius == 30
# assert param.is_linear == True
# assert param.scale_factor == 40
class TestVectorQuery:
def test_init_with_valid_id(self):
vq = VectorQuery(field_name="embedding", id="doc123")
assert vq.field_name == "embedding"
assert vq.id == "doc123"
assert vq.vector is None
assert vq.param is None
def test_init_with_valid_vector(self):
vec = [0.1, 0.2, 0.3]
param = HnswQueryParam(ef=300)
vq = VectorQuery(field_name="embedding", vector=vec, param=param)
assert vq.field_name == "embedding"
assert vq.vector == vec
assert vq.param == param
def test_init_both_id_and_vector_raises_error(self):
with pytest.raises(ValueError):
VectorQuery(field_name="embedding", id="doc123", vector=[0.1])._validate()
def test_init_without_field_name_raises_error(self):
with pytest.raises(ValueError):
VectorQuery(field_name=None)._validate()
def test_has_id_returns_true_when_id_set(self):
vq = VectorQuery(field_name="embedding", id="doc123")
assert vq.has_id()
def test_has_id_returns_false_when_no_id(self):
vq = VectorQuery(field_name="embedding", vector=[0.1])
assert not vq.has_id()
def test_has_vector_returns_true_with_non_empty_vector(self):
vq = VectorQuery(field_name="embedding", vector=[0.1])
assert vq.has_vector()
def test_validate_fails_on_both_id_and_vector(self):
vq = VectorQuery(field_name="test", id="doc123", vector=[0.1])
with pytest.raises(ValueError):
vq._validate()
+297
View File
@@ -0,0 +1,297 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import Dict, Union
from unittest.mock import MagicMock
import numpy as np
import math
from _zvec.param import _VectorQuery
import pytest
from zvec.executor.query_executor import (
MultiVectorQueryExecutor,
NoVectorQueryExecutor,
QueryContext,
QueryExecutor,
QueryExecutorFactory,
SingleVectorQueryExecutor,
VectorQuery,
)
from zvec import RrfReRanker, HnswQueryParam, CollectionSchema, VectorSchema, DataType
# ----------------------------
# Mock Vector Schema
# ----------------------------
class MockVectorSchema(VectorSchema):
def __init__(self, name="test_vector"):
self._name = name
@property
def name(self):
return self._name
def _get_object(self):
return MagicMock()
# ----------------------------
# Mock Collection Schema
# ----------------------------
class MockCollectionSchema(CollectionSchema):
def __init__(self, vectors=Union[VectorSchema, Dict[str, VectorSchema]]):
self._vectors = (
[vectors] if not isinstance(vectors, Dict) else list(vectors.values())
)
@property
def vectors(self):
return self._vectors
# ----------------------------
# VectorQuery Test Case
# ----------------------------
class TestVectorQuery:
def test_init(self):
query = VectorQuery(field_name="test_field")
assert query.field_name == "test_field"
assert query.id is None
assert query.vector is None
assert query.param is None
param = HnswQueryParam()
query = VectorQuery(
field_name="test_field", id="test_id", vector=[1, 2, 3], param=param
)
assert query.field_name == "test_field"
assert query.id == "test_id"
assert query.vector == [1, 2, 3]
assert query.param == param
def test_has_id(self):
query = VectorQuery(field_name="test_field")
assert not query.has_id()
query = VectorQuery(field_name="test_field", id="test_id")
assert query.has_id()
def test_has_vector(self):
query = VectorQuery(field_name="test_field")
assert not query.has_vector()
query = VectorQuery(field_name="test_field", vector=[])
assert not query.has_vector()
query = VectorQuery(field_name="test_field", vector=[1, 2, 3])
assert query.has_vector()
def test_validate_dense_fp16_convert(self):
v = _VectorQuery()
schema = VectorSchema(name="test", data_type=DataType.VECTOR_FP16)
vec = np.array([1.1, 2.1, 3.1], dtype=np.float16)
v.set_vector(schema._get_object(), vec)
ret = v.get_vector(schema._get_object())
assert np.array_equal(vec, ret)
def test_validate_dense_fp32_convert(self):
v = _VectorQuery()
schema = VectorSchema(name="test", data_type=DataType.VECTOR_FP32)
vec = np.array([1.1, 2.1, 3.1], dtype=np.float32)
v.set_vector(schema._get_object(), vec)
ret = v.get_vector(schema._get_object())
assert np.array_equal(vec, ret)
def test_validate_dense_fp64_convert(self):
v = _VectorQuery()
schema = VectorSchema(name="test", data_type=DataType.VECTOR_FP64)
vec = np.array([1.1, 2.1, 3.1], dtype=np.float64)
v.set_vector(schema._get_object(), vec)
ret = v.get_vector(schema._get_object())
assert np.array_equal(vec, ret)
def test_validate_dense_int8_convert(self):
v = _VectorQuery()
schema = VectorSchema(name="test", data_type=DataType.VECTOR_INT8)
vec = np.array([1, 2, 3], dtype=np.int8)
v.set_vector(schema._get_object(), vec)
ret = v.get_vector(schema._get_object())
assert np.array_equal(vec, ret)
def test_validate_sparse_fp32_convert(self):
v = _VectorQuery()
schema = VectorSchema(name="test", data_type=DataType.SPARSE_VECTOR_FP32)
vec = {1: 1.1, 2: 2.2, 3: 3.3}
v.set_vector(schema._get_object(), vec)
ret = v.get_vector(schema._get_object())
for k in vec.keys():
assert math.isclose(vec[k], ret[k], abs_tol=1e-6)
def test_validate_sparse_fp16_convert(self):
v = _VectorQuery()
schema = VectorSchema(name="test", data_type=DataType.SPARSE_VECTOR_FP16)
vec = {1: 1.1, 2: 2.2, 3: 3.3}
v.set_vector(schema._get_object(), vec)
ret = v.get_vector(schema._get_object())
for k in vec.keys():
assert math.isclose(np.float16(vec[k]), ret[k], abs_tol=1e-6)
class TestQueryContext:
def test_init(self):
ctx = QueryContext(topk=10)
assert ctx.topk == 10
assert ctx.queries == []
assert ctx.filter is None
assert ctx.reranker is None
assert ctx.output_fields is None
assert ctx.include_vector is False
assert ctx.core_vectors == []
def test_properties(self):
queries = [VectorQuery(field_name="test")]
reranker = RrfReRanker()
output_fields = ["field1", "field2"]
ctx = QueryContext(
topk=5,
filter="test_filter",
include_vector=True,
queries=queries,
output_fields=output_fields,
reranker=reranker,
)
assert ctx.topk == 5
assert ctx.queries == queries
assert ctx.filter == "test_filter"
assert ctx.reranker == reranker
assert ctx.output_fields == output_fields
assert ctx.include_vector is True
def test_core_vectors_setter(self):
ctx = QueryContext(topk=10)
core_vectors = [MagicMock()]
ctx.core_vectors = core_vectors
assert ctx.core_vectors == core_vectors
class TestNoVectorQueryExecutor:
def test_init(self):
schema = MockCollectionSchema()
executor = NoVectorQueryExecutor(schema)
assert isinstance(executor, QueryExecutor)
def test_do_validate_with_queries(self):
schema = MockCollectionSchema()
executor = NoVectorQueryExecutor(schema)
ctx = QueryContext(topk=10, queries=[VectorQuery(field_name="test")])
with pytest.raises(
ValueError, match="Collection does not support query with vector or id"
):
executor._do_validate(ctx)
def test_do_validate_without_queries(self):
schema = MockCollectionSchema()
executor = NoVectorQueryExecutor(schema)
ctx = QueryContext(topk=10)
executor._do_validate(ctx)
def test_do_build(self):
schema = MockCollectionSchema()
executor = NoVectorQueryExecutor(schema)
ctx = QueryContext(topk=5, filter="test_filter")
result = executor._do_build(ctx, MagicMock())
assert len(result) == 1
assert result[0].topk == 5
assert result[0].filter == "test_filter"
class TestSingleVectorQueryExecutor:
def test_init(self):
schema = MockCollectionSchema()
executor = SingleVectorQueryExecutor(schema)
assert isinstance(executor, NoVectorQueryExecutor)
def test_do_validate_multiple_queries(self):
schema = MockCollectionSchema()
executor = SingleVectorQueryExecutor(schema)
queries = [VectorQuery(field_name="test1"), VectorQuery(field_name="test2")]
ctx = QueryContext(topk=10, queries=queries)
with pytest.raises(
ValueError,
match="Collection has only one vector field, cannot query with multiple vectors",
):
executor._do_validate(ctx)
def test_do_build_without_queries(self):
schema = MockCollectionSchema()
executor = SingleVectorQueryExecutor(schema)
ctx = QueryContext(topk=5)
result = executor._do_build(ctx, MagicMock())
assert len(result) == 1
assert result[0].topk == 5
class TestMultiVectorQueryExecutor:
def test_init(self):
schema = MockCollectionSchema()
executor = MultiVectorQueryExecutor(schema)
assert isinstance(executor, SingleVectorQueryExecutor)
def test_do_validate_multiple_queries_without_reranker(self):
schema = MockCollectionSchema()
executor = MultiVectorQueryExecutor(schema)
queries = [VectorQuery(field_name="test1"), VectorQuery(field_name="test2")]
ctx = QueryContext(topk=10, queries=queries)
with pytest.raises(
ValueError, match="Reranker is required for multi-vector query"
):
executor._do_validate(ctx)
def test_do_validate_multiple_queries_with_reranker(self):
schema = MockCollectionSchema()
executor = MultiVectorQueryExecutor(schema)
queries = [VectorQuery(field_name="test1"), VectorQuery(field_name="test2")]
reranker = RrfReRanker()
ctx = QueryContext(topk=10, queries=queries, reranker=reranker)
executor._do_validate(ctx)
class TestQueryExecutorFactory:
def test_create_no_vectors(self):
schema = MockCollectionSchema()
executor = QueryExecutorFactory.create(schema)
assert isinstance(executor, NoVectorQueryExecutor)
def test_create_single_vector(self):
schema = MockCollectionSchema(vectors=MockVectorSchema())
executor = QueryExecutorFactory.create(schema)
assert isinstance(executor, SingleVectorQueryExecutor)
def test_create_multiple_vectors(self):
schema = MockCollectionSchema(
vectors={"test1": MockVectorSchema(), "test2": MockVectorSchema()}
)
executor = QueryExecutorFactory.create(schema)
assert isinstance(executor, MultiVectorQueryExecutor)
+184
View File
@@ -0,0 +1,184 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from unittest.mock import patch
import pytest
import math
from zvec import RrfReRanker, WeightedReRanker, Doc, MetricType
# ----------------------------
# RrfRanker Test Case
# ----------------------------
class TestRrfReRanker:
def test_init(self):
reranker = RrfReRanker(
query="test", topn=5, rerank_field="content", rank_constant=100
)
assert reranker.query == "test"
assert reranker.topn == 5
assert reranker.rerank_field == "content"
assert reranker.rank_constant == 100
def test_rrf_score(self):
reranker = RrfReRanker(query="test", rank_constant=60)
# 根据公式 1.0 / (k + rank + 1),其中k=60
assert reranker._rrf_score(0) == 1.0 / (60 + 0 + 1)
assert reranker._rrf_score(1) == 1.0 / (60 + 1 + 1)
assert reranker._rrf_score(10) == 1.0 / (60 + 10 + 1)
def test_rerank(self):
reranker = RrfReRanker(query="test", topn=3)
doc1 = Doc(id="1", score=0.8)
doc2 = Doc(id="2", score=0.7)
doc3 = Doc(id="3", score=0.9)
doc4 = Doc(id="4", score=0.6)
query_results = {"vector1": [doc1, doc2, doc3], "vector2": [doc3, doc1, doc4]}
results = reranker.rerank(query_results)
assert len(results) <= reranker.topn
for doc in results:
assert hasattr(doc, "score")
scores = [doc.score for doc in results]
assert scores == sorted(scores, reverse=True)
# ----------------------------
# WeightedRanker Test Case
# ----------------------------
class TestWeightedReRanker:
def test_init(self):
weights = {"vector1": 0.7, "vector2": 0.3}
reranker = WeightedReRanker(
query="test",
topn=5,
rerank_field="content",
metric=MetricType.L2,
weights=weights,
)
assert reranker.query == "test"
assert reranker.topn == 5
assert reranker.rerank_field == "content"
assert reranker.metric == MetricType.L2
assert reranker.weights == weights
def test_normalize_score(self):
reranker = WeightedReRanker(query="test")
score = reranker._normalize_score(1.0, MetricType.L2)
expected = 1.0 - 2 * math.atan(1.0) / math.pi
assert score == expected
score = reranker._normalize_score(1.0, MetricType.IP)
expected = 0.5 + math.atan(1.0) / math.pi
assert score == expected
score = reranker._normalize_score(1.0, MetricType.COSINE)
expected = 1.0 - 1.0 / 2.0
assert score == expected
with pytest.raises(ValueError, match="Unsupported metric type"):
reranker._normalize_score(1.0, "unsupported_metric")
def test_rerank(self):
weights = {"vector1": 0.7, "vector2": 0.3}
reranker = WeightedReRanker(
query="test", topn=3, weights=weights, metric=MetricType.L2
)
doc1 = Doc(id="1", score=0.8)
doc2 = Doc(id="2", score=0.7)
doc3 = Doc(id="3", score=0.9)
query_results = {"vector1": [doc1, doc2], "vector2": [doc2, doc3]}
results = reranker.rerank(query_results)
assert len(results) <= reranker.topn
for doc in results:
assert hasattr(doc, "score")
scores = [doc.score for doc in results]
assert scores == sorted(scores, reverse=True)
# # ----------------------------
# # QwenReRanker Test Case
# # ----------------------------
# class TestQwenReRanker:
# def test_init_without_query(self):
# with pytest.raises(ValueError):
# QwenReRanker()
#
# def test_init_without_api_key(self):
# with patch.dict(os.environ, {"DASHSCOPE_API_KEY": ""}):
# with pytest.raises(ValueError, match="DashScope API key is required"):
# QwenReRanker(query="test")
#
# @patch.dict(os.environ, {"DASHSCOPE_API_KEY": "test_key"})
# def test_init_with_env_api_key(self):
# reranker = QwenReRanker(query="test")
# assert reranker.query == "test"
# assert reranker._api_key == "test_key"
#
# def test_model_property(self):
# reranker = QwenReRanker(query="test", api_key="test_key")
# assert reranker.model == "gte-rerank-v2"
#
# reranker = QwenReRanker(query="test", model="custom-model", api_key="test_key")
# assert reranker.model == "custom-model"
#
# def test_rerank_empty_results(self):
# reranker = QwenReRanker(query="test", api_key="test_key")
# results = reranker.rerank({})
# assert results == []
#
# def test_rerank_no_documents(self):
# reranker = QwenReRanker(query="test", api_key="test_key")
# query_results = {"vector1": [Doc(id="1")]}
# with pytest.raises(ValueError, match="No documents to rerank"):
# reranker.rerank(query_results)
#
# @pytest.mark.skip(reason="Qwen ReRanker is not available in CI")
# def test_rerank_success(self):
# reranker = QwenReRanker(
# topn=3,
# query="test",
# api_key="*",
# rerank_field="content",
# )
# query_results = {
# "vector1": [
# Doc(id="1", fields={"content": "This is a test document."}),
# Doc(id="2", fields={"content": "Another test document."}),
# Doc(id="3", fields={"content": "Yet another test document."}),
# Doc(id="4", fields={"content": "One more test document."}),
# ],
# "vector2": [
# Doc(id="5", fields={"content": "This is a test document2."}),
# Doc(id="6", fields={"content": "Another test document2."}),
# Doc(id="7", fields={"content": "Yet another test document2."}),
# Doc(id="8", fields={"content": "One more test document2."}),
# ],
# }
# results = reranker.rerank(query_results)
# assert len(results) == 3
+246
View File
@@ -0,0 +1,246 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import pytest
from zvec import (
CollectionSchema,
CollectionStats,
FieldSchema,
VectorSchema,
HnswIndexParam,
InvertIndexParam,
DataType,
IndexType,
MetricType,
)
# ----------------------------
# FieldSchema Test Case
# ----------------------------
class TestFieldSchema:
def test_default(self):
field = FieldSchema("field", data_type=DataType.FLOAT)
assert field.name == "field"
assert field.data_type == DataType.FLOAT
assert field.nullable is False
assert field.index_param is None
def test_custom(self):
field_1 = FieldSchema(
name="float",
data_type=DataType.FLOAT,
nullable=True,
index_param=InvertIndexParam(),
)
assert field_1.name == "float"
assert field_1.data_type == DataType.FLOAT
assert field_1.nullable is True
assert field_1.index_param.enable_range_optimization is False
field_2 = FieldSchema(
name="str",
data_type=DataType.STRING,
nullable=True,
index_param=InvertIndexParam(enable_range_optimization=True),
)
assert field_2.name == "str"
assert field_2.data_type == DataType.STRING
assert field_2.nullable is True
assert field_2.index_param.enable_range_optimization is True
def test_readonly(self):
field = FieldSchema(
name="float",
data_type=DataType.FLOAT,
nullable=True,
index_param=InvertIndexParam(),
)
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
field.index_param = InvertIndexParam(enable_range_optimization=True)
# ----------------------------
# VectorSchema Test Case
# ----------------------------
class TestVectorSchema:
def test_default(self):
field = VectorSchema("vector", data_type=DataType.VECTOR_FP32, dimension=128)
assert field.name == "vector"
assert field.data_type == DataType.VECTOR_FP32
assert field.dimension == 128
assert field.index_param is not None
assert field.index_param.type == IndexType.FLAT
assert field.index_param.metric_type == MetricType.IP
def test_custom(self):
field = VectorSchema(
name="vector",
data_type=DataType.VECTOR_INT8,
dimension=512,
index_param=HnswIndexParam(
metric_type=MetricType.COSINE, m=15, ef_construction=300
),
)
assert field.name == "vector"
assert field.data_type == DataType.VECTOR_INT8
assert field.index_param.metric_type == MetricType.COSINE
assert field.index_param.m == 15
assert field.index_param.ef_construction == 300
def test_readonly(self):
field = VectorSchema(
name="vector",
dimension=128,
data_type=DataType.VECTOR_INT8,
)
import sys
if sys.version_info >= (3, 11):
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
else:
match_pattern = r"can't set attribute"
with pytest.raises(AttributeError, match=match_pattern):
field.dimension = 4
# ----------------------------
# CollectionSchema Test Case
# ----------------------------
class TestCollectionSchema:
def test_collection_schema_with_single_field(self):
collection_schema = CollectionSchema(
name="test_collection",
fields=FieldSchema(
name="id",
data_type=DataType.INT64,
index_param=InvertIndexParam(),
nullable=False,
),
vectors=VectorSchema(
name="vector",
data_type=DataType.VECTOR_INT8,
dimension=128,
index_param=HnswIndexParam(),
),
)
assert collection_schema is not None
assert collection_schema.name == "test_collection"
assert len(collection_schema.fields) == 1
assert len(collection_schema.vectors) == 1
field = collection_schema.field("id")
assert field is not None
assert field.name == "id"
assert field.data_type == DataType.INT64
assert not field.nullable
assert field.index_param.type == IndexType.INVERT
assert not field.index_param.enable_range_optimization
vector = collection_schema.vector("vector")
assert vector is not None
assert vector.name == "vector"
assert vector.data_type == DataType.VECTOR_INT8
assert vector.dimension == 128
assert vector.index_param.type == IndexType.HNSW
assert vector.index_param.m == 100
assert vector.index_param.ef_construction == 500
assert vector.index_param.metric_type == MetricType.IP
def test_collection_schema_with_multi_fields(self):
collection_schema = CollectionSchema(
name="test_collection",
fields=[
FieldSchema(
"id",
DataType.INT64,
nullable=False,
index_param=InvertIndexParam(enable_range_optimization=True),
),
FieldSchema(
"name",
DataType.STRING,
nullable=False,
index_param=InvertIndexParam(),
),
FieldSchema(
"weight",
DataType.INT32,
nullable=True,
),
],
vectors=[
VectorSchema(
"dense",
DataType.VECTOR_FP32,
dimension=128,
index_param=HnswIndexParam(),
),
VectorSchema(
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
),
],
)
assert collection_schema is not None
assert collection_schema.name == "test_collection"
assert len(collection_schema.fields) == 3
assert len(collection_schema.vectors) == 2
field_id = collection_schema.field("id")
assert field_id is not None
assert field_id.name == "id"
assert field_id.data_type == DataType.INT64
assert not field_id.nullable
assert field_id.index_param.type == IndexType.INVERT
dense = collection_schema.vector("dense")
assert dense is not None
assert dense.name == "dense"
assert dense.data_type == DataType.VECTOR_FP32
assert dense.dimension == 128
assert dense.index_param.type == IndexType.HNSW
sparse = collection_schema.vector("sparse")
assert sparse is not None
assert sparse.name == "sparse"
assert sparse.data_type == DataType.SPARSE_VECTOR_FP32
assert sparse.dimension == 0
assert sparse.index_param.type == IndexType.HNSW
assert str(collection_schema) is not None
# ----------------------------
# CollectionStats Test Case
# ----------------------------
class TestCollectionStats:
"""
The constructor of CollectionStats is not provided.
It can only be obtained through collection.stats()
"""
def test_collection_stats(self):
stats = CollectionStats()
assert stats is not None
+141
View File
@@ -0,0 +1,141 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import pytest
from zvec import (
DataType,
IndexType,
MetricType,
QuantizeType,
Status,
StatusCode,
)
# ----------------------------
# Enum Test Case
# ----------------------------
@pytest.mark.parametrize(
"member, name",
[
(DataType.FLOAT, "FLOAT"),
(IndexType.HNSW, "HNSW"),
(MetricType.COSINE, "COSINE"),
(QuantizeType.INT8, "INT8"),
(StatusCode.OK, "OK"),
],
)
def test_enum_names(member, name):
assert member.name == name
@pytest.mark.parametrize(
"member, value",
[
(DataType.FLOAT, 8),
(IndexType.HNSW, 1),
(MetricType.COSINE, 3),
(QuantizeType.INT8, 2),
(StatusCode.OK, 0),
],
)
def test_enum_values(member, value):
assert member.value == value
@pytest.mark.parametrize("member", ["L2", "IP", "COSINE"])
def test_metric_type_has_member(member):
assert member in MetricType.__members__
@pytest.mark.parametrize(
"member",
[
"STRING",
"BOOL",
"INT32",
"INT64",
"FLOAT",
"DOUBLE",
"UINT32",
"UINT64",
"VECTOR_FP16",
"VECTOR_FP32",
"VECTOR_FP64",
"VECTOR_INT8",
"SPARSE_VECTOR_FP32",
"SPARSE_VECTOR_FP16",
"ARRAY_STRING",
"ARRAY_INT32",
"ARRAY_INT64",
"ARRAY_FLOAT",
"ARRAY_DOUBLE",
"ARRAY_BOOL",
"ARRAY_UINT32",
"ARRAY_UINT64",
],
)
def test_data_type_has_member(member):
assert member in DataType.__members__
@pytest.mark.parametrize("member", ["HNSW", "IVF", "FLAT", "INVERT"])
def test_index_type_has_member(member):
assert member in IndexType.__members__
@pytest.mark.parametrize("member", ["FP16", "INT8", "INT4", "UNDEFINED"])
def test_quantize_type_has_member(member):
assert member in QuantizeType.__members__
@pytest.mark.parametrize(
"member",
[
"OK",
"UNKNOWN",
"NOT_FOUND",
"ALREADY_EXISTS",
"INVALID_ARGUMENT",
"PERMISSION_DENIED",
"FAILED_PRECONDITION",
"RESOURCE_EXHAUSTED",
"UNAVAILABLE",
"INTERNAL_ERROR",
"NOT_SUPPORTED",
],
)
def test_status_code_has_member(member):
assert member in StatusCode.__members__
# ----------------------------
# Status Test Case
# ----------------------------
class TestStatus:
def test_status_code(self):
status = Status(StatusCode.OK)
assert status.code() == StatusCode.OK
def test_status_message(self):
status = Status(StatusCode.OK, "OK")
assert status.message() == "OK"
status = Status(StatusCode.NOT_FOUND, "Not Found")
assert status.message() == "Not Found"
def test_status_ok(self):
status = Status(StatusCode.OK)
assert status.ok()
+94
View File
@@ -0,0 +1,94 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from unittest.mock import MagicMock, patch
import pytest
from zvec import require_module
# ----------------------------
# require_module func Test Case
# ----------------------------
def test_require_module_success():
module = require_module("os")
assert module is not None
assert hasattr(module, "path")
def test_require_module_with_submodule_success():
module = require_module("os.path")
assert module is not None
assert hasattr(module, "join")
def test_require_module_import_error():
with pytest.raises(ImportError) as exc_info:
require_module("nonexistent_module")
exception_msg = str(exc_info.value)
assert "Required package 'nonexistent_module' is not installed." in exception_msg
def test_require_module_with_mitigation_import_error():
with pytest.raises(ImportError) as exc_info:
require_module("nonexistent_module.submodule", mitigation="custom_package")
exception_msg = str(exc_info.value)
assert "Required package 'custom_package' is not installed." in exception_msg
assert (
"Module 'nonexistent_module.submodule' is part of 'nonexistent_module'"
in exception_msg
)
assert "please pip install 'custom_package'." in exception_msg
def test_require_module_submodule_import_error():
with pytest.raises(ImportError) as exc_info:
require_module("os.nonexistent_submodule")
exception_msg = str(exc_info.value)
assert (
"Required package 'os.nonexistent_submodule' is not installed." in exception_msg
)
assert "Module 'os.nonexistent_submodule' is part of 'os'" in exception_msg
assert "please pip install 'os'." in exception_msg
@patch("importlib.import_module")
def test_require_module_wraps_original_exception(mock_import_module):
original_exception = ImportError("Original error")
mock_import_module.side_effect = original_exception
with pytest.raises(ImportError) as exc_info:
require_module("some_module")
assert exc_info.value.__cause__ is original_exception
@patch("importlib.import_module")
def test_require_module_calls_importlib(mock_import_module):
mock_module = MagicMock()
mock_import_module.return_value = mock_module
result = require_module("test_module")
mock_import_module.assert_called_once_with("test_module")
assert result is mock_module
def test_require_module_with_openai():
with pytest.raises(ImportError) as exc_info:
require_module("openai")
+133
View File
@@ -0,0 +1,133 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import sys
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from importlib.metadata import PackageNotFoundError
# ==============================
# Public API — grouped by category
# ==============================
from . import model as model
# —— Extensions & typing ——
from .extension import DenseEmbeddingFunction, ReRanker, RrfReRanker, WeightedReRanker
from .model import param as param
from .model import schema as schema
# —— Core data structures ——
from .model.collection import Collection
from .model.doc import Doc
# —— Query & index parameters ——
from .model.param import (
AddColumnOption,
AlterColumnOption,
CollectionOption,
FlatIndexParam,
HnswIndexParam,
HnswQueryParam,
IndexOption,
InvertIndexParam,
IVFIndexParam,
IVFQueryParam,
OptimizeOption,
)
from .model.param.vector_query import VectorQuery
# —— Schema & field definitions ——
from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
# —— tools ——
from .tool import require_module
from .typing import (
DataType,
IndexType,
MetricType,
QuantizeType,
Status,
StatusCode,
)
from .typing.enum import LogLevel, LogType
# —— lifecycle ——
from .zvec import create_and_open, init, open
# ==============================
# Public interface declaration
# ==============================
__all__ = [
# Zvec functions
"create_and_open",
"init",
"open",
# Core classes
"Collection",
"Doc",
# Schema
"CollectionSchema",
"FieldSchema",
"VectorSchema",
"CollectionStats",
# Parameters
"VectorQuery",
"InvertIndexParam",
"HnswIndexParam",
"FlatIndexParam",
"IVFIndexParam",
"CollectionOption",
"IndexOption",
"OptimizeOption",
"AddColumnOption",
"AlterColumnOption",
"HnswQueryParam",
"IVFQueryParam",
# Extensions
"ReRanker",
"DenseEmbeddingFunction",
"RrfReRanker",
"WeightedReRanker",
# Typing
"DataType",
"MetricType",
"QuantizeType",
"IndexType",
"LogLevel",
"LogType",
"Status",
"StatusCode",
# Tools
"require_module",
]
# ==============================
# Version handling
# ==============================
__version__: str
try:
from importlib.metadata import version
except ImportError:
from importlib_metadata import version # Python < 3.8
try:
__version__ = version("zvec")
except Exception:
__version__ = "unknown"
+172
View File
@@ -0,0 +1,172 @@
"""
Zvec core module
"""
from __future__ import annotations
import collections
from . import typing
from .extension import ReRanker, RrfReRanker, WeightedReRanker
from .extension.embedding import DenseEmbeddingFunction
from .model import param, schema
from .model.collection import Collection
from .model.doc import Doc
from .model.param import (
AddColumnOption,
AlterColumnOption,
CollectionOption,
FlatIndexParam,
HnswIndexParam,
HnswQueryParam,
IndexOption,
InvertIndexParam,
IVFIndexParam,
IVFQueryParam,
OptimizeOption,
)
from .model.param.vector_query import VectorQuery
from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
from .tool import require_module
from .typing import (
DataType,
IndexType,
MetricType,
QuantizeType,
Status,
StatusCode,
)
from .typing.enum import LogLevel, LogType
from .zvec import create_and_open, init, open
__all__: list = [
"AddColumnOption",
"AlterColumnOption",
"Collection",
"CollectionOption",
"CollectionSchema",
"CollectionStats",
"DataType",
"DenseEmbeddingFunction",
"DenseEmbeddingFunction",
"Doc",
"FieldSchema",
"FlatIndexParam",
"HnswIndexParam",
"HnswQueryParam",
"IVFIndexParam",
"IVFQueryParam",
"IndexOption",
"IndexType",
"InvertIndexParam",
"LogLevel",
"LogType",
"MetricType",
"OptimizeOption",
"QuantizeType",
"ReRanker",
"ReRanker",
"RrfReRanker",
"Status",
"StatusCode",
"VectorQuery",
"VectorSchema",
"WeightedReRanker",
"create_and_open",
"init",
"open",
"require_module",
]
class _Collection:
@staticmethod
def CreateAndOpen(
arg0: str, arg1: schema._CollectionSchema, arg2: param.CollectionOption
) -> _Collection: ...
@staticmethod
def Open(arg0: str, arg1: param.CollectionOption) -> _Collection: ...
def AddColumn(
self,
arg0: str,
arg1: schema._FieldSchema,
arg2: str,
arg3: param.AddColumnOption,
) -> None: ...
def AlterColumn(
self,
arg0: str,
arg1: str,
arg2: schema._FieldSchema,
arg3: param.AlterColumnOption,
) -> None: ...
def CreateIndex(
self, arg0: str, arg1: param.IndexParam, arg2: param.IndexOption
) -> None: ...
def Delete(self, arg0: collections.abc.Sequence[str]) -> list[typing.Status]: ...
def DeleteByFilter(self, arg0: str) -> None: ...
def Destroy(self) -> None: ...
def DropColumn(self, arg0: str) -> None: ...
def DropIndex(self, arg0: str) -> None: ...
def Fetch(self, arg0: collections.abc.Sequence[str]) -> dict[str, _Doc]: ...
def Flush(self) -> None: ...
def GroupByQuery(self, arg0: ...) -> list[...]: ...
def Insert(self, arg0: collections.abc.Sequence[_Doc]) -> list[typing.Status]: ...
def Optimize(self, arg0: param.OptimizeOption) -> None: ...
def Options(self) -> param.CollectionOption: ...
def Path(self) -> str: ...
def Query(self, arg0: param._VectorQuery) -> list[_Doc]: ...
def Schema(self) -> schema._CollectionSchema: ...
def Stats(self) -> schema.CollectionStats: ...
def Update(self, arg0: collections.abc.Sequence[_Doc]) -> list[typing.Status]: ...
def Upsert(self, arg0: collections.abc.Sequence[_Doc]) -> list[typing.Status]: ...
def __getstate__(self) -> tuple: ...
def __setstate__(self, arg0: tuple) -> None: ...
class _Doc:
def __getstate__(self) -> bytes: ...
def __init__(self) -> None: ...
def __setstate__(self, arg0: bytes) -> None: ...
def field_names(self) -> list[str]: ...
def get_any(self, arg0: str, arg1: typing.DataType) -> typing.Any: ...
def has_field(self, arg0: str) -> bool: ...
def pk(self) -> str: ...
def score(self) -> float: ...
def set_any(self, arg0: str, arg1: typing.DataType, arg2: typing.Any) -> bool: ...
def set_pk(self, arg0: str) -> None: ...
def set_score(self, arg0: typing.SupportsFloat) -> None: ...
class _DocOp:
"""
Members:
INSERT
UPDATE
DELETE
UPSERT
"""
DELETE: typing.ClassVar[_DocOp] # value = <_DocOp.DELETE: 3>
INSERT: typing.ClassVar[_DocOp] # value = <_DocOp.INSERT: 0>
UPDATE: typing.ClassVar[_DocOp] # value = <_DocOp.UPDATE: 2>
UPSERT: typing.ClassVar[_DocOp] # value = <_DocOp.UPSERT: 1>
__members__: typing.ClassVar[
dict[str, _DocOp]
] # value = {'INSERT': <_DocOp.INSERT: 0>, 'UPDATE': <_DocOp.UPDATE: 2>, 'DELETE': <_DocOp.DELETE: 3>, 'UPSERT': <_DocOp.UPSERT: 1>}
def __eq__(self, other: typing.Any) -> bool: ...
def __getstate__(self) -> int: ...
def __hash__(self) -> int: ...
def __index__(self) -> int: ...
def __init__(self, value: typing.SupportsInt) -> None: ...
def __int__(self) -> int: ...
def __ne__(self, other: typing.Any) -> bool: ...
def __repr__(self) -> str: ...
def __setstate__(self, state: typing.SupportsInt) -> None: ...
def __str__(self) -> str: ...
@property
def name(self) -> str: ...
@property
def value(self) -> int: ...
+18
View File
@@ -0,0 +1,18 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from .constants import DenseVectorType, SparseVectorType, VectorType
__all__ = ["DenseVectorType", "SparseVectorType", "VectorType"]
+22
View File
@@ -0,0 +1,22 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import Optional, Union
import numpy as np
DenseVectorType = Union[list[float], list[int], np.ndarray]
SparseVectorType = dict[int, float]
VectorType = Optional[Union[DenseVectorType, SparseVectorType]]
+26
View File
@@ -0,0 +1,26 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from .query_executor import (
QueryContext,
QueryExecutor,
QueryExecutorFactory,
)
__all__ = [
"QueryContext",
"QueryExecutor",
"QueryExecutorFactory",
]
+307
View File
@@ -0,0 +1,307 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import os
from abc import ABC, abstractmethod
from concurrent.futures import ThreadPoolExecutor, as_completed
from typing import Optional, Union, final
import numpy as np
from _zvec import _Collection
from _zvec.param import _VectorQuery
from ..extension import ReRanker, RrfReRanker, WeightedReRanker
from ..model.convert import convert_to_py_doc
from ..model.doc import Doc
from ..model.param.vector_query import VectorQuery
from ..model.schema import CollectionSchema
from ..typing import DataType
__all__ = [
"QueryContext",
"QueryExecutor",
"QueryExecutorFactory",
]
DTYPE_MAP = {
DataType.VECTOR_FP16.value: np.float16,
DataType.VECTOR_FP32.value: np.float32,
DataType.VECTOR_FP64.value: np.float64,
DataType.VECTOR_INT8.value: np.int8,
}
def convert_to_numpy(vec: Union[list, np.ndarray], dtype: np.dtype) -> np.ndarray:
if isinstance(vec, np.ndarray):
if vec.dtype == dtype and vec.ndim == 1:
return vec
return np.asarray(vec, dtype=dtype).flatten()
try:
arr = np.asarray(vec, dtype=dtype)
if arr.ndim != 1:
arr = arr.flatten()
return arr
except (ValueError, TypeError) as e:
raise TypeError(
f"Cannot convert input to 1D numpy array with dtype={dtype}: {type(vec)}"
) from e
class QueryContext:
def __init__(
self,
topk: int,
filter: Optional[str] = None,
include_vector: bool = False,
queries: Optional[list[VectorQuery]] = None,
output_fields: Optional[list[str]] = None,
reranker: Optional[ReRanker] = None,
):
# query param
self._filter = filter
self._queries = queries or []
self._topk = topk
self._include_vector = include_vector
self._output_fields = output_fields
# reranker
self._reranker = reranker
# core vectors
self._core_vectors = []
@property
def topk(self):
return self._topk
@property
def queries(self):
return self._queries
@property
def filter(self):
return self._filter
@property
def reranker(self):
return self._reranker
@property
def output_fields(self):
return self._output_fields
@property
def include_vector(self):
return self._include_vector
@property
def core_vectors(self):
return self._core_vectors
@core_vectors.setter
def core_vectors(self, core_vectors: list[_VectorQuery]):
self._core_vectors = core_vectors
class QueryExecutor(ABC):
def __init__(self, schema: CollectionSchema):
self._schema = schema
self._concurrency = max(1, int(os.getenv("ZVEC_QUERY_CONCURRENCY", "1")))
@abstractmethod
def _do_validate(self, ctx: QueryContext) -> None:
pass
@abstractmethod
def _do_build(
self, ctx: QueryContext, collection: _Collection
) -> list[_VectorQuery]:
pass
def _do_build_query_wo_vector(self, ctx: QueryContext) -> _VectorQuery:
core_vector = _VectorQuery()
core_vector.topk = ctx.topk
core_vector.include_vector = ctx.include_vector
if ctx.filter:
core_vector.filter = ctx.filter
if ctx.output_fields:
core_vector.output_fields = ctx.output_fields
return core_vector
def _do_build_query_with_vector(
self, ctx: QueryContext, query: VectorQuery, collection: _Collection
) -> _VectorQuery:
core_vector = self._do_build_query_wo_vector(ctx)
core_vector.field_name = query.field_name
if query.param:
core_vector.query_params = query.param
vector_schema = (
self._schema.vector(query.field_name) if query else self._schema.vectors[0]
)
if vector_schema is None:
raise ValueError("No vector field found")
# set output_fields
core_vector.output_fields = ctx.output_fields
# set vector
if query.has_vector():
vec_data = query.vector
else:
fetched = collection.Fetch([query.id])
doc = next(iter(fetched.values()))
if not doc:
return core_vector
vec_data = doc.get_any(vector_schema.name, vector_schema.data_type)
target_dtype = DTYPE_MAP.get(vector_schema.data_type.value)
core_vector.set_vector(
vector_schema._get_object(),
convert_to_numpy(vec_data, target_dtype) if target_dtype else vec_data,
)
return core_vector
def _do_execute(
self, vectors: list[_VectorQuery], collection: _Collection
) -> dict[str, list[Doc]]:
query_cnt = len(vectors)
if query_cnt == 0:
raise ValueError("No query to execute")
if len(vectors) == 1 or self._concurrency == 1:
results = {}
for query in vectors:
docs = collection.Query(query)
results[query.field_name] = [
convert_to_py_doc(doc, self._schema) for doc in docs
]
return results
results = {}
with ThreadPoolExecutor(max_workers=self._concurrency) as executor:
future_to_query = {
executor.submit(collection.Query, query): query.field_name
for query in vectors
}
for future in as_completed(future_to_query):
field_name = future_to_query[future]
try:
docs = future.result()
results[field_name] = [
convert_to_py_doc(doc, self._schema) for doc in docs
]
except Exception as e:
raise e
return results
def _do_merge_rerank_results(
self, ctx: QueryContext, docs_map: dict[str, list[Doc]]
) -> list[Doc]:
query_result_cnt = len(docs_map) if docs_map else 0
if query_result_cnt == 0:
raise ValueError("Query results is none and dost not to rerank")
if query_result_cnt == 1:
if not ctx.reranker or isinstance(
ctx.reranker, (RrfReRanker, WeightedReRanker)
):
return next(iter(docs_map.values()))
return ctx.reranker.rerank(docs_map)
return ctx.reranker.rerank(docs_map)
@final
def execute(self, ctx: QueryContext, collection: _Collection) -> list[Doc]:
# 1. validate query
self._do_validate(ctx)
# 2. build query vector
query_vectors = self._do_build(ctx, collection)
if not query_vectors:
raise ValueError("No query to execute")
# 3. execute query
docs = self._do_execute(query_vectors, collection)
# 4. merge and rerank result
return self._do_merge_rerank_results(ctx, docs)
class NoVectorQueryExecutor(QueryExecutor):
def __init__(self, schema: CollectionSchema):
super().__init__(schema)
def _do_validate(self, ctx: QueryContext) -> None:
if len(ctx.queries) > 0:
raise ValueError("Collection does not support query with vector or id")
def _do_build(
self, ctx: QueryContext, _collection: _Collection
) -> list[_VectorQuery]:
return [self._do_build_query_wo_vector(ctx)]
class SingleVectorQueryExecutor(NoVectorQueryExecutor):
def __init__(self, schema: CollectionSchema) -> None:
super().__init__(schema)
def _do_validate(self, ctx: QueryContext) -> None:
if len(ctx.queries) > 1:
raise ValueError(
"Collection has only one vector field, cannot query with multiple vectors"
)
for query in ctx.queries:
query._validate()
def _do_build(
self, ctx: QueryContext, collection: _Collection
) -> list[_VectorQuery]:
if len(ctx.queries) == 0:
return [self._do_build_query_wo_vector(ctx)]
vectors = []
for query in ctx.queries:
vectors.append(self._do_build_query_with_vector(ctx, query, collection))
return vectors
class MultiVectorQueryExecutor(SingleVectorQueryExecutor):
def __init__(self, schema: CollectionSchema) -> None:
super().__init__(schema)
def _do_validate(self, ctx: QueryContext) -> None:
if len(ctx.queries) > 1 and ctx.reranker is None:
raise ValueError("Reranker is required for multi-vector query")
seen_fields = set()
for query in ctx.queries:
query._validate()
field = query.field_name
if field in seen_fields:
raise ValueError(f"Query field name '{field}' appears more than once")
seen_fields.add(field)
def _do_execute(
self, vectors: list[_VectorQuery], collection: _Collection
) -> dict[str, list[Doc]]:
return super()._do_execute(vectors, collection)
class QueryExecutorFactory:
@staticmethod
def create(schema: CollectionSchema) -> QueryExecutor:
vectors = schema.vectors
if len(vectors) == 0:
return NoVectorQueryExecutor(schema)
if len(vectors) == 1:
return SingleVectorQueryExecutor(schema)
return MultiVectorQueryExecutor(schema)
+26
View File
@@ -0,0 +1,26 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from .embedding import DenseEmbeddingFunction, QwenEmbeddingFunction
from .rerank import QwenReRanker, ReRanker, RrfReRanker, WeightedReRanker
__all__ = [
"DenseEmbeddingFunction",
"QwenEmbeddingFunction",
"QwenReRanker",
"ReRanker",
"RrfReRanker",
"WeightedReRanker",
]
+188
View File
@@ -0,0 +1,188 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import os
from abc import ABC, abstractmethod
from functools import lru_cache
from http import HTTPStatus
from typing import Optional, Union
from ..tool import require_module
from ..typing import DataType
class DenseEmbeddingFunction(ABC):
"""Abstract base class for dense vector embedding functions.
Dense embedding functions map text to fixed-length real-valued vectors.
Subclasses must implement the ``embed()`` method.
Args:
dimension (int): Dimensionality of the output embedding vector.
data_type (DataType, optional): Numeric type of the embedding.
Defaults to ``DataType.VECTOR_FP32``.
Note:
This class is callable: ``embedding_func("text")`` is equivalent to
``embedding_func.embed("text")``.
"""
def __init__(self, dimension: int, data_type: DataType = DataType.VECTOR_FP32):
self._dimension = dimension
self._data_type = data_type
@property
def dimension(self) -> int:
"""int: The expected dimensionality of the embedding vector."""
return self._dimension
@property
def data_type(self) -> DataType:
"""DataType: The numeric data type of the embedding (e.g., VECTOR_FP32)."""
return self._data_type
@abstractmethod
def embed(self, text: str) -> list[Union[int, float]]:
"""Generate a dense embedding vector for the input text.
Args:
text (str): Input text to embed.
Returns:
list[Union[int, float]]: A list of numbers representing the embedding.
Length must equal ``self.dimension``.
"""
raise NotImplementedError
def __call__(self, text: str) -> list[Union[int, float]]:
return self.embed(text)
class SparseEmbeddingFunction(ABC):
"""Abstract base class for sparse vector embedding functions.
Sparse embedding functions map text to a dictionary of {index: weight},
where only non-zero dimensions are stored.
Note:
Subclasses must implement the ``embed()`` method.
"""
@abstractmethod
def embed(self, text: str) -> dict[int, float]:
"""Generate a sparse embedding for the input text.
Args:
text (str): Input text to embed.
Returns:
dict[int, float]: Mapping from dimension index to non-zero weight.
"""
raise NotImplementedError
class QwenEmbeddingFunction(DenseEmbeddingFunction):
"""Dense embedding function using Qwen (DashScope) Text Embedding API.
This implementation uses the DashScope service to generate embeddings
via Qwen's text embedding models (e.g., ``text-embedding-v4``).
Args:
dimension (int): Desired embedding dimension (e.g., 1024).
model (str, optional): DashScope embedding model name.
Defaults to ``"text-embedding-v4"``.
api_key (Optional[str], optional): DashScope API key. If not provided,
reads from ``DASHSCOPE_API_KEY`` environment variable.
Raises:
ValueError: If API key is missing or input text is invalid.
Note:
Requires the ``dashscope`` Python package.
Embedding results are cached using ``functools.lru_cache`` (maxsize=10).
"""
def __init__(
self,
dimension: int,
model: str = "text-embedding-v4",
api_key: Optional[str] = None,
):
super().__init__(dimension, DataType.VECTOR_FP32)
self._model = model
self._api_key = api_key or os.environ.get("DASHSCOPE_API_KEY")
if not self._api_key:
raise ValueError("DashScope API key is required")
@property
def model(self) -> str:
"""str: The DashScope embedding model name in use."""
return self._model
def _connection(self):
dashscope = require_module("dashscope")
dashscope.api_key = self._api_key
return dashscope
@lru_cache(maxsize=10)
def embed(self, text: str) -> list[Union[int, float]]:
"""
Generate embedding for a given text using Qwen (via DashScope).
Args:
text (str): Input text to embed. Must be non-empty and valid string.
Returns:
list[Union[int, float]]: The dense embedding vector.
Raises:
ValueError: If input is invalid or API response is malformed.
RuntimeError: If network or internal error occurs during API call.
"""
if not isinstance(text, str):
raise TypeError(f"Expected 'text' to be str, got {type(text).__name__}")
text = text.strip()
if not text:
raise ValueError("Input text cannot be empty or whitespace only")
resp = self._connection().TextEmbedding.call(
model=self.model, input=text, dimension=self.dimension, output_type="dense"
)
if resp.status_code != HTTPStatus.OK:
error_msg = getattr(resp, "message", "Unknown error")
error_detail = f"Status={resp.status_code}, Message={error_msg}"
raise ValueError(f"QwenEmbedding failed: {error_detail}")
output = getattr(resp, "output", None)
if not isinstance(output, dict):
raise ValueError("Invalid response: missing or malformed 'output' field")
embeddings = output.get("embeddings")
if not isinstance(embeddings, list):
raise ValueError(
"Invalid response: 'embeddings' field is missing or not a list"
)
if len(embeddings) != 1:
raise ValueError(
f"Expected 1 embedding, got {len(embeddings)}. Response: {resp}"
)
first_emb = embeddings[0]
if not isinstance(first_emb, dict):
raise ValueError("Invalid response: embedding item is not a dictionary")
return list(first_emb.get("embedding"))
+343
View File
@@ -0,0 +1,343 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import heapq
import math
import os
from abc import ABC, abstractmethod
from collections import defaultdict
from http import HTTPStatus
from typing import Optional
from ..model.doc import Doc
from ..tool import require_module
from ..typing import MetricType
class ReRanker(ABC):
"""Abstract base class for re-ranking search results.
Re-rankers refine the output of one or more vector queries by applying
a secondary scoring strategy. They are used in the ``query()`` method of
``Collection`` via the ``reranker`` parameter.
Args:
query (Optional[str], optional): Query text used for re-ranking.
Required for LLM-based re-rankers. Defaults to None.
topn (int, optional): Number of top documents to return after re-ranking.
Defaults to 10.
rerank_field (Optional[str], optional): Field name used as input for
re-ranking (e.g., document title or body). Defaults to None.
Note:
Subclasses must implement the ``rerank()`` method.
"""
def __init__(
self,
query: Optional[str] = None,
topn: int = 10,
rerank_field: Optional[str] = None,
):
self._query = query
self._topn = topn
self._rerank_field = rerank_field
@property
def topn(self) -> int:
"""int: Number of top documents to return after re-ranking."""
return self._topn
@property
def query(self) -> str:
"""str: Query text used for re-ranking."""
return self._query
@property
def rerank_field(self) -> Optional[str]:
"""Optional[str]: Field name used as re-ranking input."""
return self._rerank_field
@abstractmethod
def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
"""Re-rank documents from one or more vector queries.
Args:
query_results (dict[str, list[Doc]]): Mapping from vector field name
to list of retrieved documents (sorted by relevance).
Returns:
list[Doc]: Re-ranked list of documents (length ≤ ``topn``),
with updated ``score`` fields.
"""
raise NotImplementedError
class RrfReRanker(ReRanker):
"""Re-ranker using Reciprocal Rank Fusion (RRF).
RRF combines results from multiple queries without requiring relevance scores.
It assigns higher weight to documents that appear early in multiple result lists.
The RRF score for a document at rank ``r`` is: ``1 / (k + r + 1)``,
where ``k`` is the rank constant.
Args:
query (Optional[str], optional): Ignored by RRF. Defaults to None.
topn (int, optional): Number of top documents to return. Defaults to 10.
rerank_field (Optional[str], optional): Ignored by RRF. Defaults to None.
rank_constant (int, optional): Smoothing constant ``k`` in RRF formula.
Larger values reduce the impact of early ranks. Defaults to 60.
"""
def __init__(
self,
query: Optional[str] = None,
topn: int = 10,
rerank_field: Optional[str] = None,
rank_constant: int = 60,
):
super().__init__(query, topn, rerank_field)
self._rank_constant = rank_constant
@property
def rank_constant(self) -> int:
return self._rank_constant
def _rrf_score(self, rank: int):
return 1.0 / (self._rank_constant + rank + 1)
def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
"""Apply Reciprocal Rank Fusion to combine multiple query results.
Args:
query_results (dict[str, list[Doc]]): Results from one or more vector queries.
Returns:
list[Doc]: Re-ranked documents with RRF scores in the ``score`` field.
"""
rrf_scores: dict[str, float] = defaultdict(float)
id_to_doc: dict[str, Doc] = {}
for _, query_result in query_results.items():
for rank, doc in enumerate(query_result):
doc_id = doc.id
rrf_score = self._rrf_score(rank)
rrf_scores[doc_id] += rrf_score
if doc_id not in id_to_doc:
id_to_doc[doc_id] = doc
top_docs = heapq.nlargest(self.topn, rrf_scores.items(), key=lambda x: x[1])
results = []
for doc_id, rrf_score in top_docs:
doc = id_to_doc[doc_id]
new_doc = doc._replace(score=rrf_score)
results.append(new_doc)
return results
class WeightedReRanker(ReRanker):
"""Re-ranker that combines scores from multiple vector fields using weights.
Each vector field's relevance score is normalized based on its metric type,
then scaled by a user-provided weight. Final scores are summed across fields.
Args:
query (Optional[str], optional): Ignored. Defaults to None.
topn (int, optional): Number of top documents to return. Defaults to 10.
rerank_field (Optional[str], optional): Ignored. Defaults to None.
metric (MetricType, optional): Distance metric used for score normalization.
Defaults to ``MetricType.L2``.
weights (Optional[dict[str, float]], optional): Weight per vector field.
Fields not listed use weight 1.0. Defaults to None.
Note:
Supported metrics: L2, IP, COSINE. Scores are normalized to [0, 1].
"""
def __init__(
self,
query: Optional[str] = None,
topn: int = 10,
rerank_field: Optional[str] = None,
metric: MetricType = MetricType.L2,
weights: Optional[dict[str, float]] = None,
):
super().__init__(query, topn, rerank_field)
self._weights = weights
self._metric = metric
@property
def weights(self) -> dict[str, float]:
"""dict[str, float]: Weight mapping for vector fields."""
return self._weights
@property
def metric(self) -> MetricType:
"""MetricType: Distance metric used for score normalization."""
return self._metric
def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
"""Combine scores from multiple vector fields using weighted sum.
Args:
query_results (dict[str, list[Doc]]): Results per vector field.
Returns:
list[Doc]: Re-ranked documents with combined scores in ``score`` field.
"""
weighted_scores: dict[str, float] = defaultdict(float)
id_to_doc: dict[str, Doc] = {}
for vector_name, query_result in query_results.items():
for _, doc in enumerate(query_result):
doc_id = doc.id
weighted_score = self._normalize_score(
doc.score, self.metric
) * self.weights.get(vector_name, 1.0)
weighted_scores[doc_id] += weighted_score
if doc_id not in id_to_doc:
id_to_doc[doc_id] = doc
top_docs = heapq.nlargest(
self.topn, weighted_scores.items(), key=lambda x: x[1]
)
results = []
for doc_id, weighted_score in top_docs:
doc = id_to_doc[doc_id]
new_doc = doc._replace(score=weighted_score)
results.append(new_doc)
return results
def _normalize_score(self, score: float, metric: MetricType) -> float:
if metric == MetricType.L2:
return 1.0 - 2 * math.atan(score) / math.pi
if metric == MetricType.IP:
return 0.5 + math.atan(score) / math.pi
if metric == MetricType.COSINE:
return 1.0 - score / 2.0
raise ValueError("Unsupported metric type")
class QwenReRanker(ReRanker):
"""Re-ranker using Qwen (DashScope) LLM-based re-ranking API.
This re-ranker sends documents to the DashScope TextReRank service for
cross-encoder style re-ranking based on semantic relevance to the query.
Args:
query (str): Query text for semantic re-ranking. **Required**.
topn (int, optional): Number of top documents to return. Defaults to 10.
rerank_field (str): Field name containing document text for re-ranking.
**Required**.
model (str, optional): DashScope re-ranking model name.
Defaults to ``"gte-rerank-v2"``.
api_key (Optional[str], optional): DashScope API key. If not provided,
reads from ``DASHSCOPE_API_KEY`` environment variable.
Raises:
ValueError: If ``query`` is missing, ``rerank_field`` is missing,
or API key is not provided.
Note:
Requires the ``dashscope`` Python package.
Documents without content in ``rerank_field`` are skipped.
"""
def __init__(
self,
query: Optional[str] = None,
topn: int = 10,
rerank_field: Optional[str] = None,
model: str = "gte-rerank-v2",
api_key: Optional[str] = None,
):
super().__init__(query, topn, rerank_field)
if not query:
raise ValueError("Query is required for reranking")
self._model = model
self._api_key = api_key or os.environ.get("DASHSCOPE_API_KEY")
if not self._api_key:
raise ValueError("DashScope API key is required")
@property
def model(self) -> str:
"""str: DashScope re-ranking model name."""
return self._model
def _connection(self):
dashscope = require_module("dashscope")
dashscope.api_key = self._api_key
return dashscope
def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
"""Re-rank documents using Qwen's TextReRank API.
Args:
query_results (dict[str, list[Doc]]): Results from vector search.
Returns:
list[Doc]: Re-ranked documents with relevance scores from Qwen.
Raises:
ValueError: If API call fails or no valid documents are found.
"""
if not query_results:
return []
id_to_doc: dict[str, Doc] = {}
doc_ids = []
contents = []
for _, query_result in query_results.items():
for doc in query_result:
doc_id = doc.id
if doc_id in id_to_doc:
continue
field_value = doc.field(self.rerank_field)
rank_content = str(field_value).strip() if field_value else ""
if not rank_content:
continue
id_to_doc[doc_id] = doc
doc_ids.append(doc_id)
contents.append(rank_content)
if not contents:
raise ValueError("No documents to rerank")
resp = self._connection().TextReRank.call(
model=self.model,
query=self.query,
documents=list(contents),
top_n=self.topn,
return_documents=False,
)
if resp.status_code != HTTPStatus.OK:
raise ValueError(
f"QwenReranker failed with status {resp.status_code}: {resp.message}"
)
results = []
for item in resp.output.results:
idx = item.index
doc_id = doc_ids[idx]
doc = id_to_doc[doc_id]
new_doc = doc._replace(score=item.relevance_score)
results.append(new_doc)
return results
+22
View File
@@ -0,0 +1,22 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from .collection import Collection
from .doc import Doc
from .param.vector_query import VectorQuery
from .schema.collection_schema import CollectionSchema
from .schema.field_schema import FieldSchema
__all__ = ["Collection", "CollectionSchema", "Doc", "FieldSchema", "VectorQuery"]
+381
View File
@@ -0,0 +1,381 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import Optional, Union
from _zvec import _Collection
from ..executor import QueryContext, QueryExecutorFactory
from ..extension import ReRanker
from ..typing import Status
from .convert import convert_to_cpp_doc, convert_to_py_doc
from .doc import Doc
from .param import (
AddColumnOption,
AlterColumnOption,
CollectionOption,
FlatIndexParam,
HnswIndexParam,
IndexOption,
InvertIndexParam,
IVFIndexParam,
OptimizeOption,
)
from .param.vector_query import VectorQuery
from .schema import CollectionSchema, CollectionStats, FieldSchema
__all__ = ["Collection"]
_VECTOR_INDEX_TYPES = (HnswIndexParam, IVFIndexParam, FlatIndexParam)
class Collection:
"""Represents an opened collection in Zvec.
A `Collection` provides methods for data definition (DDL), data manipulation (DML),
and querying (DQL). It is obtained via `create_and_open()` or `open()`.
This class is not meant to be instantiated directly; use factory functions instead.
"""
def __init__(self, obj: _Collection):
self._obj = obj
self._schema = None
self._querier = None
@classmethod
def _from_core(cls, core_collection: _Collection) -> Collection:
if not core_collection:
raise ValueError("Collection is None")
inst = cls.__new__(cls)
inst._obj = core_collection
schema = CollectionSchema._from_core(core_collection.Schema())
inst._schema = schema
inst._querier = QueryExecutorFactory.create(schema)
return inst
@property
def path(self) -> str:
"""str: The filesystem path of the collection."""
return self._obj.Path()
@property
def option(self) -> CollectionOption:
"""CollectionOption: The options used to open the collection."""
return self._obj.Options()
@property
def schema(self) -> CollectionSchema:
"""CollectionSchema: The schema defining the structure of the collection."""
return self._schema
@property
def stats(self) -> CollectionStats:
"""CollectionStats: Runtime statistics about the collection (e.g., doc count, size)."""
return self._obj.Stats()
# ========== Collection DDL Methods ==========
def destroy(self) -> None:
"""Permanently delete the collection from disk.
Warning:
This operation is irreversible. All data will be lost.
"""
self._obj.Destroy()
def flush(self) -> None:
"""Force all pending writes to disk.
Ensures durability of recent inserts/updates.
"""
self._obj.Flush()
# ========== Index DDL Methods ==========
def create_index(
self,
field_name: str,
index_param: Union[
HnswIndexParam, IVFIndexParam, FlatIndexParam, InvertIndexParam
],
option: IndexOption = IndexOption(),
) -> None:
"""Create an index on a field.
Vector index types (HNSW, IVF, FLAT) can only be applied to vector fields.
Inverted index (`InvertIndexParam`) is for scalar fields.
Args:
field_name (str): Name of the field to index.
index_param (Union[HnswIndexParam, IVFIndexParam, FlatIndexParam, InvertIndexParam]):
Index configuration.
option (Optional[IndexOption], optional): Index creation options.
Defaults to ``IndexOption()``.
Raises:
ValueError: If a vector index is applied to a non-vector field.
"""
if index_param in _VECTOR_INDEX_TYPES and not self.schema.vector(field_name):
supported_types = ", ".join(cls.__name__ for cls in _VECTOR_INDEX_TYPES)
raise ValueError(
f"Cannot apply vector index to non-vector field '{field_name}'. "
f"The field must be of vector type to use index types like {supported_types}."
)
self._obj.CreateIndex(field_name, index_param, option)
self._schema = CollectionSchema._from_core(self._obj.Schema())
def drop_index(self, field_name: str) -> None:
"""Remove the index from a field.
Args:
field_name (str): Name of the indexed field.
"""
self._obj.DropIndex(field_name)
self._schema = CollectionSchema._from_core(self._obj.Schema())
def optimize(self, option: OptimizeOption = OptimizeOption()) -> None:
"""Optimize the collection (e.g., merge segments, rebuild index).
Args:
option (Optional[OptimizeOption], optional): Optimization options.
Defaults to ``OptimizeOption()``.
"""
self._obj.Optimize(option)
# ========== COLUMN DDL Methods ==========
def add_column(
self,
field_schema: FieldSchema,
expression: str = "",
option: AddColumnOption = AddColumnOption(),
) -> None:
"""Add a new column to the collection.
The column is populated using the provided expression (e.g., SQL-like formula).
Args:
field_schema (FieldSchema): Schema definition for the new column.
expression (str): Expression to compute values for existing documents.
option (Optional[AddColumnOption], optional): Options for the operation.
Defaults to ``AddColumnOption()``.
"""
self._obj.AddColumn(
field_schema.name, field_schema._get_object(), expression, option
)
self._schema = CollectionSchema._from_core(self._obj.Schema())
def drop_column(self, field_name: str) -> None:
"""Remove a column from the collection.
Args:
field_name (str): Name of the column to drop.
"""
self._obj.DropColumn(field_name)
self._schema = CollectionSchema._from_core(self._obj.Schema())
def alter_column(
self,
old_name: str,
new_name: Optional[str] = None,
field_schema: Optional[FieldSchema] = None,
option: AlterColumnOption = AlterColumnOption(),
) -> None:
"""Rename a column, update its schema.
This method supports three atomic operations:
1. Rename only (when `field_schema` is None).
2. Modify schema only (when `new_name` is None or empty string).
Args:
old_name (str): The current name of the column to be altered.
new_name (Optional[str]): The new name for the column.
- If provided and non-empty, the column will be renamed.
- If `None` or empty string, no rename occurs.
field_schema (Optional[FieldSchema]): The new schema definition.
- If provided, the column's type, dimension, or other properties will be updated.
- If `None`, only renaming (if requested) is performed.
option (AlterColumnOption, optional): Options controlling the alteration behavior.
Defaults to ``AlterColumnOption()``.
**Limitation**: This operation **only supports scalar numeric columns**. such as:
- `DOUBLE`, `FLOAT`,
- `INT32`, `INT64`, `UINT32`, `UINT64`
Note:
- Schema modification may trigger data migration or index rebuild.
Examples:
>>> # Rename column only
>>> results = collection.alter_column(old_name="id", new_name="doc_id")
>>> # Modify schema only
>>> new_schema = FieldSchema(name="doc_id", dtype=DataType.INT64)
>>> collection.alter_column("id", field_schema=new_schema)
"""
self._obj.AlterColumn(
old_name,
new_name or "",
field_schema._get_object() if field_schema else None,
option,
)
self._schema = CollectionSchema._from_core(self._obj.Schema())
# ========== Collection DDL Methods ==========
def insert(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
"""Insert new documents into the collection.
Documents must have unique IDs and conform to the schema.
Args:
docs (Union[Doc, list[Doc]]): One or more documents to insert.
Returns:
Union[Status, list[Status]]: If a single Doc was given, returns its Status;
if a list was given, returns a list of Status objects.
"""
is_single = isinstance(docs, Doc)
doc_list = [docs] if is_single else docs
results = self._obj.Insert(
[convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
)
return results[0] if is_single else results
def upsert(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
"""Insert new documents or update existing ones by ID.
Args:
docs (Union[Doc, list[Doc]]): Documents to upsert.
Returns:
Union[Status, list[Status]]: If a single Doc was given, returns its Status;
if a list was given, returns a list of Status objects.
"""
is_single = isinstance(docs, Doc)
doc_list = [docs] if is_single else docs
results = self._obj.Upsert(
[convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
)
return results[0] if is_single else results
def update(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
"""Update existing documents by ID.
Only specified fields are updated; others remain unchanged.
Args:
docs (Union[Doc, list[Doc]]): Documents containing updated fields.
Returns:
Union[Status, list[Status]]: If a single Doc was given, returns its Status;
if a list was given, returns a list of Status objects.
"""
is_single = isinstance(docs, Doc)
doc_list = [docs] if is_single else docs
results = self._obj.Update(
[convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
)
return results[0] if is_single else results
def delete(self, ids: Union[str, list[str]]) -> Union[Status, list[Status]]:
"""Delete documents by ID.
Args:
ids (Union[str, list[str]]): One or more document IDs to delete.
Returns:
Union[Status, list[Status]]: If a single id was given, returns its Status;
if a list was given, returns a list of Status objects.
"""
is_single = isinstance(ids, str)
id_list = [ids] if isinstance(ids, str) else ids
results = self._obj.Delete(id_list)
return results[0] if is_single else results
def delete_by_filter(self, filter: str) -> None:
"""Delete documents matching a filter expression.
Args:
filter (str): Boolean expression (e.g., ``"age > 30"``).
"""
self._obj.DeleteByFilter(filter)
# ========== Collection DQL-fetch Methods ==========
def fetch(self, ids: Union[str, list[str]]) -> dict[str, Doc]:
"""Retrieve documents by ID.
Args:
ids (Union[str, list[str]]): Document IDs to fetch.
Returns:
dict[str, Doc]: Mapping from ID to document. Missing IDs are omitted.
"""
ids = [ids] if isinstance(ids, str) else ids
docs = self._obj.Fetch(ids)
return {
doc_id: py_doc
for doc_id, core_doc in docs.items()
if (py_doc := convert_to_py_doc(core_doc, self.schema)) is not None
}
# ========== Collection DQL-Query Methods ==========
def query(
self,
vectors: Optional[Union[VectorQuery, list[VectorQuery]]] = None,
*,
topk: int = 10,
filter: Optional[str] = None,
include_vector: bool = False,
output_fields: Optional[list[str]] = None,
reranker: Optional[ReRanker] = None,
) -> list[Doc]:
"""Perform vector similarity search with optional filtering and re-ranking.
At least one `VectorQuery` must be provided.
Args:
vectors (Optional[Union[VectorQuery, list[VectorQuery]]], optional):
One or more vector queries. Defaults to None.
topk (int, optional): Number of nearest neighbors to return.
Defaults to 10.
filter (Optional[str], optional): Boolean expression to pre-filter candidates.
Defaults to None.
include_vector (bool, optional): Whether to include vector data in results.
Defaults to False.
output_fields (Optional[list[str]], optional): Scalar fields to include.
If None, all fields are returned. Defaults to None.
reranker (Optional[ReRanker], optional): Re-ranker to refine results.
Defaults to None.
Returns:
list[Doc]: Top-k matching documents, sorted by relevance score.
Examples:
>>> from zvec import VectorQuery
>>> results = collection.query(
... vectors=VectorQuery("embedding", vector=[0.1, 0.2]),
... topk=5,
... filter="category == 'tech'",
... output_fields=["title", "url"]
... )
"""
ctx = QueryContext(
topk=topk,
filter=filter,
queries=[vectors] if isinstance(vectors, VectorQuery) else vectors,
include_vector=include_vector,
output_fields=output_fields,
reranker=reranker,
)
return self._querier.execute(ctx, self._obj)
+54
View File
@@ -0,0 +1,54 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from _zvec import _Doc
from .doc import Doc
from .schema import CollectionSchema
def convert_to_cpp_doc(doc: Doc, collection_schema: CollectionSchema) -> _Doc:
if not doc or not collection_schema:
return None
_doc = _Doc()
# set pk
_doc.set_pk(doc.id)
# set scalar fields
for k, v in doc.fields.items():
field_schema = collection_schema.field(k)
if not field_schema:
raise ValueError(
f"schema validate failed: {k} not found in collection schema"
)
_doc.set_any(k, field_schema._get_object(), v)
# set vector fields
for k, v in doc.vectors.items():
vector_schema = collection_schema.vector(k)
if not vector_schema:
raise ValueError(
f"schema validate failed: {k} not found in collection schema"
)
_doc.set_any(k, vector_schema._get_object(), v)
return _doc
def convert_to_py_doc(doc: _Doc, collection_schema: CollectionSchema) -> Doc:
if not doc or not collection_schema:
return None
data_tuple = doc.get_all(collection_schema._get_object())
return Doc._from_tuple(data_tuple)
+173
View File
@@ -0,0 +1,173 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import json
from typing import Any, Optional
from ..common import VectorType
__all__ = [
"Doc",
]
class Doc:
"""Represents a retrieved document with optional metadata, fields, and vectors.
This immutable data class encapsulates the result of a search or retrieval
operation. It includes the document ID, relevance score (if applicable),
scalar fields, and vector embeddings.
During initialization, any `numpy.ndarray` in `vectors` is automatically
converted to a plain Python list for JSON serialization and immutability.
Attributes:
id (str): Unique identifier of the document.
score (Optional[float], optional): Relevance score from search.
Defaults to None.
vectors (Optional[dict[str, VectorType]], optional): Named vector
embeddings associated with the document. Values are converted to
lists if originally `np.ndarray`. Defaults to None.
fields (Optional[dict[str, Any]], optional): Scalar metadata fields
(e.g., title, timestamp). Defaults to None.
Examples:
>>> import numpy as np
>>> import zvec
>>> doc = zvec.Doc(
... id="doc1",
... score=0.95,
... vectors={"emb": np.array([0.1, 0.2, 0.3])},
... fields={"title": "Hello World"}
... )
>>> print(doc.vector("emb"))
[0.1, 0.2, 0.3]
>>> print(doc.has_field("title"))
True
"""
__slots__ = ("id", "score", "vectors", "fields")
def __init__(
self,
id: str,
score: Optional[float] = None,
vectors: Optional[dict[str, VectorType]] = None,
fields: Optional[dict[str, Any]] = None,
):
self.id = id
self.score = score
self.vectors = vectors or {}
self.fields = fields or {}
def has_field(self, name: str) -> bool:
"""Check if the document contains a scalar field with the given name.
Args:
name (str): Name of the field to check.
Returns:
bool: True if the field exists, False otherwise.
"""
return name in self.fields
def has_vector(self, name: str) -> bool:
"""Check if the document contains a vector with the given name.
Args:
name (str): Name of the vector to check.
Returns:
bool: True if the vector exists, False otherwise.
"""
return name in self.vectors
def vector(self, name: str):
"""Get a vector by name.
Args:
name (str): Name of the vector.
Returns:
Any: The vector (as a list) if it exists, otherwise None.
"""
return self.vectors and self.vectors.get(name)
def field(self, name: str):
"""Get a scalar field by name.
Args:
name (str): Name of the field.
Returns:
Any: The field value if it exists, otherwise None.
"""
return self.fields and self.fields.get(name)
def vector_names(self) -> list[str]:
"""Get the list of all vector names in this document.
Returns:
list[str]: A list of vector field names. Empty if no vectors.
"""
return [] if not self.vectors else list(self.vectors.keys())
def field_names(self) -> list[str]:
"""Get the list of all scalar field names in this document.
Returns:
list[str]: A list of field names. Empty if no fields.
"""
return [] if not self.fields else list(self.fields.keys())
def __repr__(self) -> str:
try:
schema = {
"id": self.id,
"score": self.score,
"fields": self.fields,
"vectors": self.vectors,
}
return json.dumps(schema, indent=2, ensure_ascii=False)
except Exception as e:
return f"<Doc error during repr: {e}>"
def _replace(self, **changes):
new_tuple = (
changes.get("id", self.id),
changes.get("score", self.score),
changes.get("fields", self.fields.copy() if self.fields else None),
changes.get("vectors", self.vectors.copy() if self.vectors else None),
)
return type(self)._from_tuple(new_tuple)
@classmethod
def _from_tuple(
cls, data_tuple: tuple[str, float, dict[str, Any], dict[str, VectorType]]
):
obj = object.__new__(cls)
obj.id = data_tuple[0]
obj.score = data_tuple[1]
obj.fields = data_tuple[2] or {}
vectors = data_tuple[3]
if vectors is not None:
obj.vectors = {
name: (vec.tolist() if hasattr(vec, "tolist") else vec)
for name, vec in vectors.items()
}
else:
obj.vectors = {}
return obj
+42
View File
@@ -0,0 +1,42 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from _zvec.param import (
AddColumnOption,
AlterColumnOption,
CollectionOption,
FlatIndexParam,
HnswIndexParam,
HnswQueryParam,
IndexOption,
InvertIndexParam,
IVFIndexParam,
IVFQueryParam,
OptimizeOption,
)
__all__ = [
"AddColumnOption",
"AlterColumnOption",
"CollectionOption",
"FlatIndexParam",
"HnswIndexParam",
"HnswQueryParam",
"IVFIndexParam",
"IVFQueryParam",
"IndexOption",
"InvertIndexParam",
"OptimizeOption",
]
+692
View File
@@ -0,0 +1,692 @@
"""
This module contains the params of Zvec
"""
from __future__ import annotations
import collections
import typing
import _zvec.typing
__all__: list[str] = [
"AddColumnOption",
"AlterColumnOption",
"CollectionOption",
"FlatIndexParam",
"HnswIndexParam",
"HnswQueryParam",
"IVFIndexParam",
"IVFQueryParam",
"IndexOption",
"IndexParam",
"InvertIndexParam",
"OptimizeOption",
"QueryParam",
"SegmentOption",
"VectorIndexParam",
]
class AddColumnOption:
"""
Options for adding a new column to a collection.
Attributes:
concurrency (int): Number of threads to use when backfilling data
for the new column. If 0, auto-detect is used. Default is 0.
Examples:
>>> opt = AddColumnOption(concurrency=1)
>>> print(opt.concurrency)
1
"""
def __getstate__(self) -> tuple: ...
def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
"""
Constructs an AddColumnOption instance.
Args:
concurrency (int, optional): Number of threads for data backfill.
0 means auto-detect. Defaults to 0.
"""
def __setstate__(self, arg0: tuple) -> None: ...
@property
def concurrency(self) -> int:
"""
int: Number of threads used when adding a column (0 = auto).
"""
class AlterColumnOption:
"""
Options for altering an existing column (e.g., changing index settings).
Attributes:
concurrency (int): Number of threads to use during the alteration process.
If 0, the system will choose an optimal value automatically.
Default is 0.
Examples:
>>> opt = AlterColumnOption(concurrency=1)
>>> print(opt.concurrency)
1
"""
def __getstate__(self) -> tuple: ...
def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
"""
Constructs an AlterColumnOption instance.
Args:
concurrency (int, optional): Number of threads for column alteration.
0 means auto-detect. Defaults to 0.
"""
def __setstate__(self, arg0: tuple) -> None: ...
@property
def concurrency(self) -> int:
"""
int: Number of threads used when altering a column (0 = auto).
"""
class CollectionOption:
"""
Options for opening or creating a collection.
Attributes:
read_only (bool): Whether the collection is opened in read-only mode.
Default is False.
enable_mmap (bool): Whether to use memory-mapped I/O for data files.
Default is True.
Examples:
>>> opt = CollectionOption(read_only=True, enable_mmap=False)
>>> print(opt.read_only)
True
"""
def __getstate__(self) -> tuple: ...
def __init__(self, read_only: bool = False, enable_mmap: bool = True) -> None:
"""
Constructs a CollectionOption instance.
Args:
read_only (bool, optional): Open collection in read-only mode.
Defaults to False.
enable_mmap (bool, optional): Enable memory-mapped I/O.
Defaults to True.
"""
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
@property
def enable_mmap(self) -> bool: ...
@property
def read_only(self) -> bool: ...
class FlatIndexParam(VectorIndexParam):
"""
Parameters for configuring a flat (brute-force) index.
A flat index performs exact nearest neighbor search by comparing the query
vector against all vectors in the collection. It is simple, accurate, and
suitable for small to medium datasets or as a baseline.
Attributes:
metric_type (MetricType): Distance metric used for similarity computation.
Default is ``MetricType.IP`` (inner product).
quantize_type (QuantizeType): Optional quantization type for vector
compression (e.g., FP16, INT8). Use ``QuantizeType.UNDEFINED`` to
disable quantization. Default is ``QuantizeType.UNDEFINED``.
Examples:
>>> from zvec.typing import MetricType, QuantizeType
>>> params = FlatIndexParam(
... metric_type=MetricType.L2,
... quantize_type=QuantizeType.FP16
... )
>>> print(params)
{'metric_type': 'L2', 'quantize_type': 'FP16'}
"""
def __getstate__(self) -> tuple: ...
def __init__(
self,
metric_type: _zvec.typing.MetricType = ...,
quantize_type: _zvec.typing.QuantizeType = ...,
) -> None:
"""
Constructs a FlatIndexParam instance.
Args:
metric_type (MetricType, optional): Distance metric. Defaults to MetricType.IP.
quantize_type (QuantizeType, optional): Vector quantization type.
Defaults to QuantizeType.UNDEFINED (no quantization).
"""
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
def to_dict(self) -> dict:
"""
Convert to dictionary with all fields
"""
class HnswIndexParam(VectorIndexParam):
"""
Parameters for configuring an HNSW (Hierarchical Navigable Small World) index.
HNSW is a graph-based approximate nearest neighbor search index. This class
encapsulates its construction hyperparameters.
Attributes:
metric_type (MetricType): Distance metric used for similarity computation.
Default is ``MetricType.IP`` (inner product).
m (int): Number of bi-directional links created for every new element
during construction. Higher values improve accuracy but increase
memory usage and construction time. Default is 100.
ef_construction (int): Size of the dynamic candidate list for nearest
neighbors during index construction. Larger values yield better
graph quality at the cost of slower build time. Default is 500.
quantize_type (QuantizeType): Optional quantization type for vector
compression (e.g., FP16, INT8). Default is `QuantizeType.UNDEFINED` to
disable quantization.
Examples:
>>> from zvec.typing import MetricType, QuantizeType
>>> params = HnswIndexParam(
... metric_type=MetricType.COSINE,
... m=16,
... ef_construction=200,
... quantize_type=QuantizeType.INT8
... )
>>> print(params)
{'metric_type': 'IP', 'm': 16, 'ef_construction': 200, 'quantize_type': 'INT8'}
"""
def __getstate__(self) -> tuple: ...
def __init__(
self,
metric_type: _zvec.typing.MetricType = ...,
m: typing.SupportsInt = 100,
ef_construction: typing.SupportsInt = 500,
quantize_type: _zvec.typing.QuantizeType = ...,
) -> None: ...
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
def to_dict(self) -> dict:
"""
Convert to dictionary with all fields
"""
@property
def ef_construction(self) -> int:
"""
int: Candidate list size during index construction.
"""
@property
def m(self) -> int:
"""
int: Maximum number of neighbors per node in upper layers.
"""
class HnswQueryParam(QueryParam):
"""
Query parameters for HNSW (Hierarchical Navigable Small World) index.
Controls the trade-off between search speed and accuracy via the `ef` parameter.
Attributes:
type (IndexType): Always ``IndexType.HNSW``.
ef (int): Size of the dynamic candidate list during search.
Larger values improve recall but slow down search.
Default is 300.
radius (float): Search radius for range queries. Default is 0.0.
is_linear (bool): Force linear search. Default is False.
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
Examples:
>>> params = HnswQueryParam(ef=300)
>>> print(params.ef)
300
>>> print(params.to_dict() if hasattr(params, 'to_dict') else params)
{"type":"HNSW", "ef":300}
"""
def __getstate__(self) -> tuple: ...
def __init__(
self,
ef: typing.SupportsInt = 300,
radius: typing.SupportsFloat = 0.0,
is_linear: bool = False,
is_using_refiner: bool = False,
) -> None:
"""
Constructs an HnswQueryParam instance.
Args:
ef (int, optional): Search-time candidate list size.
Higher values improve accuracy. Defaults to 100.
radius (float, optional): Search radius for range queries. Default is 0.0.
is_linear (bool, optional): Force linear search. Default is False.
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
"""
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
@property
def ef(self) -> int:
"""
int: Size of the dynamic candidate list during HNSW search.
"""
class IVFIndexParam(VectorIndexParam):
"""
Parameters for configuring an IVF (Inverted File Index) index.
IVF partitions the vector space into clusters (inverted lists). At query time,
only a subset of clusters is searched, providing a trade-off between speed
and accuracy.
Attributes:
metric_type (MetricType): Distance metric used for similarity computation.
Default is ``MetricType.IP`` (inner product).
n_list (int): Number of clusters (inverted lists) to partition the dataset into.
If set to 0, the system will auto-select a reasonable value based on data size.
Default is 0 (auto).
n_iters (int): Number of iterations for k-means clustering during index training.
Higher values yield more stable centroids. Default is 10.
use_soar (bool): Whether to enable SOAR (Scalable Optimized Adaptive Routing)
for improved IVF search performance. Default is False.
quantize_type (QuantizeType): Optional quantization type for vector
compression (e.g., FP16, INT8). Default is ``QuantizeType.UNDEFINED``.
Examples:
>>> from zvec.typing import MetricType, QuantizeType
>>> params = IVFIndexParam(
... metric_type=MetricType.COSINE,
... n_list=100,
... n_iters=15,
... use_soar=True,
... quantize_type=QuantizeType.INT8
... )
>>> print(params.n_list)
100
"""
def __getstate__(self) -> tuple: ...
def __init__(
self,
metric_type: _zvec.typing.MetricType = ...,
n_list: typing.SupportsInt = 0,
n_iters: typing.SupportsInt = 10,
use_soar: bool = False,
quantize_type: _zvec.typing.QuantizeType = ...,
) -> None:
"""
Constructs an IVFIndexParam instance.
Args:
metric_type (MetricType, optional): Distance metric. Defaults to MetricType.IP.
n_list (int, optional): Number of inverted lists (clusters). Set to 0 for auto.
Defaults to 0.
n_iters (int, optional): Number of k-means iterations during training.
Defaults to 10.
use_soar (bool, optional): Enable SOAR optimization. Defaults to False.
quantize_type (QuantizeType, optional): Vector quantization type.
Defaults to QuantizeType.UNDEFINED.
"""
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
def to_dict(self) -> dict:
"""
Convert to dictionary with all fields
"""
@property
def n_iters(self) -> int:
"""
int: Number of k-means iterations during training.
"""
@property
def n_list(self) -> int:
"""
int: Number of inverted lists (0 = auto).
"""
@property
def use_soar(self) -> bool:
"""
bool: Whether SOAR optimization is enabled.
"""
class IVFQueryParam(QueryParam):
"""
Query parameters for IVF (Inverted File Index) index.
Controls how many inverted lists (`nprobe`) to visit during search.
Attributes:
type (IndexType): Always ``IndexType.IVF``.
nprobe (int): Number of closest clusters (inverted lists) to search.
Higher values improve recall but increase latency.
Default is 10.
radius (float): Search radius for range queries. Default is 0.0.
is_linear (bool): Force linear search. Default is False.
Examples:
>>> params = IVFQueryParam(nprobe=20)
>>> print(params.nprobe)
20
"""
def __getstate__(self) -> tuple: ...
def __init__(self, nprobe: typing.SupportsInt = 10) -> None:
"""
Constructs an IVFQueryParam instance.
Args:
nprobe (int, optional): Number of inverted lists to probe during search.
Higher values improve accuracy. Defaults to 10.
"""
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
@property
def nprobe(self) -> int:
"""
int: Number of inverted lists to search during IVF query.
"""
class IndexOption:
"""
Options for creating an index.
Attributes:
concurrency (int): Number of threads to use during index creation.
If 0, the system will choose an optimal value automatically.
Default is 0.
Examples:
>>> opt = IndexOption(concurrency=4)
>>> print(opt.concurrency)
4
"""
def __getstate__(self) -> tuple: ...
def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
"""
Constructs an IndexOption instance.
Args:
concurrency (int, optional): Number of concurrent threads.
0 means auto-detect. Defaults to 0.
"""
def __setstate__(self, arg0: tuple) -> None: ...
@property
def concurrency(self) -> int:
"""
int: Number of threads used for index creation (0 = auto).
"""
class IndexParam:
"""
Base class for all index parameter configurations.
This abstract base class defines the common interface for index types.
It should not be instantiated directly; use derived classes instead.
Attributes:
type (IndexType): The type of the index (e.g., HNSW, FLAT, INVERT).
"""
__hash__: typing.ClassVar[None] = None
def __eq__(self, arg0: typing.Any) -> bool: ...
def __getstate__(self) -> tuple: ...
def __setstate__(self, arg0: tuple) -> None: ...
def clone(self) -> IndexParam: ...
def to_dict(self) -> dict:
"""
Convert to dictionary with all fields
"""
@property
def type(self) -> _zvec.typing.IndexType:
"""
IndexType: The type of the index.
"""
class InvertIndexParam(IndexParam):
"""
Parameters for configuring an invert index.
This class controls whether range query
optimization is enabled for invert index structures.
Attributes:
type (IndexType): Always `IndexType.INVERTED`.
enable_range_optimization (bool): Whether range optimization is enabled.
enable_extended_wildcard (bool): Whether extended wildcard (suffix and infix) search is enabled.
Examples:
>>> params = InvertIndexParam(enable_range_optimization=True, enable_extended_wildcard=False)
>>> print(params.enable_range_optimization)
True
>>> print(params.enable_extended_wildcard)
False
>>> config = params.to_dict()
>>> print(config)
{'enable_range_optimization': True, 'enable_extended_wildcard': False}
"""
def __getstate__(self) -> tuple: ...
def __init__(
self,
enable_range_optimization: bool = False,
enable_extended_wildcard: bool = False,
) -> None:
"""
Constructs an InvertIndexParam instance.
Args:
enable_range_optimization (bool, optional): If True, enables range query
optimization for the invert index. Defaults to False.
enable_extended_wildcard (bool, optional): If True, enables extended wildcard
search including suffix and infix patterns. Defaults to False.
"""
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
def to_dict(self) -> dict:
"""
Convert to dictionary with all fields
"""
@property
def enable_extended_wildcard(self) -> bool:
"""
bool: Whether extended wildcard (suffix and infix) search is enabled.
Note: Prefix search is always enabled regardless of this setting.
"""
@property
def enable_range_optimization(self) -> bool:
"""
bool: Whether range optimization is enabled for this inverted index.
"""
class OptimizeOption:
"""
Options for optimizing a collection (e.g., merging segments).
Attributes:
concurrency (int): Number of threads to use during optimization.
If 0, the system will choose an optimal value automatically.
Default is 0.
Examples:
>>> opt = OptimizeOption(concurrency=2)
>>> print(opt.concurrency)
2
"""
def __getstate__(self) -> tuple: ...
def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
"""
Constructs an OptimizeOption instance.
Args:
concurrency (int, optional): Number of concurrent threads.
0 means auto-detect. Defaults to 0.
"""
def __setstate__(self, arg0: tuple) -> None: ...
@property
def concurrency(self) -> int:
"""
int: Number of threads used for optimization (0 = auto).
"""
class QueryParam:
"""
Base class for all query parameter configurations.
This abstract base class defines common query settings such as search radius
and whether to force linear (brute-force) search. It should not be instantiated
directly; use derived classes like `HnswQueryParam` or `IVFQueryParam`.
Attributes:
type (IndexType): The index type this query is configured for.
radius (float): Search radius for range queries. Used in combination with
top-k to filter results. Default is 0.0 (disabled).
is_linear (bool): If True, forces brute-force linear search instead of
using the index. Useful for debugging or small datasets. Default is False.
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
"""
def __getstate__(self) -> tuple: ...
def __setstate__(self, arg0: tuple) -> None: ...
@property
def is_linear(self) -> bool:
"""
bool: Whether to bypass the index and use brute-force linear search.
"""
@property
def is_using_refiner(self) -> bool:
"""
bool: Whether to use refiner for the query.
"""
@property
def radius(self) -> float:
"""
IndexType: The type of index this query targets.
"""
@property
def type(self) -> _zvec.typing.IndexType:
"""
IndexType: The type of index this query targets.
"""
class SegmentOption:
"""
Options for segment-level operations.
Currently, this class mirrors CollectionOption and is used internally.
It supports read-only mode, memory mapping, and buffer configuration.
Note:
This class is primarily for internal use. Most users should use
CollectionOption instead.
Examples:
>>> opt = SegmentOption()
>>> print(opt.enable_mmap)
True
"""
def __getstate__(self) -> tuple: ...
def __init__(self) -> None:
"""
Constructs a SegmentOption with default settings.
"""
def __repr__(self) -> str: ...
def __setstate__(self, arg0: tuple) -> None: ...
@property
def enable_mmap(self) -> bool:
"""
bool: Whether memory-mapped I/O is enabled.
"""
@property
def max_buffer_size(self) -> int:
"""
int: Maximum buffer size in bytes (internal use).
"""
@property
def read_only(self) -> bool:
"""
bool: Whether the segment is read-only.
"""
class VectorIndexParam(IndexParam):
"""
Base class for vector index parameter configurations.
Encapsulates common settings for all vector index types.
Attributes:
type (IndexType): The specific vector index type (e.g., HNSW, FLAT).
metric_type (MetricType): Distance metric used for similarity search.
quantize_type (QuantizeType): Optional vector quantization type.
"""
def __getstate__(self) -> tuple: ...
def __setstate__(self, arg0: tuple) -> None: ...
def to_dict(self) -> dict:
"""
Convert to dictionary with all fields
"""
@property
def metric_type(self) -> _zvec.typing.MetricType:
"""
MetricType: Distance metric (e.g., IP, COSINE, L2).
"""
@property
def quantize_type(self) -> _zvec.typing.QuantizeType:
"""
QuantizeType: Vector quantization type (e.g., FP16, INT8).
"""
class _VectorQuery:
field_name: str
filter: str
include_vector: bool
query_params: QueryParam
def __getstate__(self) -> tuple: ...
def __init__(self) -> None: ...
def __setstate__(self, arg0: tuple) -> None: ...
def set_vector(self, arg0: ..., arg1: typing.Any) -> None: ...
@property
def output_fields(self) -> list[str] | None: ...
@output_fields.setter
def output_fields(self, arg0: collections.abc.Sequence[str] | None) -> None: ...
@property
def topk(self) -> int: ...
@topk.setter
def topk(self, arg0: typing.SupportsInt) -> None: ...
+80
View File
@@ -0,0 +1,80 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from dataclasses import dataclass
from typing import Optional, Union
from ...common import VectorType
from . import HnswQueryParam, IVFQueryParam
__all__ = ["VectorQuery"]
@dataclass(frozen=True)
class VectorQuery:
"""Represents a vector search query for a specific field in a collection.
A `VectorQuery` can be constructed using either a document ID (to look up
its vector) or an explicit vector. It may optionally include index-specific
query parameters to control search behavior (e.g., `ef` for HNSW, `nprobe` for IVF).
Exactly one of `id` or `vector` should be provided. If both are given,
behavior is implementation-defined (typically `id` takes precedence).
Attributes:
field_name (str): Name of the vector field to query.
id (Optional[str], optional): Document ID to fetch vector from. Default is None.
vector (VectorType, optional): Explicit query vector. Default is None.
param (Optional[Union[HnswQueryParam, IVFQueryParam]], optional):
Index-specific query parameters. Default is None.
Examples:
>>> import zvec
>>> # Query by ID
>>> q1 = zvec.VectorQuery(field_name="embedding", id="doc123")
>>> # Query by vector
>>> q2 = zvec.VectorQuery(
... field_name="embedding",
... vector=[0.1, 0.2, 0.3],
... param=HnswQueryParam(ef=300)
... )
"""
field_name: str
id: Optional[str] = None
vector: VectorType = None
param: Optional[Union[HnswQueryParam, IVFQueryParam]] = None
def has_id(self) -> bool:
"""Check if the query is based on a document ID.
Returns:
bool: True if `id` is set, False otherwise.
"""
return self.id is not None
def has_vector(self) -> bool:
"""Check if the query contains an explicit vector.
Returns:
bool: True if `vector` is non-empty, False otherwise.
"""
return self.vector is not None and len(self.vector) > 0
def _validate(self) -> None:
if self.field_name is None:
raise ValueError("Field name cannot be empty")
if self.id and self.vector:
raise ValueError("Cannot provide both id and vector")
+21
View File
@@ -0,0 +1,21 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from _zvec.schema import CollectionStats
from .collection_schema import CollectionSchema
from .field_schema import FieldSchema, VectorSchema
__all__ = ["CollectionSchema", "CollectionStats", "FieldSchema", "VectorSchema"]
+109
View File
@@ -0,0 +1,109 @@
"""
This module contains the schema of Zvec
"""
from __future__ import annotations
import collections.abc
import typing
import _zvec.param
import _zvec.typing
from .collection_schema import CollectionSchema
from .field_schema import FieldSchema, VectorSchema
__all__: list[str] = [
"CollectionSchema",
"CollectionStats",
"FieldSchema",
"VectorSchema",
]
class CollectionStats:
def __init__(self) -> None: ...
def __repr__(self) -> str: ...
@property
def doc_count(self) -> int: ...
@property
def index_completeness(self) -> dict[str, float]: ...
class _CollectionSchema:
__hash__: typing.ClassVar[None] = None
def __eq__(self, arg0: _CollectionSchema) -> bool: ...
def __init__(
self, name: str, fields: collections.abc.Sequence[_FieldSchema]
) -> None:
"""
Construct with name and list of fields
"""
def __ne__(self, arg0: _CollectionSchema) -> bool: ...
def fields(self) -> list[_FieldSchema]:
"""
Return list of all field schemas.
"""
def forward_fields(self) -> list[_FieldSchema]:
"""
Return list of forward-indexed fields.
"""
def get_field(self, field_name: str) -> _FieldSchema:
"""
Get field by name (const pointer), returns None if not found.
"""
def get_forward_field(self, field_name: str) -> _FieldSchema:
"""
Get forward field (used for filtering).
"""
def get_vector_field(self, field_name: str) -> _FieldSchema:
"""
Get vector field by name.
"""
def has_field(self, field_name: str) -> bool:
"""
Check if a field exists.
"""
def vector_fields(self) -> list[_FieldSchema]:
"""
Return list of vector fields.
"""
@property
def name(self) -> str: ...
class _FieldSchema:
__hash__: typing.ClassVar[None] = None
def __eq__(self, arg0: _FieldSchema) -> bool: ...
def __init__(
self,
name: str,
data_type: _zvec.typing.DataType,
nullable: bool = False,
dimension: typing.SupportsInt = 0,
index_param: _zvec.param.IndexParam = None,
) -> None: ...
def __ne__(self, arg0: _FieldSchema) -> bool: ...
@property
def data_type(self) -> _zvec.typing.DataType: ...
@property
def dimension(self) -> int: ...
@property
def index_param(self) -> typing.Any: ...
@property
def index_type(self) -> _zvec.typing.IndexType: ...
@property
def is_dense_vector(self) -> bool: ...
@property
def is_sparse_vector(self) -> bool: ...
@property
def name(self) -> str: ...
@property
def nullable(self) -> bool: ...
@@ -0,0 +1,215 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import json
from typing import Optional, Union
from _zvec.schema import _CollectionSchema, _FieldSchema
from .field_schema import FieldSchema, VectorSchema
__all__ = [
"CollectionSchema",
]
class CollectionSchema:
"""Defines the structure of a collection in Zvec.
A collection schema specifies the name of the collection and its fields,
including both scalar fields (e.g., int, string) and vector fields.
Field names must be unique across both scalar and vector fields.
Args:
name (str): Name of the collection.
fields (Optional[Union[FieldSchema, list[FieldSchema]]], optional):
One or more scalar field definitions. Defaults to None.
vectors (Optional[Union[VectorSchema, list[VectorSchema]]], optional):
One or more vector field definitions. Defaults to None.
Raises:
TypeError: If `fields` or `vectors` are of unsupported types.
ValueError: If any field or vector name is duplicated.
Examples:
>>> from zvec import FieldSchema, VectorSchema, DataType, IndexType
>>> id_field = FieldSchema("id", DataType.INT64, is_primary=True)
>>> emb_field = VectorSchema("embedding", dim=128, data_type=DataType.VECTOR_FP32)
>>> schema = CollectionSchema(
... name="my_collection",
... fields=id_field,
... vectors=emb_field
... )
>>> print(schema.name)
my_collection
"""
def __init__(
self,
name: str,
fields: Optional[Union[FieldSchema, list[FieldSchema]]] = None,
vectors: Optional[Union[VectorSchema, list[VectorSchema]]] = None,
):
if name is None or not isinstance(name, str):
raise ValueError(
f"schema validate failed: collection name must be str, got {type(name).__name__}"
)
# handle fields
_fields_name: list[str] = []
_fields_list: list[_FieldSchema] = []
self._check_fields(fields, _fields_name, _fields_list)
self._check_vectors(vectors, _fields_name, _fields_list)
# init
self._cpp_obj = _CollectionSchema(
name=name,
fields=_fields_list,
)
def _check_fields(
self,
fields: Optional[Union[FieldSchema, list[FieldSchema]]],
_fields_name: list[str],
_fields_list: list[_FieldSchema],
) -> None:
field_items = []
if isinstance(fields, FieldSchema):
field_items = [fields]
elif isinstance(fields, list):
field_items = fields
elif fields is None:
field_items = []
else:
raise TypeError(
f"schema validate failed: invalid 'fields' type, expected FieldSchema or list[FieldSchema], "
f"got {type(fields).__name__}"
)
for idx, field in enumerate(field_items):
if not isinstance(field, FieldSchema):
raise TypeError(
f"schema validate failed: invalid field type in 'fields' list, expected FieldSchema, "
f"got {type(field).__name__} at index {idx}"
)
if field.name in _fields_name:
raise ValueError(
f"schema validate failed: duplicate field name '{field.name}': field names must be unique"
)
_fields_name.append(field.name)
_fields_list.append(field._get_object())
def _check_vectors(
self,
vectors: Optional[Union[VectorSchema, list[VectorSchema]]],
_fields_name: list[str],
_fields_list: list[_FieldSchema],
) -> None:
# handle vector
if isinstance(vectors, VectorSchema):
vectors_items = [vectors]
elif isinstance(vectors, list):
vectors_items = vectors
elif vectors is None:
vectors_items = []
else:
raise TypeError(
f"schema validate failed: invalid 'vectors' type, expected VectorSchema or list[VectorSchema], "
f"got {type(vectors).__name__}"
)
for idx, vector in enumerate(vectors_items):
if not isinstance(vector, VectorSchema):
raise TypeError(
f"schema validate failed: invalid vector type in 'vectors' list, expected VectorSchema, "
f"got {type(vector).__name__} at index {idx}"
)
if vector.name in _fields_name:
raise ValueError(
f"schema validate failed: duplicate vector name '{vector.name}', vector names must be unique "
f"(conflicts with existing field or vector)"
)
_fields_name.append(vector.name)
_fields_list.append(vector._get_object())
@classmethod
def _from_core(cls, core_collection_schema: _CollectionSchema):
inst = cls.__new__(cls)
if not core_collection_schema:
raise ValueError("schema validate failed: schema is null")
inst._cpp_obj = core_collection_schema
return inst
@property
def name(self) -> str:
"""str: The name of the collection."""
return self._cpp_obj.name
def field(self, name: str) -> Optional[FieldSchema]:
"""Retrieve a scalar field by name.
Args:
name (str): Name of the field.
Returns:
Optional[FieldSchema]: The field if found, otherwise None.
"""
_field = self._cpp_obj.get_forward_field(name)
return FieldSchema._from_core(_field) if _field else None
def vector(self, name: str) -> Optional[VectorSchema]:
"""Retrieve a vector field by name.
Args:
name (str): Name of the vector field.
Returns:
Optional[VectorSchema]: The vector field if found, otherwise None.
"""
_field = self._cpp_obj.get_vector_field(name)
return VectorSchema._from_core(_field) if _field else None
@property
def fields(self) -> list[FieldSchema]:
"""list[FieldSchema]: All scalar (non-vector) fields in the schema."""
_fields = self._cpp_obj.forward_fields()
return [FieldSchema._from_core(_field) for _field in _fields]
@property
def vectors(self) -> list[VectorSchema]:
"""list[VectorSchema]: All vector fields in the schema."""
_vectors = self._cpp_obj.vector_fields()
return [VectorSchema._from_core(_vector) for _vector in _vectors]
def _get_object(self) -> _CollectionSchema:
return self._cpp_obj
def __repr__(self) -> str:
try:
schema = {
"name": self.name,
"fields": {field.name: field.__dict__() for field in self.fields},
"vectors": {vector.name: vector.__dict__() for vector in self.vectors},
}
return json.dumps(schema, indent=2, ensure_ascii=False)
except Exception as e:
return f"<CollectionSchema error during repr: {e}>"
def __str__(self) -> str:
return self.__repr__()
+300
View File
@@ -0,0 +1,300 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import json
from typing import Any, Optional, Union
from _zvec.schema import _FieldSchema
from zvec.model.param import (
FlatIndexParam,
HnswIndexParam,
InvertIndexParam,
IVFIndexParam,
)
from zvec.typing import DataType
__all__ = [
"FieldSchema",
"VectorSchema",
]
SUPPORT_VECTOR_DATA_TYPE = [
DataType.VECTOR_FP16,
DataType.VECTOR_FP32,
DataType.VECTOR_FP64,
DataType.VECTOR_INT8,
DataType.SPARSE_VECTOR_FP16,
DataType.SPARSE_VECTOR_FP32,
]
SUPPORT_SCALAR_DATA_TYPE = [
DataType.INT32,
DataType.INT64,
DataType.UINT32,
DataType.UINT64,
DataType.FLOAT,
DataType.DOUBLE,
DataType.STRING,
DataType.BOOL,
DataType.ARRAY_INT32,
DataType.ARRAY_INT64,
DataType.ARRAY_UINT32,
DataType.ARRAY_UINT64,
DataType.ARRAY_FLOAT,
DataType.ARRAY_DOUBLE,
DataType.ARRAY_STRING,
DataType.ARRAY_BOOL,
]
class FieldSchema:
"""Represents a scalar (non-vector) field in a collection schema.
A `FieldSchema` defines the name, data type, nullability, and optional
inverted index configuration for a regular field (e.g., ID, timestamp, category).
Args:
name (str): Name of the field. Must be unique within the collection.
data_type (DataType): Data type of the field (e.g., INT64, STRING).
nullable (bool, optional): Whether the field can contain null values.
Defaults to False.
index_param (Optional[InvertIndexParam], optional): Inverted index
parameters for this field. Only applicable to fields that support
indexing (e.g., scalar fields used in filtering). Defaults to None.
Examples:
>>> from zvec.typing import DataType
>>> from zvec.model.param import InvertIndexParam
>>> id_field = FieldSchema(
... name="id",
... data_type=DataType.INT64,
... nullable=False,
... index_param=InvertIndexParam(enable_range_optimization=True)
... )
"""
def __init__(
self,
name: str,
data_type: DataType,
nullable: bool = False,
index_param: Optional[InvertIndexParam] = None,
):
if name is None or not isinstance(name, str):
raise ValueError(
f"schema validate failed: field name must be str, got {type(name).__name__}"
)
if data_type not in SUPPORT_SCALAR_DATA_TYPE:
raise ValueError(
f"schema validate failed: scalar_field's data_type must be one of "
f"{', '.join(str(dt) for dt in SUPPORT_SCALAR_DATA_TYPE)}, "
f"but field[{name}]'s data_type is {data_type}"
)
self._cpp_obj = _FieldSchema(
name=name,
data_type=data_type,
dimension=0,
nullable=nullable,
index_param=index_param,
)
@classmethod
def _from_core(cls, core_field_schema: _FieldSchema):
if core_field_schema is None:
raise ValueError("schema validate failed: field schema is None")
inst = cls.__new__(cls)
inst._cpp_obj = core_field_schema
return inst
def _get_object(self) -> _FieldSchema:
return self._cpp_obj
@property
def name(self) -> str:
"""str: The name of the field."""
return self._cpp_obj.name
@property
def data_type(self) -> DataType:
"""DataType: The data type of the field (e.g., INT64, STRING)."""
return self._cpp_obj.data_type
@property
def nullable(self) -> bool:
"""bool: Whether the field allows null values."""
return self._cpp_obj.nullable
@property
def index_param(self) -> Optional[InvertIndexParam]:
"""Optional[InvertIndexParam]: Inverted index configuration, if any."""
return self._cpp_obj.index_param
def __dict__(self) -> dict[str, Any]:
return {
"name": self.name,
"data_type": (
self.data_type.name
if hasattr(self.data_type, "name")
else str(self.data_type)
),
"nullable": self.nullable,
"index_param": (
self.index_param.to_dict() if self.index_param is not None else None
),
}
def __repr__(self) -> str:
try:
schema = self.__dict__()
return json.dumps(schema, indent=2, ensure_ascii=False)
except Exception as e:
return f"<FieldSchema error during repr: {e}>"
def __str__(self) -> str:
return self.__repr__()
def __eq__(self, other: object) -> bool:
if not isinstance(other, FieldSchema):
return False
return self._cpp_obj == other._cpp_obj
def __hash__(self) -> int:
return hash((self.name, self.data_type, self.nullable))
class VectorSchema:
"""Represents a vector field in a collection schema.
A `VectorSchema` defines the name, data type, dimensionality, and index
configuration for a vector field used in similarity search.
Args:
name (str): Name of the vector field. Must be unique within the collection.
data_type (DataType): Vector data type (e.g., VECTOR_FP32, VECTOR_INT8).
dimension (int, optional): Dimensionality of the vector. Must be > 0 for dense vectors;
may be `None` for sparse vectors.
index_param (Union[HnswIndexParam, IVFIndexParam, FlatIndexParam], optional):
Index configuration for this vector field. Defaults to
``HnswIndexParam()``.
Examples:
>>> from zvec.typing import DataType
>>> from zvec.model.param import HnswIndexParam
>>> emb_field = VectorSchema(
... name="embedding",
... data_type=DataType.VECTOR_FP32,
... dimension=128,
... index_param=HnswIndexParam(ef_construction=200, m=16)
... )
"""
def __init__(
self,
name: str,
data_type: DataType,
dimension: Optional[int] = 0,
index_param: Optional[
Union[HnswIndexParam, FlatIndexParam, IVFIndexParam]
] = None,
):
if name is None or not isinstance(name, str):
raise ValueError(
f"schema validate failed: field name must be str, got {type(name).__name__}"
)
if not isinstance(dimension, int) or dimension < 0:
raise ValueError("schema validate failed: vector's dimension must be >= 0")
if data_type not in SUPPORT_VECTOR_DATA_TYPE:
raise ValueError(
f"schema validate failed: vector's data_type must be one of "
f"{', '.join(str(dt) for dt in SUPPORT_VECTOR_DATA_TYPE)}, "
f"but field[{name}]'s data_type is {data_type}"
)
if index_param is None:
index_param = FlatIndexParam()
self._cpp_obj = _FieldSchema(
name=name,
data_type=data_type,
dimension=dimension,
nullable=False,
index_param=index_param,
)
@classmethod
def _from_core(cls, core_field_schema: _FieldSchema):
inst = cls.__new__(cls)
inst._cpp_obj = core_field_schema
return inst
def _get_object(self) -> _FieldSchema:
return self._cpp_obj
@property
def name(self) -> str:
"""str: The name of the vector field."""
return self._cpp_obj.name
@property
def data_type(self) -> DataType:
"""DataType: The vector data type (e.g., VECTOR_FP32)."""
return self._cpp_obj.data_type
@property
def dimension(self) -> int:
"""int: The dimensionality of the vector."""
return self._cpp_obj.dimension
@property
def index_param(self) -> Union[HnswIndexParam, IVFIndexParam, FlatIndexParam]:
"""Union[HnswIndexParam, IVFIndexParam, FlatIndexParam]: Index configuration for the vector."""
return self._cpp_obj.index_param
def __dict__(self) -> dict[str, Any]:
return {
"name": self.name,
"data_type": (
self.data_type.name
if hasattr(self.data_type, "name")
else str(self.data_type)
),
"dimension": self.dimension,
"index_param": (
self.index_param.to_dict() if self.index_param is not None else None
),
}
def __repr__(self) -> str:
try:
schema = self.__dict__()
return json.dumps(schema, indent=2, ensure_ascii=False)
except Exception as e:
return f"<FieldSchema error during repr: {e}>"
def __str__(self) -> str:
return self.__repr__()
def __eq__(self, other: object) -> bool:
if not isinstance(other, VectorSchema):
return False
return self._cpp_obj == other._cpp_obj
def __hash__(self) -> int:
return hash((self.name, self.data_type, self.dimension))
View File
+18
View File
@@ -0,0 +1,18 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from .util import require_module
__all__ = ["require_module"]
+63
View File
@@ -0,0 +1,63 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import importlib
from typing import Any, Optional
def require_module(module: str, mitigation: Optional[str] = None) -> Any:
"""Import a Python module and raise a user-friendly error if it is not available.
This utility helps provide actionable error messages when optional dependencies
are missing. It attempts to import the given module and, on failure, suggests
a `pip install` command based on either the module name or an optional
mitigation package name.
Args:
module (str): The full module name to import (e.g., ``"numpy"``, ``"pandas.io.parquet"``).
mitigation (Optional[str], optional): The package name to suggest for installation
if the import fails. If not provided, the top-level package of `module`
will be used (e.g., ``"pandas"`` for ``"pandas.io.parquet"``).
Returns:
Any: The imported module object.
Raises:
ImportError: If the module cannot be imported, with a clear installation hint.
Examples:
>>> import zvec
>>> np = zvec.require_module("numpy")
>>> pq = zvec.require_module("pyarrow.parquet", mitigation="pyarrow")
Note:
This function is intended for lazy-loading optional dependencies
with helpful error messages, not for core dependencies.
"""
try:
return importlib.import_module(module)
except ImportError as e:
package = mitigation or module
msg = f"Required package '{package}' is not installed. "
if "." in module:
top_level = module.split(".")[0]
msg += f"Module '{module}' is part of '{top_level}', "
if mitigation:
msg += f"please pip install '{mitigation}'."
else:
msg += f"please pip install '{top_level}'."
else:
msg += f"Please pip install '{package}."
raise ImportError(msg) from e
+32
View File
@@ -0,0 +1,32 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from _zvec.typing import (
DataType,
IndexType,
MetricType,
QuantizeType,
Status,
StatusCode,
)
__all__ = [
"DataType",
"IndexType",
"MetricType",
"QuantizeType",
"Status",
"StatusCode",
]
+404
View File
@@ -0,0 +1,404 @@
"""
This module contains the basic data types of Zvec
"""
from __future__ import annotations
import typing
__all__: list[str] = [
"DataType",
"IndexType",
"MetricType",
"QuantizeType",
"Status",
"StatusCode",
]
class DataType:
"""
Enumeration of supported data types in Zvec.
Includes scalar types, dense/sparse vector types, and array types.
Examples:
>>> import zvec
>>> print(zvec.DataType.FLOAT)
DataType.FLOAT
>>> print(zvec.DataType.VECTOR_FP32)
DataType.VECTOR_FP32
Members:
STRING
BOOL
INT32
INT64
FLOAT
DOUBLE
UINT32
UINT64
VECTOR_FP16
VECTOR_FP32
VECTOR_FP64
VECTOR_INT8
SPARSE_VECTOR_FP32
SPARSE_VECTOR_FP16
ARRAY_STRING
ARRAY_INT32
ARRAY_INT64
ARRAY_FLOAT
ARRAY_DOUBLE
ARRAY_BOOL
ARRAY_UINT32
ARRAY_UINT64
"""
ARRAY_BOOL: typing.ClassVar[DataType] # value = <DataType.ARRAY_BOOL: 42>
ARRAY_DOUBLE: typing.ClassVar[DataType] # value = <DataType.ARRAY_DOUBLE: 48>
ARRAY_FLOAT: typing.ClassVar[DataType] # value = <DataType.ARRAY_FLOAT: 47>
ARRAY_INT32: typing.ClassVar[DataType] # value = <DataType.ARRAY_INT32: 43>
ARRAY_INT64: typing.ClassVar[DataType] # value = <DataType.ARRAY_INT64: 44>
ARRAY_STRING: typing.ClassVar[DataType] # value = <DataType.ARRAY_STRING: 41>
ARRAY_UINT32: typing.ClassVar[DataType] # value = <DataType.ARRAY_UINT32: 45>
ARRAY_UINT64: typing.ClassVar[DataType] # value = <DataType.ARRAY_UINT64: 46>
BOOL: typing.ClassVar[DataType] # value = <DataType.BOOL: 3>
DOUBLE: typing.ClassVar[DataType] # value = <DataType.DOUBLE: 9>
FLOAT: typing.ClassVar[DataType] # value = <DataType.FLOAT: 8>
INT32: typing.ClassVar[DataType] # value = <DataType.INT32: 4>
INT64: typing.ClassVar[DataType] # value = <DataType.INT64: 5>
SPARSE_VECTOR_FP16: typing.ClassVar[
DataType
] # value = <DataType.SPARSE_VECTOR_FP16: 30>
SPARSE_VECTOR_FP32: typing.ClassVar[
DataType
] # value = <DataType.SPARSE_VECTOR_FP32: 31>
STRING: typing.ClassVar[DataType] # value = <DataType.STRING: 2>
UINT32: typing.ClassVar[DataType] # value = <DataType.UINT32: 6>
UINT64: typing.ClassVar[DataType] # value = <DataType.UINT64: 7>
VECTOR_FP16: typing.ClassVar[DataType] # value = <DataType.VECTOR_FP16: 22>
VECTOR_FP32: typing.ClassVar[DataType] # value = <DataType.VECTOR_FP32: 23>
VECTOR_FP64: typing.ClassVar[DataType] # value = <DataType.VECTOR_FP64: 24>
VECTOR_INT8: typing.ClassVar[DataType] # value = <DataType.VECTOR_INT8: 26>
__members__: typing.ClassVar[
dict[str, DataType]
] # value = {'STRING': <DataType.STRING: 2>, 'BOOL': <DataType.BOOL: 3>, 'INT32': <DataType.INT32: 4>, 'INT64': <DataType.INT64: 5>, 'FLOAT': <DataType.FLOAT: 8>, 'DOUBLE': <DataType.DOUBLE: 9>, 'UINT32': <DataType.UINT32: 6>, 'UINT64': <DataType.UINT64: 7>, 'VECTOR_FP16': <DataType.VECTOR_FP16: 22>, 'VECTOR_FP32': <DataType.VECTOR_FP32: 23>, 'VECTOR_FP64': <DataType.VECTOR_FP64: 24>, 'VECTOR_INT8': <DataType.VECTOR_INT8: 26>, 'SPARSE_VECTOR_FP32': <DataType.SPARSE_VECTOR_FP32: 31>, 'SPARSE_VECTOR_FP16': <DataType.SPARSE_VECTOR_FP16: 30>, 'ARRAY_STRING': <DataType.ARRAY_STRING: 41>, 'ARRAY_INT32': <DataType.ARRAY_INT32: 43>, 'ARRAY_INT64': <DataType.ARRAY_INT64: 44>, 'ARRAY_FLOAT': <DataType.ARRAY_FLOAT: 47>, 'ARRAY_DOUBLE': <DataType.ARRAY_DOUBLE: 48>, 'ARRAY_BOOL': <DataType.ARRAY_BOOL: 42>, 'ARRAY_UINT32': <DataType.ARRAY_UINT32: 45>, 'ARRAY_UINT64': <DataType.ARRAY_UINT64: 46>}
def __eq__(self, other: typing.Any) -> bool: ...
def __getstate__(self) -> int: ...
def __hash__(self) -> int: ...
def __index__(self) -> int: ...
def __init__(self, value: typing.SupportsInt) -> None: ...
def __int__(self) -> int: ...
def __ne__(self, other: typing.Any) -> bool: ...
def __repr__(self) -> str: ...
def __setstate__(self, state: typing.SupportsInt) -> None: ...
def __str__(self) -> str: ...
@property
def name(self) -> str: ...
@property
def value(self) -> int: ...
class IndexType:
"""
Enumeration of supported index types in Zvec.
Examples:
>>> import zvec
>>> print(zvec.IndexType.HNSW)
IndexType.HNSW
Members:
UNDEFINED
HNSW
IVF
FLAT
INVERT
"""
FLAT: typing.ClassVar[IndexType] # value = <IndexType.FLAT: 4>
HNSW: typing.ClassVar[IndexType] # value = <IndexType.HNSW: 1>
INVERT: typing.ClassVar[IndexType] # value = <IndexType.INVERT: 10>
IVF: typing.ClassVar[IndexType] # value = <IndexType.IVF: 3>
UNDEFINED: typing.ClassVar[IndexType] # value = <IndexType.UNDEFINED: 0>
__members__: typing.ClassVar[
dict[str, IndexType]
] # value = {'UNDEFINED': <IndexType.UNDEFINED: 0>, 'HNSW': <IndexType.HNSW: 1>, 'IVF': <IndexType.IVF: 3>, 'FLAT': <IndexType.FLAT: 4>, 'INVERT': <IndexType.INVERT: 10>}
def __eq__(self, other: typing.Any) -> bool: ...
def __getstate__(self) -> int: ...
def __hash__(self) -> int: ...
def __index__(self) -> int: ...
def __init__(self, value: typing.SupportsInt) -> None: ...
def __int__(self) -> int: ...
def __ne__(self, other: typing.Any) -> bool: ...
def __repr__(self) -> str: ...
def __setstate__(self, state: typing.SupportsInt) -> None: ...
def __str__(self) -> str: ...
@property
def name(self) -> str: ...
@property
def value(self) -> int: ...
class MetricType:
"""
Enumeration of supported distance/similarity metrics.
- COSINE: Cosine similarity.
- IP: Inner product (dot product).
- L2: Euclidean distance (L2 norm).
Examples:
>>> import zvec
>>> print(zvec.MetricType.COSINE)
MetricType.COSINE
Members:
COSINE
IP
L2
"""
COSINE: typing.ClassVar[MetricType] # value = <MetricType.COSINE: 3>
IP: typing.ClassVar[MetricType] # value = <MetricType.IP: 2>
L2: typing.ClassVar[MetricType] # value = <MetricType.L2: 1>
__members__: typing.ClassVar[
dict[str, MetricType]
] # value = {'COSINE': <MetricType.COSINE: 3>, 'IP': <MetricType.IP: 2>, 'L2': <MetricType.L2: 1>}
def __eq__(self, other: typing.Any) -> bool: ...
def __getstate__(self) -> int: ...
def __hash__(self) -> int: ...
def __index__(self) -> int: ...
def __init__(self, value: typing.SupportsInt) -> None: ...
def __int__(self) -> int: ...
def __ne__(self, other: typing.Any) -> bool: ...
def __repr__(self) -> str: ...
def __setstate__(self, state: typing.SupportsInt) -> None: ...
def __str__(self) -> str: ...
@property
def name(self) -> str: ...
@property
def value(self) -> int: ...
class QuantizeType:
"""
Enumeration of supported quantization types for vector compression.
Examples:
>>> import zvec
>>> print(zvec.QuantizeType.INT8)
QuantizeType.INT8
Members:
UNDEFINED
FP16
INT8
INT4
"""
FP16: typing.ClassVar[QuantizeType] # value = <QuantizeType.FP16: 1>
INT4: typing.ClassVar[QuantizeType] # value = <QuantizeType.INT4: 3>
INT8: typing.ClassVar[QuantizeType] # value = <QuantizeType.INT8: 2>
UNDEFINED: typing.ClassVar[QuantizeType] # value = <QuantizeType.UNDEFINED: 0>
__members__: typing.ClassVar[
dict[str, QuantizeType]
] # value = {'UNDEFINED': <QuantizeType.UNDEFINED: 0>, 'FP16': <QuantizeType.FP16: 1>, 'INT8': <QuantizeType.INT8: 2>, 'INT4': <QuantizeType.INT4: 3>}
def __eq__(self, other: typing.Any) -> bool: ...
def __getstate__(self) -> int: ...
def __hash__(self) -> int: ...
def __index__(self) -> int: ...
def __init__(self, value: typing.SupportsInt) -> None: ...
def __int__(self) -> int: ...
def __ne__(self, other: typing.Any) -> bool: ...
def __repr__(self) -> str: ...
def __setstate__(self, state: typing.SupportsInt) -> None: ...
def __str__(self) -> str: ...
@property
def name(self) -> str: ...
@property
def value(self) -> int: ...
class Status:
"""
Represents the outcome of a Zvec operation.
A `Status` object is either OK (success) or carries an error code and message.
Examples:
>>> from zvec.typing import Status, StatusCode
>>> s = Status()
>>> print(s.ok())
True
>>> s = Status(StatusCode.INVALID_ARGUMENT, "Field not found")
>>> print(s.code() == StatusCode.INVALID_ARGUMENT)
True
>>> print(s.message())
Field not found
"""
__hash__: typing.ClassVar[None] = None
@staticmethod
def AlreadyExists(message: str) -> Status: ...
@staticmethod
def InternalError(message: str) -> Status: ...
@staticmethod
def InvalidArgument(message: str) -> Status: ...
@staticmethod
def NotFound(message: str) -> Status: ...
@staticmethod
def OK() -> Status:
"""
Create an OK status.
"""
@staticmethod
def PermissionDenied(message: str) -> Status: ...
def __eq__(self, arg0: Status) -> bool: ...
@typing.overload
def __init__(self) -> None: ...
@typing.overload
def __init__(self, code: StatusCode, message: str = "") -> None:
"""
Construct a status with the given code and optional message.
Args:
code (StatusCode): The status code.
message (str, optional): Error message. Defaults to empty string.
"""
def __ne__(self, arg0: Status) -> bool: ...
def __repr__(self) -> str: ...
def code(self) -> StatusCode:
"""
StatusCode: Returns the status code.
"""
def message(self) -> str:
"""
str: Returns the error message (may be empty).
"""
def ok(self) -> bool:
"""
bool: Returns True if the status is OK.
"""
class StatusCode:
"""
Enumeration of possible status codes for Zvec operations.
Used by the `Status` class to indicate success or failure reason.
Members:
OK
NOT_FOUND
ALREADY_EXISTS
INVALID_ARGUMENT
PERMISSION_DENIED
FAILED_PRECONDITION
RESOURCE_EXHAUSTED
UNAVAILABLE
INTERNAL_ERROR
NOT_SUPPORTED
UNKNOWN
"""
ALREADY_EXISTS: typing.ClassVar[
StatusCode
] # value = <StatusCode.ALREADY_EXISTS: 2>
FAILED_PRECONDITION: typing.ClassVar[
StatusCode
] # value = <StatusCode.FAILED_PRECONDITION: 5>
INTERNAL_ERROR: typing.ClassVar[
StatusCode
] # value = <StatusCode.INTERNAL_ERROR: 8>
INVALID_ARGUMENT: typing.ClassVar[
StatusCode
] # value = <StatusCode.INVALID_ARGUMENT: 3>
NOT_FOUND: typing.ClassVar[StatusCode] # value = <StatusCode.NOT_FOUND: 1>
NOT_SUPPORTED: typing.ClassVar[StatusCode] # value = <StatusCode.NOT_SUPPORTED: 9>
OK: typing.ClassVar[StatusCode] # value = <StatusCode.OK: 0>
PERMISSION_DENIED: typing.ClassVar[
StatusCode
] # value = <StatusCode.PERMISSION_DENIED: 4>
RESOURCE_EXHAUSTED: typing.ClassVar[
StatusCode
] # value = <StatusCode.RESOURCE_EXHAUSTED: 6>
UNAVAILABLE: typing.ClassVar[StatusCode] # value = <StatusCode.UNAVAILABLE: 7>
UNKNOWN: typing.ClassVar[StatusCode] # value = <StatusCode.UNKNOWN: 10>
__members__: typing.ClassVar[
dict[str, StatusCode]
] # value = {'OK': <StatusCode.OK: 0>, 'NOT_FOUND': <StatusCode.NOT_FOUND: 1>, 'ALREADY_EXISTS': <StatusCode.ALREADY_EXISTS: 2>, 'INVALID_ARGUMENT': <StatusCode.INVALID_ARGUMENT: 3>, 'PERMISSION_DENIED': <StatusCode.PERMISSION_DENIED: 4>, 'FAILED_PRECONDITION': <StatusCode.FAILED_PRECONDITION: 5>, 'RESOURCE_EXHAUSTED': <StatusCode.RESOURCE_EXHAUSTED: 6>, 'UNAVAILABLE': <StatusCode.UNAVAILABLE: 7>, 'INTERNAL_ERROR': <StatusCode.INTERNAL_ERROR: 8>, 'NOT_SUPPORTED': <StatusCode.NOT_SUPPORTED: 9>, 'UNKNOWN': <StatusCode.UNKNOWN: 10>}
def __eq__(self, other: typing.Any) -> bool: ...
def __getstate__(self) -> int: ...
def __hash__(self) -> int: ...
def __index__(self) -> int: ...
def __init__(self, value: typing.SupportsInt) -> None: ...
def __int__(self) -> int: ...
def __ne__(self, other: typing.Any) -> bool: ...
def __repr__(self) -> str: ...
def __setstate__(self, state: typing.SupportsInt) -> None: ...
def __str__(self) -> str: ...
@property
def name(self) -> str: ...
@property
def value(self) -> int: ...
+62
View File
@@ -0,0 +1,62 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from enum import IntEnum
__all__ = ["LogLevel", "LogType"]
class LogLevel(IntEnum):
"""Enumeration of logging severity levels, ordered from lowest to highest priority.
Used to control verbosity and filtering of log messages. Higher numeric values
indicate more severe conditions.
Note:
``WARNING`` is an alias for ``WARN`` to match Python's built-in :mod:`logging`
module convention.
Attributes:
DEBUG (int): Detailed information, typically of interest only when diagnosing problems.
INFO (int): Confirmation that things are working as expected.
WARN (int): An indication that something unexpected happened, or indicative of
potential future problems. (Alias: ``WARNING``)
WARNING (int): Same as ``WARN``.
ERROR (int): Due to a more serious problem, the software has not been able
to perform some function.
FATAL (int): A serious error, indicating that the program itself may be unable
to continue running.
"""
DEBUG = 0
INFO = 1
WARN = 2
WARNING = 2
ERROR = 3
FATAL = 4
class LogType(IntEnum):
"""Enumeration of supported log output destinations.
Specifies where log messages should be written.
Attributes:
CONSOLE (int): Output logs to standard output/error (e.g., terminal or IDE console).
FILE (int): Write logs to a persistent file on disk.
"""
CONSOLE = 0
FILE = 1
+226
View File
@@ -0,0 +1,226 @@
# Copyright 2025-present the zvec project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import Optional
from _zvec import Initialize, _Collection
from .model import Collection
from .model.param import CollectionOption
from .model.schema import CollectionSchema
__all__ = ["create_and_open", "init", "open"]
from .typing.enum import LogLevel, LogType
def init(
*,
log_type: Optional[LogType] = LogType.CONSOLE,
log_level: Optional[LogLevel] = LogLevel.WARN,
log_dir: Optional[str] = "./logs",
log_basename: Optional[str] = "zvec.log",
log_file_size: Optional[int] = 2048,
log_overdue_days: Optional[int] = 7,
query_threads: Optional[int] = None,
optimize_threads: Optional[int] = None,
invert_to_forward_scan_ratio: Optional[float] = None,
brute_force_by_keys_ratio: Optional[float] = None,
memory_limit_mb: Optional[int] = None,
) -> None:
"""Initialize Zvec with configuration options.
This function must be called before any other operation.
It can only be called once — subsequent calls raise a ``RuntimeError``.
Parameters set to ``None`` are **omitted** from the configuration and
fall back to Zvec's internal defaults, which may be derived from the runtime
environment (e.g., cgroup CPU/memory limits). Explicitly provided values
always override defaults.
Args:
log_type (Optional[LogType], optional): Logger destination.
- ``LogType.CONSOLE`` (default if omitted or set to this)
- ``LogType.FILE``
- If ``None``, uses internal default (currently ``CONSOLE``).
log_level (Optional[LogLevel], optional): Minimum log severity.
Default: ``LogLevel.WARN``.
Accepted values: ``DEBUG``, ``INFO``, ``WARN``, ``ERROR``, ``FATAL``.
If ``None``, uses internal default (``WARN``).
log_dir (Optional[str], optional):
Directory for log files (only used when ``log_type=FILE``).
Parent directories are **not** created automatically.
Default: ``"./logs"``.
If ``None``, internal default is used.
log_basename (Optional[str], optional):
Base name for rotated log files (e.g., ``zvec.log.1``, ``zvec.log.2``).
Default: ``"zvec.log"``.
log_file_size (Optional[int], optional):
Max size per log file in **MB** before rotation.
Default: ``2048`` MB (2 GB).
log_overdue_days (Optional[int], optional):
Days to retain rotated log files before deletion.
Default: ``7`` days.
query_threads (Optional[int], optional):
Number of threads for query execution.
If ``None`` (default), inferred from available CPU cores (via cgroup).
Must be ≥ 1 if provided.
optimize_threads (Optional[int], optional):
Threads for background tasks (e.g., compaction, indexing).
If ``None``, defaults to same as ``query_threads`` or CPU count.
invert_to_forward_scan_ratio (Optional[float], optional):
Threshold to switch from inverted index to full forward scan.
Range: [0.0, 1.0]. Higher → more aggressive index skipping.
Default: ``0.9`` (if omitted).
brute_force_by_keys_ratio (Optional[float], optional):
Threshold to use brute-force key lookup over index.
Lower → prefer index; higher → prefer brute-force.
Range: [0.0, 1.0]. Default: ``0.1``.
memory_limit_mb (Optional[int], optional):
Soft memory cap in MB. Zvec may throttle or fail operations
approaching this limit.
If ``None``, inferred from cgroup memory limit * 0.8 (e.g., in Docker).
Must be > 0 if provided.
Raises:
RuntimeError: If Zvec is already initialized.
ValueError: On invalid values (e.g., negative thread count, log level out of range).
TypeError: If a value has incorrect type (e.g., string for ``query_threads``).
Note:
- All ``None`` arguments are **excluded** from the configuration payload,
allowing the core library to apply environment-aware defaults.
- This design ensures container-friendliness: in Kubernetes/Docker,
omitting ``memory_limit_mb`` and thread counts lets Zvec auto-adapt.
Examples:
Initialize with defaults (log to console, auto-detect resources):
>>> import zvec
>>> zvec.init()
Customize logging to file with rotation:
>>> zvec.init(
... log_type=LogType.FILE,
... log_dir="/var/log/zvec",
... log_file_size=1024,
... log_overdue_days=30
... )
Limit resources explicitly:
>>> zvec.init(
... memory_limit_mb=2048,
... query_threads=4,
... optimize_threads=2
... )
Fine-tune query heuristics:
>>> zvec.init(
... invert_to_forward_scan_ratio=0.95,
... brute_force_by_keys_ratio=0.05
... )
"""
# Build config dict, skipping None values
config_dict = {}
if log_type is not None:
if not isinstance(log_type, LogType):
raise TypeError("log_type must be LogType")
config_dict["log_type"] = log_type.name
if log_level is not None:
if not isinstance(log_level, LogLevel):
raise TypeError("log_level must be LogLevel")
config_dict["log_level"] = log_level.name
if log_dir is not None:
config_dict["log_dir"] = log_dir
if log_basename is not None:
config_dict["log_basename"] = log_basename
if log_file_size is not None:
config_dict["log_file_size"] = log_file_size
if log_overdue_days is not None:
config_dict["log_overdue_days"] = log_overdue_days
if query_threads is not None:
config_dict["query_threads"] = query_threads
if optimize_threads is not None:
config_dict["optimize_threads"] = optimize_threads
if invert_to_forward_scan_ratio is not None:
config_dict["invert_to_forward_scan_ratio"] = invert_to_forward_scan_ratio
if brute_force_by_keys_ratio is not None:
config_dict["brute_force_by_keys_ratio"] = brute_force_by_keys_ratio
if memory_limit_mb is not None:
config_dict["memory_limit_mb"] = memory_limit_mb
Initialize(config_dict)
def create_and_open(
path: str,
schema: CollectionSchema,
option: Optional[CollectionOption] = None,
) -> Collection:
"""Create a new collection and open it for use.
If a collection already exists at the given path, it may raise an error
depending on the underlying implementation.
Args:
path (str): Path or name of the collection to create.
schema (CollectionSchema): Schema defining the structure of the collection.
option (CollectionOption): Configuration options
for opening the collection. Defaults to a default-constructed
``CollectionOption()`` if not provided.
Returns:
Collection: An opened collection instance ready for operations.
Examples:
>>> import zvec
>>> schema = zvec.CollectionSchema(
... name="my_collection",
... fields=[zvec.FieldSchema("id", zvec.DataType.INT64, nullable=True)]
... )
>>> coll = create_and_open("./my_collection", schema)
"""
if not isinstance(path, str):
raise TypeError("path must be a string")
if not isinstance(schema, CollectionSchema):
raise TypeError("schema must be a CollectionSchema")
option = option or CollectionOption()
if not isinstance(option, CollectionOption):
raise TypeError("option must be a CollectionOption")
_collection = _Collection.CreateAndOpen(path, schema._get_object(), option)
return Collection._from_core(_collection)
def open(path: str, option: CollectionOption = CollectionOption()) -> Collection:
"""Open an existing collection from disk.
The collection must have been previously created with ``create_and_open``.
Args:
path (str): Path or name of the existing collection.
option (CollectionOption): Configuration options
for opening the collection. Defaults to a default-constructed
``CollectionOption()`` if not provided.
Returns:
Collection: An opened collection instance.
Examples:
>>> import zvec
>>> coll = zvec.open("./my_collection")
"""
_collection = _Collection.Open(path, option)
return Collection._from_core(_collection)
View File
+41
View File
@@ -0,0 +1,41 @@
#!/bin/bash
project_name=proxima-zvec
gcov_tool=gcov
zip_html=false
output_name=html
keep_info=false
script_dir=$(cd "$(dirname "$0")"; pwd)
source_base=$(dirname "$script_dir")
filter_list="'*/tests/*' '*/thirdparty/*' '*/deps/*' '*/proto/*' '*/external/*' '*/sqlengine/antlr/gen/*'"
while getopts t:p:o:zk option; do
case "$option" in
t)
gcov_tool=$OPTARG;;
p)
project_name=$OPTARG;;
o)
output_name=$OPTARG;;
z)
zip_html=true;;
k)
keep_info=true;;
esac
done
# Process sources
lcov -c -b "$source_base" -d . -o $project_name.lcov.info --gcov-tool=$gcov_tool --no-external || exit 1
eval $(echo lcov -r $project_name.lcov.info -o $project_name-filtered.lcov.info $filter_list) || exit 1
# Gather HTML files
genhtml -t "$project_name" -o $output_name $project_name-filtered.lcov.info || exit 1
if [ "$keep_info" = false ]; then
rm -rf *.lcov.info
fi
# Zip HTML files
if $zip_html ; then
zip -r $output_name.zip $output_name/
fi
+13
View File
@@ -0,0 +1,13 @@
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
# Retrieve version from git repository
git_version(ZVEC_VERSION ${CMAKE_CURRENT_SOURCE_DIR})
# Add repository
cc_directory(ailego)
cc_directory(core)
cc_directory(db)
if(BUILD_PYTHON_BINDINGS)
cc_directory(binding)
endif()
+28
View File
@@ -0,0 +1,28 @@
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
find_package(Threads REQUIRED)
if(UNIX AND NOT APPLE)
find_library(LIB_RT NAMES rt)
else()
set(LIB_RT "")
endif()
git_version(GIT_SRCS_VER ${CMAKE_CURRENT_SOURCE_DIR})
file(GLOB_RECURSE ALL_SRCS *.cc *.c *.h)
set(EXTRA_LIBS ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
if(UNIX AND NOT APPLE)
list(APPEND EXTRA_LIBS ${LIB_RT})
endif()
cc_library(
NAME zvec_ailego STATIC STRICT PACKED
SRCS ${ALL_SRCS}
LIBS ${EXTRA_LIBS}
Arrow::arrow_static
Arrow::parquet_static
VERSION "${GIT_SRCS_VER}"
)
+77
View File
@@ -0,0 +1,77 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "binary_quantizer.h"
#include <algorithm>
#include <cmath>
#include <cstring>
#include <iostream>
#include <numeric>
#include <ailego/math/normalizer.h>
namespace zvec {
namespace ailego {
//! Feed the training data
bool BinaryQuantizer::feed(const float *vec, size_t dim) {
for (size_t i = 0; i < dim; ++i) {
data_.emplace_back(vec[i]);
}
return true;
}
//! Train the quantizer
bool BinaryQuantizer::train(void) {
return true;
}
//! Quantize data: encode the float input to uint32_t output
void BinaryQuantizer::encode(const float *in, size_t dim, uint32_t *out) const {
for (size_t i = 0; i < dim; i += 32) {
size_t remain = i + 32 <= dim ? 32 : dim - i;
uint32_t data = 0;
uint32_t mask = 1;
for (size_t j = 0; j < remain; j++) {
if (in[i + j] >= threshold_) {
data |= mask;
}
mask <<= 1;
}
*out = data;
out++;
}
}
//! De-quantize data: decode the input uint32_t to float output
//! bit value 1 will be mapped to 1.0
//! bit value 0 will be mapped to -1.0
void BinaryQuantizer::decode(const uint32_t *in, size_t dim, float *out) const {
for (size_t i = 0; i < dim; ++i) {
uint8_t bit = (in[i >> 5] >> (i & 31)) & 0x01;
if (bit == 1) {
out[i] = 1.0f;
} else {
out[i] = -1.0f;
}
// std::cout << "dim: " << i << ", value: " << (size_t)bit << std::endl;
}
}
} // namespace ailego
} // namespace zvec
+69
View File
@@ -0,0 +1,69 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <vector>
#include <ailego/internal/platform.h>
namespace zvec {
namespace ailego {
/*! Binary Quantization Algorithm
*/
class BinaryQuantizer {
public:
//! Constructor
BinaryQuantizer(void) {}
//! Feed the training data
bool feed(const float *vec, size_t dim);
//! Train the quantizer
bool train(void);
//! Quantize data: encode the float input to uint32_t output
void encode(const float *in, size_t dim, uint32_t *out) const;
//! De-quantize data: decode the input uint32_t to float output
void decode(const uint32_t *in, size_t dim, float *out) const;
//! Get encoded elements in type of uint32_t
static size_t EncodedSizeInBinary32(size_t dim) {
return (dim + 31) / 32;
}
//! Set quantization threshold
void set_threshold(float threshold) {
threshold_ = threshold;
}
//! Get quantization threshold
float threshold(void) const {
return threshold_;
}
private:
//! Disable them
BinaryQuantizer(const BinaryQuantizer &) = delete;
BinaryQuantizer &operator=(const BinaryQuantizer &) = delete;
private:
//! Members
std::vector<float> data_{};
float threshold_{0.0f};
};
} // namespace ailego
} // namespace zvec
+466
View File
@@ -0,0 +1,466 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "integer_quantizer.h"
#include <algorithm>
#include <cmath>
#include <cstring>
#include <numeric>
#include <stdexcept>
#include <ailego/math/normalizer.h>
namespace zvec {
namespace ailego {
//! Make smooth the distribution to eliminate zero in hist
static inline void MakeSmooth(std::vector<float> &dist) {
constexpr float epsilon = std::numeric_limits<float>::epsilon();
// L1 Normalize first
float norm = 1.0f;
Normalizer<float>::L1(dist.data(), dist.size(), &norm);
size_t zero_count = std::count_if(dist.begin(), dist.end(), [](float val) {
return (std::abs(val) < std::numeric_limits<float>::epsilon());
});
size_t nonzero_count = dist.size() - zero_count;
// Double check
if (nonzero_count == 0 || zero_count == 0) {
return;
}
float y = epsilon * zero_count / static_cast<float>(nonzero_count);
for (auto &it : dist) {
if (std::abs(it) < epsilon) {
it += epsilon;
} else {
it -= y;
}
} // end of for
}
//! Compute the Entropy of distribution p/q by Kullback-Leibler Divergence
static inline double ComputeKlDivergence(const std::vector<float> &p,
const std::vector<float> &q) {
if (p.size() != q.size() || p.size() == 0) {
return std::numeric_limits<float>::max();
}
double v = 0.0f;
for (size_t i = 0; i != p.size(); ++i) {
if (p[i] == 0 || q[i] == 0) {
return std::numeric_limits<double>::max();
}
v += p[i] * std::log(static_cast<double>(p[i]) / static_cast<double>(q[i]));
}
return v;
}
//! Expand the quantization distribution to origin distribution in
//! [-threshold, threshold]
static inline void ExpandCandidateDistribution(
const std::vector<uint32_t> &distribution,
const std::vector<float> &quantized_distribution, size_t threshold,
std::vector<float> *expand_distribution) {
expand_distribution->resize(threshold * 2, 0);
float merged_cnt = static_cast<float>(expand_distribution->size()) /
quantized_distribution.size();
size_t left_boundary = distribution.size() / 2 - threshold;
for (size_t i = 0; i < quantized_distribution.size(); ++i) {
float start = i * merged_cnt;
float end = start + merged_cnt;
const size_t start_ceil = static_cast<size_t>(std::ceil(start));
const size_t end_floor = static_cast<size_t>(std::floor(end));
float left_ratio = static_cast<float>(start_ceil) - start;
float right_ratio = end - static_cast<float>(end_floor);
float nonzero_count = 0;
//! Count the non-zeros bins, if the histogram bin is partially included,
//! non-zero bins is also partially counted
if (left_ratio > 0 && left_boundary + start_ceil > 0) {
if (distribution[left_boundary + start_ceil - 1] != 0) {
nonzero_count += left_ratio;
}
}
if (right_ratio > 0 && left_boundary + end_floor < distribution.size()) {
if (distribution[left_boundary + end_floor] != 0) {
nonzero_count += right_ratio;
}
}
for (size_t j = start_ceil; j < end_floor; j++) {
nonzero_count += distribution[left_boundary + j] != 0;
}
if (nonzero_count == 0) {
continue;
}
//! expand the quantized value
float value = quantized_distribution[i] / nonzero_count;
if (left_ratio > 0 && start_ceil > 0) {
(*expand_distribution)[start_ceil - 1] += value * left_ratio;
}
if (right_ratio > 0 && end_floor < expand_distribution->size()) {
(*expand_distribution)[end_floor] += value * right_ratio;
}
for (size_t j = start_ceil; j < end_floor; j++) {
if (distribution[left_boundary + j] != 0) {
(*expand_distribution)[j] = value;
}
} // end of for
} // end of for
}
/*! Compute quantization threshold bins
* Implement Int8 Quantization Algorithm ref:
* http://on-demand.gputechconf.com/gtc/2017/presentation/s7310-8-bit-inference-with-tensorrt.pdf
*/
static inline size_t ComputeThreshold(const std::vector<uint32_t> &hist,
const size_t target_bins) {
std::vector<float> P_distribution(hist.size());
size_t zero_point_index = hist.size() / 2;
size_t start_bin = target_bins / 2;
size_t end_bin = hist.size() / 2;
size_t negative_outliers_count = 0;
size_t positive_outliers_count = 0;
double min_divergence = std::numeric_limits<double>::max();
size_t target_threshold = end_bin;
for (size_t threshold = start_bin; threshold <= end_bin; ++threshold) {
negative_outliers_count += hist[zero_point_index - threshold];
positive_outliers_count += hist[zero_point_index + threshold - 1];
}
//! for each zero-axised quantization range: [-threshold, threshold], search
//! the best solution
for (size_t threshold = start_bin; threshold <= end_bin; ++threshold) {
P_distribution.resize(threshold * 2);
auto p_hist = &hist[zero_point_index - threshold];
for (size_t i = 0; i != P_distribution.size(); ++i) {
P_distribution[i] = static_cast<float>(p_hist[i]);
}
negative_outliers_count -= hist[zero_point_index - threshold];
positive_outliers_count -= hist[zero_point_index + threshold - 1];
P_distribution[0] += negative_outliers_count;
P_distribution[P_distribution.size() - 1] += positive_outliers_count;
//! Quantize the bins in range [-threshold, threshold] to target_bins
std::vector<float> Q_distribution(target_bins, 0);
float merged_cnt = static_cast<float>(threshold * 2) / target_bins;
size_t left_boundary = zero_point_index - threshold;
for (size_t i = 0; i < target_bins; ++i) {
float start = i * merged_cnt;
float end = start + merged_cnt;
const size_t start_ceil = static_cast<size_t>(std::ceil(start));
const size_t end_floor = static_cast<size_t>(std::floor(end));
if (left_boundary + start_ceil > 0) {
Q_distribution[i] +=
((float)start_ceil - start) * hist[left_boundary + start_ceil - 1];
}
if (left_boundary + end_floor < hist.size()) {
Q_distribution[i] +=
(end - (float)end_floor) * hist[left_boundary + end_floor];
}
for (size_t j = start_ceil; j < end_floor; j++) {
Q_distribution[i] += hist[left_boundary + j];
}
}
std::vector<float> Q_expand_distribution;
ExpandCandidateDistribution(hist, Q_distribution, threshold,
&Q_expand_distribution);
//! Compute Kullback-Leibler Divergence, normalize the smooth the data
//! first. Ref: http://hanj.cs.illinois.edu/cs412/bk3/KL-divergence.pdf
MakeSmooth(P_distribution);
MakeSmooth(Q_expand_distribution);
double divergence =
ComputeKlDivergence(P_distribution, Q_expand_distribution);
if (divergence < min_divergence) {
min_divergence = divergence;
target_threshold = threshold;
}
}
return target_threshold;
}
// Quantize the value in range
template <int RANGE_MIN, int RANGE_MAX>
static inline float QuantizeValue(float val, float scale, float bias) {
val = (val + bias) * scale;
if (val > RANGE_MAX) {
val = RANGE_MAX;
} else if (val < RANGE_MIN) {
val = RANGE_MIN;
}
return val;
}
// Init the historgram params
#define INIT_HISTOGRAM() \
{ \
if (histogram_bins_ == 0) { \
size_t range = non_bias_ \
? std::max(std::abs(MIN_VALUE), std::abs(MAX_VALUE)) \
: (MAX_VALUE - MIN_VALUE); \
histogram_bins_ = std::max<size_t>(4096u, range * 8); \
} \
histogram_.resize((histogram_bins_ + 1) >> 1 << 1); \
if (non_bias_) { \
bias_ = 0.0f; \
auto val = std::max(std::abs(max_), std::abs(min_)); \
left_boundary_ = -val; \
hist_interval_ = (val * 2) / static_cast<float>(histogram_.size()); \
} else { \
bias_ = -static_cast<float>(min_ + (max_ - min_) * 0.5); \
left_boundary_ = min_; \
hist_interval_ = (max_ - min_) / static_cast<float>(histogram_.size()); \
} \
}
// Feed vector and update the historgram
#define UPDATE_HISTOGRAM(vec, dim) \
{ \
if (max_ < min_) { \
return false; \
} \
if (histogram_.size() == 0) { \
INIT_HISTOGRAM() \
} \
for (size_t i = 0; i < dim; ++i) { \
ssize_t index = 0; \
if (hist_interval_ > 0.0) { \
index = \
static_cast<ssize_t>((vec[i] - left_boundary_) / hist_interval_); \
} \
if (index < 0) { \
index = 0; \
} else if ((size_t)index >= histogram_.size()) { \
index = histogram_.size() - 1; \
} \
ailego_assert_with((size_t)index < histogram_.size(), "Invalid index"); \
histogram_[index] += 1; \
} \
return true; \
}
// Train the quantizer
#define TRAIN_QUANTIZER() \
{ \
auto sum = std::accumulate(histogram_.begin(), histogram_.end(), 0); \
if (sum == 0) { \
return false; \
} \
size_t target_bins = \
ailego_align(static_cast<size_t>(MAX_VALUE - MIN_VALUE), 2); \
auto threshold_bins = ComputeThreshold(histogram_, target_bins); \
auto threshold = \
(static_cast<float>(threshold_bins) + 0.5f) * hist_interval_; \
scale_ = target_bins / 2 / threshold; \
if (!non_bias_) { \
bias_ += (MAX_VALUE + MIN_VALUE) * 0.5f / scale_; \
} \
scale_reciprocal_ = 1 / scale_; \
return true; \
}
// Feed the INT16 quantizer
bool EntropyInt16Quantizer::feed(const float *vec, size_t dim) {
UPDATE_HISTOGRAM(vec, dim)
}
// Train the INT16 quantizer
bool EntropyInt16Quantizer::train(void) {
TRAIN_QUANTIZER()
}
// Encode to INT16
void EntropyInt16Quantizer::encode(const float *in, size_t dim,
int16_t *out) const {
for (size_t i = 0; i < dim; ++i) {
out[i] = static_cast<int16_t>(
std::round(QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_)));
}
}
// Decode from INT16
void EntropyInt16Quantizer::decode(const int16_t *in, size_t dim,
float *out) const {
for (size_t i = 0; i < dim; ++i) {
out[i] = in[i] * this->scale_reciprocal() - this->bias();
}
}
// Feed the UINT16 quantizer
bool EntropyUInt16Quantizer::feed(const float *vec, size_t dim) {
UPDATE_HISTOGRAM(vec, dim)
}
// Train the UINT16 quantizer
bool EntropyUInt16Quantizer::train(void) {
TRAIN_QUANTIZER()
}
// Encode to UINT16
void EntropyUInt16Quantizer::encode(const float *in, size_t dim,
uint16_t *out) const {
for (size_t i = 0; i < dim; ++i) {
out[i] = static_cast_from_float_to_uint16(
std::round(QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_)));
}
}
// Decode from INT16
void EntropyUInt16Quantizer::decode(const uint16_t *in, size_t dim,
float *out) const {
for (size_t i = 0; i < dim; ++i) {
out[i] = in[i] * this->scale_reciprocal() - this->bias();
}
}
// Feed the INT8 quantizer
bool EntropyInt8Quantizer::feed(const float *vec, size_t dim) {
UPDATE_HISTOGRAM(vec, dim)
}
// Train the INT8 quantizer
bool EntropyInt8Quantizer::train(void) {
TRAIN_QUANTIZER()
}
// Encode to INT8
void EntropyInt8Quantizer::encode(const float *in, size_t dim,
int8_t *out) const {
for (size_t i = 0; i < dim; ++i) {
out[i] = static_cast<int8_t>(
std::round(QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_)));
}
}
// Decode from INT8
void EntropyInt8Quantizer::decode(const int8_t *in, size_t dim,
float *out) const {
for (size_t i = 0; i < dim; ++i) {
out[i] = in[i] * this->scale_reciprocal() - this->bias();
}
}
// Feed the UINT8 quantizer
bool EntropyUInt8Quantizer::feed(const float *vec, size_t dim) {
UPDATE_HISTOGRAM(vec, dim)
}
// Train the UINT8 quantizer
bool EntropyUInt8Quantizer::train(void) {
TRAIN_QUANTIZER()
}
// Encode to INT8
void EntropyUInt8Quantizer::encode(const float *in, size_t dim,
uint8_t *out) const {
for (size_t i = 0; i < dim; ++i) {
out[i] = static_cast_from_float_to_uint8(
std::round(QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_)));
}
}
// Decode from UINT8
void EntropyUInt8Quantizer::decode(const uint8_t *in, size_t dim,
float *out) const {
for (size_t i = 0; i < dim; ++i) {
out[i] = in[i] * this->scale_reciprocal() - this->bias();
}
}
// Feed the INT4 quantizer
bool EntropyInt4Quantizer::feed(const float *vec, size_t dim) {
UPDATE_HISTOGRAM(vec, dim)
}
// Train the INT4 quantizer
bool EntropyInt4Quantizer::train(void) {
TRAIN_QUANTIZER()
}
// Encode to INT4
void EntropyInt4Quantizer::encode(const float *in, size_t dim,
uint8_t *out) const {
ailego_assert_with(dim % 2 == 0, "Dimension must be aligned with 2");
for (size_t i = 0; i < dim; i += 2) {
float lo = QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_);
float hi = QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i + 1], scale_, bias_);
out[i / 2] = (static_cast_from_float_to_uint8(std::round(hi)) << 4) |
(static_cast_from_float_to_uint8(std::round(lo)) & 0xF);
}
}
// Decode from INT4
void EntropyInt4Quantizer::decode(const uint8_t *in, size_t dim,
float *out) const {
ailego_assert_with(dim % 2 == 0, "Dimension must be aligned with 2");
size_t size = dim / 2;
for (size_t i = 0; i < size; i += 1) {
uint8_t v = in[i];
int8_t lo = (static_cast<int8_t>(v << 4) >> 4);
int8_t hi = (static_cast<int8_t>(v & 0xf0) >> 4);
out[2 * i] = lo * this->scale_reciprocal() - this->bias();
out[2 * i + 1] = hi * this->scale_reciprocal() - this->bias();
}
}
// Feed the UINT4 quantizer
bool EntropyUInt4Quantizer::feed(const float *vec, size_t dim) {
UPDATE_HISTOGRAM(vec, dim)
}
// Train the UINT4 quantizer
bool EntropyUInt4Quantizer::train(void) {
TRAIN_QUANTIZER()
}
// Encode to INT4
void EntropyUInt4Quantizer::encode(const float *in, size_t dim,
uint8_t *out) const {
ailego_assert_with(dim % 2 == 0, "Dimension must be aligned with 2");
for (size_t i = 0; i < dim; i += 2) {
float lo = QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_);
float hi = QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i + 1], scale_, bias_);
out[i / 2] = (static_cast_from_float_to_uint8(std::round(hi)) << 4) |
(static_cast_from_float_to_uint8(std::round(lo)) & 0xF);
}
}
// Decode from INT4
void EntropyUInt4Quantizer::decode(const uint8_t *in, size_t dim,
float *out) const {
ailego_assert_with(dim % 2 == 0, "Dimension must be aligned with 2");
size_t size = dim / 2;
for (size_t i = 0; i < size; i += 1) {
uint8_t v = in[i];
out[2 * i] = (v & 0xf) * this->scale_reciprocal() - this->bias();
out[2 * i + 1] = (v >> 4) * this->scale_reciprocal() - this->bias();
}
}
} // namespace ailego
} // namespace zvec
+238
View File
@@ -0,0 +1,238 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <limits>
#include <vector>
#include <ailego/internal/platform.h>
namespace zvec {
namespace ailego {
/*! Entropy-based Integer Quantization Algorithm
*/
template <typename T, int RANGE_MIN, int RANGE_MAX>
class EntropyIntegerQuantizer {
public:
//! Primitive Built-in Types to store the quantized data
using ValueType = typename std::remove_cv<T>::type;
//! Constants
constexpr static int MIN_VALUE = RANGE_MIN;
constexpr static int MAX_VALUE = RANGE_MAX;
// Check supporting type
static_assert(std::is_integral<T>::value, "ValueType must be integral");
// Check template values
static_assert(RANGE_MIN < RANGE_MAX, "Invalid value range");
//! Constructor
EntropyIntegerQuantizer(void) {}
//! Set histogram bins in train
void set_histogram_bins(size_t bins) {
if (bins > (RANGE_MAX - RANGE_MIN)) {
histogram_bins_ = bins;
}
}
//! Set quantization params scale
void set_scale(float val) {
if (val > 0.0f) {
scale_ = val;
scale_reciprocal_ = 1 / scale_;
}
}
//! Set quantization params bias
void set_bias(float val) {
bias_ = val;
}
//! Set quantization params max
void set_max(float val) {
max_ = val;
}
//! Set quantization params min
void set_min(float val) {
min_ = val;
}
//! Set quantization params non bias
void set_non_bias(bool val) {
non_bias_ = val;
}
//! Get histogram bins in train
size_t histogram_bins(void) const {
return histogram_bins_;
}
//! Get quantization params scale
float scale(void) const {
return scale_;
}
//! Get quantization params bias
float bias(void) const {
return bias_;
}
//! Get quantization params max
float max(void) const {
return max_;
}
//! Get quantization params min
float min(void) const {
return min_;
}
//! Get quantization params non bias
bool non_bias(void) const {
return non_bias_;
}
//! Retrieve the scale reciprocal for decoding
float scale_reciprocal(void) const {
return scale_reciprocal_;
}
protected:
//! Disable them
EntropyIntegerQuantizer(const EntropyIntegerQuantizer &) = delete;
EntropyIntegerQuantizer &operator=(const EntropyIntegerQuantizer &) = delete;
//! Members
size_t histogram_bins_{0};
float hist_interval_{1.0f};
float max_{std::numeric_limits<float>::min()};
float min_{std::numeric_limits<float>::max()};
float bias_{0.0f};
float scale_{0.0f};
float scale_reciprocal_{0.0f};
float left_boundary_{0.0f};
bool non_bias_{false};
std::vector<uint32_t> histogram_{};
};
/*! INT16 Quantizer
*/
class EntropyInt16Quantizer
: public EntropyIntegerQuantizer<int16_t, -32767, 32767> {
public:
//! Feed the training data
bool feed(const float *vec, size_t dim);
//! Train the quantizer
bool train(void);
//! Encode float vector to int16
void encode(const float *in, size_t dim, ValueType *out) const;
//! Decode to float vector from int16
void decode(const ValueType *in, size_t dim, float *out) const;
};
/*! UINT16 Quantizer
*/
class EntropyUInt16Quantizer
: public EntropyIntegerQuantizer<uint16_t, 0, 65535> {
public:
//! Feed the training data
bool feed(const float *vec, size_t dim);
//! Train the quantizer
bool train(void);
//! Encode float vector to uint16
void encode(const float *in, size_t dim, ValueType *out) const;
//! Decode to float vector from uint16
void decode(const ValueType *in, size_t dim, float *out) const;
};
/*! INT8 Quantizer
*/
class EntropyInt8Quantizer : public EntropyIntegerQuantizer<int8_t, -127, 127> {
public:
//! Feed the training data
bool feed(const float *vec, size_t dim);
//! Train the quantizer
bool train(void);
//! Encode float vector to int8
void encode(const float *in, size_t dim, ValueType *out) const;
//! Decode to float vector from int8
void decode(const ValueType *in, size_t dim, float *out) const;
};
/*! UINT8 Quantizer
*/
class EntropyUInt8Quantizer : public EntropyIntegerQuantizer<uint8_t, 0, 255> {
public:
//! Feed the training data
bool feed(const float *vec, size_t dim);
//! Train the quantizer
bool train(void);
//! Encode float vector to uint8
void encode(const float *in, size_t dim, ValueType *out) const;
//! Decode to float vector from uint8
void decode(const ValueType *in, size_t dim, float *out) const;
};
/*! INT4 Quantizer
*/
class EntropyInt4Quantizer : public EntropyIntegerQuantizer<uint8_t, -8, 7> {
public:
//! Feed the training data
bool feed(const float *vec, size_t dim);
//! Train the quantizer
bool train(void);
//! Encode float vector to int4
void encode(const float *in, size_t dim, ValueType *out) const;
//! Decode to float vector from int4
void decode(const ValueType *in, size_t dim, float *out) const;
};
/*! UINT4 Quantizer
*/
class EntropyUInt4Quantizer : public EntropyIntegerQuantizer<uint8_t, 0, 15> {
public:
//! Feed the training data
bool feed(const float *vec, size_t dim);
//! Train the quantizer
bool train(void);
//! Encode float vector to uint4
void encode(const float *in, size_t dim, ValueType *out) const;
//! Decode to float vector from uint4
void decode(const ValueType *in, size_t dim, float *out) const;
};
} // namespace ailego
} // namespace zvec
File diff suppressed because it is too large Load Diff
+363
View File
@@ -0,0 +1,363 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <algorithm>
#include <random>
#include <ailego/parallel/lock.h>
#include <ailego/parallel/thread_pool.h>
#include <ailego/utility/type_helper.h>
namespace zvec {
namespace ailego {
/*! Random Centroids Generator
*/
template <typename T, typename TPool>
struct RandomCentroidsGenerator {
//! Type of values
using OwnerType = typename std::decay<T>::type;
using ContainerType = typename OwnerType::ContainerType;
using ContextType = typename OwnerType::ContextType;
using ThreadPoolType = TPool;
//! constexpr variables
constexpr static size_t BatchCount = OwnerType::BatchCount;
//! Generate centroids
void operator()(OwnerType *owner, ThreadPoolType &) const {
const auto &matrix = owner->feature_matrix();
const auto &cache = owner->feature_cache();
auto *centroids = owner->mutable_centroids();
ContainerType rows(cache.dimension());
size_t m = matrix.count();
size_t n = m + cache.count();
size_t k = owner->k_value();
std::mt19937 mt((std::random_device())());
rows.resize(BatchCount);
centroids->reset(cache.dimension());
centroids->reserve(k);
for (size_t i = 0; k > 0 && i < n; ++i) {
if (mt() % (n - i) >= k) {
continue;
}
// Selected a feature
if (i < m) {
ContextType::MatrixReverseTranspose(matrix[i / BatchCount * BatchCount],
matrix.dimension(), rows.data());
centroids->append(rows[i & (BatchCount - 1u)], matrix.dimension());
} else {
centroids->append(cache[i - m], cache.dimension());
}
--k;
}
}
};
/*! Lloyd's algorithm cluster
*/
template <typename T, typename TPool, typename TContext, typename TContainer>
class LloydCluster {
public:
//! constexpr variables
constexpr static size_t BatchCount = TContext::BatchCount;
//! Type of values
using ThreadPoolType = TPool;
using ContainerType = TContainer;
using ContextType = TContext;
using ValueType = typename TContext::ValueType;
using StoreType = typename TContext::StoreType;
//! Constructor
LloydCluster(size_t k, size_t dim)
: k_value_(k),
feature_cache_(dim),
feature_matrix_(dim),
centroids_matrix_(dim),
centroids_(dim) {}
//! Constructor
LloydCluster(size_t k, size_t dim, bool spherical)
: k_value_(k),
feature_cache_(dim),
feature_matrix_(dim),
centroids_matrix_(dim),
centroids_(dim),
spherical_{spherical} {}
//! Constructor
LloydCluster(void) {}
//! Destructor
~LloydCluster(void) {}
//! Append a feature
void append(const StoreType *arr, size_t dim) {
feature_cache_.append(arr, dim);
if (feature_cache_.count() == BatchCount) {
size_t pos = feature_matrix_.count();
feature_matrix_.resize(pos + BatchCount);
ContextType::MatrixTranspose(feature_cache_.data(), dim,
feature_matrix_[pos]);
feature_cache_.clear();
}
}
//! Reset cluster
void reset(size_t k, size_t dim) {
k_value_ = k;
feature_cache_.reset(dim);
feature_matrix_.reset(dim);
centroids_.reset(dim);
centroids_matrix_.reset(dim);
context_.clear();
}
//! Reset cluster
void reset(size_t k, size_t dim, bool spherical) {
k_value_ = k;
feature_cache_.reset(dim);
feature_matrix_.reset(dim);
centroids_.reset(dim);
centroids_matrix_.reset(dim);
context_.clear();
spherical_ = spherical;
}
//! Initialize centroids
template <typename G = RandomCentroidsGenerator<LloydCluster, ThreadPoolType>>
void init_centroids(ThreadPoolType &pool, const G &g = G()) {
g(this, pool);
}
//! Cluster one time
template <typename ThreadPoolType>
bool cluster_once(ThreadPoolType &pool, double *cost) {
if (centroids_.empty()) {
RandomCentroidsGenerator<LloydCluster, ThreadPoolType> g;
this->init_centroids(pool, g);
}
if (centroids_.count() != k_value_) {
return false;
}
context_.reset(centroids_.count(), centroids_.dimension());
size_t count = centroids_.count() / BatchCount * BatchCount;
centroids_matrix_.resize(count);
for (size_t i = 0; i != count; i += BatchCount) {
ContextType::MatrixTranspose(centroids_[i], centroids_.dimension(),
centroids_matrix_[i]);
}
size_t remain = static_cast<uint32_t>(centroids_.count() - count);
if (remain > 0) {
centroids_matrix_.append(centroids_[count], centroids_.dimension(),
remain);
}
// Using thread pool
auto group = pool.make_group();
if (!feature_matrix_.empty()) {
size_t n = feature_matrix_.count() / BatchCount;
size_t c = std::max<size_t>(n / pool.count() / 2u, 1u);
size_t m = n / c * c;
for (size_t i = 0; i != m; i += c) {
group->submit(Closure::New(this, &LloydCluster::cluster_matrix_features,
i, i + c));
}
for (size_t i = m; i != n; i += 1) {
group->submit(Closure::New(this, &LloydCluster::cluster_matrix_features,
i, i + 1));
}
}
if (!feature_cache_.empty()) {
group->submit(Closure::New(this, &LloydCluster::cluster_cache_features));
}
group->wait_finish();
*cost = 0.0;
for (size_t i = 0, n = centroids_.count(); i != n; ++i) {
const auto &item = context_[i];
item.centroid(centroids_[i], centroids_.dimension());
*cost += item.cost();
}
if (spherical_) {
for (size_t i = 0, n = centroids_.count(); i != n; ++i) {
float norm;
ContextType::Norm2(centroids_[i], centroids_.dimension(), &norm);
}
}
return true;
}
//! Retrieve the controids
ContainerType *mutable_centroids(void) {
return &centroids_;
}
//! Retrieve the controids
const ContainerType &centroids(void) const {
return centroids_;
}
//! Retrieve the K value
size_t k_value(void) const {
return k_value_;
}
//! Retrieve context
const ContextType &context(void) const {
return context_;
}
//! Retrieve the feature cache
const ContainerType &feature_cache(void) const {
return feature_cache_;
}
//! Retrieve the feature matrix
const ContainerType &feature_matrix(void) const {
return feature_matrix_;
}
//! Reserve the feature matrix
void feature_matrix_reserve(size_t count) {
feature_matrix_.reserve(count);
}
protected:
//! Cluster the cache features
void cluster_cache_features(void) {
float scores[BatchCount];
for (size_t i = 0, n = feature_cache_.count(); i != n; ++i) {
size_t count = centroids_matrix_.count() / BatchCount * BatchCount;
const StoreType *feature = feature_cache_[i];
float nearest_score = std::numeric_limits<float>::max();
size_t nearest_index = 0;
for (size_t j = 0; j != count; j += BatchCount) {
ContextType::template BatchDistance<1>(centroids_matrix_[j], feature,
centroids_matrix_.dimension(),
scores);
for (size_t k = 0; k < BatchCount; ++k) {
if (scores[k] < nearest_score) {
nearest_score = scores[k];
nearest_index = j + k;
}
}
} // end of for
for (size_t j = count, total = centroids_matrix_.count(); j != total;
++j) {
ContextType::Distance(centroids_matrix_[j], feature,
centroids_matrix_.dimension(), scores);
if (scores[0] < nearest_score) {
nearest_score = scores[0];
nearest_index = j;
}
}
context_[nearest_index].append(feature, feature_cache_.dimension(),
nearest_score);
} // end of for
}
//! Cluster the matrix features
void cluster_matrix_features(size_t first, size_t last) {
std::vector<float> scores(BatchCount * BatchCount, 0);
ContainerType rows(centroids_matrix_.dimension());
auto comp = [](float i, float j) {
if (std::isnan(i)) return false;
if (std::isnan(j)) return true;
return i < j;
};
float nearest_scores[BatchCount];
size_t nearest_indexes[BatchCount];
rows.resize(BatchCount);
for (size_t i = first * BatchCount; i != last * BatchCount;
i += BatchCount) {
size_t count = centroids_matrix_.count() / BatchCount * BatchCount;
const StoreType *block = feature_matrix_[i];
std::fill(nearest_indexes, nearest_indexes + BatchCount, 0);
std::fill(nearest_scores, nearest_scores + BatchCount,
std::numeric_limits<float>::max());
for (size_t j = 0; j != count; j += BatchCount) {
ContextType::template BatchDistance<BatchCount>(
centroids_matrix_[j], block, centroids_matrix_.dimension(),
&scores[0]);
for (size_t k = 0; k < BatchCount; ++k) {
const float *start = &scores[k * BatchCount];
const float *result =
std::min_element(start, start + BatchCount, comp);
if (*result < nearest_scores[k]) {
nearest_scores[k] = *result;
nearest_indexes[k] = j + (result - start);
}
}
} // end of for
for (size_t j = count, total = centroids_matrix_.count(); j != total;
++j) {
ContextType::template BatchDistance<1>(block, centroids_matrix_[j],
centroids_matrix_.dimension(),
&scores[0]);
for (size_t k = 0; k < BatchCount; ++k) {
float score = scores[k];
if (score < nearest_scores[k]) {
nearest_scores[k] = score;
nearest_indexes[k] = j;
}
}
} // end of for
ContextType::MatrixReverseTranspose(block, feature_matrix_.dimension(),
rows.data());
for (size_t k = 0; k < BatchCount; ++k) {
context_[nearest_indexes[k]].append(
rows[k], feature_matrix_.dimension(), nearest_scores[k]);
}
} // end of for
}
private:
//! Members
size_t k_value_{0u};
ContainerType feature_cache_{};
ContainerType feature_matrix_{};
ContainerType centroids_matrix_{};
ContainerType centroids_{};
ContextType context_{};
bool spherical_{false};
};
} // namespace ailego
} // namespace zvec
+579
View File
@@ -0,0 +1,579 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "buffer_manager.h"
#include <atomic>
#include <mutex>
#include <ailego/logger/logger.h>
#include <ailego/pattern/defer.h>
#include <arrow/io/api.h>
#include <parquet/arrow/reader.h>
namespace zvec {
namespace ailego {
namespace {
struct IDHash {
size_t operator()(const BufferID &buffer_id) const {
size_t hash = std::hash<int>{}(static_cast<int>(buffer_id.type));
hash = hash ^ (std::hash<uint64_t>{}(buffer_id.file_id));
if (buffer_id.type == BufferID::TYPE::PARQUET) {
hash = hash * 31 + std::hash<int>{}(buffer_id.parquet().column);
hash = hash * 31 + std::hash<int>{}(buffer_id.parquet().row_group);
} else if (buffer_id.type == BufferID::TYPE::VECTOR) {
hash = hash * 31 + std::hash<uint32_t>{}(buffer_id.vector().offset);
}
return hash;
}
};
struct IDEqual {
bool operator()(const BufferID &a, const BufferID &b) const {
if (a.type != b.type) {
return false;
}
if (a.file_name != b.file_name) {
return false;
}
if (a.file_id != b.file_id) {
return false;
}
if (a.mtime != b.mtime) {
return false;
}
if (a.type == BufferID::TYPE::PARQUET) {
return a.parquet().column == b.parquet().column &&
a.parquet().row_group == b.parquet().row_group;
} else if (a.type == BufferID::TYPE::VECTOR) {
return a.vector().offset == b.vector().offset;
} else {
return false;
}
}
};
} // namespace
struct BufferManager::BufferContext {
BufferContext(const BufferID &id, BufferPool *p) : id(id), pool(p) {};
BufferContext(const BufferContext &) = delete;
BufferContext(BufferContext &&) = delete;
BufferContext &operator=(const BufferContext &) = delete;
BufferContext &operator=(BufferContext &&) = delete;
~BufferContext() {
if (vector) {
free(vector);
}
}
typedef std::unique_ptr<BufferManager::BufferContext> Pointer;
enum State : uint32_t {
IDLE = 0, // Empty and not held by any users, not in LRU
RESERVED = 1, // Pinned by a user but no data yet, not in LRU
IN_USE = 2, // Pinned by a user and data is present, not in LRU
CACHED = 3, // Data is present but not held by any users, in LRU
ERROR = 4 // Something went wrong, not in LRU
};
// Identifier for the buffer
BufferID id;
// Current state
State state{IDLE};
// The size of the buffer
uint32_t size{0};
// Handle of the file backing this buffer
File file;
// The number of external references to this buffer (via pin/unpin)
std::atomic<uint32_t> refs_buf{0};
// The number of external references to this context (via BufferHandle)
std::atomic<uint32_t> refs_context{0};
BufferPool *pool{nullptr};
// A shared pointer to the buffers allocated for arrow parquet data
std::shared_ptr<arrow::ChunkedArray> arrow{nullptr};
// Guard original arrow buffers to prevent premature deletion
std::vector<std::shared_ptr<arrow::Buffer>> arrow_refs{};
// A pointer to the buffer allocated for vector data
void *vector{nullptr};
// Doubly linked LRU list
BufferContext *next{nullptr};
BufferContext *prev{nullptr};
// Return a string representation of the status
const std::string status_string() const;
// Populate the buffer with parquet data
arrow::Status read_arrow_parquet();
// Populate the buffer with vector data
bool read_vector();
};
const std::string BufferManager::BufferContext::status_string() const {
std::string msg{id.to_string() + ": "};
switch (state) {
case State::IDLE: {
msg += "Idle";
break;
}
case State::RESERVED: {
msg += "Reserved";
break;
}
case State::IN_USE: {
msg += "In use";
break;
}
case State::CACHED: {
msg += "Cached";
break;
}
case State::ERROR: {
msg += "Error";
break;
}
}
return msg;
}
arrow::Status BufferManager::BufferContext::read_arrow_parquet() {
// TODO: file handler and memory pool can be optimized
arrow::MemoryPool *mem_pool = arrow::default_memory_pool();
// Open file
std::shared_ptr<arrow::io::RandomAccessFile> input;
const auto &file_name = id.file_name;
ARROW_ASSIGN_OR_RAISE(input, arrow::io::ReadableFile::Open(file_name));
// Open reader
std::unique_ptr<parquet::arrow::FileReader> reader;
ARROW_ASSIGN_OR_RAISE(reader, parquet::arrow::OpenFile(input, mem_pool));
// Perform read
int row_group = id.parquet().row_group;
int column = id.parquet().column;
auto s = reader->RowGroup(row_group)->Column(column)->Read(&arrow);
if (!s.ok()) {
LOG_ERROR("Failed to read parquet file[%s]", file_name.c_str());
arrow = nullptr;
return s;
}
// Compute the memory usage and hijack Arrow's buffers with our implementation
for (auto &array : arrow->chunks()) {
auto &buffers = array->data()->buffers;
for (size_t buf_idx = 0; buf_idx < buffers.size(); ++buf_idx) {
if (buffers[buf_idx] == nullptr) {
continue;
}
// Keep references to original buffers to prevent premature deletion
arrow_refs.emplace_back(buffers[buf_idx]);
size += buffers[buf_idx]->capacity();
// Create hijacked buffer with custom deleter that notifies us when Arrow
// is finished with the buffer
std::shared_ptr<arrow::Buffer> hijacked_buffer(
buffers[buf_idx].get(), BufferManager::ArrowBufferDeleter(this));
buffers[buf_idx] = hijacked_buffer;
}
}
return arrow::Status::OK();
}
bool BufferManager::BufferContext::read_vector() {
const auto &file_name = id.file_name;
if (!file.is_valid()) {
if (!File::IsExist(file_name)) {
LOG_ERROR("File[%s] does not exist", file_name.c_str());
return false;
}
if (!File::IsRegular(file_name)) {
LOG_ERROR("[%s] is not a regular file", file_name.c_str());
return false;
}
if (!file.open(file_name.c_str(), true, false)) {
LOG_ERROR("Failed to open file[%s]", file_name.c_str());
return false;
}
}
AILEGO_DEFER([this] { file.close(); });
uint32_t len = id.vector().length;
auto ret = posix_memalign((void **)&vector, 64, len); // 64-byte alignment
if (ret != 0 || vector == nullptr) {
LOG_ERROR("Failed to allocate buffer for file[%s]", file_name.c_str());
return false;
}
uint32_t offset = id.vector().offset;
if (file.read(offset, vector, len) != len) {
LOG_ERROR("Failed to read file[%s]", file_name.c_str());
free(vector);
vector = nullptr;
return false;
}
size = len;
return true;
}
// Thread-safe buffer pool implementation.
//
// BufferContext states:
// 1. Must exist in the lookup (hash) table.
// 2. LRU list presence:
// - In LRU: holds memory but not pinned by any users
// - Not in LRU: either holds memory pinned by users, or doesn't hold memory
// 3. External references: when an external user acquires a context and pins the
// memory, that context is removed from LRU list; when they unpins the
// memory, that context is moved to LRU list if it was the last reference.
//
// Any operation on the hash table is protected by mutex_table_.
// Any change to context state and LRU list is protected by mutex_context_.
//
class BufferManager::BufferPool {
public:
explicit BufferPool(uint64_t limit) : limit_(limit) {
sentinel_.next = &sentinel_;
sentinel_.prev = &sentinel_;
}
BufferContext *acquire_locked(BufferID &id) {
std::lock_guard<std::mutex> lock(mutex_context_);
if (auto iter = table_.find(id); iter != table_.end()) {
return iter->second.get();
}
auto [iter, _] =
table_.emplace(id, std::make_unique<BufferContext>(id, this));
return iter->second.get();
}
void try_release_context_locked(BufferContext *context) {
if (context->refs_context.load() != 0) {
return;
}
std::lock_guard<std::mutex> lock(mutex_table_);
if (context->refs_context.load() != 0) {
return;
}
if (context->state == BufferContext::State::IDLE) {
table_.erase(context->id);
}
}
void pin_locked(BufferContext *ctx) {
std::lock_guard<std::mutex> lock(mutex_context_);
if (ctx->state == BufferContext::State::IDLE) {
return pin_at_IDLE(ctx);
}
if (ctx->state == BufferContext::State::IN_USE) {
return pin_at_IN_USE(ctx);
}
if (ctx->state == BufferContext::State::CACHED) {
return pin_at_CACHED(ctx);
}
if (ctx->state == BufferContext::State::ERROR) {
return;
}
}
bool unpin_locked(BufferContext *ctx) {
uint32_t prev_refs = ctx->refs_buf.fetch_sub(1);
if (prev_refs > 1) {
return false;
}
std::lock_guard<std::mutex> lock(mutex_context_);
if (ctx->refs_buf.load() == 0 &&
ctx->state != BufferContext::State::CACHED) {
ctx->state = BufferContext::State::CACHED;
LRU_insert(ctx);
return true;
} else {
return false;
}
}
void LRU_insert_locked(BufferContext *context) {
std::lock_guard<std::mutex> lock(mutex_context_);
LRU_insert(context);
}
void LRU_remove_locked(BufferContext *context) {
std::lock_guard<std::mutex> lock(mutex_context_);
LRU_remove(context);
}
uint64_t usage() const {
return usage_;
}
private:
void pin_at_IDLE(BufferContext *ctx) {
ctx->state = BufferContext::State::RESERVED;
while (usage_ >= limit_) {
// The tail of LRU list is the least recently used context
BufferContext *victim = sentinel_.prev;
if (victim == &sentinel_) { // No victim could be found
ctx->state = BufferContext::State::ERROR;
return;
}
if (victim->state == BufferContext::State::ERROR) {
LRU_remove(victim);
try_release_context_locked(ctx);
continue;
}
if (victim->id.type == BufferID::TYPE::PARQUET) {
victim->arrow_refs.clear();
} else {
free(victim->vector);
victim->vector = nullptr;
}
victim->state = BufferContext::State::IDLE;
LRU_remove(victim);
try_release_context_locked(ctx);
usage_ -= victim->size;
}
if (ctx->id.type == BufferID::TYPE::PARQUET) {
if (ctx->read_arrow_parquet().ok()) {
ctx->state = BufferContext::State::IN_USE;
ctx->refs_buf.fetch_add(ctx->arrow_refs.size());
usage_ += ctx->size;
} else {
LOG_ERROR("Failed to read to %s", ctx->id.to_string().c_str());
ctx->state = BufferContext::State::ERROR;
}
} else {
if (ctx->read_vector()) {
ctx->state = BufferContext::State::IN_USE;
ctx->refs_buf.fetch_add(1);
usage_ += ctx->size;
} else {
LOG_ERROR("Failed to read to %s", ctx->id.to_string().c_str());
ctx->state = BufferContext::State::ERROR;
}
}
}
void pin_at_IN_USE(BufferContext *ctx) {
if (ctx->id.type == BufferID::TYPE::PARQUET) {
ctx->refs_buf.fetch_add(ctx->arrow_refs.size());
} else {
ctx->refs_buf.fetch_add(1);
}
}
void pin_at_CACHED(BufferContext *ctx) {
if (ctx->id.type == BufferID::TYPE::PARQUET) {
ctx->refs_buf.fetch_add(ctx->arrow_refs.size());
} else {
ctx->refs_buf.fetch_add(1);
}
LRU_remove(ctx);
ctx->state = BufferContext::State::IN_USE;
}
void LRU_insert(BufferContext *context) {
if (context->refs_buf > 0) {
return; // Already pinned, should not be evicted
}
if (context->next != nullptr || context->prev != nullptr) {
return;
}
// Insert the context to the head of LRU list
context->next = sentinel_.next;
context->prev = &sentinel_;
sentinel_.next = context;
context->next->prev = context;
inactive_ += context->size;
}
void LRU_remove(BufferContext *context) {
if (context->next == nullptr) {
return; // Not in LRU list
}
context->next->prev = context->prev;
context->prev->next = context->next;
context->next = nullptr;
context->prev = nullptr;
inactive_ -= context->size;
}
private:
using Table =
std::unordered_map<BufferID, BufferContext::Pointer, IDHash, IDEqual>;
uint64_t limit_;
std::atomic<uint64_t> usage_{0};
std::atomic<uint64_t> inactive_{0};
Table table_{};
std::mutex mutex_table_{};
BufferContext sentinel_{BufferID{}, this}; // LRU list sentinel
std::mutex mutex_context_{};
};
BufferManager::ArrowBufferDeleter::ArrowBufferDeleter(BufferContext *c)
: context(c) {}
void BufferManager::ArrowBufferDeleter::operator()(arrow::Buffer *) {
context->pool->unpin_locked(context);
}
BufferHandle::BufferHandle(BufferContext *context) : context_(context) {
if (context_ != nullptr) {
pool_ = context_->pool;
context_->refs_context.fetch_add(1);
}
}
BufferHandle::~BufferHandle() {
if (context_ != nullptr) {
uint32_t prev_refs = context_->refs_context.fetch_sub(1);
if (prev_refs > 1) {
return;
}
if (context_->state == BufferContext::State::IDLE) {
pool_->try_release_context_locked(context_);
}
}
}
std::shared_ptr<arrow::ChunkedArray> BufferHandle::pin_parquet_data() {
pool_->pin_locked(context_);
return context_->arrow;
}
void *BufferHandle::pin_vector_data() {
if (!context_) {
return nullptr;
}
pool_->pin_locked(context_);
return context_->vector;
}
bool BufferHandle::unpin_vector_data() {
if (!context_) {
return true;
}
return pool_->unpin_locked(context_);
}
uint32_t BufferHandle::references() const {
return context_->refs_buf.load();
}
uint32_t BufferHandle::size() const {
return context_->size;
}
void BufferManager::init(uint64_t limit, uint32_t num_shards) {
pools_.clear();
uint64_t limit_per_shard = ailego_align(limit / num_shards, 4096);
for (uint32_t i = 0; i < num_shards; ++i) {
auto pool = new BufferPool(limit_per_shard);
pools_.push_back(pool);
}
LOG_INFO(
"BufferManager initialized with [%u] buffer pools, [%zu] bytes memory "
"limit per pool, total memory limit [%zu] bytes",
num_shards, (size_t)limit_per_shard, (size_t)limit);
}
BufferHandle BufferManager::acquire(BufferID &buffer_id) {
static IDHash id_hash{};
auto hash_val = id_hash(buffer_id);
auto ctx = pools_[hash_val % pools_.size()]->acquire_locked(buffer_id);
return BufferHandle(ctx);
}
std::unique_ptr<BufferHandle> BufferManager::acquire_ptr(BufferID &buffer_id) {
static IDHash id_hash{};
auto hash_val = id_hash(buffer_id);
auto ctx = pools_[hash_val % pools_.size()]->acquire_locked(buffer_id);
return std::make_unique<BufferHandle>(ctx);
}
uint64_t BufferManager::total_size_in_bytes() const {
uint64_t total_usage = 0;
for (auto pool : pools_) {
total_usage += pool->usage();
}
return total_usage;
}
BufferManager::~BufferManager() {
for (auto pool : pools_) {
delete pool;
}
}
} // namespace ailego
} // namespace zvec
+269
View File
@@ -0,0 +1,269 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <sys/stat.h>
#include <chrono>
#include <cstdint>
#include <filesystem>
#include <memory>
#include <ailego/io/file.h>
#ifdef __clang__
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wunused-parameter"
#pragma clang diagnostic ignored "-Wshadow"
#elif defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wunused-parameter"
#pragma GCC diagnostic ignored "-Wshadow"
#endif
#include <arrow/api.h>
#ifdef __clang__
#pragma clang diagnostic pop
#elif defined(__GNUC__) || defined(__GNUG__)
#pragma GCC diagnostic pop
#endif
#include "ailego/pattern/singleton.h"
namespace zvec {
namespace ailego {
struct BufferID;
class BufferManager;
class BufferHandle;
struct BufferID {
struct ParquetPos {
int column;
int row_group;
};
struct VectorPos {
uint32_t offset;
uint32_t length;
};
union Position {
explicit Position() = default;
ParquetPos forward;
VectorPos vector;
};
enum TYPE {
PARQUET = 1,
VECTOR = 2,
UNKNOWN = 0,
};
static std::uint64_t getLastModifiedNs(const std::filesystem::path &p) {
auto ftime = std::filesystem::last_write_time(p);
return static_cast<std::uint64_t>(ftime.time_since_epoch().count());
}
// Cross-platform helper to get nanosecond modification time
// static long get_st_mtime_nsec(const struct stat &file_stat) {
// #ifdef __APPLE__
// return file_stat.st_mtim.tv_nsec;
// #else
// return file_stat.st_mtim.tv_nsec;
// #endif
// }
static BufferID ParquetID(const std::string &file_name, int column,
int row_group) {
BufferID buffer_id{};
buffer_id.type = TYPE::PARQUET;
buffer_id.file_name = file_name;
buffer_id.pos.forward.column = column;
buffer_id.pos.forward.row_group = row_group;
struct stat file_stat;
if (stat(file_name.c_str(), &file_stat) == 0) {
// file_stat.st_ino contains the inode number
// file_stat.st_dev contains the device ID
// Together they uniquely identify a file
buffer_id.file_id = file_stat.st_ino;
std::filesystem::path p(file_name);
buffer_id.mtime = getLastModifiedNs(p);
}
return buffer_id;
}
static BufferID VectorID(const std::string &file_name, uint32_t offset,
uint32_t length) {
BufferID buffer_id{};
buffer_id.type = TYPE::VECTOR;
buffer_id.file_name = file_name;
struct stat file_stat;
if (stat(file_name.c_str(), &file_stat) == 0) {
buffer_id.file_id = file_stat.st_ino;
std::filesystem::path p(file_name);
buffer_id.mtime = getLastModifiedNs(p);
}
buffer_id.pos.vector.offset = offset;
buffer_id.pos.vector.length = length;
return buffer_id;
}
explicit BufferID() = default;
// Type of the file backing this buffer
TYPE type{UNKNOWN};
// Name of the file backing this buffer
std::string file_name{};
// Unique file id
uint64_t file_id{};
long mtime{};
// To identify which part of the backing file should be loaded into the buffer
Position pos{};
// Get the forward ID
const inline struct ParquetPos &parquet() const {
return pos.forward;
}
// Get the vector ID
const inline struct VectorPos &vector() const {
return pos.vector;
}
// Get debug string
const std::string to_string() const {
std::string msg{"Buffer["};
if (type == TYPE::PARQUET) {
msg += "parquet: " + file_name + "[" + std::to_string(file_id) + "]" +
", column: " + std::to_string(parquet().column) +
", row_group: " + std::to_string(parquet().row_group);
} else if (type == TYPE::VECTOR) {
msg += "vector: " + file_name + "[" + std::to_string(file_id) + "]" +
", offset: " + std::to_string(vector().offset);
} else {
msg += "unknown";
}
msg += ", mtime: " + std::to_string(mtime);
msg += "]";
return msg;
}
};
// Thread-safe LRU buffer implementation.
class BufferManager : public Singleton<BufferManager> {
friend BufferHandle;
public:
void init(uint64_t limit, uint32_t num_shards = 1);
BufferHandle acquire(BufferID &buffer_id);
std::unique_ptr<BufferHandle> acquire_ptr(BufferID &buffer_id);
uint64_t total_size_in_bytes() const;
~BufferManager();
private:
struct BufferContext;
class BufferPool;
// Custom deleter for Arrow buffer that automatically notifies us when the
// buffer is no longer referenced by Arrow
struct ArrowBufferDeleter {
explicit ArrowBufferDeleter(BufferContext *c);
BufferContext *context;
// Only reduces the reference count but does not actually release the
// buffer, since the buffer memory is managed by the BufferManager.
void operator()(arrow::Buffer *);
};
std::vector<BufferPool *> pools_;
};
class BufferHandle {
public:
typedef std::unique_ptr<BufferHandle> Pointer;
explicit BufferHandle(BufferManager::BufferContext *context = nullptr);
BufferHandle(const BufferHandle &) = delete;
BufferHandle(BufferHandle &&) = default;
BufferHandle &operator=(const BufferHandle &) = delete;
BufferHandle &operator=(BufferHandle &&) = default;
~BufferHandle();
// Pin parquet data in memory by allocating arrow buffers of appropriate size
// and reading data from the backing file.
// The lifecycle of the allocated memory is automatically managed through
// shared pointers. The buffers are guaranteed to be held until they are not
// referenced.
// Returns a pointer to the loaded ChunkedArray in Arrow format.
std::shared_ptr<arrow::ChunkedArray> pin_parquet_data();
// Pin vector data in memory by allocating a buffer of appropriate size and
// loading data from the backing file.
// The memory is guaranteed to be held until unpin() is called. The caller
// must call unpin() to release the memory when it is no longer needed.
// Returns a raw memory address.
void *pin_vector_data();
// Reduce the reference count for this vector buffer.
// Returns true if this was the last reference.
// When reference count is zero, the buffer is moved to the eviction list and
// becomes eligible for removal under memory pressure.
bool unpin_vector_data();
// Get the current reference count.
uint32_t references() const;
// Get the buffer size.
uint32_t size() const;
private:
using BufferContext = BufferManager::BufferContext;
using BufferPool = BufferManager::BufferPool;
BufferContext *context_;
BufferPool *pool_;
};
} // namespace ailego
} // namespace zvec
+443
View File
@@ -0,0 +1,443 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "bitmap.h"
namespace zvec {
namespace ailego {
size_t Bitset::BitwiseAndCardinality(const Bitset &lhs, const Bitset &rhs) {
return BitsetHelper::BitwiseAndCardinality(
lhs.array_.data(), rhs.array_.data(),
std::min(lhs.array_.size(), rhs.array_.size()));
}
size_t Bitset::BitwiseAndnotCardinality(const Bitset &lhs, const Bitset &rhs) {
size_t lsize = lhs.array_.size();
size_t rsize = rhs.array_.size();
if (lsize > rsize) {
return (
BitsetHelper::BitwiseAndnotCardinality(lhs.array_.data(),
rhs.array_.data(), rsize) +
BitsetHelper::Cardinality(lhs.array_.data() + rsize, lsize - rsize));
}
return BitsetHelper::BitwiseAndnotCardinality(lhs.array_.data(),
rhs.array_.data(), lsize);
}
size_t Bitset::BitwiseXorCardinality(const Bitset &lhs, const Bitset &rhs) {
size_t lsize = lhs.array_.size();
size_t rsize = rhs.array_.size();
if (lsize < rsize) {
return (
BitsetHelper::BitwiseXorCardinality(lhs.array_.data(),
rhs.array_.data(), lsize) +
BitsetHelper::Cardinality(rhs.array_.data() + lsize, rsize - lsize));
} else if (lsize > rsize) {
return (
BitsetHelper::BitwiseXorCardinality(lhs.array_.data(),
rhs.array_.data(), rsize) +
BitsetHelper::Cardinality(lhs.array_.data() + rsize, lsize - rsize));
}
return BitsetHelper::BitwiseXorCardinality(lhs.array_.data(),
rhs.array_.data(), lsize);
}
size_t Bitset::BitwiseOrCardinality(const Bitset &lhs, const Bitset &rhs) {
size_t lsize = lhs.array_.size();
size_t rsize = rhs.array_.size();
if (lsize < rsize) {
return (
BitsetHelper::BitwiseOrCardinality(lhs.array_.data(), rhs.array_.data(),
lsize) +
BitsetHelper::Cardinality(rhs.array_.data() + lsize, rsize - lsize));
} else if (lsize > rsize) {
return (
BitsetHelper::BitwiseOrCardinality(lhs.array_.data(), rhs.array_.data(),
rsize) +
BitsetHelper::Cardinality(lhs.array_.data() + rsize, lsize - rsize));
}
return BitsetHelper::BitwiseOrCardinality(lhs.array_.data(),
rhs.array_.data(), lsize);
}
void Bitmap::clear(void) {
for (std::vector<Bucket *>::iterator iter = array_.begin();
iter != array_.end(); ++iter) {
delete (*iter);
}
array_.clear();
}
void Bitmap::copy(const Bitmap &rhs) {
this->clear();
for (std::vector<Bucket *>::const_iterator iter = rhs.array_.begin();
iter != rhs.array_.end(); ++iter) {
Bucket *bucket = NULL;
if (*iter) {
bucket = new Bucket(*(*iter));
}
array_.push_back(bucket);
}
}
void Bitmap::shrink_to_fit(void) {
size_t shrink_count = 0;
std::vector<Bucket *>::reverse_iterator iter;
for (iter = array_.rbegin(); iter != array_.rend(); ++iter) {
if (*iter) {
if (!(*iter)->test_none()) {
break;
}
delete (*iter);
*iter = NULL;
}
++shrink_count;
}
for (; iter != array_.rend(); ++iter) {
if ((*iter) && (*iter)->test_none()) {
delete (*iter);
*iter = NULL;
}
}
if (shrink_count != 0) {
array_.resize(array_.size() - shrink_count);
}
}
bool Bitmap::test(size_t num) const {
// High 16 bits
size_t offset = num >> 16;
if (offset < array_.size()) {
const Bucket *bucket = array_[offset];
if (bucket) {
// Low 16 bits
return bucket->test(static_cast<uint16_t>(num));
}
}
return false;
}
void Bitmap::set(size_t num) {
// High 16 bits
size_t offset = num >> 16;
if (offset >= array_.size()) {
array_.resize(offset + 1, NULL);
}
Bucket *&bucket = array_[offset];
if (!bucket) {
bucket = new Bucket;
}
// Low 16 bits
bucket->set(static_cast<uint16_t>(num));
}
void Bitmap::reset(size_t num) {
// High 16 bits
size_t offset = num >> 16;
if (offset >= array_.size()) {
array_.resize(offset + 1, NULL);
}
if (offset < array_.size()) {
Bucket *bucket = array_[offset];
if (bucket) {
// Low 16 bits
bucket->reset(static_cast<uint16_t>(num));
}
}
}
void Bitmap::flip(size_t num) {
// High 16 bits
uint16_t offset = (uint16_t)(num >> 16);
if (offset >= array_.size()) {
array_.resize(offset + 1, NULL);
}
Bucket *&bucket = array_[offset];
if (!bucket) {
bucket = new Bucket;
}
// Low 16 bits
bucket->flip(static_cast<uint16_t>(num));
}
void Bitmap::bitwise_and(const Bitmap &rhs) {
size_t overlap = std::min(array_.size(), rhs.array_.size());
for (size_t i = 0; i < overlap; ++i) {
Bucket *&dst = array_[i];
if (dst) {
const Bucket *src = rhs.array_[i];
if (src) {
dst->bitwise_and(*src);
} else {
delete dst;
dst = NULL;
}
}
}
for (size_t i = overlap; i < array_.size(); ++i) {
Bucket *&dst = array_[i];
delete dst;
dst = NULL;
}
}
void Bitmap::bitwise_andnot(const Bitmap &rhs) {
size_t overlap = std::min(array_.size(), rhs.array_.size());
for (size_t i = 0; i < overlap; ++i) {
Bucket *&dst = array_[i];
if (dst) {
const Bucket *src = rhs.array_[i];
if (src) {
dst->bitwise_andnot(*src);
}
}
}
}
void Bitmap::bitwise_or(const Bitmap &rhs) {
size_t overlap = std::min(array_.size(), rhs.array_.size());
for (size_t i = 0; i < overlap; ++i) {
const Bucket *src = rhs.array_[i];
if (src) {
Bucket *&dst = array_[i];
if (dst) {
dst->bitwise_or(*src);
} else {
dst = new Bucket(*src);
}
}
}
for (size_t i = overlap; i < rhs.array_.size(); ++i) {
const Bucket *src = rhs.array_[i];
Bucket *bucket = NULL;
if (src) {
bucket = new Bucket(*src);
}
array_.push_back(bucket);
}
}
void Bitmap::bitwise_xor(const Bitmap &rhs) {
size_t overlap = std::min(array_.size(), rhs.array_.size());
for (size_t i = 0; i < overlap; ++i) {
const Bucket *src = rhs.array_[i];
if (src) {
Bucket *&dst = array_[i];
if (dst) {
dst->bitwise_xor(*src);
} else {
dst = new Bucket(*src);
}
}
}
for (size_t i = overlap; i < rhs.array_.size(); ++i) {
const Bucket *src = rhs.array_[i];
Bucket *bucket = NULL;
if (src) {
bucket = new Bucket(*src);
}
array_.push_back(bucket);
}
}
void Bitmap::bitwise_not(void) {
for (std::vector<Bucket *>::iterator iter = array_.begin();
iter != array_.end(); ++iter) {
Bucket *&bucket = *iter;
if (!bucket) {
bucket = new Bucket;
}
bucket->bitwise_not();
}
}
bool Bitmap::test_all(void) const {
if (array_.empty()) {
return false;
}
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
iter != array_.end(); ++iter) {
if (!(*iter) || !(*iter)->test_all()) {
return false;
}
}
return true;
}
bool Bitmap::test_any(void) const {
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
iter != array_.end(); ++iter) {
if (*iter && (*iter)->test_any()) {
return true;
}
}
return false;
}
bool Bitmap::test_none(void) const {
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
iter != array_.end(); ++iter) {
if (*iter && !(*iter)->test_none()) {
return false;
}
}
return true;
}
size_t Bitmap::cardinality(void) const {
size_t result = 0;
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
iter != array_.end(); ++iter) {
if (*iter) {
result += (*iter)->cardinality();
}
}
return result;
}
void Bitmap::extract(size_t base, std::vector<size_t> *out) const {
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
iter != array_.end(); ++iter) {
if (*iter) {
(*iter)->extract(base, out);
}
base += Bucket::MAX_SIZE;
}
}
size_t Bitmap::BitwiseAndCardinality(const Bitmap &lhs, const Bitmap &rhs) {
size_t overlap = std::min(lhs.array_.size(), rhs.array_.size());
size_t dist = 0;
for (size_t i = 0; i < overlap; ++i) {
const Bucket *l = lhs.array_[i];
const Bucket *r = rhs.array_[i];
if (l && r) {
dist += Bucket::BitwiseAndCardinality(*l, *r);
}
}
return dist;
}
size_t Bitmap::BitwiseAndnotCardinality(const Bitmap &lhs, const Bitmap &rhs) {
size_t overlap = std::min(lhs.array_.size(), rhs.array_.size());
size_t dist = 0;
for (size_t i = 0; i < overlap; ++i) {
const Bucket *l = lhs.array_[i];
if (l) {
const Bucket *r = rhs.array_[i];
if (r) {
dist += Bucket::BitwiseAndnotCardinality(*l, *r);
} else {
dist += l->cardinality();
}
}
}
for (size_t i = overlap; i < lhs.array_.size(); ++i) {
const Bucket *l = lhs.array_[i];
if (l) {
dist += l->cardinality();
}
}
return dist;
}
size_t Bitmap::BitwiseXorCardinality(const Bitmap &lhs, const Bitmap &rhs) {
size_t overlap = std::min(lhs.array_.size(), rhs.array_.size());
size_t dist = 0;
for (size_t i = 0; i < overlap; ++i) {
const Bucket *l = lhs.array_[i];
const Bucket *r = rhs.array_[i];
if (l && r) {
dist += Bucket::BitwiseXorCardinality(*l, *r);
} else if (l) {
dist += l->cardinality();
} else if (r) {
dist += r->cardinality();
}
}
for (size_t i = overlap; i < lhs.array_.size(); ++i) {
const Bucket *l = lhs.array_[i];
if (l) {
dist += l->cardinality();
}
}
for (size_t i = overlap; i < rhs.array_.size(); ++i) {
const Bucket *r = rhs.array_[i];
if (r) {
dist += r->cardinality();
}
}
return dist;
}
size_t Bitmap::BitwiseOrCardinality(const Bitmap &lhs, const Bitmap &rhs) {
size_t overlap = std::min(lhs.array_.size(), rhs.array_.size());
size_t dist = 0;
for (size_t i = 0; i < overlap; ++i) {
const Bucket *l = lhs.array_[i];
const Bucket *r = rhs.array_[i];
if (l && r) {
dist += Bucket::BitwiseOrCardinality(*l, *r);
} else if (l) {
dist += l->cardinality();
} else if (r) {
dist += r->cardinality();
}
}
for (size_t i = overlap; i < lhs.array_.size(); ++i) {
const Bucket *l = lhs.array_[i];
if (l) {
dist += l->cardinality();
}
}
for (size_t i = overlap; i < rhs.array_.size(); ++i) {
const Bucket *r = rhs.array_[i];
if (r) {
dist += r->cardinality();
}
}
return dist;
}
} // namespace ailego
} // namespace zvec
+503
View File
@@ -0,0 +1,503 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <algorithm>
#include <vector>
#include <ailego/utility/bitset_helper.h>
namespace zvec {
namespace ailego {
/*! Fixed Bitset
*/
template <size_t N, typename = typename std::enable_if<N % 32 == 0>::type>
class FixedBitset {
public:
enum { MAX_SIZE = N };
//! Constructor
FixedBitset(void) {
memset(array_, 0, sizeof(array_));
}
//! Constructor
FixedBitset(const FixedBitset &rhs) {
memcpy(array_, rhs.array_, sizeof(array_));
}
//! Destructor
~FixedBitset(void) {}
//! Assignment
FixedBitset &operator=(const FixedBitset &rhs) {
memcpy(array_, rhs.array_, sizeof(array_));
return *this;
}
//! Retrieve data pointer
uint32_t *data(void) {
return reinterpret_cast<uint32_t *>(array_);
}
//! Retrieve data pointer
const uint32_t *data(void) const {
return reinterpret_cast<const uint32_t *>(array_);
}
//! Retrieve count of bits in set
constexpr size_t size(void) const {
return MAX_SIZE;
}
// !Clear the bitset
void clear(void) {
memset(array_, 0, sizeof(array_));
}
//! Test a bit in bitset
bool test(size_t num) const {
ailego_assert_with(N > num, "overflow argument");
return ((array_[num >> 5] & (1u << (num & 0x1f))) != 0);
}
//! Set a bit in bitset
void set(size_t num) {
ailego_assert_with(N > num, "overflow argument");
uint32_t mask = (1u << (num & 0x1f));
array_[num >> 5] |= mask;
}
//! Clear a bit in bitset
void reset(size_t num) {
ailego_assert_with(N > num, "overflow argument");
uint32_t mask = (1u << (num & 0x1f));
array_[num >> 5] &= ~mask;
}
//! Toggle a bit in bitset
void flip(size_t num) {
ailego_assert_with(N > num, "overflow argument");
uint32_t mask = (1u << (num & 0x1f));
array_[num >> 5] ^= mask;
}
//! Perform binary AND
void bitwise_and(const FixedBitset &rhs) {
BitsetHelper::BitwiseAnd(array_, rhs.array_, ((N + 0x1f) >> 5));
}
//! Perform binary AND NOT
void bitwise_andnot(const FixedBitset &rhs) {
BitsetHelper::BitwiseAndnot(array_, rhs.array_, ((N + 0x1f) >> 5));
}
//! Perform binary OR
void bitwise_or(const FixedBitset &rhs) {
BitsetHelper::BitwiseOr(array_, rhs.array_, ((N + 0x1f) >> 5));
}
//! Perform binary XOR
void bitwise_xor(const FixedBitset &rhs) {
BitsetHelper::BitwiseXor(array_, rhs.array_, ((N + 0x1f) >> 5));
}
//! Perform binary NOT
void bitwise_not(void) {
BitsetHelper::BitwiseNot(array_, ((N + 0x1f) >> 5));
}
//! Check if all bits are set to true
bool test_all(void) const {
return BitsetHelper::TestAll(array_, ((N + 0x1f) >> 5));
}
//! Check if any bits are set to true
bool test_any(void) const {
return BitsetHelper::TestAny(array_, ((N + 0x1f) >> 5));
}
//! Check if none of the bits are set to true
bool test_none(void) const {
return BitsetHelper::TestNone(array_, ((N + 0x1f) >> 5));
}
//! Compute the cardinality of a bitset
size_t cardinality(void) const {
return BitsetHelper::Cardinality(array_, ((N + 0x1f) >> 5));
}
//! Extract the bitset to an array
void extract(size_t base, std::vector<size_t> *out) const {
const uint32_t *iter = array_;
const uint32_t *last = array_ + ((N + 0x1f) >> 5);
for (; iter != last; ++iter) {
uint32_t w = *iter;
while (w != 0) {
uint32_t c = ailego_ctz32(w);
w &= ~(1u << c);
out->push_back(base + c);
}
base += 32u;
}
}
//! Extract the bitset to an array
void extract(std::vector<size_t> *out) const {
this->extract(0, out);
}
//! Compute the AND cardinality between two bitsets
static size_t BitwiseAndCardinality(const FixedBitset &lhs,
const FixedBitset &rhs) {
return BitsetHelper::BitwiseAndCardinality(lhs.array_, rhs.array_,
((N + 0x1f) >> 5));
}
//! Compute the ANDNOT cardinality between two bitsets
static size_t BitwiseAndnotCardinality(const FixedBitset &lhs,
const FixedBitset &rhs) {
return BitsetHelper::BitwiseAndnotCardinality(lhs.array_, rhs.array_,
((N + 0x1f) >> 5));
}
//! Compute the XOR cardinality between two bitsets
static size_t BitwiseXorCardinality(const FixedBitset &lhs,
const FixedBitset &rhs) {
return BitsetHelper::BitwiseXorCardinality(lhs.array_, rhs.array_,
((N + 0x1f) >> 5));
}
//! Compute the OR cardinality between two bitsets
static size_t BitwiseOrCardinality(const FixedBitset &lhs,
const FixedBitset &rhs) {
return BitsetHelper::BitwiseOrCardinality(lhs.array_, rhs.array_,
((N + 0x1f) >> 5));
}
//! Convert a array pointer to bitset pointer
static FixedBitset *Cast(uint32_t *arr) {
return reinterpret_cast<FixedBitset<N> *>(arr);
}
//! Convert a array pointer to bitset pointer
static const FixedBitset *Cast(const uint32_t *arr) {
return reinterpret_cast<const FixedBitset<N> *>(arr);
}
//! Convert a array pointer to bitset pointer
static FixedBitset *Cast(uint64_t *arr) {
return reinterpret_cast<FixedBitset<N> *>(arr);
}
//! Convert a array pointer to bitset pointer
static const FixedBitset *Cast(const uint64_t *arr) {
return reinterpret_cast<const FixedBitset<N> *>(arr);
}
private:
uint32_t array_[(N + 0x1f) >> 5];
};
/*! Fixed Bitset (Special)
*/
template <>
class FixedBitset<0> {
public:
enum { MAX_SIZE = 0 };
//! Retrieve max size of bitset
constexpr size_t size(void) const {
return MAX_SIZE;
}
};
/*! Bitset
*/
class Bitset {
public:
//! Constructor
Bitset(void) : array_() {}
//! Constructor
Bitset(size_t bits) : array_((bits + 0x1f) >> 5) {}
//! Constructor
Bitset(const Bitset &rhs) : array_(rhs.array_) {}
//! Constructor
Bitset(Bitset &&rhs) : array_(std::move(rhs.array_)) {}
//! Destructor
~Bitset(void) {}
//! Assignment
Bitset &operator=(const Bitset &rhs) {
array_ = rhs.array_;
return *this;
}
//! Assignment
Bitset &operator=(Bitset &&rhs) {
array_ = std::move(rhs.array_);
return *this;
}
//! Retrieve data pointer
uint32_t *data(void) {
return array_.data();
}
//! Retrieve data pointer
const uint32_t *data(void) const {
return array_.data();
}
//! Retrieve count of bits in set
size_t size(void) const {
return (array_.size() << 5);
}
//! Resize the bitset
void resize(size_t bits) {
array_.resize((bits + 0x1f) >> 5);
}
// !Clear the bitset
void clear(void) {
array_.clear();
}
//! Test a bit in bitset
bool test(size_t num) const {
ailego_assert_with(this->size() > num, "overflow argument");
return ((array_[num >> 5] & (1u << (num & 0x1f))) != 0);
}
//! Set a bit in bitset
void set(size_t num) {
ailego_assert_with(this->size() > num, "overflow argument");
uint32_t mask = (1u << (num & 0x1f));
array_[num >> 5] |= mask;
}
//! Clear a bit in bitset
void reset(size_t num) {
ailego_assert_with(this->size() > num, "overflow argument");
uint32_t mask = (1u << (num & 0x1f));
array_[num >> 5] &= ~mask;
}
//! Toggle a bit in bitset
void flip(size_t num) {
ailego_assert_with(this->size() > num, "overflow argument");
uint32_t mask = (1u << (num & 0x1f));
array_[num >> 5] ^= mask;
}
//! Perform binary AND
void bitwise_and(const Bitset &rhs) {
BitsetHelper::BitwiseAnd(array_.data(), rhs.array_.data(),
std::min(array_.size(), rhs.array_.size()));
}
//! Perform binary AND NOT
void bitwise_andnot(const Bitset &rhs) {
BitsetHelper::BitwiseAndnot(array_.data(), rhs.array_.data(),
std::min(array_.size(), rhs.array_.size()));
}
//! Perform binary OR
void bitwise_or(const Bitset &rhs) {
BitsetHelper::BitwiseOr(array_.data(), rhs.array_.data(),
std::min(array_.size(), rhs.array_.size()));
}
//! Perform binary XOR
void bitwise_xor(const Bitset &rhs) {
BitsetHelper::BitwiseXor(array_.data(), rhs.array_.data(),
std::min(array_.size(), rhs.array_.size()));
}
//! Perform binary NOT
void bitwise_not(void) {
BitsetHelper::BitwiseNot(array_.data(), array_.size());
}
//! Check if all bits are set to true
bool test_all(void) const {
return BitsetHelper::TestAll(array_.data(), array_.size());
}
//! Check if any bits are set to true
bool test_any(void) const {
return BitsetHelper::TestAny(array_.data(), array_.size());
}
//! Check if none of the bits are set to true
bool test_none(void) const {
return BitsetHelper::TestNone(array_.data(), array_.size());
}
//! Compute the cardinality of a bitset
size_t cardinality(void) const {
return BitsetHelper::Cardinality(array_.data(), array_.size());
}
//! Extract the bitset to an array
void extract(size_t base, std::vector<size_t> *out) const {
const uint32_t *iter = array_.data();
const uint32_t *last = array_.data() + array_.size();
for (; iter != last; ++iter) {
uint32_t w = *iter;
while (w != 0) {
uint32_t c = ailego_ctz32(w);
w &= ~(1u << c);
out->push_back(base + c);
}
base += 32u;
}
}
//! Extract the bitset to an array
void extract(std::vector<size_t> *out) const {
this->extract(0, out);
}
//! Compute the AND cardinality between two bitsets
static size_t BitwiseAndCardinality(const Bitset &lhs, const Bitset &rhs);
//! Compute the ANDNOT cardinality between two bitsets
static size_t BitwiseAndnotCardinality(const Bitset &lhs, const Bitset &rhs);
//! Compute the XOR cardinality between two bitsets
static size_t BitwiseXorCardinality(const Bitset &lhs, const Bitset &rhs);
//! Compute the OR cardinality between two bitsets
static size_t BitwiseOrCardinality(const Bitset &lhs, const Bitset &rhs);
private:
std::vector<uint32_t> array_;
};
/*! Bitmap
*/
class Bitmap {
public:
typedef FixedBitset<65536u> Bucket;
//! Constructor
Bitmap(void) : array_() {}
//! Constructor
Bitmap(const Bitmap &rhs) {
this->copy(rhs);
}
//! Destructor
~Bitmap(void) {
this->clear();
}
//! Assignment
Bitmap &operator=(const Bitmap &rhs) {
this->copy(rhs);
return *this;
}
//! Retrieve bucket size of bitmap
size_t bucket_size(void) const {
return array_.size();
}
// !Clear the bitmap
void clear(void);
//! Remove the none buckets
void shrink_to_fit(void);
//! Test a bit in bitmap
bool test(size_t num) const;
//! Set a bit in bitmap
void set(size_t num);
//! Reset a bit in bitmap
void reset(size_t num);
//! Toggle a bit in bitmap
void flip(size_t num);
//! Perform binary AND
void bitwise_and(const Bitmap &rhs);
//! Perform binary AND NOT
void bitwise_andnot(const Bitmap &rhs);
//! Perform binary OR
void bitwise_or(const Bitmap &rhs);
//! Perform binary XOR
void bitwise_xor(const Bitmap &rhs);
//! Perform binary NOT (It will expand the whole map)
void bitwise_not(void);
//! Check if all bits are set to true
bool test_all(void) const;
//! Check if any bits are set to true
bool test_any(void) const;
//! Check if none of the bits are set to true
bool test_none(void) const;
//! Compute the cardinality of a bitmap
size_t cardinality(void) const;
//! Extract the bitmap to an array
void extract(size_t base, std::vector<size_t> *out) const;
//! Extract the bitmap to an array
void extract(std::vector<size_t> *out) const {
this->extract(0, out);
}
//! Compute the AND cardinality between two bitmaps
static size_t BitwiseAndCardinality(const Bitmap &lhs, const Bitmap &rhs);
//! Compute the ANDNOT cardinality between two bitmaps
static size_t BitwiseAndnotCardinality(const Bitmap &lhs, const Bitmap &rhs);
//! Compute the XOR cardinality between two bitmaps
static size_t BitwiseXorCardinality(const Bitmap &lhs, const Bitmap &rhs);
//! Compute the OR cardinality between two bitmaps
static size_t BitwiseOrCardinality(const Bitmap &lhs, const Bitmap &rhs);
protected:
//! Copy the content from another bitmap
void copy(const Bitmap &rhs);
private:
std::vector<Bucket *> array_;
};
} // namespace ailego
} // namespace zvec
+131
View File
@@ -0,0 +1,131 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <algorithm>
#include <string>
#include <ailego/internal/platform.h>
namespace zvec {
namespace ailego {
/*! AiLego Blob Wrap
*/
class BlobWrap {
public:
//! Constructor
BlobWrap(void) : buffer_(nullptr), size_(0u) {}
//! Constructor
BlobWrap(const BlobWrap &rhs) : buffer_(rhs.buffer_), size_(rhs.size_) {}
//! Constructor
BlobWrap(BlobWrap &&rhs) : buffer_(rhs.buffer_), size_(rhs.size_) {
rhs.buffer_ = nullptr;
rhs.size_ = 0u;
}
//! Constructor
BlobWrap(const void *buf, size_t len)
: buffer_(const_cast<void *>(buf)), size_(len) {}
//! Constructor
BlobWrap(const std::string &buf)
: buffer_(const_cast<char *>(buf.data())), size_(buf.size()) {}
//! Destructor
~BlobWrap(void) {}
//! Assignment
BlobWrap &operator=(const BlobWrap &rhs) {
buffer_ = rhs.buffer_;
size_ = rhs.size_;
return *this;
}
//! Assignment
BlobWrap &operator=(BlobWrap &&rhs) {
buffer_ = rhs.buffer_;
size_ = rhs.size_;
rhs.buffer_ = nullptr;
rhs.size_ = 0u;
return *this;
}
//! Test if the blob is valid
bool is_valid(void) const {
return (buffer_ && size_);
}
//! Mount a buffer as blob
void mount(void *buf, size_t len) {
buffer_ = buf;
size_ = len;
}
//! Mount a string as blob
void mount(std::string &buf) {
buffer_ = const_cast<char *>(buf.data());
size_ = buf.size();
}
//! Umount the buffer of blob
void umount(void) {
buffer_ = nullptr;
size_ = 0u;
}
//! Retrieve buffer of blob
void *buffer(void) {
return buffer_;
}
//! Retrieve buffer of blob
const void *buffer(void) const {
return buffer_;
}
//! Retrieve size of blob
size_t size(void) const {
return size_;
}
//! Copy a buffer into blob
void copy(const void *buf, size_t len) {
memcpy(buffer_, buf, std::min(size_, len));
}
//! Copy a blob to blob
void copy(const BlobWrap &rhs) {
memcpy(buffer_, rhs.buffer_, std::min(size_, rhs.size_));
}
//! Copy a string to blob
void copy(const std::string &str) {
memcpy(buffer_, str.data(), std::min(size_, str.size()));
}
//! Zero the buffer of blob
void zero(void) {
memset(buffer_, 0, size_);
}
private:
void *buffer_;
size_t size_;
};
} // namespace ailego
} // namespace zvec
+279
View File
@@ -0,0 +1,279 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <cmath>
#include <ailego/internal/platform.h>
#include <ailego/utility/type_helper.h>
namespace zvec {
namespace ailego {
/*! Bloom Filter Calculator
*/
struct BloomFilterCalculator {
/**
* \brief Calculate probability of false positives
* \param n Number of items in the filter
* \param m Number of bits in the filter
* \param k Number of hash functions
* \return Probability of false positives
*/
static double Probability(size_t n, size_t m, size_t k) {
return std::pow(1.0 - std::exp(-((double)k / (double)m * (double)n)), k);
}
/**
* \brief Calculate number of items in the filter
* \param m Number of bits in the filter
* \param k Number of hash functions
* \param p Probability of false positives
* \return Number of items in the filter
*/
static size_t NumberOfItems(size_t m, size_t k, double p) {
return (size_t)std::ceil(
-((double)m / (double)k *
std::log(1.0 - std::exp(std::log(p) / (double)k))));
}
/**
* \brief Calculate number of bits in the filter
* \param n Number of items in the filter
* \param p Probability of false positives
* \return Number of bits in the filter
*/
static size_t NumberOfBits(size_t n, double p) {
return (size_t)std::ceil((double)n * std::log(p) /
std::log(1.0 / std::pow(2.0, std::log(2.0))));
}
/**
* \brief Calculate number of bits in the filter
* \param n Number of items in the filter
* \param k Number of hash functions
* \param p Probability of false positives
* \return Number of bits in the filter
*/
static size_t NumberOfBits(size_t n, size_t k, double p) {
return (size_t)std::ceil(-((double)k * (double)n /
std::log(1.0 - std::pow(p, 1.0 / (double)k))));
}
/**
* \brief Calculate number of bytes in the filter
* \param n Number of items in the filter
* \param p Probability of false positives
* \return Number of bytes in the filter
*/
static size_t NumberOfBytes(size_t n, double p) {
return ((NumberOfBits(n, p) + 7) >> 3);
}
/**
* \brief Calculate number of bits in the filter
* \param n Number of items in the filter
* \param k Number of hash functions
* \param p Probability of false positives
* \return Number of bits in the filter
*/
static size_t NumberOfBytes(size_t n, size_t k, double p) {
return ((NumberOfBits(n, k, p) + 7) >> 3);
}
/**
* \brief Calculate number of hash functions
* \param n Number of items in the filter
* \param m Number of bits in the filter
* \return Number of hash functions
*/
static size_t NumberOfHash(size_t n, size_t m) {
return (size_t)std::round((double)m / (double)n * std::log(2.0));
}
};
/*! Bloom Filter
*/
template <size_t K>
class BloomFilter {
public:
//! Constructor
BloomFilter(void) {}
//! Constructor
BloomFilter(size_t n, double p) {
if (n > 0 && p > 0.0 && p < 1.0) {
capacity_ = n;
bits_count_ = BloomFilterCalculator::NumberOfBits(n, K, p);
bits_count_ = ((bits_count_ + 31) >> 5) << 5;
probability_ = BloomFilterCalculator::Probability(n, bits_count_, K);
bitset_ = new uint32_t[bits_count_ >> 5];
memset(bitset_, 0, (bits_count_ >> 3));
}
}
//! Constructor
BloomFilter(BloomFilter &&rhs)
: bitset_(rhs.bitset_),
bits_count_(rhs.bits_count_),
capacity_(rhs.capacity_),
count_(rhs.count_),
probability_(rhs.probability_) {
rhs.bitset_ = nullptr;
rhs.bits_count_ = 0u;
rhs.capacity_ = 0u;
rhs.count_ = 0u;
rhs.probability_ = 0u;
}
//! Destructor
~BloomFilter(void) {
delete[] bitset_;
}
//! Test if the filter is valid
bool is_valid(void) const {
return (bitset_ != nullptr);
}
//! Reset the bloom filter
bool reset(size_t n, double p) {
if (n <= 0 || p <= 0.0 || p >= 1.0) {
return false;
}
delete[] bitset_;
capacity_ = n;
count_ = 0u;
bits_count_ = BloomFilterCalculator::NumberOfBits(n, K, p);
bits_count_ = ((bits_count_ + 31) >> 5) << 5;
probability_ = BloomFilterCalculator::Probability(n, bits_count_, K);
bitset_ = new (std::nothrow) uint32_t[bits_count_ >> 5];
if (!bitset_) {
return false;
}
memset(bitset_, 0, (bits_count_ >> 3));
return true;
}
//! Clear the bloom filter
void clear(void) {
if (bitset_) {
memset(bitset_, 0, (bits_count_ >> 3));
count_ = 0u;
}
}
//! Insert a item into bloom filter
template <typename... TArgs,
typename = typename std::enable_if<
Conjunction<std::is_integral<TArgs>...>::value &&
sizeof...(TArgs) == K>::type>
bool insert(TArgs... vals) {
if (count_ >= capacity_) {
return false;
}
this->set_bits(vals...);
++count_;
return true;
}
//! Force insert a item into bloom filter
template <typename... TArgs,
typename = typename std::enable_if<
Conjunction<std::is_integral<TArgs>...>::value &&
sizeof...(TArgs) == K>::type>
void force_insert(TArgs... vals) {
this->set_bits(vals...);
++count_;
}
//! Insert a item into bloom filter
template <typename... TArgs,
typename = typename std::enable_if<
Conjunction<std::is_integral<TArgs>...>::value &&
sizeof...(TArgs) == K>::type>
bool has(TArgs... vals) const {
return this->test_bits(vals...);
}
//! Retrieve count of bits in bloom filter
size_t bits_count(void) const {
return bits_count_;
}
//! Retrieve capacity of bloom filter
size_t capacity(void) const {
return capacity_;
}
//! Retrieve count of items in bloom filter
size_t count(void) const {
return count_;
}
//! Retrieve probability of false positives
double probability(void) const {
return probability_;
}
protected:
//! Disable them
BloomFilter(const BloomFilter &) = delete;
BloomFilter &operator=(const BloomFilter &) = delete;
//! Set bits in bloom filter
template <typename TArg>
void set_bits(TArg val) {
size_t num = static_cast<size_t>(val) % bits_count_;
bitset_[num >> 5] |= (1u << (num & 0x1f));
}
//! Set bits in bloom filter
template <typename TArg, typename... TArgs>
void set_bits(TArg val, TArgs... vals) {
this->set_bits(val);
this->set_bits(vals...);
}
//! Test bits in bloom filter
template <typename TArg>
bool test_bits(TArg val) const {
size_t num = static_cast<size_t>(val) % bits_count_;
return ((bitset_[num >> 5] & (1u << (num & 0x1f))) != 0);
}
//! Test bits in bloom filter
template <typename TArg, typename... TArgs>
bool test_bits(TArg val, TArgs... vals) const {
if (!this->test_bits(val)) {
return false;
}
return this->test_bits(vals...);
}
private:
uint32_t *bitset_{nullptr};
size_t bits_count_{0u};
size_t capacity_{0u};
size_t count_{0u};
double probability_{0.0};
};
/*! Bloom Filter (Special)
*/
template <>
struct BloomFilter<0> {};
} // namespace ailego
} // namespace zvec
+414
View File
@@ -0,0 +1,414 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <algorithm>
#include <string>
#include <typeinfo>
#include <ailego/utility/type_helper.h>
namespace zvec {
namespace ailego {
namespace internal {
/*! Cube Policy
*/
struct CubePolicy {
//! Destructor
virtual ~CubePolicy(void) {}
//! Assign `src` to `dst`
virtual void assign(const void *src, void **dst) = 0;
//! Cleanup value
virtual void cleanup(void **val) = 0;
//! Clone value of `src` to `dst`
virtual void clone(void *const *src, void **dst) = 0;
//! Move `src` to `dst`
virtual void move(void *src, void **dst) = 0;
//! Retrieve size
virtual size_t size(void) const = 0;
//! Retrieve type information
virtual const std::type_info &type(void) const = 0;
//! Retrieve value
virtual void *value(void **src) = 0;
//! Retrieve value
virtual const void *value(void *const *src) const = 0;
};
/*! Small Cube Policy
*/
template <typename T>
struct SmallCubePolicy : public CubePolicy {
//! Assign `src` to `dst`
void assign(const void *src, void **dst) {
new (dst) T(*reinterpret_cast<const T *>(src));
}
//! Cleanup value
void cleanup(void **val) {
reinterpret_cast<T *>(val)->~T();
}
//! Clone value of `src` to `dst`
void clone(void *const *src, void **dst) {
new (dst) T(*reinterpret_cast<const T *>(src));
}
//! Move `src` to `dst`
void move(void *src, void **dst) {
new (dst) T(std::move(*reinterpret_cast<T *>(src)));
}
//! Retrieve size
size_t size(void) const {
return sizeof(T);
}
//! Retrieve type information
const std::type_info &type(void) const {
return typeid(T);
}
//! Retrieve value
void *value(void **src) {
return reinterpret_cast<void *>(src);
}
//! Retrieve value
const void *value(void *const *src) const {
return reinterpret_cast<const void *>(src);
}
};
/*! Large Cube Policy
*/
template <typename T>
struct LargeCubePolicy : public CubePolicy {
//! Assign `src` to `dst`
void assign(const void *src, void **dst) {
*dst = new T(*reinterpret_cast<const T *>(src));
}
//! Cleanup value
void cleanup(void **val) {
delete (reinterpret_cast<T *>(*val));
}
//! Clone value of `src` to `dst`
void clone(void *const *src, void **dst) {
*dst = new T(**reinterpret_cast<T *const *>(src));
}
//! Move `src` to `dst`
void move(void *src, void **dst) {
*dst = new T(std::move(*reinterpret_cast<T *>(src)));
}
//! Retrieve size
size_t size(void) const {
return sizeof(T);
}
//! Retrieve type information
const std::type_info &type(void) const {
return typeid(T);
}
//! Retrieve value
void *value(void **src) {
return *src;
}
//! Retrieve value
const void *value(void *const *src) const {
return *src;
}
};
/*! Policy Selector
*/
template <typename T, typename = void>
struct PolicySelector {
typedef LargeCubePolicy<T> Type;
};
/*! Policy Selector
*/
template <typename T>
struct PolicySelector<
T, typename std::enable_if<sizeof(T) <= sizeof(void *)>::type> {
typedef SmallCubePolicy<T> Type;
};
} // namespace internal
/*! Cube class
*/
class Cube {
public:
//! Constructor
Cube(void) : policy_(Cube::Policy<Cube::EmptyPolicy>()), object_(nullptr) {}
//! Constructor
template <typename T>
Cube(const T &rhs) : policy_(Cube::Policy<T>()), object_(nullptr) {
policy_->assign(&rhs, &object_);
}
//! Constructor
template <typename T, typename = typename std::enable_if<
!std::is_same<Cube &, T>::value &&
!std::is_same<T &, T>::value>::type>
Cube(T &&rhs) : policy_(Cube::Policy<T>()), object_(nullptr) {
policy_->move(&rhs, &object_);
}
//! Constructor
Cube(const char *str)
: policy_(Cube::Policy<std::string>()), object_(nullptr) {
std::string rhs(str);
policy_->move(&rhs, &object_);
}
//! Constructor
Cube(char str[]) : policy_(Cube::Policy<std::string>()), object_(nullptr) {
std::string rhs(str);
policy_->move(&rhs, &object_);
}
//! Constructor
Cube(const Cube &rhs) : policy_(rhs.policy_), object_(nullptr) {
policy_->clone(&rhs.object_, &object_);
}
//! Constructor
Cube(Cube &&rhs) : policy_(rhs.policy_), object_(rhs.object_) {
rhs.policy_ = Cube::Policy<Cube::EmptyPolicy>();
rhs.object_ = nullptr;
}
//! Destructor
~Cube(void) {
policy_->cleanup(&object_);
}
//! Assignment
template <typename T>
Cube &operator=(const T &rhs) {
this->assign(rhs);
return *this;
}
//! Assignment
template <typename T, typename = typename std::enable_if<
!std::is_same<Cube &, T>::value &&
!std::is_same<T &, T>::value>::type>
Cube &operator=(T &&rhs) {
this->assign(std::forward<T>(rhs));
return *this;
}
//! Assignment
Cube &operator=(const Cube &rhs) {
this->assign(rhs);
return *this;
}
//! Assignment
Cube &operator=(Cube &&rhs) {
this->assign(std::forward<Cube>(rhs));
return *this;
}
//! Assignment
Cube &operator=(const char *str) {
this->assign(str);
return *this;
}
//! Assignment
Cube &operator=(char str[]) {
this->assign(str);
return *this;
}
//! Retrieve object in original type
template <typename T>
operator T &() {
return this->cast<T>();
}
//! Retrieve object in original type
template <typename T>
operator const T &() const {
return this->cast<T>();
}
//! Assign content
template <typename T>
void assign(const T &rhs) {
policy_->cleanup(&object_);
policy_ = Cube::Policy<T>();
policy_->assign(&rhs, &object_);
}
//! Assign content
template <typename T, typename = typename std::enable_if<
!std::is_same<Cube &, T>::value &&
!std::is_same<T &, T>::value>::type>
void assign(T &&rhs) {
policy_->cleanup(&object_);
policy_ = Cube::Policy<T>();
policy_->move(&rhs, &object_);
}
//! Assign content from another Cube
void assign(const Cube &rhs) {
policy_->cleanup(&object_);
policy_ = rhs.policy_;
policy_->clone(&rhs.object_, &object_);
}
//! Assign content from another Cube
void assign(Cube &&rhs) {
if (this != &rhs) {
policy_->cleanup(&object_);
policy_ = rhs.policy_;
object_ = rhs.object_;
rhs.policy_ = Cube::Policy<Cube::EmptyPolicy>();
rhs.object_ = nullptr;
}
}
//! Assign content
void assign(const char *str) {
policy_->cleanup(&object_);
policy_ = Cube::Policy<std::string>();
std::string rhs(str);
policy_->move(&rhs, &object_);
}
//! Assign content
void assign(char str[]) {
policy_->cleanup(&object_);
policy_ = Cube::Policy<std::string>();
std::string rhs(str);
policy_->move(&rhs, &object_);
}
//! Swap the content with another Cube
Cube &swap(Cube &rhs) {
std::swap(policy_, rhs.policy_);
std::swap(object_, rhs.object_);
return *this;
}
//! Cast to the original type
template <typename T>
T &cast(void) {
if (policy_ != Cube::Policy<T>()) {
throw std::bad_cast();
}
return *reinterpret_cast<T *>(policy_->value(&object_));
}
//! Cast to the original type
template <typename T>
const T &cast(void) const {
if (policy_ != Cube::Policy<T>()) {
throw std::bad_cast();
}
return *reinterpret_cast<const T *>(policy_->value(&object_));
}
//! Cast to the original type (unsafe)
template <typename T>
T &unsafe_cast(void) {
return *reinterpret_cast<T *>(policy_->value(&object_));
}
//! Cast to the original type (unsafe)
template <typename T>
const T &unsafe_cast(void) const {
return *reinterpret_cast<const T *>(policy_->value(&object_));
}
//! Test if the Cube is empty
bool empty(void) const {
return (policy_ == Cube::Policy<Cube::EmptyPolicy>());
}
//! Reset Cube allocated memory
void reset(void) {
policy_->cleanup(&object_);
policy_ = Cube::Policy<Cube::EmptyPolicy>();
object_ = nullptr;
}
//! Test if the Cube is compatible with another one
bool compatible(const Cube &rhs) const {
return (policy_ == rhs.policy_ || policy_->type() == rhs.policy_->type());
}
//! Test if the Cube is compatible with another one
template <typename T>
bool compatible(void) const {
return (policy_ == Cube::Policy<T>() ||
policy_->type() == Cube::Policy<T>()->type());
}
//! Retrieve size
size_t size(void) const {
return (!this->empty() ? policy_->size() : 0u);
}
//! Retrieve type information
const std::type_info &type(void) const {
return (!this->empty() ? policy_->type() : typeid(void));
}
protected:
/*! Empty Policy
*/
struct EmptyPolicy {};
//! Make a static policy object
template <typename T>
static internal::CubePolicy *MakePolicy(void) {
static typename internal::PolicySelector<T>::Type policy;
return (&policy);
}
//! Retrieve a static policy object
template <typename T>
static internal::CubePolicy *Policy(void) {
return MakePolicy<typename UnderlyingType<T>::type>();
}
private:
//! Members
internal::CubePolicy *policy_;
void *object_;
};
} // namespace ailego
} // namespace zvec
+234
View File
@@ -0,0 +1,234 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <algorithm>
#include <functional>
#include <limits>
#include <utility>
#include <vector>
namespace zvec {
namespace ailego {
/*! Heap Adapter
*/
template <typename T, typename TCompare = std::less<T>,
typename TBase = std::vector<T>>
class Heap : public TBase {
public:
//! Constructor
Heap(void)
: TBase(), limit_(std::numeric_limits<size_t>::max()), compare_() {}
//! Constructor
template <typename... Args>
Heap(size_t max, Args &&...args)
: TBase(),
limit_(std::max<size_t>(max, 1u)),
compare_(std::forward<Args>(args)...) {
TBase::reserve(limit_);
}
//! Constructor
Heap(const Heap &rhs)
: TBase(rhs), limit_(rhs.limit_), compare_(rhs.compare_) {}
//! Constructor
Heap(Heap &&rhs)
: TBase(std::move(rhs)),
limit_(rhs.limit_),
compare_(std::move(rhs.compare_)) {}
//! Constructor
Heap(const TBase &rhs)
: TBase(rhs), limit_(std::numeric_limits<size_t>::max()), compare_() {
std::make_heap(TBase::begin(), TBase::end(), compare_);
}
//! Constructor
Heap(TBase &&rhs)
: TBase(std::move(rhs)),
limit_(std::numeric_limits<size_t>::max()),
compare_() {
std::make_heap(TBase::begin(), TBase::end(), compare_);
}
//! Assignment
Heap &operator=(const Heap &rhs) {
TBase::operator=(static_cast<const TBase &>(rhs));
limit_ = rhs.limit_;
compare_ = rhs.compare_;
return *this;
}
//! Assignment
Heap &operator=(Heap &&rhs) {
TBase::operator=(std::move(static_cast<TBase &&>(rhs)));
limit_ = rhs.limit_;
compare_ = std::move(rhs.compare_);
return *this;
}
//! Exchange the content
void swap(Heap &rhs) {
TBase::swap(static_cast<TBase &>(rhs));
std::swap(limit_, rhs.limit_);
std::swap(compare_, rhs.compare_);
}
//! Pop the front element
void pop(void) {
if (TBase::size() > 1) {
auto last = TBase::end() - 1;
this->replace_heap(TBase::begin(), last, std::move(*last));
}
TBase::pop_back();
}
//! Insert a new element into the heap
template <class... TArgs>
void emplace(TArgs &&...args) {
if (this->full()) {
typename std::remove_reference<T>::type val(std::forward<TArgs>(args)...);
auto first = TBase::begin();
if (compare_(val, *first)) {
this->replace_heap(first, TBase::end(), std::move(val));
}
} else {
TBase::emplace_back(std::forward<TArgs>(args)...);
std::push_heap(TBase::begin(), TBase::end(), compare_);
}
}
//! Insert a new element into the heap
void push(const T &val) {
if (this->full()) {
auto first = TBase::begin();
if (compare_(val, *first)) {
this->replace_heap(first, TBase::end(), val);
}
} else {
TBase::push_back(val);
std::push_heap(TBase::begin(), TBase::end(), compare_);
}
}
//! Insert a new element into the heap
void push(T &&val) {
if (this->full()) {
auto first = TBase::begin();
if (compare_(val, *first)) {
this->replace_heap(first, TBase::end(), std::move(val));
}
} else {
TBase::push_back(std::move(val));
std::push_heap(TBase::begin(), TBase::end(), compare_);
}
}
//! Retrieve the limit of heap
size_t limit(void) const {
return limit_;
}
//! Limit the heap with max size
void limit(size_t max) {
limit_ = std::max<size_t>(max, 1u);
TBase::reserve(limit_);
}
//! Unlimit the size of heap
void unlimit(void) {
limit_ = std::numeric_limits<size_t>::max();
}
//! Check whether the heap is full
bool full(void) const {
return (TBase::size() == limit_);
}
//! Update the heap
void update(void) {
std::make_heap(TBase::begin(), TBase::end(), compare_);
while (limit_ < TBase::size()) {
this->pop();
}
}
//! Sort the elements in the heap
void sort(void) {
std::sort(TBase::begin(), TBase::end(), compare_);
}
protected:
//! Replace the top element of heap
template <typename TRandomIterator, typename TValue>
void replace_heap(TRandomIterator first, TRandomIterator last, TValue &&val) {
using _DistanceType =
typename std::iterator_traits<TRandomIterator>::difference_type;
_DistanceType hole = 0;
_DistanceType count = _DistanceType(last - first);
if (count > 1) {
_DistanceType child = (hole << 1) + 1;
while (child < count) {
_DistanceType right_child = child + 1;
if (right_child < count &&
compare_(*(first + child), *(first + right_child))) {
child = right_child;
}
if (!compare_(val, *(first + child))) {
break;
}
*(first + hole) = std::move(*(first + child));
hole = child;
child = (hole << 1) + 1;
}
}
*(first + hole) = std::forward<TValue>(val);
}
private:
size_t limit_;
TCompare compare_;
};
/*! Key Value Heap Comparer
*/
template <typename TKey, typename TValue, typename TCompare = std::less<TValue>>
struct KeyValueHeapComparer {
//! Function call
bool operator()(const std::pair<TKey, TValue> &lhs,
const std::pair<TKey, TValue> &rhs) const {
return compare_(lhs.second, rhs.second);
}
private:
TCompare compare_;
};
/*! Key Value Heap
*/
template <typename TKey, typename TValue, typename TCompare = std::less<TValue>>
using KeyValueHeap =
Heap<std::pair<TKey, TValue>, KeyValueHeapComparer<TKey, TValue, TCompare>>;
} // namespace ailego
} // namespace zvec
+268
View File
@@ -0,0 +1,268 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <map>
#include <string>
#include "cube.h"
namespace zvec {
namespace ailego {
/*! Hypercube
*/
class Hypercube {
public:
//! Constructor
Hypercube(void) : cubes_() {}
//! Constructor
Hypercube(const Hypercube &rhs) : cubes_(rhs.cubes_) {}
//! Constructor
Hypercube(Hypercube &&rhs) : cubes_() {
cubes_.swap(rhs.cubes_);
}
//! Destructor
~Hypercube(void) {}
//! Assignment
Hypercube &operator=(const Hypercube &rhs) {
cubes_ = rhs.cubes_;
return *this;
}
//! Assignment
Hypercube &operator=(Hypercube &&rhs) {
cubes_ = std::move(rhs.cubes_);
return *this;
}
//! Overloaded operator []
Cube &operator[](const std::string &key) {
return cubes_[key];
}
//! Overloaded operator []
Cube &operator[](std::string &&key) {
return cubes_[std::forward<std::string>(key)];
}
//! Test if the element is exist
bool has(const std::string &key) const {
return (cubes_.find(key) != cubes_.end());
}
//! Test if the hyper cube is empty
bool empty(void) const {
return cubes_.empty();
}
//! Insert a key-value pair into map
bool insert(const std::string &key, Cube &&val) {
return cubes_.emplace(key, std::forward<Cube>(val)).second;
}
//! Insert a key-value pair into map
bool insert(std::string &&key, Cube &&val) {
return cubes_
.emplace(std::forward<std::string>(key), std::forward<Cube>(val))
.second;
}
//! Insert a key-value pair into map
template <typename T>
bool insert(const std::string &key, T &&val) {
return cubes_.emplace(key, Cube(std::forward<T>(val))).second;
}
//! Insert a key-value pair into map
template <typename T>
bool insert(std::string &&key, T &&val) {
return cubes_
.emplace(std::forward<std::string>(key), Cube(std::forward<T>(val)))
.second;
}
//! Insert or assign a key-value pair to map
void insert_or_assign(const std::string &key, Cube &&val) {
auto it = cubes_.lower_bound(key);
if (it != cubes_.end() && it->first == key) {
it->second = std::forward<Cube>(val);
} else {
cubes_.emplace_hint(it, key, std::forward<Cube>(val));
}
}
//! Insert or assign a key-value pair to map
void insert_or_assign(std::string &&key, Cube &&val) {
auto it = cubes_.lower_bound(key);
if (it != cubes_.end() && it->first == key) {
it->second = std::forward<Cube>(val);
} else {
cubes_.emplace_hint(it, std::forward<std::string>(key),
std::forward<Cube>(val));
}
}
//! Insert or assign a key-value pair to map
template <typename T>
void insert_or_assign(const std::string &key, T &&val) {
auto it = cubes_.lower_bound(key);
if (it != cubes_.end() && it->first == key) {
it->second = Cube(std::forward<T>(val));
} else {
cubes_.emplace_hint(it, key, Cube(std::forward<T>(val)));
}
}
//! Insert or assign a key-value pair to map
template <typename T>
void insert_or_assign(std::string &&key, T &&val) {
auto it = cubes_.lower_bound(key);
if (it != cubes_.end() && it->first == key) {
it->second = Cube(std::forward<T>(val));
} else {
cubes_.emplace_hint(it, std::forward<std::string>(key),
Cube(std::forward<T>(val)));
}
}
//! Clear the map
void clear(void) {
cubes_.clear();
}
//! Swap the map
void swap(Hypercube &rhs) {
cubes_.swap(rhs.cubes_);
}
//! Erase the pair via a key
bool erase(const std::string &key) {
auto iter = cubes_.find(key);
if (iter != cubes_.end()) {
cubes_.erase(iter);
return true;
}
return false;
}
//! Retrieve the value via a key
bool get(const std::string &key, Cube *out) const {
auto iter = cubes_.find(key);
if (iter != cubes_.end()) {
*out = iter->second;
return true;
}
return false;
}
//! Retrieve the value via a key
Cube *get(const std::string &key) {
auto iter = cubes_.find(key);
if (iter != cubes_.end()) {
return &iter->second;
}
return nullptr;
}
//! Retrieve the value via a key
const Cube *get(const std::string &key) const {
auto iter = cubes_.find(key);
if (iter != cubes_.end()) {
return &iter->second;
}
return nullptr;
}
//! Retrieve the value via a key
template <typename T>
bool get(const std::string &key, T *out) const {
auto iter = cubes_.find(key);
if (iter != cubes_.end()) {
if (iter->second.compatible<T>()) {
*out = iter->second.unsafe_cast<T>();
return true;
}
}
return false;
}
//! Retrieve the value via a key
template <typename T>
T &get(const std::string &key, T &def) {
auto iter = cubes_.find(key);
if (iter != cubes_.end()) {
if (iter->second.compatible<T>()) {
return iter->second.unsafe_cast<T>();
}
}
return def;
}
//! Retrieve the value via a key
template <typename T>
const T &get(const std::string &key, const T &def) const {
auto iter = cubes_.find(key);
if (iter != cubes_.end()) {
if (iter->second.compatible<T>()) {
return iter->second.unsafe_cast<T>();
}
}
return def;
}
//! Merge another hyper cube
void merge(const Hypercube &rhs) {
for (const auto &it : rhs.cubes_) {
auto iter = cubes_.find(it.first);
if (iter != cubes_.end()) {
iter->second = it.second;
} else {
cubes_.emplace(it.first, it.second);
}
}
}
//! Merge another hyper cube
void merge(Hypercube &&rhs) {
for (auto &it : rhs.cubes_) {
auto iter = cubes_.find(it.first);
if (iter != cubes_.end()) {
iter->second = std::move(it.second);
} else {
cubes_.emplace(std::move(it.first), std::move(it.second));
}
}
}
//! Retrieve the cubes
const std::map<std::string, Cube> &cubes(void) const {
return cubes_;
}
//! Retrieve the cubes
std::map<std::string, Cube> *mutable_cubes(void) {
return &cubes_;
}
private:
std::map<std::string, Cube> cubes_;
};
} // namespace ailego
} // namespace zvec
+139
View File
@@ -0,0 +1,139 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "params.h"
#include <cstring>
#include <ailego/encoding/json.h>
#include "ailego/logger/logger.h"
//! Global environ variable
extern char **environ;
namespace zvec {
namespace ailego {
static void ParseFromJsonObject(const ailego::JsonObject &obj, Params *params) {
for (ailego::JsonObject::const_iterator it = obj.begin(); it != obj.end();
++it) {
const ailego::JsonValue &val = it->value();
if (val.is_boolean()) {
params->set(it->key().as_stl_string(), val.as_bool());
} else if (val.is_integer()) {
params->set(it->key().as_stl_string(),
static_cast<int64_t>(val.as_integer()));
} else if (val.is_float()) {
params->set(it->key().as_stl_string(), val.as_float());
} else if (val.is_string()) {
params->set(it->key().as_stl_string(),
val.as_string().decode().as_stl_string());
} else if (val.is_object()) {
Params subparams;
ParseFromJsonObject(val.as_object(), &subparams);
params->set(it->key().as_stl_string(), std::move(subparams));
}
}
}
bool Params::ParseFromBuffer(const std::string &buf, Params *params) {
ailego::JsonValue val;
ailego::JsonParser parser;
parser.set_comment(true);
parser.set_simple(true);
parser.set_squote(true);
parser.set_unstrict(false);
if (!parser.parse(buf.c_str(), &val)) {
return false;
}
if (!val.is_object()) {
return false;
}
ParseFromJsonObject(val.as_object(), params);
return true;
}
void Params::ParseFromEnvironment(Params *params) {
// Dump all environ string
for (size_t i = 0; environ[i]; ++i) {
const char *env = environ[i];
const char *p = std::strchr(env, '=');
if (p) {
params->set(std::string(env, p - env), std::string(p + 1));
}
}
}
static void SerializeToJsonObject(const Params &params,
ailego::JsonObject *obj) {
for (const auto &it : params.hypercube().cubes()) {
const ailego::Cube &cube = it.second;
const char *key = it.first.c_str();
if (cube.compatible<std::string>()) {
const auto &val = cube.unsafe_cast<std::string>();
ailego::JsonString str(val.data(), val.size());
obj->set(key, ailego::JsonValue(str.encode()));
} else if (cube.compatible<unsigned long long int>()) {
obj->set(key,
ailego::JsonValue(cube.unsafe_cast<unsigned long long int>()));
} else if (cube.compatible<long long int>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<long long int>()));
} else if (cube.compatible<unsigned long int>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<unsigned long int>()));
} else if (cube.compatible<long int>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<long int>()));
} else if (cube.compatible<unsigned int>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<unsigned int>()));
} else if (cube.compatible<int>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<int>()));
} else if (cube.compatible<unsigned short int>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<unsigned short int>()));
} else if (cube.compatible<short int>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<short int>()));
} else if (cube.compatible<unsigned char>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<unsigned char>()));
} else if (cube.compatible<char>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<char>()));
} else if (cube.compatible<signed char>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<signed char>()));
} else if (cube.compatible<bool>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<bool>()));
} else if (cube.compatible<float>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<float>()));
} else if (cube.compatible<double>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<double>()));
} else if (cube.compatible<long double>()) {
obj->set(key, ailego::JsonValue(cube.unsafe_cast<long double>()));
} else if (cube.compatible<Params>()) {
ailego::JsonObject subobj;
SerializeToJsonObject(cube.unsafe_cast<Params>(), &subobj);
obj->set(key, ailego::JsonValue(subobj));
} else {
LOG_WARN("Unsupported serializing \'%s\' <%s>.", key, cube.type().name());
}
}
}
void Params::SerializeToBuffer(const Params &params, std::string *buf) {
if (buf != nullptr) {
ailego::JsonObject obj;
SerializeToJsonObject(params, &obj);
buf->assign(ailego::JsonValue(obj).as_json_string().as_stl_string());
}
}
} // namespace ailego
} // namespace zvec
+776
View File
@@ -0,0 +1,776 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ailego/container/hypercube.h>
namespace zvec {
namespace ailego {
//! Trying compatible with T
#define _TRYING_COMPATIBLE(cube, T, out) \
if (cube->compatible<T>()) \
return ( \
*out = static_cast<typename std::remove_pointer<decltype(out)>::type>( \
cube->unsafe_cast<T>()), \
true)
//! Trying compatible with T (Boolean)
#define _TRYING_COMPATIBLE_BOOL(cube, T, out) \
if (cube->compatible<T>()) return (*out = !!cube->unsafe_cast<T>(), true)
//! Trying compatible with T (String)
#define _TRYING_COMPATIBLE_STRING(cube, T, out) \
if (cube->compatible<T>()) \
return (out->assign(std::to_string(cube->unsafe_cast<T>())), true)
//! Trying convert string
#define _TRYING_CONVERT_STRING(cube, out) \
if (cube->compatible<std::string>()) \
return (*out = Params::StringCast<std::remove_pointer<decltype(out)>::type>( \
cube->unsafe_cast<std::string>()), \
true)
/*! Index Params
*/
class Params {
public:
//! Constructor
Params(void) : hypercube_() {}
//! Constructor
Params(const Params &rhs) : hypercube_(rhs.hypercube_) {}
//! Constructor
Params(Params &&rhs) : hypercube_() {
hypercube_.swap(rhs.hypercube_);
}
//! Destructor
~Params(void) {}
//! Assignment
Params &operator=(const Params &rhs) {
hypercube_ = rhs.hypercube_;
return *this;
}
//! Assignment
Params &operator=(Params &&rhs) {
hypercube_.swap(rhs.hypercube_);
return *this;
}
//! Overloaded operator []
ailego::Cube &operator[](const std::string &key) {
return hypercube_[key];
}
//! Overloaded operator []
ailego::Cube &operator[](std::string &&key) {
return hypercube_[std::move(key)];
}
//! Test if the element is exist
bool has(const std::string &key) const {
return hypercube_.has(key);
}
//! Test if the map is empty
bool empty(void) const {
return hypercube_.empty();
}
//! Clear the map
void clear(void) {
hypercube_.clear();
}
//! Erase the pair via a key
bool erase(const std::string &key) {
return hypercube_.erase(key);
}
//! Merge another index params
void merge(const Params &rhs) {
hypercube_.merge(rhs.hypercube_);
}
//! Merge another index params
void merge(Params &&rhs) {
hypercube_.merge(std::move(rhs.hypercube_));
}
//! Set the value of key in T
template <typename T>
bool insert(const std::string &key, T &&val) {
return hypercube_.insert<T>(key, std::forward<T>(val));
}
//! Set the value of key in T
template <typename T>
bool insert(std::string &&key, T &&val) {
return hypercube_.insert<T>(std::forward<std::string>(key),
std::forward<T>(val));
}
//! Set the value of key in T
template <typename T>
void set(const std::string &key, T &&val) {
hypercube_.insert_or_assign<T>(key, std::forward<T>(val));
}
//! Set the value of key in T
template <typename T>
void set(std::string &&key, T &&val) {
hypercube_.insert_or_assign<T>(std::forward<std::string>(key),
std::forward<T>(val));
}
//! Retrieve the value in boolean
bool get(const std::string &key, bool *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE_BOOL(cube, char, out);
_TRYING_COMPATIBLE_BOOL(cube, unsigned char, out);
_TRYING_COMPATIBLE_BOOL(cube, signed char, out);
_TRYING_COMPATIBLE_BOOL(cube, short int, out);
_TRYING_COMPATIBLE_BOOL(cube, unsigned short int, out);
_TRYING_COMPATIBLE_BOOL(cube, int, out);
_TRYING_COMPATIBLE_BOOL(cube, unsigned int, out);
_TRYING_COMPATIBLE_BOOL(cube, long int, out);
_TRYING_COMPATIBLE_BOOL(cube, unsigned long int, out);
_TRYING_COMPATIBLE_BOOL(cube, long long int, out);
_TRYING_COMPATIBLE_BOOL(cube, unsigned long long int, out);
_TRYING_COMPATIBLE_BOOL(cube, float, out);
_TRYING_COMPATIBLE_BOOL(cube, double, out);
_TRYING_COMPATIBLE_BOOL(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'char'
bool get(const std::string &key, char *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'unsigned char'
bool get(const std::string &key, unsigned char *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'signed char'
bool get(const std::string &key, signed char *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'short int'
bool get(const std::string &key, short int *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'unsigned short int'
bool get(const std::string &key, unsigned short int *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'int'
bool get(const std::string &key, int *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'unsigned int'
bool get(const std::string &key, unsigned int *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'long int'
bool get(const std::string &key, long int *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'unsigned long int'
bool get(const std::string &key, unsigned long int *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'long long int'
bool get(const std::string &key, long long int *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'unsigned long long int'
bool get(const std::string &key, unsigned long long int *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'float'
bool get(const std::string &key, float *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'double'
bool get(const std::string &key, double *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in 'long double'
bool get(const std::string &key, long double *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, long double, out);
_TRYING_COMPATIBLE(cube, double, out);
_TRYING_COMPATIBLE(cube, float, out);
_TRYING_COMPATIBLE(cube, long long int, out);
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
_TRYING_COMPATIBLE(cube, long int, out);
_TRYING_COMPATIBLE(cube, unsigned long int, out);
_TRYING_COMPATIBLE(cube, int, out);
_TRYING_COMPATIBLE(cube, unsigned int, out);
_TRYING_COMPATIBLE(cube, short int, out);
_TRYING_COMPATIBLE(cube, unsigned short int, out);
_TRYING_COMPATIBLE(cube, char, out);
_TRYING_COMPATIBLE(cube, unsigned char, out);
_TRYING_COMPATIBLE(cube, signed char, out);
_TRYING_COMPATIBLE(cube, bool, out);
_TRYING_CONVERT_STRING(cube, out);
}
return false;
}
//! Retrieve the value in string
bool get(const std::string &key, std::string *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, std::string, out);
_TRYING_COMPATIBLE_STRING(cube, bool, out);
_TRYING_COMPATIBLE_STRING(cube, char, out);
_TRYING_COMPATIBLE_STRING(cube, unsigned char, out);
_TRYING_COMPATIBLE_STRING(cube, signed char, out);
_TRYING_COMPATIBLE_STRING(cube, short int, out);
_TRYING_COMPATIBLE_STRING(cube, unsigned short int, out);
_TRYING_COMPATIBLE_STRING(cube, int, out);
_TRYING_COMPATIBLE_STRING(cube, unsigned int, out);
_TRYING_COMPATIBLE_STRING(cube, long int, out);
_TRYING_COMPATIBLE_STRING(cube, unsigned long int, out);
_TRYING_COMPATIBLE_STRING(cube, long long int, out);
_TRYING_COMPATIBLE_STRING(cube, unsigned long long int, out);
_TRYING_COMPATIBLE_STRING(cube, float, out);
_TRYING_COMPATIBLE_STRING(cube, double, out);
_TRYING_COMPATIBLE_STRING(cube, long double, out);
}
return false;
}
//! Retrieve the value in T
template <typename T>
bool get(const std::string &key, T *out) const {
const ailego::Cube *cube = hypercube_.get(key);
if (cube) {
_TRYING_COMPATIBLE(cube, T, out);
}
return false;
}
//! Retrieve the value in boolean
bool get_as_bool(const std::string &key) const {
bool result = false;
this->get(key, &result);
return result;
}
//! Retrieve the value in int8
int8_t get_as_int8(const std::string &key) const {
int8_t result = 0;
this->get(key, &result);
return result;
}
//! Retrieve the value in int16
int16_t get_as_int16(const std::string &key) const {
int16_t result = 0;
this->get(key, &result);
return result;
}
//! Retrieve the value in int32
int32_t get_as_int32(const std::string &key) const {
int32_t result = 0;
this->get(key, &result);
return result;
}
//! Retrieve the value in int64
int64_t get_as_int64(const std::string &key) const {
int64_t result = 0;
this->get(key, &result);
return result;
}
//! Retrieve the value in uint8
uint8_t get_as_uint8(const std::string &key) const {
uint8_t result = 0;
this->get(key, &result);
return result;
}
//! Retrieve the value in uint16
uint16_t get_as_uint16(const std::string &key) const {
uint16_t result = 0;
this->get(key, &result);
return result;
}
//! Retrieve the value in uint32
uint32_t get_as_uint32(const std::string &key) const {
uint32_t result = 0;
this->get(key, &result);
return result;
}
//! Retrieve the value in uint64
uint64_t get_as_uint64(const std::string &key) const {
uint64_t result = 0;
this->get(key, &result);
return result;
}
//! Retrieve the value in float
float get_as_float(const std::string &key) const {
float result = 0.0f;
this->get(key, &result);
return result;
}
//! Retrieve the value in double
double get_as_double(const std::string &key) const {
double result = 0.0f;
this->get(key, &result);
return result;
}
//! Retrieve the value in string
std::string get_as_string(const std::string &key) const {
std::string result;
this->get(key, &result);
return result;
}
//! Retrieve the debug string
std::string debug_string(void) const {
std::string str;
SerializeToBuffer(*this, &str);
return str;
}
//! Retrieve the map of parameters
const ailego::Hypercube &hypercube(void) const {
return hypercube_;
}
//! Retrieve the map of parameters
ailego::Hypercube *mutable_hypercube(void) {
return &hypercube_;
}
//! Parse parameters from buffer (Json format)
static bool ParseFromBuffer(const std::string &buf, Params *params);
//! Parse parameters from OS environment
static void ParseFromEnvironment(Params *params);
//! Serialize parameters into buffer
static void SerializeToBuffer(const Params &params, std::string *buf);
protected:
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, float>::value, T>::type {
return std::strtof(str.c_str(), nullptr);
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, double>::value, T>::type {
return std::strtod(str.c_str(), nullptr);
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, long double>::value, T>::type {
return std::strtold(str.c_str(), nullptr);
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, char>::value, T>::type {
return static_cast<char>(std::strtol(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, signed char>::value, T>::type {
return static_cast<signed char>(std::strtol(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, unsigned char>::value, T>::type {
return static_cast<unsigned char>(std::strtoul(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, short int>::value, T>::type {
return static_cast<short int>(std::strtol(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, int>::value, T>::type {
return static_cast<int>(std::strtol(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, long int>::value, T>::type {
return static_cast<long int>(std::strtol(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, long long int>::value, T>::type {
return static_cast<long long int>(std::strtoll(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, unsigned short int>::value,
T>::type {
return static_cast<unsigned short int>(
std::strtoul(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, unsigned int>::value, T>::type {
return static_cast<unsigned int>(std::strtoul(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, unsigned long int>::value,
T>::type {
return static_cast<unsigned long int>(
std::strtoul(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, unsigned long long int>::value,
T>::type {
return static_cast<unsigned long long int>(
std::strtoull(str.c_str(), nullptr, 0));
}
//! Convert string type to another type
template <typename T>
static auto StringCast(const std::string &str) ->
typename std::enable_if<std::is_same<T, bool>::value, T>::type {
if (str.empty()) {
return false;
}
char c = str[0];
if (c == 'Y' || c == 'T' || c == 'y' || c == 't') {
return true;
}
return !!std::strtof(str.c_str(), nullptr);
}
private:
ailego::Hypercube hypercube_;
};
#undef _TRYING_COMPATIBLE
#undef _TRYING_COMPATIBLE
#undef _TRYING_COMPATIBLE_BOOL
#undef _TRYING_COMPATIBLE_STRING
#undef _TRYING_CONVERT_STRING
} // namespace ailego
} // namespace zvec
+140
View File
@@ -0,0 +1,140 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <random>
#include <vector>
namespace zvec {
namespace ailego {
/*! Sampling Reservoir
*/
template <typename T, typename Allocator = std::allocator<T>>
class Reservoir {
public:
//! Constructor
Reservoir(size_t cnt)
: samples_(cnt), total_(0), mt_(std::random_device()()), pool_() {
pool_.reserve(samples_);
}
//! Constructor
Reservoir(const Reservoir &rhs)
: samples_(rhs.samples_),
total_(rhs.total_),
mt_(std::random_device()()),
pool_(rhs.pool_) {}
//! Constructor
Reservoir(Reservoir &&rhs)
: samples_(rhs.samples_),
total_(rhs.total_),
mt_(std::random_device()()),
pool_(std::move(rhs.pool_)) {}
//! Destructor
~Reservoir(void) {}
//! Assignment
Reservoir &operator=(const Reservoir &rhs) {
samples_ = rhs.samples_;
total_ = rhs.total_;
pool_ = rhs.pool_;
return *this;
}
//! Assignment
Reservoir &operator=(Reservoir &&rhs) {
samples_ = rhs.samples_;
total_ = rhs.total_;
pool_ = std::move(rhs.pool_);
return *this;
}
//! Retrieve pool of reservoir
std::vector<T, Allocator> *mutable_pool(void) {
return &pool_;
}
//! Retrieve pool of reservoir
const std::vector<T, Allocator> &pool(void) const {
return pool_;
}
//! Retrieve count of samples
size_t samples(void) const {
return samples_;
}
//! Retrieve total count of filling
size_t total(void) const {
return total_;
}
//! Reset the reservoir
void reset(void) {
total_ = 0;
pool_.clear();
pool_.reserve(samples_);
}
//! Fill the reservoir
void fill(const T &item) {
if (samples_ > 0) {
if (pool_.size() >= samples_) {
std::uniform_int_distribution<size_t> dt(0, total_);
size_t i = dt(mt_);
if (i < samples_) {
pool_[i] = item;
}
} else {
pool_.push_back(item);
}
}
++total_;
}
//! Fill the reservoir
void fill(T &&item) {
if (samples_ > 0) {
if (pool_.size() >= samples_) {
std::uniform_int_distribution<size_t> dt(0, total_);
size_t i = dt(mt_);
if (i < samples_) {
pool_[i] = std::move(item);
}
} else {
pool_.push_back(std::move(item));
}
}
++total_;
}
private:
//! Disable them
Reservoir(void) = delete;
//! Members
size_t samples_;
size_t total_;
std::mt19937 mt_;
std::vector<T, Allocator> pool_;
};
} // namespace ailego
} // namespace zvec
File diff suppressed because it is too large Load Diff
+555
View File
@@ -0,0 +1,555 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "vector.h"
namespace zvec {
namespace ailego {
/*! Numerical Vector Array
*/
template <typename T,
typename =
typename std::enable_if<IsTriviallyCopyable<T>::value>::type>
class NumericalVectorArray {
public:
//! Type of value
using ValueType = typename NumericalVector<T>::ValueType;
//! Constructor
NumericalVectorArray(void) {}
//! Constructor
explicit NumericalVectorArray(size_t dim) : dimension_(dim) {}
//! Constructor
NumericalVectorArray(const NumericalVectorArray &rhs)
: dimension_(rhs.dimension_), buffer_(rhs.buffer_) {}
//! Constructor
NumericalVectorArray(NumericalVectorArray &&rhs)
: dimension_(rhs.dimension_), buffer_(std::move(rhs.buffer_)) {}
//! Assignment
NumericalVectorArray &operator=(const NumericalVectorArray &rhs) {
dimension_ = rhs.dimension_;
buffer_ = rhs.buffer_;
return *this;
}
//! Assignment
NumericalVectorArray &operator=(NumericalVectorArray &&rhs) {
dimension_ = rhs.dimension_;
buffer_ = std::move(rhs.buffer_);
return *this;
}
//! Overloaded operator []
ValueType *operator[](size_t i) {
return (reinterpret_cast<ValueType *>(&buffer_[0]) + i * dimension_);
}
//! Overloaded operator []
const ValueType *operator[](size_t i) const {
return (reinterpret_cast<const ValueType *>(buffer_.data()) +
i * dimension_);
}
//! Append a vector
void append(const ValueType *vec, size_t dim) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
buffer_.append(reinterpret_cast<const char *>(vec),
dim * sizeof(ValueType));
}
//! Append vectors
void append(const ValueType *vec, size_t dim, size_t cnt) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
buffer_.append(reinterpret_cast<const char *>(vec),
cnt * dim * sizeof(ValueType));
}
//! Append a vector
void append(const NumericalVector<ValueType> &vec) {
this->append(vec.data(), vec.dimension());
}
//! Replace a vector
void replace(size_t index, const ValueType *vec, size_t dim) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
size_t element_size = dim * sizeof(ValueType);
buffer_.replace(index * element_size, element_size,
reinterpret_cast<const char *>(vec), element_size);
}
//! Replace a vector
void replace(size_t index, const ValueType *vec, size_t dim, size_t cnt) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
size_t element_size = dim * sizeof(ValueType);
size_t total = element_size * cnt;
buffer_.replace(index * element_size, total,
reinterpret_cast<const char *>(vec), total);
}
//! Replace a vector
void replace(size_t index, const NumericalVector<ValueType> &vec) {
this->replace(index, vec.data(), vec.dimension());
}
//! Request a change in capacity
void reserve(size_t n) {
buffer_.reserve(n * dimension_ * sizeof(ValueType));
}
//! Resize the array to a length of n elements
void resize(size_t n) {
buffer_.resize(n * dimension_ * sizeof(ValueType));
}
//! Clear the vector array
void clear(void) {
buffer_.clear();
}
//! Reset the vector array
void reset(size_t dim) {
dimension_ = dim;
buffer_.clear();
}
//! Requests the removal of unused capacity.
void shrink_to_fit(void) {
buffer_.shrink_to_fit();
}
//! Retrieve pointer of data
ValueType *data(void) {
return reinterpret_cast<ValueType *>(&buffer_[0]);
}
//! Retrieve pointer of data
const ValueType *data(void) const {
return reinterpret_cast<const ValueType *>(buffer_.data());
}
//! Retrieve pointer of data
ValueType *at(size_t i) {
if (ailego_unlikely(i >= this->count())) {
throw std::out_of_range("Index overflow");
}
return (reinterpret_cast<ValueType *>(&buffer_[0]) + i * dimension_);
}
//! Retrieve pointer of data
const ValueType *at(size_t i) const {
if (ailego_unlikely(i >= this->count())) {
throw std::out_of_range("Index overflow");
}
return (reinterpret_cast<const ValueType *>(buffer_.data()) +
i * dimension_);
}
//! Test if the array is empty
bool empty(void) const {
return buffer_.empty();
}
//! Retrieve count of vectors
size_t count(void) const {
return (dimension_ > 0 ? buffer_.size() / (dimension_ * sizeof(ValueType))
: 0u);
}
//! Retrieve dimension of vector
size_t dimension(void) const {
return dimension_;
}
//! Retrieve size of array in bytes
size_t bytes(void) const {
return buffer_.size();
}
private:
size_t dimension_{0u};
std::string buffer_{};
};
/*! Nibble Vector Array
*/
template <typename T,
typename = typename std::enable_if<std::is_integral<T>::value>::type>
class NibbleVectorArray {
public:
//! Type of value
using ValueType = typename NibbleVector<T>::ValueType;
using StoreType = typename NibbleVector<T>::StoreType;
//! Constructor
NibbleVectorArray(void) {}
//! Constructor
explicit NibbleVectorArray(size_t dim)
: dimension_((dim + (sizeof(ValueType) << 1) - 1) /
(sizeof(ValueType) << 1) * sizeof(ValueType)
<< 1) {}
//! Constructor
NibbleVectorArray(const NibbleVectorArray &rhs)
: dimension_(rhs.dimension_), buffer_(rhs.buffer_) {}
//! Constructor
NibbleVectorArray(NibbleVectorArray &&rhs)
: dimension_(rhs.dimension_), buffer_(std::move(rhs.buffer_)) {}
//! Assignment
NibbleVectorArray &operator=(const NibbleVectorArray &rhs) {
dimension_ = rhs.dimension_;
buffer_ = rhs.buffer_;
return *this;
}
//! Assignment
NibbleVectorArray &operator=(NibbleVectorArray &&rhs) {
dimension_ = rhs.dimension_;
buffer_ = std::move(rhs.buffer_);
return *this;
}
//! Overloaded operator []
StoreType *operator[](size_t i) {
return reinterpret_cast<StoreType *>(&buffer_[0] + i * (dimension_ >> 1));
}
//! Overloaded operator []
const StoreType *operator[](size_t i) const {
return reinterpret_cast<const StoreType *>(&buffer_[0] +
i * (dimension_ >> 1));
}
//! Append a vector
void append(const StoreType *vec, size_t dim) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
buffer_.append(reinterpret_cast<const char *>(vec), dim >> 1);
}
//! Append vectors
void append(const StoreType *vec, size_t dim, size_t cnt) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
buffer_.append(reinterpret_cast<const char *>(vec), cnt * (dim >> 1));
}
//! Append a vector
void append(const NibbleVector<ValueType> &vec) {
this->append(vec.data(), vec.dimension());
}
//! Replace a vector
void replace(size_t index, const StoreType *vec, size_t dim) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
size_t element_size = (dim >> 1);
buffer_.replace(index * element_size, element_size,
reinterpret_cast<const char *>(vec), element_size);
}
//! Replace a vector
void replace(size_t index, const StoreType *vec, size_t dim, size_t cnt) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
size_t element_size = (dim >> 1);
size_t total = element_size * cnt;
buffer_.replace(index * element_size, total,
reinterpret_cast<const char *>(vec), total);
}
//! Replace a vector
void replace(size_t index, const NibbleVector<ValueType> &vec) {
this->replace(index, vec.data(), vec.dimension());
}
//! Request a change in capacity
void reserve(size_t n) {
buffer_.reserve(n * (dimension_ >> 1));
}
//! Resize the array to a length of n elements
void resize(size_t n) {
buffer_.resize(n * (dimension_ >> 1));
}
//! Clear the vector array
void clear(void) {
buffer_.clear();
}
//! Reset the vector array
void reset(size_t dim) {
dimension_ = (dim + (sizeof(ValueType) << 1) - 1) /
(sizeof(ValueType) << 1) * sizeof(ValueType)
<< 1;
buffer_.clear();
}
//! Requests the removal of unused capacity.
void shrink_to_fit(void) {
buffer_.shrink_to_fit();
}
//! Retrieve pointer of data
StoreType *data(void) {
return reinterpret_cast<StoreType *>(&buffer_[0]);
}
//! Retrieve pointer of data
const StoreType *data(void) const {
return reinterpret_cast<const StoreType *>(buffer_.data());
}
//! Retrieve pointer of data
StoreType *at(size_t i) {
if (ailego_unlikely(i >= this->count())) {
throw std::out_of_range("Index overflow");
}
return reinterpret_cast<StoreType *>(&buffer_[0] + i * (dimension_ >> 1));
}
//! Retrieve pointer of data
const StoreType *at(size_t i) const {
if (ailego_unlikely(i >= this->count())) {
throw std::out_of_range("Index overflow");
}
return reinterpret_cast<const StoreType *>(buffer_.data() +
i * (dimension_ >> 1));
}
//! Test if the array is empty
bool empty(void) const {
return buffer_.empty();
}
//! Retrieve count of vectors
size_t count(void) const {
return (dimension_ > 1 ? buffer_.size() / (dimension_ >> 1) : 0u);
}
//! Retrieve dimension of vector
size_t dimension(void) const {
return dimension_;
}
//! Retrieve size of array in bytes
size_t bytes(void) const {
return buffer_.size();
}
private:
size_t dimension_{0u};
std::string buffer_{};
};
/*! Binary Vector Array
*/
template <typename T,
typename = typename std::enable_if<std::is_integral<T>::value>::type>
class BinaryVectorArray {
public:
//! Type of value
using ValueType = typename BinaryVector<T>::ValueType;
//! Constructor
BinaryVectorArray(void) {}
//! Constructor
explicit BinaryVectorArray(size_t dim)
: dimension_((dim + (sizeof(ValueType) << 3) - 1) /
(sizeof(ValueType) << 3) * (sizeof(ValueType) << 3)) {}
//! Constructor
BinaryVectorArray(const BinaryVectorArray &rhs)
: dimension_(rhs.dimension_), buffer_(rhs.buffer_) {}
//! Constructor
BinaryVectorArray(BinaryVectorArray &&rhs)
: dimension_(rhs.dimension_), buffer_(std::move(rhs.buffer_)) {}
//! Assignment
BinaryVectorArray &operator=(const BinaryVectorArray &rhs) {
dimension_ = rhs.dimension_;
buffer_ = rhs.buffer_;
return *this;
}
//! Assignment
BinaryVectorArray &operator=(BinaryVectorArray &&rhs) {
dimension_ = rhs.dimension_;
buffer_ = std::move(rhs.buffer_);
return *this;
}
//! Overloaded operator []
ValueType *operator[](size_t i) {
return reinterpret_cast<ValueType *>(&buffer_[0] + i * (dimension_ >> 3));
}
//! Overloaded operator []
const ValueType *operator[](size_t i) const {
return reinterpret_cast<const ValueType *>(buffer_.data() +
i * (dimension_ >> 3));
}
//! Append a vector
void append(const ValueType *vec, size_t dim) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
buffer_.append(reinterpret_cast<const char *>(vec), (dim >> 3));
}
//! Append vectors
void append(const ValueType *vec, size_t dim, size_t cnt) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
buffer_.append(reinterpret_cast<const char *>(vec), cnt * (dim >> 3));
}
//! Append a vector
void append(const BinaryVector<ValueType> &vec) {
this->append(vec.data(), vec.dimension());
}
//! Replace a vector
void replace(size_t index, const ValueType *vec, size_t dim) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
size_t element_size = (dim >> 3);
buffer_.replace(index * element_size, element_size,
reinterpret_cast<const char *>(vec), element_size);
}
//! Replace a vector
void replace(size_t index, const ValueType *vec, size_t dim, size_t cnt) {
if (ailego_unlikely(dim != dimension_)) {
throw std::length_error("Unmatched dimension");
}
size_t element_size = (dim >> 3);
size_t total = element_size * cnt;
buffer_.replace(index * element_size, total,
reinterpret_cast<const char *>(vec), total);
}
//! Replace a vector
void replace(size_t index, const BinaryVector<ValueType> &vec) {
this->replace(index, vec.data(), vec.dimension());
}
//! Request a change in capacity
void reserve(size_t n) {
buffer_.reserve(n * (dimension_ >> 3));
}
//! Resize the array to a length of n elements
void resize(size_t n) {
buffer_.resize(n * (dimension_ >> 3));
}
//! Clear the vector array
void clear(void) {
buffer_.clear();
}
//! Reset the vector array
void reset(size_t dim) {
dimension_ = (dim + (sizeof(ValueType) << 3) - 1) /
(sizeof(ValueType) << 3) * (sizeof(ValueType) << 3);
buffer_.clear();
}
//! Requests the removal of unused capacity.
void shrink_to_fit(void) {
buffer_.shrink_to_fit();
}
//! Retrieve pointer of data
ValueType *data(void) {
return reinterpret_cast<ValueType *>(&buffer_[0]);
}
//! Retrieve pointer of data
const ValueType *data(void) const {
return reinterpret_cast<const ValueType *>(buffer_.data());
}
//! Retrieve pointer of data
ValueType *at(size_t i) {
if (ailego_unlikely(i >= this->count())) {
throw std::out_of_range("Index overflow");
}
return reinterpret_cast<ValueType *>(&buffer_[0] + i * (dimension_ >> 3));
}
//! Retrieve pointer of data
const ValueType *at(size_t i) const {
if (ailego_unlikely(i >= this->count())) {
throw std::out_of_range("Index overflow");
}
return reinterpret_cast<const ValueType *>(buffer_.data() +
i * (dimension_ >> 3));
}
//! Test if the array is empty
bool empty(void) const {
return buffer_.empty();
}
//! Retrieve count of vectors
size_t count(void) const {
return (dimension_ > 0 ? buffer_.size() / (dimension_ >> 3) : 0u);
}
//! Retrieve dimension of vector
size_t dimension(void) const {
return dimension_;
}
//! Retrieve size of array in bytes
size_t bytes(void) const {
return buffer_.size();
}
private:
size_t dimension_{0u};
std::string buffer_{};
};
} // namespace ailego
} // namespace zvec
+17
View File
@@ -0,0 +1,17 @@
// Copyright 2025-present the zvec project
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include "json/mod_json_plus.h"
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More