Initial commit
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
# http://clang.llvm.org/docs/ClangFormatStyleOptions.html
|
||||
# Defines the Google C++ style for automatic reformatting.
|
||||
BasedOnStyle: Google
|
||||
MaxEmptyLinesToKeep: 2
|
||||
DerivePointerAlignment: false
|
||||
PointerAlignment: Right
|
||||
AllowShortFunctionsOnASingleLine: Empty
|
||||
IncludeBlocks: Merge
|
||||
IncludeCategories:
|
||||
- Regex: '^<linux/[0-9A-Za-z]+'
|
||||
Priority: 100
|
||||
- Regex: '^<mach/[0-9A-Za-z]+'
|
||||
Priority: 101
|
||||
- Regex: '^<mach-o/[0-9A-Za-z]+'
|
||||
Priority: 102
|
||||
- Regex: '^<sys/[0-9A-Za-z]+'
|
||||
Priority: 103
|
||||
- Regex: '^<[0-9A-Za-z]+\.h>$'
|
||||
Priority: 200
|
||||
- Regex: '^<[0-9A-Za-z_]+>$'
|
||||
Priority: 201
|
||||
- Regex: '^<[0-9A-Za-z_]+\.[0-9A-Za-z]+>$'
|
||||
Priority: 202
|
||||
- Regex: '^<[0-9A-Za-z_]+/[0-9A-Za-z]+'
|
||||
Priority: 203
|
||||
- Regex: '^\"[0-9A-Za-z_]+/[0-9A-Za-z]+'
|
||||
Priority: 300
|
||||
- Regex: '^\"[0-9A-Za-z_]+\.[0-9A-Za-z]+\"$'
|
||||
Priority: 301
|
||||
- Regex: '.*'
|
||||
Priority: 1000
|
||||
@@ -0,0 +1,73 @@
|
||||
name: Bug Report
|
||||
description: Report a bug or unexpected behavior (e.g., crash, incorrect vector query, memory leak)
|
||||
title: "[Bug]: "
|
||||
labels: ["bug", "triage"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thank you for reporting! Please provide detailed info so we can reproduce and fix it quickly.
|
||||
|
||||
- type: textarea
|
||||
id: description
|
||||
attributes:
|
||||
label: Description
|
||||
description: What happened? What did you expect?
|
||||
placeholder: |
|
||||
e.g. "Query with vector field crashes when using Zvec Python API"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: steps_to_reproduce
|
||||
attributes:
|
||||
label: Steps to Reproduce
|
||||
description: Exact steps to trigger the issue (code snippets welcome)
|
||||
placeholder: |
|
||||
1. Build Zvec with CMake (Debug/Release)
|
||||
2. Run Python script: `python test.py`
|
||||
3. Call `collection.query(VectorQuery())`
|
||||
4. Process segfaults / hangs / returns wrong results
|
||||
render: python
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: logs_or_trace
|
||||
attributes:
|
||||
label: Logs / Stack Trace
|
||||
description: Paste relevant logs, LLDB/GDB backtrace, or CI failures
|
||||
placeholder: |
|
||||
Thread 1 "python" received signal SIGSEGV, Segmentation fault.
|
||||
0x0000000104a2c3f0 in std::__1::shared_ptr<...>::...
|
||||
render: shell
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: input
|
||||
id: os
|
||||
attributes:
|
||||
label: Operating System
|
||||
placeholder: macOS 14 (M1), Ubuntu 22.04, Windows 11 (WSL2)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: input
|
||||
id: build_env
|
||||
attributes:
|
||||
label: Build & Runtime Environment
|
||||
description: Compiler, CMake, Python, key dependencies
|
||||
placeholder: |
|
||||
clang 15.0.0, CMake 4.1.2, Python 3.11.9, magic_enum v0.9.7 (via git submodule)
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: checkboxes
|
||||
id: additional_context
|
||||
attributes:
|
||||
label: Additional Context
|
||||
options:
|
||||
- label: I've checked `git status` — no uncommitted submodule changes
|
||||
- label: I built with `CMAKE_BUILD_TYPE=Debug`
|
||||
- label: This occurs with or without `COVERAGE=ON`
|
||||
- label: The issue involves Python ↔ C++ integration (pybind11)
|
||||
@@ -0,0 +1,9 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Documentation
|
||||
url: https://github.com/your-org/zvec#readme
|
||||
about: Check the README, build guide, and API docs first.
|
||||
|
||||
- name: Python API Examples
|
||||
url: https://github.com/your-org/zvec/tree/main/examples
|
||||
about: See working usage examples.
|
||||
@@ -0,0 +1,55 @@
|
||||
name: Feature Request
|
||||
description: Suggest a new feature or improvement (e.g., better memory control, new query option)
|
||||
title: "[Feature]: "
|
||||
labels: ["enhancement"]
|
||||
body:
|
||||
- type: markdown
|
||||
attributes:
|
||||
value: |
|
||||
Thanks for your idea! Help us understand the motivation and scope.
|
||||
|
||||
- type: textarea
|
||||
id: problem_or_motivation
|
||||
attributes:
|
||||
label: Problem / Motivation
|
||||
description: What problem does this solve? Why is it needed?
|
||||
placeholder: |
|
||||
e.g. "Current vector queries don't allow filtering by metadata + distance threshold at once"
|
||||
validations:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: proposed_solution
|
||||
attributes:
|
||||
label: Proposed Solution
|
||||
description: How should it work? API sketch or pseudocode welcome.
|
||||
placeholder: |
|
||||
Add `filter=` and `max_distance=` args to `Zvec.query()`:
|
||||
```python
|
||||
results = db.query(vector, filter="category == 'A'", max_distance=0.5)
|
||||
```
|
||||
render: python
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: textarea
|
||||
id: alternatives
|
||||
attributes:
|
||||
label: Alternatives Considered
|
||||
description: Are there workarounds? Why not use them?
|
||||
validations:
|
||||
required: false
|
||||
|
||||
- type: dropdown
|
||||
id: impact_area
|
||||
attributes:
|
||||
label: Affected Area
|
||||
multiple: true
|
||||
options:
|
||||
- label: C++ Core (storage, indexing)
|
||||
- label: Python API / Bindings
|
||||
- label: Build System (CMake, Homebrew pkg)
|
||||
- label: Testing / CI / Coverage
|
||||
- label: Documentation
|
||||
validations:
|
||||
required: false
|
||||
@@ -0,0 +1,32 @@
|
||||
codecov:
|
||||
require_ci_to_pass: true
|
||||
|
||||
coverage:
|
||||
precision: 2
|
||||
round: down
|
||||
range: "60...75"
|
||||
|
||||
status:
|
||||
project:
|
||||
default: false
|
||||
patch:
|
||||
default: false
|
||||
|
||||
|
||||
parsers:
|
||||
gcov:
|
||||
branch_detection:
|
||||
conditional: true
|
||||
loop: true
|
||||
method: false
|
||||
macro: false
|
||||
|
||||
comment:
|
||||
require_changes: false
|
||||
layout: "reach,diff,flags,tree"
|
||||
behavior: default
|
||||
|
||||
|
||||
ignore:
|
||||
- "thirdparty/"
|
||||
- "tests/"
|
||||
@@ -0,0 +1,57 @@
|
||||
name: Build Wheels
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build_wheels:
|
||||
name: Build wheels on self-hosted manylinux_2_28
|
||||
runs-on: linux_x64
|
||||
|
||||
steps:
|
||||
- name: Checkout code
|
||||
uses: actions/checkout@v4
|
||||
with:
|
||||
submodules: recursive
|
||||
|
||||
- name: Set up Python (for cibuildwheel controller)
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.11'
|
||||
|
||||
- name: Install cibuildwheel
|
||||
run: |
|
||||
pip install --upgrade pip
|
||||
pip install cibuildwheel==2.17.0
|
||||
|
||||
- name: Build wheels using cibuildwheel
|
||||
run: |
|
||||
python -m cibuildwheel --output-dir wheelhouse
|
||||
# Save list of built wheels for publishing
|
||||
ls wheelhouse/*.whl | tee $GITHUB_STEP_SUMMARY
|
||||
echo "wheels=$(ls wheelhouse/*.whl | tr '\n' ' ')" >> $GITHUB_ENV
|
||||
|
||||
- name: Publish to TestPyPI
|
||||
if: success() && github.event_name == 'workflow_dispatch'
|
||||
env:
|
||||
TWINE_USERNAME: __token__
|
||||
TWINE_PASSWORD: ${{ secrets.TEST_PYPI_API_TOKEN }}
|
||||
TWINE_REPOSITORY_URL: https://test.pypi.org/legacy/
|
||||
run: |
|
||||
pip install twine
|
||||
twine upload --skip-existing --verbose wheelhouse/*.whl
|
||||
|
||||
- name: (Optional) Install and test from TestPyPI
|
||||
if: success() && github.event_name == 'workflow_dispatch'
|
||||
run: |
|
||||
# Create a clean venv
|
||||
python -m venv test_env
|
||||
source test_env/bin/activate
|
||||
pip install --upgrade pip
|
||||
# Install from TestPyPI (must allow pre-releases if version has dev/alpha)
|
||||
pip install --index-url https://test.pypi.org/simple/ \
|
||||
--extra-index-url https://pypi.org/simple/ \
|
||||
zvec
|
||||
# Run a simple smoke test
|
||||
python -c "import zvec; print('Import OK:', zvec.__version__)"
|
||||
shell: bash
|
||||
@@ -0,0 +1,83 @@
|
||||
# =============================================================================
|
||||
# Dockerfile.linux_x64_glibc228
|
||||
# Purpose: Ubuntu 18.10 gcc-9 + glibc 2.28 + CMake 3.30.0 + PyBind11 build environment
|
||||
# Warning: ubuntu:18.10 is EOL; use only for glibc 2.28 compatibility testing.
|
||||
# =============================================================================
|
||||
|
||||
# Use official Ubuntu 18.10 (Cosmic Cuttlefish)
|
||||
# glibc version: 2.28 (confirmed via `ldd --version`)
|
||||
FROM ubuntu:18.10
|
||||
|
||||
# Replace Ubuntu mirror with old-releases.ubuntu.com for older glibc compatibility
|
||||
RUN sed -i 's|http://\(.*\)/ubuntu|http://old-releases.ubuntu.com/ubuntu|g' /etc/apt/sources.list && \
|
||||
sed -i 's|http://security.ubuntu.com/ubuntu|http://old-releases.ubuntu.com/ubuntu|g' /etc/apt/sources.list
|
||||
|
||||
# Add Ubuntu 20.04 (focal) repo for GCC 9 ONLY
|
||||
RUN echo "deb http://archive.ubuntu.com/ubuntu/ focal main universe" >> /etc/apt/sources.list && \
|
||||
echo "deb http://security.ubuntu.com/ubuntu/ focal-security main universe" >> /etc/apt/sources.list
|
||||
|
||||
# Prevent interactive prompts & set non-root user
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
TZ=Etc/UTC
|
||||
|
||||
# Create non-root user for safety (optional but recommended)
|
||||
RUN useradd -m -u 1000 builder && \
|
||||
mkdir -p /workspace && chown builder:builder /workspace
|
||||
|
||||
# Install base system dependencies
|
||||
RUN apt-get update && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
gcc-9 g++-9 \
|
||||
ninja-build git curl ca-certificates vim wget lcov gnupg clang-format-18\
|
||||
rsync lsb-release \
|
||||
uuid-dev zlib1g-dev libssl-dev libffi-dev \
|
||||
pybind11-dev && \
|
||||
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 90 \
|
||||
--slave /usr/bin/g++ g++ /usr/bin/g++-9 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Miniforge (Conda) as root, then assign to builder
|
||||
ENV MINIFORGE_VERSION="latest"
|
||||
ENV MINIFORGE_HOME="/opt/miniforge3"
|
||||
|
||||
RUN curl -sSL "https://github.com/conda-forge/miniforge/releases/${MINIFORGE_VERSION}/download/Miniforge3-Linux-x86_64.sh" -o miniforge.sh && \
|
||||
bash miniforge.sh -b -p ${MINIFORGE_HOME} && \
|
||||
rm miniforge.sh && \
|
||||
chown -R builder:builder ${MINIFORGE_HOME}
|
||||
|
||||
# Switch to non-root user
|
||||
USER builder
|
||||
ENV PATH="${MINIFORGE_HOME}/bin:${PATH}"
|
||||
WORKDIR /workspace
|
||||
|
||||
# Create conda envs for supported Python versions
|
||||
RUN conda create -n py310 python=3.10 -y && \
|
||||
conda create -n py311 python=3.11 -y && \
|
||||
conda create -n py312 python=3.12 -y
|
||||
RUN conda clean --all -f -y
|
||||
|
||||
# Install CMake 3.30.0 from Kitware official binary
|
||||
# Ref: https://github.com/Kitware/CMake/releases/tag/v3.30.0
|
||||
RUN mkdir -p /tmp/cmake && cd /tmp/cmake && \
|
||||
curl -sSL -o cmake.tar.gz \
|
||||
"https://github.com/Kitware/CMake/releases/download/v3.30.0/cmake-3.30.0-linux-x86_64.tar.gz" && \
|
||||
tar -xzf cmake.tar.gz --strip-components=1 -C /tmp/cmake && \
|
||||
mkdir -p /home/builder/.local && \
|
||||
mv * /home/builder/.local/ && \
|
||||
chown -R builder:builder /home/builder/.local && \
|
||||
rm -rf /tmp/cmake
|
||||
|
||||
# Add CMake to PATH
|
||||
ENV PATH="/home/builder/.local/bin:${PATH}"
|
||||
|
||||
# Verify installations
|
||||
RUN cmake --version && \
|
||||
conda info && \
|
||||
conda env list && \
|
||||
python --version && \
|
||||
gcc --version && \
|
||||
ldd --version | head -n1
|
||||
|
||||
# Final setup
|
||||
WORKDIR /workspace
|
||||
@@ -0,0 +1,124 @@
|
||||
name: Zvec LinuxX64 CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: linux_x64
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ['3.10']
|
||||
fail-fast: false
|
||||
|
||||
container:
|
||||
image: zvec-registry.cn-hongkong.cr.aliyuncs.com/zvec/zvec:0.0.2
|
||||
options: --user root
|
||||
|
||||
steps:
|
||||
- name: Activate Conda environment
|
||||
run: |
|
||||
# Map version to env name
|
||||
if [[ "${{ matrix.python-version }}" == "3.10" ]]; then
|
||||
ENV_NAME="py310"
|
||||
elif [[ "${{ matrix.python-version }}" == "3.11" ]]; then
|
||||
ENV_NAME="py311"
|
||||
elif [[ "${{ matrix.python-version }}" == "3.12" ]]; then
|
||||
ENV_NAME="py312"
|
||||
else
|
||||
echo "Unsupported Python version"
|
||||
exit 1
|
||||
fi
|
||||
echo "CONDA_ENV_NAME=$ENV_NAME" >> $GITHUB_ENV
|
||||
source /opt/miniforge3/bin/activate $ENV_NAME
|
||||
|
||||
pip install --upgrade pip pytest pytest-cov ruff
|
||||
|
||||
python --version
|
||||
shell: bash
|
||||
|
||||
- name: Prepare clean build directory
|
||||
run: |
|
||||
export CLEAN_WORKSPACE="/tmp/zvec"
|
||||
mkdir -p "$CLEAN_WORKSPACE"
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
|
||||
git config --global --add safe.directory "$CLEAN_WORKSPACE"
|
||||
git clone --recursive "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" .
|
||||
|
||||
if [ -n "${{ github.event.pull_request.head.sha }}" ]; then
|
||||
git checkout "${{ github.event.pull_request.head.sha }}"
|
||||
else
|
||||
git checkout "${{ github.sha }}"
|
||||
fi
|
||||
|
||||
echo "CLEAN_WORKSPACE=$CLEAN_WORKSPACE" >> $GITHUB_ENV
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
shell: bash
|
||||
|
||||
- name: Run Ruff Linter
|
||||
run: |
|
||||
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
ruff check .
|
||||
shell: bash
|
||||
|
||||
- name: Run Ruff Formatter Check (ensure code is formatted)
|
||||
run: |
|
||||
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
ruff format --check .
|
||||
shell: bash
|
||||
|
||||
- name: Run clang-format Check
|
||||
run: |
|
||||
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
|
||||
CPP_FILES=$(find . -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.hpp" -o -name "*.cc" -o -name "*.cxx" \) \
|
||||
! -path "./build/*" \
|
||||
! -path "./tests/*" \
|
||||
! -path "./scripts/*" \
|
||||
! -path "./python/*" \
|
||||
! -path "./thirdparty/*" \
|
||||
! -path "./.git/*")
|
||||
|
||||
if [ -z "$CPP_FILES" ]; then
|
||||
echo "No C++ files found to check."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
clang-format-18 --dry-run --Werror $CPP_FILES
|
||||
shell: bash
|
||||
|
||||
- name: Install Python dependencies and build package
|
||||
run: |
|
||||
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
|
||||
NPROC=$(nproc 2>/dev/null || echo $(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 2))
|
||||
echo "ParallelGroup: Using $NPROC parallel jobs for build"
|
||||
|
||||
CMAKE_GENERATOR="Unix Makefiles" \
|
||||
CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \
|
||||
pip install -v .
|
||||
shell: bash
|
||||
|
||||
- name: Run Python Tests with Coverage
|
||||
run: |
|
||||
source /opt/miniforge3/bin/activate ${{ env.CONDA_ENV_NAME }}
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
python -m pytest python/tests/ --cov=zvec --cov-report=xml --no-cov-on-fail
|
||||
shell: bash
|
||||
|
||||
- name: Run Cpp Tests with Coverage
|
||||
run: |
|
||||
cd "$CLEAN_WORKSPACE/build"
|
||||
make unittest -j$(nproc)
|
||||
shell: bash
|
||||
@@ -0,0 +1,85 @@
|
||||
name: Zvec MacArm64 CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: mac_m1_arm
|
||||
|
||||
steps:
|
||||
- name: Run Ruff Linter
|
||||
run: |
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
ruff check .
|
||||
shell: bash
|
||||
|
||||
- name: Run Ruff Formatter Check (ensure code is formatted)
|
||||
run: |
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
ruff format --check .
|
||||
shell: bash
|
||||
|
||||
- name: Run clang-format Check
|
||||
run: |
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
|
||||
CPP_FILES=$(find . -type f \( -name "*.cpp" -o -name "*.h" -o -name "*.hpp" -o -name "*.cc" -o -name "*.cxx" \) \
|
||||
! -path "./build/*" \
|
||||
! -path "./tests/*" \
|
||||
! -path "./scripts/*" \
|
||||
! -path "./python/*" \
|
||||
! -path "./thirdparty/*" \
|
||||
! -path "./.git/*")
|
||||
|
||||
if [ -z "$CPP_FILES" ]; then
|
||||
echo "No C++ files found to check."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
clang-format --dry-run --Werror $CPP_FILES
|
||||
shell: bash
|
||||
|
||||
- name: Prepare clean build directory
|
||||
run: |
|
||||
export CLEAN_WORKSPACE="/tmp/zvec"
|
||||
rm -rf "$CLEAN_WORKSPACE"
|
||||
mkdir -p "$CLEAN_WORKSPACE"
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
|
||||
git config --global --add safe.directory "$CLEAN_WORKSPACE"
|
||||
git clone --recursive "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" .
|
||||
|
||||
if [ -n "${{ github.event.pull_request.head.sha }}" ]; then
|
||||
git checkout "${{ github.event.pull_request.head.sha }}"
|
||||
else
|
||||
git checkout "${{ github.sha }}"
|
||||
fi
|
||||
|
||||
echo "CLEAN_WORKSPACE=$CLEAN_WORKSPACE" >> $GITHUB_ENV
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
shell: bash
|
||||
|
||||
- name: Install Python dependencies and build package
|
||||
run: |
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
pip install --upgrade pip pytest pytest-cov ruff
|
||||
pip install -v .
|
||||
shell: bash
|
||||
|
||||
- name: Run Python Tests with Coverage
|
||||
run: |
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
python -m pytest python/tests/ --cov=zvec --cov-report=xml --no-cov-on-fail
|
||||
shell: bash
|
||||
|
||||
- name: Run Cpp Tests with Coverage
|
||||
run: |
|
||||
cd "$CLEAN_WORKSPACE/build"
|
||||
make unittest -j 16
|
||||
shell: bash
|
||||
@@ -0,0 +1,95 @@
|
||||
name: Nightly Coverage Report
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Runs daily at 00:00 CST (China Standard Time) = 16:00 UTC
|
||||
- cron: '0 16 * * *'
|
||||
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
coverage:
|
||||
runs-on: linux_x64
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
python-version: ['3.10']
|
||||
fail-fast: false
|
||||
|
||||
container:
|
||||
image: zvec-registry.cn-hongkong.cr.aliyuncs.com/zvec/zvec:0.0.2
|
||||
options: --user root
|
||||
|
||||
steps:
|
||||
- name: Activate Conda environment
|
||||
run: |
|
||||
if [[ "${{ matrix.python-version }}" == "3.10" ]]; then
|
||||
ENV_NAME="py310"
|
||||
elif [[ "${{ matrix.python-version }}" == "3.11" ]]; then
|
||||
ENV_NAME="py311"
|
||||
elif [[ "${{ matrix.python-version }}" == "3.12" ]]; then
|
||||
ENV_NAME="py312"
|
||||
else
|
||||
echo "Unsupported Python version"
|
||||
exit 1
|
||||
fi
|
||||
echo "CONDA_ENV_NAME=$ENV_NAME" >> $GITHUB_ENV
|
||||
source /opt/miniforge3/bin/activate "$ENV_NAME"
|
||||
python --version
|
||||
shell: bash
|
||||
|
||||
- name: Prepare clean build directory
|
||||
run: |
|
||||
export CLEAN_WORKSPACE="/tmp/zvec"
|
||||
mkdir -p "$CLEAN_WORKSPACE"
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
|
||||
git config --global --add safe.directory "$CLEAN_WORKSPACE"
|
||||
git clone --recursive "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" .
|
||||
|
||||
git checkout main # Always use main for nightly
|
||||
|
||||
echo "CLEAN_WORKSPACE=$CLEAN_WORKSPACE" >> $GITHUB_ENV
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
shell: bash
|
||||
|
||||
- name: Build with COVERAGE config
|
||||
run: |
|
||||
source /opt/miniforge3/bin/activate "${{ env.CONDA_ENV_NAME }}"
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
pip install --upgrade pip pytest pytest-cov
|
||||
|
||||
NPROC=$(nproc 2>/dev/null || echo 2)
|
||||
echo "Using $NPROC parallel jobs"
|
||||
|
||||
CMAKE_GENERATOR="Unix Makefiles" \
|
||||
CMAKE_BUILD_PARALLEL_LEVEL="$NPROC" \
|
||||
pip install -v . \
|
||||
--config-settings="cmake.build-type=COVERAGE"
|
||||
shell: bash
|
||||
|
||||
- name: Run Python Tests with Coverage
|
||||
run: |
|
||||
source /opt/miniforge3/bin/activate "${{ env.CONDA_ENV_NAME }}"
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
python -m pytest python/tests/ --cov=zvec --cov-report=xml
|
||||
shell: bash
|
||||
|
||||
- name: Run C++ Tests and Generate Coverage
|
||||
run: |
|
||||
cd "$CLEAN_WORKSPACE/build"
|
||||
make unittest -j$(nproc) # Run all (nightly can afford it)
|
||||
cd "$CLEAN_WORKSPACE"
|
||||
# Ensure gcov.sh is executable
|
||||
chmod +x scripts/gcov.sh
|
||||
bash scripts/gcov.sh -k
|
||||
shell: bash
|
||||
|
||||
- name: Upload Coverage to Codecov
|
||||
uses: codecov/codecov-action@v4
|
||||
with:
|
||||
files: ${{ env.CLEAN_WORKSPACE }}/proxima-zvec-filtered.lcov.info,${{ env.CLEAN_WORKSPACE }}/coverage.xml
|
||||
flags: python,cpp,nightly
|
||||
name: nightly-linux-py${{ matrix.python-version }}
|
||||
token: ${{ secrets.CODECOV_TOKEN }}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
.*
|
||||
*~
|
||||
bazel-*
|
||||
/*build*
|
||||
bin/*
|
||||
lib/*
|
||||
var/*
|
||||
thirdparty
|
||||
venv*
|
||||
tests/integration/conf/*
|
||||
tests/de_integration/conf/*
|
||||
**/__pycache__/*
|
||||
tests/bench/log/*
|
||||
tests/integration/integration
|
||||
tests/integration/log
|
||||
tests/integration/*.log
|
||||
tests/de_integration/log
|
||||
tests/de_integration/*.log
|
||||
!.git*
|
||||
!.clang-format
|
||||
!.circleci
|
||||
!.drone.yml
|
||||
sdk/python/dist/
|
||||
compile_commands.json
|
||||
dist
|
||||
html
|
||||
*.lcov.info
|
||||
|
||||
# Dependencies
|
||||
/node_modules
|
||||
|
||||
# Production
|
||||
/build
|
||||
|
||||
# Generated files
|
||||
.docusaurus
|
||||
.cache-loader
|
||||
|
||||
# Misc
|
||||
.DS_Store
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
|
||||
npm-debug.log*
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
|
||||
allure-*
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
[submodule "thirdparty/googletest/googletest-1.10.0"]
|
||||
path = thirdparty/googletest/googletest-1.10.0
|
||||
url = https://github.com/google/googletest.git
|
||||
[submodule "thirdparty/sparsehash/sparsehash-2.0.4"]
|
||||
path = thirdparty/sparsehash/sparsehash-2.0.4
|
||||
url = https://github.com/sparsehash/sparsehash.git
|
||||
ignore = untracked
|
||||
[submodule "thirdparty/gflags/gflags-2.2.2"]
|
||||
path = thirdparty/gflags/gflags-2.2.2
|
||||
url = https://github.com/gflags/gflags.git
|
||||
[submodule "thirdparty/rocksdb/rocksdb-8.1.1"]
|
||||
path = thirdparty/rocksdb/rocksdb-8.1.1
|
||||
url = https://github.com/facebook/rocksdb.git
|
||||
[submodule "thirdparty/yaml-cpp/yaml-cpp-0.6.3"]
|
||||
path = thirdparty/yaml-cpp/yaml-cpp-0.6.3
|
||||
url = https://github.com/jbeder/yaml-cpp.git
|
||||
[submodule "thirdparty/arrow/apache-arrow-21.0.0"]
|
||||
path = thirdparty/arrow/apache-arrow-21.0.0
|
||||
url = https://github.com/apache/arrow.git
|
||||
ignore = all
|
||||
[submodule "thirdparty/CRoaring/CRoaring-2.0.4"]
|
||||
path = thirdparty/CRoaring/CRoaring-2.0.4
|
||||
url = https://github.com/RoaringBitmap/CRoaring.git
|
||||
[submodule "thirdparty/glog/glog-0.5.0"]
|
||||
path = thirdparty/glog/glog-0.5.0
|
||||
url = https://github.com/google/glog.git
|
||||
ignore = all
|
||||
[submodule "thirdparty/protobuf/protobuf-3.21.12"]
|
||||
path = thirdparty/protobuf/protobuf-3.21.12
|
||||
url = https://github.com/protocolbuffers/protobuf.git
|
||||
[submodule "thirdparty/lz4/lz4-1.9.4"]
|
||||
path = thirdparty/lz4/lz4-1.9.4
|
||||
url = https://github.com/lz4/lz4.git
|
||||
[submodule "thirdparty/antlr/antlr4"]
|
||||
path = thirdparty/antlr/antlr4
|
||||
url = https://github.com/antlr/antlr4.git
|
||||
ignore = all
|
||||
[submodule "thirdparty/magic_enum/magic_enum-0.9.7"]
|
||||
path = thirdparty/magic_enum/magic_enum-0.9.7
|
||||
url = https://github.com/Neargye/magic_enum.git
|
||||
ignore = all
|
||||
@@ -0,0 +1,30 @@
|
||||
repos:
|
||||
- repo: https://github.com/astral-sh/ruff-pre-commit
|
||||
rev: v0.14.4
|
||||
hooks:
|
||||
- id: ruff-check
|
||||
name: Ruff (lint + fix)
|
||||
args: [--config=pyproject.toml, --fix]
|
||||
types_or: [python, pyi]
|
||||
- id: ruff-format
|
||||
name: Ruff (format)
|
||||
args: [--config=pyproject.toml]
|
||||
types_or: [python, pyi]
|
||||
|
||||
|
||||
- repo: https://github.com/pre-commit/mirrors-clang-format
|
||||
rev: v18.1.8
|
||||
hooks:
|
||||
- id: clang-format
|
||||
name: clang-format (C++)
|
||||
args: [--style=file, --fallback-style=none]
|
||||
types_or: [c++, c]
|
||||
exclude: ^thirdparty/.*$
|
||||
|
||||
|
||||
- repo: https://github.com/gitleaks/gitleaks
|
||||
rev: v8.21.4
|
||||
hooks:
|
||||
- id: gitleaks
|
||||
name: Detect hardcoded secrets (gitleaks)
|
||||
description: Scans staged files for secrets using gitleaks rules
|
||||
@@ -0,0 +1,60 @@
|
||||
cmake_minimum_required(VERSION 3.13)
|
||||
cmake_policy(SET CMP0077 NEW)
|
||||
project(zvec)
|
||||
set(CC_CXX_STANDARD 17)
|
||||
|
||||
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wall -Werror=return-type")
|
||||
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror=return-type")
|
||||
|
||||
if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
|
||||
set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--no-as-needed")
|
||||
set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-as-needed")
|
||||
endif()
|
||||
|
||||
if(NOT DEFINED PROJECT_ROOT_DIR OR NOT PROJECT_ROOT_DIR)
|
||||
set(PROJECT_ROOT_DIR ${CMAKE_SOURCE_DIR} CACHE PATH "Root directory of the project" FORCE)
|
||||
endif()
|
||||
|
||||
message(STATUS "PROJECT_ROOT_DIR = ${PROJECT_ROOT_DIR}")
|
||||
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
|
||||
include_directories(${PROJECT_ROOT_DIR}/src)
|
||||
|
||||
option(BUILD_PYTHON_BINDINGS "Build Python bindings using pybind11" OFF)
|
||||
message(STATUS "BUILD_PYTHON_BINDINGS:${BUILD_PYTHON_BINDINGS}")
|
||||
|
||||
option(BUILD_TOOLS "Build tools" ON)
|
||||
message(STATUS "BUILD_TOOLS:${BUILD_TOOLS}")
|
||||
|
||||
cc_directory(thirdparty)
|
||||
cc_directories(src)
|
||||
cc_directories(tests)
|
||||
|
||||
if(BUILD_TOOLS)
|
||||
cc_directories(tools)
|
||||
endif ()
|
||||
|
||||
git_version(GIT_SRCS_VER ${PROJECT_ROOT_DIR})
|
||||
set(CPACK_PACKAGE_VERSION ${GIT_SRCS_VER})
|
||||
set(CPACK_PACKAGE_NAME zvec)
|
||||
include(CPack)
|
||||
|
||||
if(BUILD_PYTHON_BINDINGS)
|
||||
if(APPLE)
|
||||
set(CMAKE_STRIP "")
|
||||
message(STATUS "Disabled strip on macOS to preserve code signature")
|
||||
endif()
|
||||
|
||||
include(GNUInstallDirs)
|
||||
if(DEFINED SKBUILD_PLATLIB_DIR)
|
||||
set(ZVEC_PY_INSTALL_DIR "${SKBUILD_PLATLIB_DIR}")
|
||||
elseif(DEFINED Python_SITEARCH)
|
||||
set(ZVEC_PY_INSTALL_DIR "${Python_SITEARCH}")
|
||||
else()
|
||||
set(ZVEC_PY_INSTALL_DIR "${CMAKE_INSTALL_LIBDIR}")
|
||||
endif()
|
||||
|
||||
MESSAGE(STATUS "Zvec install path: ${ZVEC_PY_INSTALL_DIR}")
|
||||
install(TARGETS _zvec LIBRARY DESTINATION ${ZVEC_PY_INSTALL_DIR})
|
||||
endif()
|
||||
@@ -0,0 +1,41 @@
|
||||
# Zvec Code of Conduct
|
||||
|
||||
## Our Pledge
|
||||
|
||||
We pledge to foster an open, respectful, and harassment-free environment for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation.
|
||||
|
||||
## Expected Behavior
|
||||
|
||||
- Use welcoming and inclusive language
|
||||
- Respect differing viewpoints and experiences
|
||||
- Gracefully accept constructive criticism
|
||||
- Focus on what is best for the community
|
||||
- Show empathy and kindness toward others
|
||||
|
||||
## Unacceptable Behavior
|
||||
|
||||
- Harassment, intimidation, or discriminatory conduct
|
||||
- Trolling, insulting, or derogatory comments
|
||||
- Public or private harassment
|
||||
- Publishing others’ private information without consent
|
||||
- Any conduct that would reasonably be considered inappropriate in a professional setting
|
||||
|
||||
## Enforcement
|
||||
|
||||
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team
|
||||
at **zvec@alibaba-inc.com** (replace with your contact). All complaints will be reviewed and investigated promptly
|
||||
and fairly.
|
||||
|
||||
The project team is obligated to respect the privacy and security of the reporter.
|
||||
|
||||
Consequences may include:
|
||||
- A formal warning
|
||||
- Temporary or permanent ban from project spaces
|
||||
- Removal of contributions (e.g. comments, PRs)
|
||||
|
||||
## Attribution
|
||||
|
||||
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.1, available at
|
||||
https://www.contributor-covenant.org/version/2/1/code_of_conduct.html
|
||||
|
||||
[homepage]: https://www.contributor-covenant.org
|
||||
+104
@@ -0,0 +1,104 @@
|
||||
# Contributing to Zvec
|
||||
|
||||
First off, thank you for considering contributing to Zvec! 🙌
|
||||
Whether you're reporting a bug, proposing a feature, improving documentation, or submitting code — every contribution helps make Zvec better.
|
||||
|
||||
## Code of Conduct
|
||||
|
||||
By participating, you agree to abide by our [Code of Conduct](CODE_OF_CONDUCT.md). Please be respectful, collaborative, and inclusive.
|
||||
|
||||
---
|
||||
|
||||
## Development Setup
|
||||
|
||||
### Prerequisites
|
||||
- Python ≥ 3.9
|
||||
- CMake ≥ 3.15 (`cmake --version`)
|
||||
- A C++17-compatible compiler (e.g., `g++-11+`, `clang++`, Apple Clang on macOS)
|
||||
|
||||
### Clone & Initialize
|
||||
```bash
|
||||
git clone --recursive https://github.com/your-org/zvec.git
|
||||
cd zvec
|
||||
```
|
||||
|
||||
> 💡 **Submodule tip**: If you forgot `--recursive`, run:
|
||||
> `git submodule update --init --recursive`
|
||||
|
||||
### Build from Source (Editable Install)
|
||||
```bash
|
||||
pip install -e ".[dev]"
|
||||
# This installs dev dependencies (pytest, ruff, etc.) and builds the C++ extension in-place
|
||||
```
|
||||
|
||||
> ✅ Verify:
|
||||
> ```bash
|
||||
> python -c "import zvec; print('Success!')"
|
||||
> ```
|
||||
|
||||
---
|
||||
|
||||
## Testing
|
||||
|
||||
### Run All Tests
|
||||
```bash
|
||||
pytest python/tests/ -v
|
||||
```
|
||||
|
||||
### Run with Coverage (Debug/CI)
|
||||
```bash
|
||||
pytest python/tests/ --cov=zvec --cov-report=term-missing
|
||||
```
|
||||
|
||||
> 🔎 See full rules in `[tool.ruff]` section of `pyproject.toml`.
|
||||
|
||||
---
|
||||
|
||||
## Build Customization
|
||||
|
||||
You can control build behavior via environment variables or `pyproject.toml`:
|
||||
|
||||
| Option | How to Set | Description |
|
||||
|--------|------------|-------------|
|
||||
| **Build Type** | `CMAKE_BUILD_TYPE=Debug` | `Debug`, `Release`, or `Coverage` (for gcov/lcov) |
|
||||
| **Generator** | `CMAKE_GENERATOR="Unix Makefiles"` | Default: `Ninja`; use Make if preferred |
|
||||
| **AVX-512** | `ENABLE_SKYLAKE_AVX512=ON` | Enable AVX-512 optimizations (x86_64 only) |
|
||||
|
||||
Example (Debug + Make):
|
||||
```bash
|
||||
CMAKE_BUILD_TYPE=Debug CMAKE_GENERATOR="Unix Makefiles" pip install -v .
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Submitting Changes
|
||||
|
||||
1. Fork the repo and create a feature branch (`feat/...`, `fix/...`, `docs/...`)
|
||||
2. Write clear commit messages (e.g., `fix(query): handle null vector in dense_fp32`)
|
||||
3. Ensure tests pass & linter is clean
|
||||
4. Open a Pull Request to `main`
|
||||
5. Link related issue (e.g., `Closes #123`)
|
||||
|
||||
✅ **PRs should include**:
|
||||
- Test coverage for new behavior
|
||||
- Updates to documentation (if applicable)
|
||||
- Reasoning behind non-obvious design choices
|
||||
|
||||
---
|
||||
|
||||
## Documentation
|
||||
|
||||
- User guides: `docs/` (built with MkDocs)
|
||||
- API reference: generated from docstrings (follow [Google style](https://google.github.io/styleguide/pyguide.html#38-comments-and-docstrings))
|
||||
- Build & deploy: `mkdocs serve` / `mkdocs build`
|
||||
|
||||
---
|
||||
|
||||
## Need Help
|
||||
|
||||
- Browse [existing issues](https://github.com/graphscope/zvec/issues)
|
||||
- For sensitive/security issues: email `zvec@alibaba-inc.com`
|
||||
|
||||
---
|
||||
|
||||
✨ Thanks again for being part of Zvec!
|
||||
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,99 @@
|
||||
<h1 align=" center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" srcset="https://zvec.oss-cn-hongkong.aliyuncs.com/logo/github_log_2.svg" />
|
||||
<img src="https://zvec.oss-cn-hongkong.aliyuncs.com/logo/github_logo_1.svg" width="400" alt="zvec logo" />
|
||||
</picture>
|
||||
</h1>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://codecov.io/github/GraphScope/zvec" style="text-decoration: none;"><img src="https://codecov.io/github/GraphScope/zvec/graph/badge.svg?token=PN16Q9PSZX" alt="Code Coverage"/></a>
|
||||
<a href="https://github.com/GraphScope/zvec/actions" style="text-decoration:none;"><img src="https://github.com/GraphScope/zvec/actions/workflows/linux_x64_docker_ci.yml/badge.svg" alt="Linux x64 CI"/></a>
|
||||
<a href="https://github.com/GraphScope/zvec/releases" style="text-decoration: none;"><img src="https://img.shields.io/github/v/release/GraphScope/zvec" alt="Latest Release"/></a>
|
||||
<a href="https://github.com/GraphScope/zvec/blob/main/LICENSE" style="text-decoration: none;"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License"/></a>
|
||||
</p>
|
||||
|
||||
# Zvec
|
||||
|
||||
**Zvec** is an open-source, in-process vector database — lightweight, lightning-fast, and designed to embed directly into applications. Built on **Proxima** (Alibaba's battle-tested vector search engine), it delivers production-grade, low-latency, scalable similarity search with minimal setup.
|
||||
|
||||
[📚 Quick Start](https://zvec.org/en/docs/quickstart/) | 🌐 [Website](https://zvec.org/en/) | [📖 Documentation](https://zvec.org/en/docs/) | [📊 Benchmarks](https://zvec.org/en/docs/benchmarks/)
|
||||
|
||||
## 💫 Features
|
||||
|
||||
- **Blazing Fast**: Searches billions of vectors in milliseconds.
|
||||
- **Simple, Just Works**: Install with `pip install zvec` and start searching in seconds. No servers, no config, no fuss.
|
||||
- **Dense + Sparse Vectors**: Work with both dense and sparse embeddings, with native support for multi-vector queries in a single call.
|
||||
- **Hybrid Search**: Combine semantic similarity with structured filters for precise results.
|
||||
- **Runs Anywhere**: As an in-process library, Zvec runs wherever your code runs — notebooks, servers, CLI tools, or even edge devices.
|
||||
|
||||
## 📦 Installation
|
||||
|
||||
Install Zvec from PyPI with a single command:
|
||||
|
||||
```bash
|
||||
pip install zvec
|
||||
```
|
||||
|
||||
**Requirements**:
|
||||
|
||||
- Python 3.9
|
||||
- **Supported platforms**: Linux (x86_64) or macOS (ARM64/x86_64)
|
||||
|
||||
### Building from Source
|
||||
|
||||
If you prefer to build Zvec from source, you can clone the repository and install it locally.
|
||||
**Note**: *The repository includes Git submodules, so cloning may take a few minutes depending on your network*.
|
||||
|
||||
```bash
|
||||
# Clone the repository
|
||||
git clone --recurse-submodules https://github.com/GraphScope/zvec.git
|
||||
cd zvec
|
||||
|
||||
# Install from source
|
||||
pip install .
|
||||
```
|
||||
|
||||
**Build Requirements**:
|
||||
|
||||
- Python 3.9 or higher
|
||||
- **Supported platforms**: Linux (x86_64) or macOS (ARM64/x86_64)
|
||||
- C++17 compatible compiler
|
||||
|
||||
For detailed build instructions, check the [Building from Source](https://zvec.org/en/docs/build/) guide.
|
||||
|
||||
## ⚡ One-Minute Example
|
||||
|
||||
```python
|
||||
import zvec
|
||||
|
||||
# Define collection schema
|
||||
schema = zvec.CollectionSchema(
|
||||
name="example",
|
||||
vectors=zvec.VectorSchema("embedding", zvec.DataType.VECTOR_FP32, 4),
|
||||
)
|
||||
|
||||
# Create collection
|
||||
collection = zvec.create_and_open(path="./zvec_example", schema=schema,)
|
||||
|
||||
# Insert documents
|
||||
collection.insert([
|
||||
zvec.Doc(id="doc_1", vectors={"embedding": [0.1, 0.2, 0.3, 0.4]}),
|
||||
zvec.Doc(id="doc_2", vectors={"embedding": [0.2, 0.3, 0.4, 0.1]}),
|
||||
])
|
||||
|
||||
# Search by vector similarity
|
||||
results = collection.query(
|
||||
zvec.VectorQuery("embedding", vector=[0.4, 0.3, 0.3, 0.1]),
|
||||
topk=10
|
||||
)
|
||||
|
||||
# Results: list of {'id': str, 'score': float, ...}, sorted by relevance
|
||||
print(results)
|
||||
```
|
||||
|
||||
## ❤️ Contributing
|
||||
|
||||
We welcome and appreciate contributions from the community!
|
||||
Whether you're fixing a bug, adding a feature, or improving documentation, your help makes Zvec better for everyone.
|
||||
|
||||
Check out our [Contributing Guide](./CONTRIBUTING.md) to get started!
|
||||
+2338
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,585 @@
|
||||
## https://en.wikipedia.org/wiki/List_of_Intel_CPU_microarchitectures
|
||||
## https://en.wikipedia.org/wiki/List_of_AMD_CPU_microarchitectures
|
||||
## https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html
|
||||
|
||||
## Intel Microarchitectures
|
||||
option(ENABLE_NEHALEM "Enable Intel Nehalem CPU microarchitecture" OFF)
|
||||
option(ENABLE_SANDYBRIDGE "Enable Intel Sandy Bridge CPU microarchitecture" OFF)
|
||||
option(ENABLE_HASWELL "Enable Intel Haswell CPU microarchitecture" OFF)
|
||||
option(ENABLE_BROADWELL "Enable Intel Broadwell CPU microarchitecture" OFF)
|
||||
option(ENABLE_SKYLAKE "Enable Intel Skylake CPU microarchitecture" OFF)
|
||||
option(ENABLE_SKYLAKE_AVX512 "Enable Intel Skylake Server CPU microarchitecture" OFF)
|
||||
option(ENABLE_SAPPHIRERAPIDS "Enable Intel Sapphire Rapids Server CPU microarchitecture" OFF)
|
||||
option(ENABLE_EMERALDRAPIDS "Enable Intel Emerald Rapids Server CPU microarchitecture" OFF)
|
||||
option(ENABLE_GRANITERAPIDS "Enable Intel Granite Rapids Server CPU microarchitecture" OFF)
|
||||
|
||||
## AMD Microarchitectures
|
||||
option(ENABLE_ZEN1 "Enable AMD Zen+ Family 17h CPU microarchitecture" OFF)
|
||||
option(ENABLE_ZEN2 "Enable AMD Zen 2 Family 17h CPU microarchitecture" OFF)
|
||||
option(ENABLE_ZEN3 "Enable AMD Zen 3 Family 19h CPU microarchitecture" OFF)
|
||||
|
||||
## Intel Instructions
|
||||
option(ENABLE_SSE "Enable Intel SSE instructions" OFF)
|
||||
option(ENABLE_SSE2 "Enable Intel SSE2 instructions" OFF)
|
||||
option(ENABLE_SSE3 "Enable Intel SSE3 instructions" OFF)
|
||||
option(ENABLE_SSSE3 "Enable Intel SSSE3 instructions" OFF)
|
||||
option(ENABLE_SSE4.1 "Enable Intel SSE4.1 instructions" OFF)
|
||||
option(ENABLE_SSE4.2 "Enable Intel SSE4.2 instructions" OFF)
|
||||
option(ENABLE_AVX "Enable Intel AVX instructions" OFF)
|
||||
option(ENABLE_AVX2 "Enable Intel AVX2 instructions" OFF)
|
||||
option(ENABLE_AVX512F "Enable Intel AVX512F instructions" OFF)
|
||||
option(ENABLE_AVX512BW "Enable Intel AVX512BW instructions" OFF)
|
||||
option(ENABLE_AVX512CD "Enable Intel AVX512CD instructions" OFF)
|
||||
option(ENABLE_AVX512DQ "Enable Intel AVX512DQ instructions" OFF)
|
||||
option(ENABLE_AVX512ER "Enable Intel AVX512ER instructions" OFF)
|
||||
option(ENABLE_AVX512IFMA52 "Enable Intel AVX512IFMA52 instructions" OFF)
|
||||
option(ENABLE_AVX512PF "Enable Intel AVX512PF instructions" OFF)
|
||||
option(ENABLE_AVX512VL "Enable Intel AVX512VL instructions" OFF)
|
||||
option(ENABLE_AVX512VPOPCNTDQ "Enable Intel AVX512VPOPCNTDQ instructions" OFF)
|
||||
option(ENABLE_AVX5124FMAPS "Enable Intel AVX5124FMAPS instructions" OFF)
|
||||
option(ENABLE_AVX5124VNNIW "Enable Intel AVX5124VNNIW instructions" OFF)
|
||||
option(ENABLE_AVX512BF16 "Enable Intel AVX512BF16 instructions" OFF)
|
||||
option(ENABLE_AVX512BITALG "Enable Intel AVX512BITALG instructions" OFF)
|
||||
option(ENABLE_AVX512VBMI "Enable Intel AVX512VBMI instructions" OFF)
|
||||
option(ENABLE_AVX512VBMI2 "Enable Intel AVX512VBMI2 instructions" OFF)
|
||||
option(ENABLE_AVX512VNNI "Enable Intel AVX512VNNI instructions" OFF)
|
||||
option(ENABLE_AVX512VP2INTERSECT "Enable Intel AVX512VP2INTERSECT instructions" OFF)
|
||||
option(ENABLE_AVX512FP16 "Enable Intel AVX512FP16 instructions" OFF)
|
||||
|
||||
option(ENABLE_AVXVNNI "Enable Intel AVXVNNI instructions" OFF)
|
||||
option(ENABLE_FMA "Enable Intel FMA instructions" OFF)
|
||||
option(ENABLE_F16C "Enable Intel F16C instructions" OFF)
|
||||
|
||||
## ARM architectures
|
||||
option(ENABLE_ARMV8A "Enable ARMv8-a architecture" OFF)
|
||||
option(ENABLE_ARMV8.1A "Enable ARMv8.1-a architecture" OFF)
|
||||
option(ENABLE_ARMV8.2A "Enable ARMv8.2-a architecture" OFF)
|
||||
option(ENABLE_ARMV8.3A "Enable ARMv8.3-a architecture" OFF)
|
||||
option(ENABLE_ARMV8.4A "Enable ARMv8.4-a architecture" OFF)
|
||||
option(ENABLE_ARMV8.5A "Enable ARMv8.5-a architecture" OFF)
|
||||
option(ENABLE_ARMV8.6A "Enable ARMv8.6-a architecture" OFF)
|
||||
|
||||
## OpenMP option
|
||||
option(ENABLE_OPENMP "Enable OpenMP support" OFF)
|
||||
|
||||
function(_AppendFlags _RESULT _FLAG)
|
||||
if(${_RESULT} AND NOT "${${_RESULT}}" MATCHES "${_FLAG}")
|
||||
set(${_RESULT} "${${_RESULT}} ${_FLAG}" PARENT_SCOPE)
|
||||
else()
|
||||
set(${_RESULT} "${_FLAG}" PARENT_SCOPE)
|
||||
endif()
|
||||
endfunction()
|
||||
|
||||
include(CheckCCompilerFlag)
|
||||
if(NOT MSVC)
|
||||
if(ENABLE_ARMV8.6A)
|
||||
CHECK_C_COMPILER_FLAG("-march=armv8.6-a" COMPILER_SUPPORT_ARMV8.6A)
|
||||
if(COMPILER_SUPPORT_ARMV8.6A)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=armv8.6-a)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.6-a)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.6-a\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_ARMV8.5A)
|
||||
CHECK_C_COMPILER_FLAG("-march=armv8.5-a" COMPILER_SUPPORT_ARMV8.5A)
|
||||
if(COMPILER_SUPPORT_ARMV8.5A)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=armv8.5-a)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.5-a)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.5-a\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_ARMV8.4A)
|
||||
CHECK_C_COMPILER_FLAG("-march=armv8.4-a" COMPILER_SUPPORT_ARMV8.4A)
|
||||
if(COMPILER_SUPPORT_ARMV8.4A)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=armv8.4-a)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.4-a)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.4-a\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_ARMV8.3A)
|
||||
CHECK_C_COMPILER_FLAG("-march=armv8.3-a" COMPILER_SUPPORT_ARMV8.3A)
|
||||
if(COMPILER_SUPPORT_ARMV8.3A)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=armv8.3-a)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.3-a)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.3-a\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_ARMV8.2A)
|
||||
CHECK_C_COMPILER_FLAG("-march=armv8.2-a" COMPILER_SUPPORT_ARMV8.2A)
|
||||
if(COMPILER_SUPPORT_ARMV8.2A)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=armv8.2-a)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.2-a)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.2-a\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_ARMV8.1A)
|
||||
CHECK_C_COMPILER_FLAG("-march=armv8.1-a" COMPILER_SUPPORT_ARMV8.1A)
|
||||
if(COMPILER_SUPPORT_ARMV8.1A)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=armv8.1-a)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8.1-a)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8.1-a\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_ARMV8A)
|
||||
CHECK_C_COMPILER_FLAG("-march=armv8-a" COMPILER_SUPPORT_ARMV8A)
|
||||
if(COMPILER_SUPPORT_ARMV8A)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=armv8-a)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=armv8-a)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=armv8-a\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_ZEN1)
|
||||
CHECK_C_COMPILER_FLAG("-march=znver1" COMPILER_SUPPORT_ZEN1)
|
||||
if(COMPILER_SUPPORT_ZEN1)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=znver1)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=znver1)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=znver1\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_ZEN2)
|
||||
CHECK_C_COMPILER_FLAG("-march=znver2" COMPILER_SUPPORT_ZEN2)
|
||||
if(COMPILER_SUPPORT_ZEN2)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=znver2)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=znver2)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=znver2\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_ZEN3)
|
||||
CHECK_C_COMPILER_FLAG("-march=znver3" COMPILER_SUPPORT_ZEN3)
|
||||
if(COMPILER_SUPPORT_ZEN3)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=znver3)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=znver3)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=znver3\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_NEHALEM)
|
||||
CHECK_C_COMPILER_FLAG("-march=nehalem" COMPILER_SUPPORT_NEHALEM)
|
||||
if(COMPILER_SUPPORT_NEHALEM)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=nehalem)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=nehalem)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=nehalem\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_SANDYBRIDGE)
|
||||
CHECK_C_COMPILER_FLAG("-march=sandybridge" COMPILER_SUPPORT_SANDYBRIDGE)
|
||||
if(COMPILER_SUPPORT_SANDYBRIDGE)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=sandybridge)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=sandybridge)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=sandybridge\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_HASWELL)
|
||||
CHECK_C_COMPILER_FLAG("-march=haswell" COMPILER_SUPPORT_HASWELL)
|
||||
if(COMPILER_SUPPORT_HASWELL)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=haswell)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=haswell)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=haswell\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_BROADWELL)
|
||||
CHECK_C_COMPILER_FLAG("-march=broadwell" COMPILER_SUPPORT_BROADWELL)
|
||||
if(COMPILER_SUPPORT_BROADWELL)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=broadwell)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=broadwell)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=broadwell\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_SKYLAKE)
|
||||
CHECK_C_COMPILER_FLAG("-march=skylake" COMPILER_SUPPORT_SKYLAKE)
|
||||
if(COMPILER_SUPPORT_SKYLAKE)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=skylake)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=skylake)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=skylake\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_SKYLAKE_AVX512)
|
||||
CHECK_C_COMPILER_FLAG("-march=skylake-avx512" COMPILER_SUPPORT_SKYLAKE_AVX512)
|
||||
if(COMPILER_SUPPORT_SKYLAKE_AVX512)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=skylake-avx512)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=skylake-avx512)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=skylake-avx512\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_SAPPHIRERAPIDS)
|
||||
CHECK_C_COMPILER_FLAG("-march=sapphirerapids" COMPILER_SUPPORT_SAPPHIRERAPIDS)
|
||||
if(COMPILER_SUPPORT_SAPPHIRERAPIDS)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=sapphirerapids)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=sapphirerapids)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=sapphirerapids\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_EMERALDRAPIDS)
|
||||
CHECK_C_COMPILER_FLAG("-march=emeraldrapids" COMPILER_SUPPORT_EMERALDRAPID)
|
||||
if(COMPILER_SUPPORT_EMERALDRAPID)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=emeraldrapids)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=emeraldrapids)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=emeraldrapids\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_GRANITERAPIDS)
|
||||
CHECK_C_COMPILER_FLAG("-march=graniterapids" COMPILER_SUPPORT_GRANITERAPIDS)
|
||||
if(COMPILER_SUPPORT_GRANITERAPIDS)
|
||||
_AppendFlags(CMAKE_C_FLAGS -march=graniterapids)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -march=graniterapids)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-march=graniterapids\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512BW)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512bw" COMPILER_SUPPORT_AVX512BW)
|
||||
if(COMPILER_SUPPORT_AVX512BW)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512bw)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512bw)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512bw\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512CD)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512cd" COMPILER_SUPPORT_AVX512CD)
|
||||
if(COMPILER_SUPPORT_AVX512CD)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512cd)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512cd)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512cd\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512DQ)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512dq" COMPILER_SUPPORT_AVX512DQ)
|
||||
if(COMPILER_SUPPORT_AVX512DQ)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512dq)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512dq)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512dq\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512ER)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512er" COMPILER_SUPPORT_AVX512ER)
|
||||
if(COMPILER_SUPPORT_AVX512ER)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512er)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512er)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512er\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512IFMA52)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512ifma52" COMPILER_SUPPORT_AVX512IFMA52)
|
||||
if(COMPILER_SUPPORT_AVX512IFMA52)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512ifma52)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512ifma52)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512ifma52\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512PF)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512pf" COMPILER_SUPPORT_AVX512PF)
|
||||
if(COMPILER_SUPPORT_AVX512PF)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512pf)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512pf)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512pf\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512VL)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512vl" COMPILER_SUPPORT_AVX512VL)
|
||||
if(COMPILER_SUPPORT_AVX512VL)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512vl)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vl)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vl\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512VPOPCNTDQ)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512vpopcntdq" COMPILER_SUPPORT_AVX512VPOPCNTDQ)
|
||||
if(COMPILER_SUPPORT_AVX512VPOPCNTDQ)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512vpopcntdq)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vpopcntdq)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vpopcntdq\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX5124FMAPS)
|
||||
CHECK_C_COMPILER_FLAG("-mavx5124fmaps" COMPILER_SUPPORT_AVX5124FMAPS)
|
||||
if(COMPILER_SUPPORT_AVX5124FMAPS)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx5124fmaps)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx5124fmaps)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx5124fmaps\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX5124VNNIW)
|
||||
CHECK_C_COMPILER_FLAG("-mavx5124vnniw" COMPILER_SUPPORT_AVX5124VNNIW)
|
||||
if(COMPILER_SUPPORT_AVX5124VNNIW)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx5124vnniw)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx5124vnniw)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx5124vnniw\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512BF16)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512bf16" COMPILER_SUPPORT_AVX512BF16)
|
||||
if(COMPILER_SUPPORT_AVX512BF16)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512bf16)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512bf16)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512bf16\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512BITALG)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512bitalg" COMPILER_SUPPORT_AVX512BITALG)
|
||||
if(COMPILER_SUPPORT_AVX512BITALG)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512bitalg)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512bitalg)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512bitalg\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512VBMI)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512vbmi" COMPILER_SUPPORT_AVX512VBMI)
|
||||
if(COMPILER_SUPPORT_AVX512VBMI)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512vbmi)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vbmi)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vbmi\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512VBMI2)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512vbmi2" COMPILER_SUPPORT_AVX512VBMI2)
|
||||
if(COMPILER_SUPPORT_AVX512VBMI2)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512vbmi2)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vbmi2)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vbmi2\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512VNNI)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512vnni" COMPILER_SUPPORT_AVX512VNNI)
|
||||
if(COMPILER_SUPPORT_AVX512VNNI)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512vnni)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vnni)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vnni\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512VP2INTERSECT)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512vp2intersect" COMPILER_SUPPORT_AVX512VP2INTERSECT)
|
||||
if(COMPILER_SUPPORT_AVX512VP2INTERSECT)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512vp2intersect)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512vp2intersect)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512vp2intersect\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512FP16)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512fp16" COMPILER_SUPPORT_AVX512FP16)
|
||||
if(COMPILER_SUPPORT_AVX512FP16)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512fp16)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512fp16)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512fp16\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVXVNNI)
|
||||
CHECK_C_COMPILER_FLAG("-mavxvnni" COMPILER_SUPPORT_AVXVNNI)
|
||||
if(COMPILER_SUPPORT_AVXVNNI)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavxvnni)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavxvnni)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavxvnni\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_AVX512F)
|
||||
CHECK_C_COMPILER_FLAG("-mavx512f" COMPILER_SUPPORT_AVX512F)
|
||||
if(COMPILER_SUPPORT_AVX512F)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx512f)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx512f)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx512f\'.")
|
||||
endif()
|
||||
elseif(ENABLE_AVX2)
|
||||
CHECK_C_COMPILER_FLAG("-mavx2" COMPILER_SUPPORT_AVX2)
|
||||
if(COMPILER_SUPPORT_AVX2)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx2)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx2)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx2\'.")
|
||||
endif()
|
||||
elseif(ENABLE_AVX)
|
||||
CHECK_C_COMPILER_FLAG("-mavx" COMPILER_SUPPORT_AVX)
|
||||
if(COMPILER_SUPPORT_AVX)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mavx)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mavx)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mavx\'.")
|
||||
endif()
|
||||
elseif(ENABLE_SSE4.2)
|
||||
CHECK_C_COMPILER_FLAG("-msse4.2" COMPILER_SUPPORT_SSE4.2)
|
||||
if(COMPILER_SUPPORT_SSE4.2)
|
||||
_AppendFlags(CMAKE_C_FLAGS -msse4.2)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -msse4.2)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-msse4.2\'.")
|
||||
endif()
|
||||
elseif(ENABLE_SSE4.1)
|
||||
CHECK_C_COMPILER_FLAG("-msse4.1" COMPILER_SUPPORT_SSE4.1)
|
||||
if(COMPILER_SUPPORT_SSE4.1)
|
||||
_AppendFlags(CMAKE_C_FLAGS -msse4.1)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -msse4.1)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-msse4.1\'.")
|
||||
endif()
|
||||
elseif(ENABLE_SSSE3)
|
||||
CHECK_C_COMPILER_FLAG("-mssse3" COMPILER_SUPPORT_SSSE3)
|
||||
if(COMPILER_SUPPORT_SSSE3)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mssse3)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mssse3)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mssse3\'.")
|
||||
endif()
|
||||
elseif(ENABLE_SSE3)
|
||||
CHECK_C_COMPILER_FLAG("-msse3" COMPILER_SUPPORT_SSE3)
|
||||
if(COMPILER_SUPPORT_SSE3)
|
||||
_AppendFlags(CMAKE_C_FLAGS -msse3)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -msse3)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-msse3\'.")
|
||||
endif()
|
||||
elseif(ENABLE_SSE2)
|
||||
CHECK_C_COMPILER_FLAG("-msse2" COMPILER_SUPPORT_SSE2)
|
||||
if(COMPILER_SUPPORT_SSE2)
|
||||
_AppendFlags(CMAKE_C_FLAGS -msse2)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -msse2)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-msse2\'.")
|
||||
endif()
|
||||
elseif(ENABLE_SSE)
|
||||
CHECK_C_COMPILER_FLAG("-msse" COMPILER_SUPPORT_SSE)
|
||||
if(COMPILER_SUPPORT_SSE)
|
||||
_AppendFlags(CMAKE_C_FLAGS -msse)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -msse)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-msse\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_FMA)
|
||||
CHECK_C_COMPILER_FLAG("-mfma" COMPILER_SUPPORT_FMA)
|
||||
if(COMPILER_SUPPORT_FMA)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mfma)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mfma)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mfma\'.")
|
||||
endif()
|
||||
endif()
|
||||
|
||||
if(ENABLE_F16C)
|
||||
CHECK_C_COMPILER_FLAG("-mf16c" COMPILER_SUPPORT_F16C)
|
||||
if(COMPILER_SUPPORT_F16C)
|
||||
_AppendFlags(CMAKE_C_FLAGS -mf16c)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS -mf16c)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'-mf16c\'.")
|
||||
endif()
|
||||
endif()
|
||||
else()
|
||||
if(ENABLE_AVX512F OR ENABLE_AVX512CD OR ENABLE_AVX512BW OR ENABLE_AVX512DQ OR ENABLE_AVX512VL)
|
||||
CHECK_C_COMPILER_FLAG("/arch:AVX512" COMPILER_SUPPORT_AVX512F)
|
||||
if(COMPILER_SUPPORT_AVX512F)
|
||||
_AppendFlags(CMAKE_C_FLAGS /arch:AVX512)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS /arch:AVX512)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'/arch:AVX512\'.")
|
||||
endif()
|
||||
elseif(ENABLE_AVX2)
|
||||
CHECK_C_COMPILER_FLAG("/arch:AVX2" COMPILER_SUPPORT_AVX2)
|
||||
if(COMPILER_SUPPORT_AVX2)
|
||||
_AppendFlags(CMAKE_C_FLAGS /arch:AVX2)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS /arch:AVX2)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'/arch:AVX2\'.")
|
||||
endif()
|
||||
elseif(ENABLE_AVX)
|
||||
CHECK_C_COMPILER_FLAG("/arch:AVX" COMPILER_SUPPORT_AVX)
|
||||
if(COMPILER_SUPPORT_AVX)
|
||||
_AppendFlags(CMAKE_C_FLAGS /arch:AVX)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS /arch:AVX)
|
||||
else()
|
||||
message(SEND_ERROR "Compiler cannot support flag \'/arch:AVX\'.")
|
||||
endif()
|
||||
elseif(ENABLE_SSE4.2 OR ENABLE_SSE4.1 OR ENABLE_SSE3 OR ENABLE_SSSE3 OR ENABLE_SSE2)
|
||||
CHECK_C_COMPILER_FLAG("/arch:SSE2" COMPILER_SUPPORT_SSE2)
|
||||
if(COMPILER_SUPPORT_SSE2)
|
||||
_AppendFlags(CMAKE_C_FLAGS /arch:SSE2)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS /arch:SSE2)
|
||||
endif()
|
||||
elseif(ENABLE_SSE)
|
||||
CHECK_C_COMPILER_FLAG("/arch:SSE" COMPILER_SUPPORT_SSE)
|
||||
if(COMPILER_SUPPORT_SSE)
|
||||
_AppendFlags(CMAKE_C_FLAGS /arch:SSE)
|
||||
_AppendFlags(CMAKE_CXX_FLAGS /arch:SSE)
|
||||
endif()
|
||||
endif()
|
||||
endif()
|
||||
|
||||
# OpenMP support
|
||||
if(ENABLE_OPENMP)
|
||||
find_package(OpenMP REQUIRED)
|
||||
if(OPENMP_FOUND)
|
||||
_AppendFlags(CMAKE_C_FLAGS ${OpenMP_C_FLAGS})
|
||||
_AppendFlags(CMAKE_CXX_FLAGS ${OpenMP_CXX_FLAGS})
|
||||
endif()
|
||||
endif()
|
||||
@@ -0,0 +1,28 @@
|
||||
function(apply_patch_once patch_name target_dir patch_file)
|
||||
set(mark_file "${target_dir}/.${patch_name}_patched")
|
||||
|
||||
if(EXISTS "${mark_file}")
|
||||
#message(STATUS "Patch '${patch_name}' already applied to ${target_dir}, skipping.")
|
||||
return()
|
||||
endif()
|
||||
|
||||
if(NOT EXISTS "${patch_file}")
|
||||
message(FATAL_ERROR "Patch file '${patch_file}' not found!")
|
||||
endif()
|
||||
|
||||
#message(STATUS "Applying patch '${patch_name}' to ${target_dir} ...")
|
||||
execute_process(
|
||||
COMMAND patch -p1 -i "${patch_file}"
|
||||
WORKING_DIRECTORY "${target_dir}"
|
||||
RESULT_VARIABLE patch_result
|
||||
OUTPUT_VARIABLE patch_stdout
|
||||
ERROR_VARIABLE patch_stderr
|
||||
)
|
||||
|
||||
if(NOT patch_result EQUAL 0)
|
||||
message(FATAL_ERROR "Failed to apply patch '${patch_name}' to ${target_dir}:\n${patch_stderr}")
|
||||
else()
|
||||
#message(STATUS "Patch '${patch_name}' applied successfully:\n${patch_stdout}")
|
||||
file(WRITE "${mark_file}" "patched")
|
||||
endif()
|
||||
endfunction()
|
||||
+242
@@ -0,0 +1,242 @@
|
||||
######################################################################################################
|
||||
# Zvec: High-Performance Vector Database with PyBind11 & C++ Backend
|
||||
######################################################################################################
|
||||
[project]
|
||||
name = "zvec"
|
||||
dynamic = ["version"]
|
||||
description = "A high-performance vector database engine with native C++ backend and Python bindings"
|
||||
readme = "README.md"
|
||||
license = { text = "Apache-2.0" }
|
||||
authors = [
|
||||
{ name = "zvec", email = "zvec@alibaba-inc.com" },
|
||||
]
|
||||
maintainers = [
|
||||
{ name = "Zvec Core Team", email = "zvec@alibaba-inc.com" },
|
||||
]
|
||||
requires-python = ">=3.9"
|
||||
classifiers = [
|
||||
"Development Status :: 3 - Alpha",
|
||||
"Intended Audience :: Developers",
|
||||
"Intended Audience :: Education",
|
||||
"Intended Audience :: Science/Research",
|
||||
"License :: OSI Approved :: Apache Software License",
|
||||
"Operating System :: POSIX :: Linux",
|
||||
"Operating System :: MacOS",
|
||||
"Programming Language :: C++",
|
||||
"Programming Language :: Python :: 3 :: Only",
|
||||
"Programming Language :: Python :: 3.10",
|
||||
"Programming Language :: Python :: 3.11",
|
||||
"Programming Language :: Python :: 3.12",
|
||||
"Topic :: Database",
|
||||
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
||||
"Topic :: Software Development :: Libraries :: Python Modules",
|
||||
]
|
||||
keywords = [
|
||||
"vector-database", "ann", "nearest-neighbor"
|
||||
]
|
||||
dependencies = [
|
||||
"numpy >=1.23",
|
||||
]
|
||||
|
||||
[project.urls]
|
||||
Homepage = "https://github.com/graphscope/zvec"
|
||||
Repository = "https://github.com/graphscope/zvec"
|
||||
"Bug Tracker" = "https://github.com/graphscope/zvec/issues"
|
||||
"Documentation" = "https://zvec.org"
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = [
|
||||
"pytest >=8.0",
|
||||
"pytest-cov >=4.1",
|
||||
"pytest-mock >=3.12",
|
||||
"cibuildwheel == 2.17.0",
|
||||
]
|
||||
docs = [
|
||||
"mkdocs >=1.5",
|
||||
"mkdocs-material >=9.5",
|
||||
"mkdocstrings[python] >=0.24",
|
||||
]
|
||||
dev = [
|
||||
"ruff >=0.4",
|
||||
"black >=24.0",
|
||||
"mypy >=1.8",
|
||||
"pre-commit >=3.6",
|
||||
"build >=1.0",
|
||||
"twine >=4.0",
|
||||
"numpy >=1.23",
|
||||
# Inherit test deps
|
||||
"pytest >=8.0",
|
||||
"pytest-cov >=4.1",
|
||||
"pytest-mock >=3.12",
|
||||
"cibuildwheel == 2.17.0",
|
||||
# Inherit docs deps
|
||||
"mkdocs >=1.5",
|
||||
"mkdocs-material >=9.5",
|
||||
"mkdocstrings[python] >=0.24",
|
||||
"pybind11-stubgen>=2.5.5",
|
||||
"pybind11 >=3.0",
|
||||
]
|
||||
######################################################################################################
|
||||
# BUILD SYSTEM CONFIGURATION (scikit-build-core)
|
||||
######################################################################################################
|
||||
[build-system]
|
||||
requires = ["scikit-build-core >=0.11", "pybind11 >=3.0", "setuptools_scm>=8.0"]
|
||||
build-backend = "scikit_build_core.build"
|
||||
|
||||
[tool.scikit-build]
|
||||
# Core settings
|
||||
minimum-version = "0.11"
|
||||
metadata.version.provider = "scikit_build_core.metadata.setuptools_scm"
|
||||
|
||||
# CMake configuration
|
||||
cmake.version = ">=3.15,<4.0"
|
||||
ninja.version = ">=1.11"
|
||||
cmake.build-type = "Release"
|
||||
install.strip = true # Strip symbols in release builds to reduce wheel size
|
||||
|
||||
# Build directory
|
||||
build-dir = "build"
|
||||
|
||||
# Platform support
|
||||
wheel.expand-macos-universal-tags = true
|
||||
wheel.packages = ["python/zvec"]
|
||||
|
||||
# Source distribution
|
||||
sdist.include = [
|
||||
"README.md",
|
||||
"LICENSE",
|
||||
"pyproject.toml",
|
||||
"CMakeLists.txt",
|
||||
"src/**/*",
|
||||
"stub/zvec/**/*",
|
||||
"python/zvec/py.typed",
|
||||
]
|
||||
|
||||
# CMake defines (env-overridable)
|
||||
[tool.scikit-build.cmake.define]
|
||||
BUILD_TOOLS = "OFF"
|
||||
BUILD_PYTHON_BINDINGS = "ON"
|
||||
ENABLE_SKYLAKE_AVX512 = { env = "ENABLE_SKYLAKE_AVX512", default = "OFF" }
|
||||
ENABLE_SAPPHIRERAPIDS = { env = "ENABLE_SAPPHIRERAPIDS", default = "OFF" }
|
||||
|
||||
######################################################################################################
|
||||
# TESTING & QUALITY
|
||||
######################################################################################################
|
||||
[tool.pytest.ini_options]
|
||||
minversion = "8.0"
|
||||
addopts = [
|
||||
"-ra",
|
||||
"--showlocals",
|
||||
"--strict-markers",
|
||||
"--strict-config",
|
||||
"--tb=short",
|
||||
]
|
||||
xfail_strict = true
|
||||
log_cli_level = "INFO"
|
||||
filterwarnings = [
|
||||
"error",
|
||||
"ignore::pytest.PytestCacheWarning",
|
||||
# Ignore numpy deprecation warnings in tests (if any)
|
||||
"ignore:.*numpy.*:DeprecationWarning",
|
||||
]
|
||||
testpaths = ["python/tests"]
|
||||
markers = [
|
||||
"title: Custom marker for test title/description",
|
||||
# "slow: marks tests as slow",
|
||||
]
|
||||
|
||||
######################################################################################################
|
||||
# BUILD WHEEL
|
||||
######################################################################################################
|
||||
[tool.cibuildwheel]
|
||||
build = [
|
||||
"cp310-*",
|
||||
"cp311-*",
|
||||
"cp312-*",
|
||||
]
|
||||
build-frontend = "build"
|
||||
test-requires = ["pytest", "numpy"]
|
||||
build-verbosity = 1
|
||||
|
||||
[tool.cibuildwheel.linux]
|
||||
archs = ["x86_64"]
|
||||
test-command = "cd {project} && pytest python/tests -v --tb=short"
|
||||
manylinux-x86_64-image = "manylinux_2_28"
|
||||
skip = "*musllinux*"
|
||||
|
||||
[tool.cibuildwheel.macos]
|
||||
archs = ["arm64"]
|
||||
environment = { MACOSX_DEPLOYMENT_TARGET = "11.0" }
|
||||
######################################################################################################
|
||||
# CODE QUALITY & FORMATTING (Ruff)
|
||||
######################################################################################################
|
||||
[tool.ruff]
|
||||
target-version = "py310"
|
||||
line-length = 88
|
||||
exclude = [
|
||||
"build/",
|
||||
"dist/",
|
||||
".git/",
|
||||
".venv/",
|
||||
"venv/",
|
||||
]
|
||||
|
||||
[tool.ruff.lint]
|
||||
extend-select = [
|
||||
"B", # flake8-bugbear
|
||||
"I", # isort
|
||||
"ARG", # flake8-unused-arguments
|
||||
"C4", # flake8-comprehensions
|
||||
"EM", # flake8-errmsg
|
||||
"ICN", # flake8-import-conventions
|
||||
"G", # flake8-logging-format
|
||||
"PGH", # pygrep-hooks
|
||||
"PIE", # flake8-pie
|
||||
"PL", # pylint
|
||||
"PT", # flake8-pytest-style
|
||||
"PTH", # flake8-use-pathlib
|
||||
"RET", # flake8-return
|
||||
"RUF", # Ruff-specific
|
||||
"SIM", # flake8-simplify
|
||||
"T20", # flake8-print
|
||||
"UP", # pyupgrade
|
||||
"YTT", # flake8-2020
|
||||
"EXE", # flake8-executable
|
||||
"NPY", # NumPy-specific
|
||||
"PD", # pandas-vet
|
||||
]
|
||||
ignore = [
|
||||
"PLR0913", # Too many arguments (common in bindings)
|
||||
"PLR2004", # Magic value used in comparison
|
||||
"UP045", "UP007", # Use list() instead of [] (breaks C++ init)
|
||||
"EM101", "EM102", # Exception messages as literals (ok in tests/utils)
|
||||
"B008", # Mutable default args (cautiously allowed in config)
|
||||
"E731", # Lambda assignment (used in callbacks)
|
||||
"B019", # `functools.lru_cache` on methods (handled manually)
|
||||
"PLR0912", # Too many branches
|
||||
]
|
||||
fixable = ["ALL"]
|
||||
unfixable = []
|
||||
|
||||
[tool.ruff.lint.isort]
|
||||
required-imports = ["from __future__ import annotations"]
|
||||
known-first-party = ["zvec"]
|
||||
|
||||
[tool.ruff.lint.per-file-ignores]
|
||||
"python/tests/**" = ["ALL"]
|
||||
"bench/core/**" = ["ALL"]
|
||||
"python/zvec/__init__.py" = [
|
||||
"F401", # Unused import (for __all__)
|
||||
"E402", # Module level import not at top (C++ module init order)
|
||||
"PLE0605", # Invalid format for __all__
|
||||
"RUF022", # __all__ is not sorted
|
||||
]
|
||||
"python/zvec/model/doc.py" = [
|
||||
"RUF023", # Unused sort (for __slot__)
|
||||
]
|
||||
|
||||
[tool.ruff.format]
|
||||
indent-style = "space"
|
||||
quote-style = "double"
|
||||
line-ending = "lf"
|
||||
skip-magic-trailing-comma = false
|
||||
@@ -0,0 +1,302 @@
|
||||
import logging
|
||||
import math
|
||||
import numpy as np
|
||||
|
||||
from zvec import (
|
||||
MetricType,
|
||||
DataType,
|
||||
QuantizeType,
|
||||
Doc,
|
||||
CollectionSchema,
|
||||
FieldSchema,
|
||||
VectorSchema,
|
||||
)
|
||||
|
||||
from typing import Dict
|
||||
|
||||
|
||||
def is_float_equal(actual, expected, rel_tol=1e-5, abs_tol=1e-8):
|
||||
if actual is None and expected is None:
|
||||
return True
|
||||
return math.isclose(actual, expected, rel_tol=rel_tol, abs_tol=abs_tol)
|
||||
|
||||
|
||||
def is_dense_vector_equal(vec1, vec2, rtol=1e-5, atol=1e-8):
|
||||
"""Compare two dense vectors with tolerance."""
|
||||
return np.allclose(vec1, vec2, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
def is_sparse_vector_equal(vec1, vec2, rtol=1e-5, atol=1e-8):
|
||||
"""Compare two sparse vectors with tolerance."""
|
||||
# Check if they have the same keys
|
||||
if set(vec1.keys()) != set(vec2.keys()):
|
||||
return False
|
||||
|
||||
# Check if all values are close
|
||||
for key in vec1:
|
||||
if not math.isclose(vec1[key], vec2[key], rel_tol=rtol, abs_tol=atol):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def is_float_array_equal(arr1, arr2, rtol=1e-5, atol=1e-8):
|
||||
"""Compare two float arrays with tolerance."""
|
||||
return np.allclose(arr1, arr2, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
def is_double_array_equal(arr1, arr2, rtol=1e-9, atol=1e-12):
|
||||
"""Compare two double arrays with tolerance."""
|
||||
return np.allclose(arr1, arr2, rtol=rtol, atol=atol)
|
||||
|
||||
|
||||
def is_int_array_equal(arr1, arr2):
|
||||
"""Compare two integer arrays with exact equality."""
|
||||
return np.array_equal(arr1, arr2)
|
||||
|
||||
|
||||
def cosine_distance_dense(
|
||||
vec1,
|
||||
vec2,
|
||||
dtype: DataType = DataType.VECTOR_FP32,
|
||||
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
|
||||
):
|
||||
if dtype == DataType.VECTOR_FP16 or quantize_type == QuantizeType.FP16:
|
||||
vec1 = [np.float16(a) for a in vec1]
|
||||
vec2 = [np.float16(b) for b in vec2]
|
||||
|
||||
dot_product = sum(a * b for a, b in zip(vec1, vec2))
|
||||
|
||||
magnitude1 = math.sqrt(sum(a * a for a in vec1))
|
||||
magnitude2 = math.sqrt(sum(b * b for b in vec2))
|
||||
|
||||
if magnitude1 == 0 or magnitude2 == 0:
|
||||
return 0.0
|
||||
|
||||
return 1 - dot_product / (magnitude1 * magnitude2)
|
||||
|
||||
|
||||
def dp_distance_dense(
|
||||
vec1,
|
||||
vec2,
|
||||
dtype: DataType = DataType.VECTOR_FP32,
|
||||
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
|
||||
):
|
||||
if dtype == DataType.VECTOR_FP16 or quantize_type == QuantizeType.FP16:
|
||||
return sum(np.float16(a) * np.float16(b) for a, b in zip(vec1, vec2))
|
||||
return sum(a * b for a, b in zip(vec1, vec2))
|
||||
|
||||
|
||||
def euclidean_distance_dense(
|
||||
vec1,
|
||||
vec2,
|
||||
dtype: DataType = DataType.VECTOR_FP32,
|
||||
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
|
||||
):
|
||||
if dtype == DataType.VECTOR_FP16 or quantize_type == QuantizeType.FP16:
|
||||
return sum((np.float16(a) - np.float16(b)) ** 2 for a, b in zip(vec1, vec2))
|
||||
return sum((a - b) ** 2 for a, b in zip(vec1, vec2))
|
||||
|
||||
|
||||
def distance_dense(
|
||||
vec1,
|
||||
vec2,
|
||||
metric: MetricType,
|
||||
data_type: DataType = DataType.VECTOR_FP32,
|
||||
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
|
||||
):
|
||||
if metric == MetricType.COSINE:
|
||||
return cosine_distance_dense(vec1, vec2, data_type, quantize_type)
|
||||
elif metric == MetricType.L2:
|
||||
return euclidean_distance_dense(vec1, vec2, data_type, quantize_type)
|
||||
elif metric == MetricType.IP:
|
||||
return dp_distance_dense(vec1, vec2, data_type, quantize_type)
|
||||
else:
|
||||
raise ValueError("Unsupported metric type")
|
||||
|
||||
|
||||
def dp_distance_sparse(
|
||||
vec1,
|
||||
vec2,
|
||||
data_type: DataType = DataType.SPARSE_VECTOR_FP32,
|
||||
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
|
||||
):
|
||||
dot_product = 0.0
|
||||
for dim in set(vec1.keys()) & set(vec2.keys()):
|
||||
if (
|
||||
data_type == DataType.SPARSE_VECTOR_FP16
|
||||
or quantize_type == QuantizeType.FP16
|
||||
):
|
||||
vec1[dim] = np.float16(vec1[dim])
|
||||
vec2[dim] = np.float16(vec2[dim])
|
||||
dot_product += vec1[dim] * vec2[dim]
|
||||
return dot_product
|
||||
|
||||
|
||||
def distance(
|
||||
vec1,
|
||||
vec2,
|
||||
metric: MetricType,
|
||||
data_type: DataType,
|
||||
quantize_type: QuantizeType = QuantizeType.UNDEFINED,
|
||||
):
|
||||
is_sparse = (
|
||||
data_type == DataType.SPARSE_VECTOR_FP32
|
||||
or data_type == DataType.SPARSE_VECTOR_FP16
|
||||
)
|
||||
|
||||
if is_sparse:
|
||||
if metric != MetricType.IP:
|
||||
raise ValueError("Unsupported metric type for sparse vectors")
|
||||
|
||||
if is_sparse:
|
||||
return dp_distance_sparse(vec1, vec2, data_type, quantize_type)
|
||||
else:
|
||||
return distance_dense(vec1, vec2, metric, data_type, quantize_type)
|
||||
|
||||
|
||||
def calculate_rrf_score(rank, k=60):
|
||||
return 1.0 / (k + rank + 1)
|
||||
|
||||
|
||||
def calculate_multi_vector_rrf_scores(query_results: Dict[str, Doc], k=60):
|
||||
rrf_scores = {}
|
||||
|
||||
for vector_name, docs in query_results.items():
|
||||
for rank, doc in enumerate(docs):
|
||||
doc_id = doc.id
|
||||
rrf_score = calculate_rrf_score(rank, k)
|
||||
if doc_id in rrf_scores:
|
||||
rrf_scores[doc_id] += rrf_score
|
||||
else:
|
||||
rrf_scores[doc_id] = rrf_score
|
||||
|
||||
return rrf_scores
|
||||
|
||||
|
||||
def calculate_multi_vector_weighted_scores(
|
||||
query_results: Dict[str, Doc], weights: Dict[str, float], metric: MetricType
|
||||
):
|
||||
def _normalize_score(score: float, metric: MetricType) -> float:
|
||||
if metric == MetricType.L2:
|
||||
return 1.0 - 2 * math.atan(score) / math.pi
|
||||
if metric == MetricType.IP:
|
||||
return 0.5 + math.atan(score) / math.pi
|
||||
if metric == MetricType.COSINE:
|
||||
return 1.0 - score / 2.0
|
||||
raise ValueError("Unsupported metric type")
|
||||
|
||||
weighted_scores = {}
|
||||
|
||||
for vector_name, docs in query_results.items():
|
||||
weight = weights.get(vector_name, 1.0)
|
||||
|
||||
for doc in docs:
|
||||
doc_id = doc.id
|
||||
weighted_score = (_normalize_score(doc.score, metric)) * weight
|
||||
if doc_id in weighted_scores:
|
||||
weighted_scores[doc_id] += weighted_score
|
||||
else:
|
||||
weighted_scores[doc_id] = weighted_score
|
||||
|
||||
return weighted_scores
|
||||
|
||||
|
||||
def is_field_equal(field1, field2, schema: FieldSchema) -> bool:
|
||||
if field1 is None and field2 is None:
|
||||
return True
|
||||
if field1 is None or field2 is None:
|
||||
return False
|
||||
|
||||
if schema.data_type == DataType.ARRAY_FLOAT:
|
||||
return is_float_array_equal(field1, field2)
|
||||
elif schema.data_type == DataType.ARRAY_DOUBLE:
|
||||
return is_double_array_equal(field1, field2)
|
||||
elif schema.data_type in [
|
||||
DataType.ARRAY_INT32,
|
||||
DataType.ARRAY_INT64,
|
||||
DataType.ARRAY_BOOL,
|
||||
DataType.ARRAY_STRING,
|
||||
DataType.ARRAY_UINT32,
|
||||
DataType.ARRAY_UINT64,
|
||||
DataType.ARRAY_INT64,
|
||||
]:
|
||||
return is_int_array_equal(field1, field2)
|
||||
elif schema.data_type in [DataType.FLOAT, DataType.DOUBLE]:
|
||||
return is_float_equal(field1, field2)
|
||||
|
||||
return field1 == field2
|
||||
|
||||
|
||||
def is_vector_equal(vec1, vec2, schema: VectorSchema) -> bool:
|
||||
if (
|
||||
schema.data_type == DataType.SPARSE_VECTOR_FP16
|
||||
or schema.data_type == DataType.VECTOR_FP16
|
||||
):
|
||||
# skip fp16 vector equal
|
||||
return True
|
||||
|
||||
is_sparse = (
|
||||
schema.data_type == DataType.SPARSE_VECTOR_FP32
|
||||
or schema.data_type == DataType.SPARSE_VECTOR_FP16
|
||||
)
|
||||
|
||||
if is_sparse:
|
||||
return is_sparse_vector_equal(vec1, vec2)
|
||||
else:
|
||||
return is_dense_vector_equal(vec1, vec2)
|
||||
|
||||
|
||||
def is_doc_equal(
|
||||
doc1: Doc,
|
||||
doc2: Doc,
|
||||
schema: CollectionSchema,
|
||||
except_score: bool = True,
|
||||
include_vector: bool = True,
|
||||
):
|
||||
if doc1.id != doc2.id:
|
||||
logging.error("doc ids are not equal")
|
||||
return False
|
||||
|
||||
reduce_field_names = set(doc1.field_names() + doc2.field_names())
|
||||
reduce_vector_names = set(doc1.vector_names() + doc2.vector_names())
|
||||
|
||||
is_doc1_fields_empty = doc1.fields is None or doc1.fields == {}
|
||||
is_doc2_fields_empty = doc2.fields is None or doc2.fields == {}
|
||||
|
||||
if is_doc1_fields_empty or is_doc2_fields_empty:
|
||||
if is_doc1_fields_empty != is_doc2_fields_empty:
|
||||
return False
|
||||
else:
|
||||
for field_name in reduce_field_names:
|
||||
field_schema = schema.field(field_name)
|
||||
if field_schema is None:
|
||||
return False
|
||||
if is_field_equal(
|
||||
doc1.field(field_name), doc2.field(field_name), field_schema
|
||||
):
|
||||
continue
|
||||
else:
|
||||
logging.error(f"{field_name} are not equal")
|
||||
return False
|
||||
|
||||
if include_vector:
|
||||
is_doc1_vectors_empty = doc1.vectors is None or doc1.vectors == {}
|
||||
is_doc2_vectors_empty = doc2.vectors is None or doc2.vectors == {}
|
||||
|
||||
if is_doc1_vectors_empty or is_doc2_vectors_empty:
|
||||
if is_doc1_fields_empty != is_doc2_vectors_empty:
|
||||
return False
|
||||
else:
|
||||
for vector_name in reduce_vector_names:
|
||||
vector_schema = schema.vector(vector_name)
|
||||
if vector_schema is None:
|
||||
return False
|
||||
if is_vector_equal(
|
||||
doc1.vector(vector_name), doc2.vector(vector_name), vector_schema
|
||||
):
|
||||
continue
|
||||
else:
|
||||
return False
|
||||
|
||||
return True
|
||||
@@ -0,0 +1,377 @@
|
||||
from zvec import CollectionSchema, Doc
|
||||
|
||||
from support_helper import *
|
||||
|
||||
import numpy as np
|
||||
from typing import Literal, Optional, Union, Tuple
|
||||
|
||||
import random
|
||||
import string
|
||||
|
||||
|
||||
def generate_constant_vector(
|
||||
i: int, dimension: int, dtype: Literal["int8", "float16", "float32"] = "float32"
|
||||
):
|
||||
if dtype == "int8":
|
||||
vec = [i % 128] * dimension
|
||||
vec[i % dimension] = (i + 1) % 128
|
||||
else:
|
||||
vec = [i / 256.0] * dimension
|
||||
vec[i % dimension] = (i + 1) / 256.0
|
||||
|
||||
return vec
|
||||
|
||||
|
||||
def generate_sparse_vector(i: int):
|
||||
return {i: i + 0.1}
|
||||
|
||||
|
||||
def generate_vectordict(i: int, schema: CollectionSchema) -> Doc:
|
||||
doc_fields = {}
|
||||
doc_vectors = {}
|
||||
doc_fields = {}
|
||||
doc_vectors = {}
|
||||
for field in schema.fields:
|
||||
if field.data_type == DataType.BOOL:
|
||||
doc_fields[field.name] = i % 2 == 0
|
||||
elif field.data_type == DataType.INT32:
|
||||
doc_fields[field.name] = i
|
||||
elif field.data_type == DataType.UINT32:
|
||||
doc_fields[field.name] = i
|
||||
elif field.data_type == DataType.INT64:
|
||||
doc_fields[field.name] = i
|
||||
elif field.data_type == DataType.UINT64:
|
||||
doc_fields[field.name] = i
|
||||
elif field.data_type == DataType.FLOAT:
|
||||
doc_fields[field.name] = float(i) + 0.1
|
||||
elif field.data_type == DataType.DOUBLE:
|
||||
doc_fields[field.name] = float(i) + 0.11
|
||||
elif field.data_type == DataType.STRING:
|
||||
doc_fields[field.name] = f"test_{i}"
|
||||
elif field.data_type == DataType.ARRAY_BOOL:
|
||||
doc_fields[field.name] = [i % 2 == 0, i % 3 == 0]
|
||||
elif field.data_type == DataType.ARRAY_INT32:
|
||||
doc_fields[field.name] = [i, i + 1, i + 2]
|
||||
elif field.data_type == DataType.ARRAY_UINT32:
|
||||
doc_fields[field.name] = [i, i + 1, i + 2]
|
||||
elif field.data_type == DataType.ARRAY_INT64:
|
||||
doc_fields[field.name] = [i, i + 1, i + 2]
|
||||
elif field.data_type == DataType.ARRAY_UINT64:
|
||||
doc_fields[field.name] = [i, i + 1, i + 2]
|
||||
elif field.data_type == DataType.ARRAY_FLOAT:
|
||||
doc_fields[field.name] = [float(i + 0.1), float(i + 1.1), float(i + 2.1)]
|
||||
elif field.data_type == DataType.ARRAY_DOUBLE:
|
||||
doc_fields[field.name] = [float(i + 0.11), float(i + 1.11), float(i + 2.11)]
|
||||
elif field.data_type == DataType.ARRAY_STRING:
|
||||
doc_fields[field.name] = [f"test_{i}", f"test_{i + 1}", f"test_{i + 2}"]
|
||||
else:
|
||||
raise ValueError(f"Unsupported field type: {field.data_type}")
|
||||
for vector in schema.vectors:
|
||||
if vector.data_type == DataType.VECTOR_FP16:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
i, vector.dimension, "float16"
|
||||
)
|
||||
elif vector.data_type == DataType.VECTOR_FP32:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
i, vector.dimension, "float32"
|
||||
)
|
||||
elif vector.data_type == DataType.VECTOR_INT8:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
i,
|
||||
vector.dimension,
|
||||
"int8",
|
||||
)
|
||||
elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
|
||||
doc_vectors[vector.name] = generate_sparse_vector(i)
|
||||
elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
|
||||
doc_vectors[vector.name] = generate_sparse_vector(i)
|
||||
else:
|
||||
raise ValueError(f"Unsupported vector type: {vector.data_type}")
|
||||
return doc_fields, doc_vectors
|
||||
|
||||
|
||||
def generate_doc(i: int, schema: CollectionSchema) -> Doc:
|
||||
doc_fields = {}
|
||||
doc_vectors = {}
|
||||
doc_fields, doc_vectors = generate_vectordict(i, schema)
|
||||
doc = Doc(id=str(i), fields=doc_fields, vectors=doc_vectors)
|
||||
return doc
|
||||
|
||||
|
||||
def generate_update_doc(i: int, schema: CollectionSchema) -> Doc:
|
||||
doc_fields = {}
|
||||
doc_vectors = {}
|
||||
for field in schema.fields:
|
||||
if field.data_type == DataType.BOOL:
|
||||
doc_fields[field.name] = (i + 1) % 2 == 0
|
||||
elif field.data_type == DataType.INT32:
|
||||
doc_fields[field.name] = i + 1
|
||||
elif field.data_type == DataType.UINT32:
|
||||
doc_fields[field.name] = i + 1
|
||||
elif field.data_type == DataType.INT64:
|
||||
doc_fields[field.name] = i + 1
|
||||
elif field.data_type == DataType.UINT64:
|
||||
doc_fields[field.name] = i + 1
|
||||
elif field.data_type == DataType.FLOAT:
|
||||
doc_fields[field.name] = float(i + 1) + 0.1
|
||||
elif field.data_type == DataType.DOUBLE:
|
||||
doc_fields[field.name] = float(i) + 0.11
|
||||
elif field.data_type == DataType.STRING:
|
||||
doc_fields[field.name] = f"test_{i + 1}"
|
||||
elif field.data_type == DataType.ARRAY_BOOL:
|
||||
doc_fields[field.name] = [(i + 1) % 2 == 0, (i + 1) % 3 == 0]
|
||||
elif field.data_type == DataType.ARRAY_INT32:
|
||||
doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
|
||||
elif field.data_type == DataType.ARRAY_UINT32:
|
||||
doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
|
||||
elif field.data_type == DataType.ARRAY_INT64:
|
||||
doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
|
||||
elif field.data_type == DataType.ARRAY_UINT64:
|
||||
doc_fields[field.name] = [i + 1, (i + 1) + 1, (i + 1) + 2]
|
||||
elif field.data_type == DataType.ARRAY_FLOAT:
|
||||
doc_fields[field.name] = [
|
||||
float((i + 1) + 0.1),
|
||||
float((i + 1) + 1.1),
|
||||
float((i + 1) + 2.1),
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_DOUBLE:
|
||||
doc_fields[field.name] = [
|
||||
float((i + 1) + 0.11),
|
||||
float((i + 1) + 1.11),
|
||||
float((i + 1) + 2.11),
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_STRING:
|
||||
doc_fields[field.name] = [
|
||||
f"test_{i + 1}",
|
||||
f"test_{(i + 1) + 1}",
|
||||
f"test_{(i + 1) + 2}",
|
||||
]
|
||||
else:
|
||||
raise ValueError(f"Unsupported field type: {field.data_type}")
|
||||
for vector in schema.vectors:
|
||||
if vector.data_type == DataType.VECTOR_FP16:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
i + 1, DEFAULT_VECTOR_DIMENSION, "float16"
|
||||
)
|
||||
elif vector.data_type == DataType.VECTOR_FP32:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
i + 1, DEFAULT_VECTOR_DIMENSION, "float32"
|
||||
)
|
||||
elif vector.data_type == DataType.VECTOR_INT8:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
i + 1,
|
||||
DEFAULT_VECTOR_DIMENSION,
|
||||
"int8",
|
||||
)
|
||||
elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
|
||||
doc_vectors[vector.name] = generate_sparse_vector(i)
|
||||
elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
|
||||
doc_vectors[vector.name] = generate_sparse_vector(i)
|
||||
else:
|
||||
raise ValueError(f"Unsupported vector type: {vector.data_type}")
|
||||
doc = Doc(id=str(i), fields=doc_fields, vectors=doc_vectors)
|
||||
return doc
|
||||
|
||||
|
||||
def generate_doc_random(i, schema: CollectionSchema) -> Doc:
|
||||
doc_fields = {}
|
||||
doc_vectors = {}
|
||||
|
||||
random.seed(i)
|
||||
|
||||
for field in schema.fields:
|
||||
if field.data_type == DataType.BOOL:
|
||||
doc_fields[field.name] = random.choice([True, False])
|
||||
elif field.data_type == DataType.INT32:
|
||||
doc_fields[field.name] = random.randint(-2147483648, 2147483647)
|
||||
elif field.data_type == DataType.UINT32:
|
||||
doc_fields[field.name] = random.randint(0, 4294967295)
|
||||
elif field.data_type == DataType.INT64:
|
||||
doc_fields[field.name] = random.randint(
|
||||
-9223372036854775808, 9223372036854775807
|
||||
)
|
||||
elif field.data_type == DataType.UINT64:
|
||||
doc_fields[field.name] = random.randint(0, 18446744073709551615)
|
||||
elif field.data_type == DataType.FLOAT:
|
||||
doc_fields[field.name] = random.uniform(-3.4028235e38, 3.4028235e38)
|
||||
elif field.data_type == DataType.DOUBLE:
|
||||
doc_fields[field.name] = random.uniform(
|
||||
-1.7976931348623157e308, 1.7976931348623157e308
|
||||
)
|
||||
elif field.data_type == DataType.STRING:
|
||||
length = random.randint(1, 999)
|
||||
doc_fields[field.name] = "".join(
|
||||
random.choices(string.ascii_letters + string.digits, k=length)
|
||||
)
|
||||
elif field.data_type == DataType.ARRAY_BOOL:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.choice([True, False]) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_INT32:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.randint(-2147483648, 2147483647) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_UINT32:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.randint(0, 4294967295) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_INT64:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.randint(-9223372036854775808, 9223372036854775807)
|
||||
for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_UINT64:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.randint(0, 18446744073709551615) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_FLOAT:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.uniform(-3.4028235e38, 3.4028235e38) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_DOUBLE:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.uniform(-1.7976931348623157e308, 1.7976931348623157e308)
|
||||
for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_STRING:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
"".join(
|
||||
random.choices(
|
||||
string.ascii_letters + string.digits, k=random.randint(1, 100)
|
||||
)
|
||||
)
|
||||
for _ in range(array_length)
|
||||
]
|
||||
else:
|
||||
raise ValueError(f"Unsupported field type: {field.data_type}")
|
||||
|
||||
for vector in schema.vectors:
|
||||
if vector.data_type == DataType.VECTOR_FP16:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float16"
|
||||
)
|
||||
elif vector.data_type == DataType.VECTOR_FP32:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float32"
|
||||
)
|
||||
elif vector.data_type == DataType.VECTOR_INT8:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "int8"
|
||||
)
|
||||
elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
|
||||
doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
|
||||
elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
|
||||
doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
|
||||
else:
|
||||
raise ValueError(f"Unsupported vector type: {vector.data_type}")
|
||||
|
||||
doc = Doc(id=i, fields=doc_fields, vectors=doc_vectors)
|
||||
return doc
|
||||
|
||||
|
||||
def generate_vectordict_random(schema: CollectionSchema):
|
||||
doc_fields = {}
|
||||
doc_vectors = {}
|
||||
for field in schema.fields:
|
||||
if field.data_type == DataType.BOOL:
|
||||
doc_fields[field.name] = random.choice([True, False])
|
||||
elif field.data_type == DataType.INT32:
|
||||
doc_fields[field.name] = random.randint(-2147483648, 2147483647)
|
||||
elif field.data_type == DataType.UINT32:
|
||||
doc_fields[field.name] = random.randint(0, 4294967295)
|
||||
elif field.data_type == DataType.INT64:
|
||||
doc_fields[field.name] = random.randint(
|
||||
-9223372036854775808, 9223372036854775807
|
||||
)
|
||||
elif field.data_type == DataType.UINT64:
|
||||
doc_fields[field.name] = random.randint(0, 18446744073709551615)
|
||||
elif field.data_type == DataType.FLOAT:
|
||||
doc_fields[field.name] = random.uniform(-3.4028235e38, 3.4028235e38)
|
||||
elif field.data_type == DataType.DOUBLE:
|
||||
doc_fields[field.name] = random.uniform(
|
||||
-1.7976931348623157e308, 1.7976931348623157e308
|
||||
)
|
||||
elif field.data_type == DataType.STRING:
|
||||
length = random.randint(1, 999)
|
||||
doc_fields[field.name] = "".join(
|
||||
random.choices(string.ascii_letters + string.digits, k=length)
|
||||
)
|
||||
elif field.data_type == DataType.ARRAY_BOOL:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.choice([True, False]) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_INT32:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.randint(-2147483648, 2147483647) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_UINT32:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.randint(0, 4294967295) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_INT64:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.randint(-9223372036854775808, 9223372036854775807)
|
||||
for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_UINT64:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.randint(0, 18446744073709551615) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_FLOAT:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.uniform(-3.4028235e38, 3.4028235e38) for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_DOUBLE:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
random.uniform(-1.7976931348623157e308, 1.7976931348623157e308)
|
||||
for _ in range(array_length)
|
||||
]
|
||||
elif field.data_type == DataType.ARRAY_STRING:
|
||||
array_length = random.randint(0, 10)
|
||||
doc_fields[field.name] = [
|
||||
"".join(
|
||||
random.choices(
|
||||
string.ascii_letters + string.digits, k=random.randint(1, 100)
|
||||
)
|
||||
)
|
||||
for _ in range(array_length)
|
||||
]
|
||||
else:
|
||||
raise ValueError(f"Unsupported field type: {field.data_type}")
|
||||
|
||||
for vector in schema.vectors:
|
||||
if vector.data_type == DataType.VECTOR_FP16:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float16"
|
||||
)
|
||||
elif vector.data_type == DataType.VECTOR_FP32:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "float32"
|
||||
)
|
||||
elif vector.data_type == DataType.VECTOR_INT8:
|
||||
doc_vectors[vector.name] = generate_constant_vector(
|
||||
random.randint(1, 100), DEFAULT_VECTOR_DIMENSION, "int8"
|
||||
)
|
||||
elif vector.data_type == DataType.SPARSE_VECTOR_FP32:
|
||||
doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
|
||||
elif vector.data_type == DataType.SPARSE_VECTOR_FP16:
|
||||
doc_vectors[vector.name] = generate_sparse_vector(random.randint(1, 100))
|
||||
else:
|
||||
raise ValueError(f"Unsupported vector type: {vector.data_type}")
|
||||
|
||||
return doc_fields, doc_vectors
|
||||
@@ -0,0 +1,319 @@
|
||||
import pytest
|
||||
import logging
|
||||
|
||||
from typing import Any, Generator
|
||||
|
||||
import zvec
|
||||
from zvec import (
|
||||
CollectionOption,
|
||||
InvertIndexParam,
|
||||
HnswIndexParam,
|
||||
IVFIndexParam,
|
||||
FieldSchema,
|
||||
VectorSchema,
|
||||
CollectionSchema,
|
||||
Collection,
|
||||
Doc,
|
||||
VectorQuery,
|
||||
)
|
||||
|
||||
from support_helper import *
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def basic_schema(collection_name="test_collection") -> CollectionSchema:
|
||||
return CollectionSchema(
|
||||
name=collection_name if len(collection_name) > 0 else "test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name", DataType.STRING, nullable=False, index_param=InvertIndexParam()
|
||||
),
|
||||
FieldSchema("weight", DataType.FLOAT, nullable=True),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
),
|
||||
VectorSchema(
|
||||
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def full_schema(
|
||||
nullable: bool = False,
|
||||
has_index: bool = False,
|
||||
) -> CollectionSchema:
|
||||
scalar_index_param = None
|
||||
vector_index_param = None
|
||||
if has_index:
|
||||
scalar_index_param = InvertIndexParam(enable_range_optimization=True)
|
||||
vector_index_param = HnswIndexParam()
|
||||
|
||||
fields = []
|
||||
for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
|
||||
fields.append(
|
||||
FieldSchema(
|
||||
v,
|
||||
k,
|
||||
nullable=nullable,
|
||||
index_param=scalar_index_param,
|
||||
)
|
||||
)
|
||||
vetors = []
|
||||
for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
|
||||
vetors.append(
|
||||
VectorSchema(
|
||||
v,
|
||||
k,
|
||||
dimension=DEFAULT_VECTOR_DIMENSION,
|
||||
index_param=vector_index_param,
|
||||
)
|
||||
)
|
||||
|
||||
return CollectionSchema(
|
||||
name="full_collection",
|
||||
fields=fields,
|
||||
vectors=vetors,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def full_schema_new(request) -> CollectionSchema:
|
||||
if hasattr(request, "param"):
|
||||
nullable, has_index, vector_index = request.param
|
||||
else:
|
||||
nullable, has_index, vector_index = True, False, HnswIndexParam()
|
||||
|
||||
scalar_index_param = None
|
||||
vector_index_param = None
|
||||
if has_index:
|
||||
scalar_index_param = InvertIndexParam(enable_range_optimization=True)
|
||||
vector_index_param = vector_index
|
||||
|
||||
fields = []
|
||||
for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
|
||||
fields.append(
|
||||
FieldSchema(
|
||||
v,
|
||||
k,
|
||||
nullable=nullable,
|
||||
index_param=scalar_index_param,
|
||||
)
|
||||
)
|
||||
vectors = []
|
||||
for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
|
||||
vectors.append(
|
||||
VectorSchema(
|
||||
v,
|
||||
k,
|
||||
dimension=DEFAULT_VECTOR_DIMENSION,
|
||||
index_param=vector_index_param,
|
||||
)
|
||||
)
|
||||
|
||||
return CollectionSchema(
|
||||
name="full_collection_new",
|
||||
fields=fields,
|
||||
vectors=vectors,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def full_schema_ivf(request) -> CollectionSchema:
|
||||
if hasattr(request, "param"):
|
||||
nullable, has_index, vector_index = request.param
|
||||
else:
|
||||
nullable, has_index, vector_index = True, False, IVFIndexParam()
|
||||
|
||||
scalar_index_param = None
|
||||
vector_index_param = None
|
||||
if has_index:
|
||||
scalar_index_param = InvertIndexParam(enable_range_optimization=True)
|
||||
vector_index_param = vector_index
|
||||
|
||||
fields = []
|
||||
for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
|
||||
fields.append(
|
||||
FieldSchema(
|
||||
v,
|
||||
k,
|
||||
nullable=nullable,
|
||||
index_param=scalar_index_param,
|
||||
)
|
||||
)
|
||||
vectors = []
|
||||
for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
|
||||
if v in ["vector_fp16_field", "vector_fp32_field"]:
|
||||
vectors.append(
|
||||
VectorSchema(
|
||||
v,
|
||||
k,
|
||||
dimension=DEFAULT_VECTOR_DIMENSION,
|
||||
index_param=vector_index_param,
|
||||
)
|
||||
)
|
||||
|
||||
return CollectionSchema(
|
||||
name="full_collection_ivf",
|
||||
fields=fields,
|
||||
vectors=vectors,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def single_vector_schema(
|
||||
data_type: DataType,
|
||||
) -> CollectionSchema:
|
||||
vector_schema = [
|
||||
VectorSchema(
|
||||
DEFAULT_VECTOR_FIELD_NAME[data_type],
|
||||
data_type,
|
||||
DEFAULT_VECTOR_DIMENSION,
|
||||
)
|
||||
]
|
||||
|
||||
return CollectionSchema(
|
||||
name="full_collection",
|
||||
vectors=vector_schema,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def single_vector_schema_with_index_param(
|
||||
data_type: DataType, index_param
|
||||
) -> CollectionSchema:
|
||||
vector_schema = [
|
||||
VectorSchema(
|
||||
DEFAULT_VECTOR_FIELD_NAME[data_type],
|
||||
data_type,
|
||||
DEFAULT_VECTOR_DIMENSION,
|
||||
index_param,
|
||||
)
|
||||
]
|
||||
|
||||
return CollectionSchema(
|
||||
name="full_collection",
|
||||
vectors=vector_schema,
|
||||
)
|
||||
|
||||
|
||||
def create_collection_fixture(
|
||||
collection_temp_dir, schema: CollectionSchema, collection_option: CollectionOption
|
||||
) -> Generator[Any, Any, Collection]:
|
||||
"""Common helper function to create and manage collection fixtures."""
|
||||
coll = zvec.create_and_open(
|
||||
path=str(collection_temp_dir),
|
||||
schema=schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert coll is not None, "Failed to create and open collection"
|
||||
assert coll.path == str(collection_temp_dir)
|
||||
assert coll.schema.name == schema.name
|
||||
assert list(coll.schema.fields) == list(schema.fields)
|
||||
assert list(coll.schema.vectors) == list(schema.vectors)
|
||||
assert coll.option.read_only == collection_option.read_only
|
||||
assert coll.option.enable_mmap == collection_option.enable_mmap
|
||||
|
||||
try:
|
||||
yield coll
|
||||
finally:
|
||||
if hasattr(coll, "destroy") and coll is not None:
|
||||
try:
|
||||
coll.destroy()
|
||||
except Exception as e:
|
||||
logging.warning(f"Warning: failed to destroy collection: {e}")
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def basic_collection(
|
||||
collection_temp_dir, basic_schema, collection_option
|
||||
) -> Generator[Any, Any, Collection]:
|
||||
yield from create_collection_fixture(
|
||||
collection_temp_dir, basic_schema, collection_option
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def collection_option():
|
||||
return CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def collection_temp_dir(tmp_path_factory):
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection_path"
|
||||
return str(collection_path)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def full_collection(
|
||||
collection_temp_dir,
|
||||
full_schema,
|
||||
collection_option,
|
||||
nullable: bool = True,
|
||||
has_index: bool = False,
|
||||
) -> Generator[Any, Any, Collection]:
|
||||
yield from create_collection_fixture(
|
||||
collection_temp_dir, full_schema, collection_option
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def full_collection_new(
|
||||
collection_temp_dir, full_schema_new, collection_option
|
||||
) -> Generator[Any, Any, Collection]:
|
||||
yield from create_collection_fixture(
|
||||
collection_temp_dir, full_schema_new, collection_option
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def full_collection_ivf(
|
||||
collection_temp_dir, full_schema_ivf, collection_option
|
||||
) -> Generator[Any, Any, Collection]:
|
||||
yield from create_collection_fixture(
|
||||
collection_temp_dir, full_schema_ivf, collection_option
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_field_list(nullable: bool = True, scalar_index_param=None, name_prefix=""):
|
||||
field_list = []
|
||||
for k, v in DEFAULT_SCALAR_FIELD_NAME.items():
|
||||
field_list.append(
|
||||
FieldSchema(
|
||||
f"{name_prefix}_{v}" if len(name_prefix) > 0 else v,
|
||||
k,
|
||||
nullable=nullable,
|
||||
index_param=scalar_index_param,
|
||||
)
|
||||
)
|
||||
return field_list
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sample_vector_list(vector_index_param=None, name_prefix=""):
|
||||
vector_list = []
|
||||
for k, v in DEFAULT_VECTOR_FIELD_NAME.items():
|
||||
vector_list.append(
|
||||
VectorSchema(
|
||||
f"{name_prefix}_{v}" if len(name_prefix) > 0 else v,
|
||||
k,
|
||||
dimension=DEFAULT_VECTOR_DIMENSION,
|
||||
index_param=vector_index_param,
|
||||
)
|
||||
)
|
||||
return vector_list
|
||||
@@ -0,0 +1,209 @@
|
||||
from zvec import (
|
||||
CollectionOption,
|
||||
IndexOption,
|
||||
OptimizeOption,
|
||||
InvertIndexParam,
|
||||
HnswIndexParam,
|
||||
IVFIndexParam,
|
||||
FlatIndexParam,
|
||||
AlterColumnOption,
|
||||
AddColumnOption,
|
||||
DataType,
|
||||
MetricType,
|
||||
QuantizeType,
|
||||
)
|
||||
|
||||
|
||||
VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP = {
|
||||
DataType.VECTOR_FP32: [
|
||||
HnswIndexParam(),
|
||||
HnswIndexParam(
|
||||
metric_type=MetricType.IP,
|
||||
m=16,
|
||||
ef_construction=100,
|
||||
quantize_type=QuantizeType.INT8,
|
||||
),
|
||||
HnswIndexParam(
|
||||
metric_type=MetricType.COSINE,
|
||||
m=24,
|
||||
ef_construction=150,
|
||||
quantize_type=QuantizeType.INT4,
|
||||
),
|
||||
HnswIndexParam(
|
||||
metric_type=MetricType.L2,
|
||||
m=32,
|
||||
ef_construction=200,
|
||||
quantize_type=QuantizeType.FP16,
|
||||
),
|
||||
FlatIndexParam(),
|
||||
FlatIndexParam(metric_type=MetricType.IP, quantize_type=QuantizeType.INT4),
|
||||
FlatIndexParam(metric_type=MetricType.L2, quantize_type=QuantizeType.INT8),
|
||||
FlatIndexParam(metric_type=MetricType.COSINE, quantize_type=QuantizeType.FP16),
|
||||
IVFIndexParam(),
|
||||
IVFIndexParam(
|
||||
metric_type=MetricType.IP,
|
||||
quantize_type=QuantizeType.INT4,
|
||||
n_list=100,
|
||||
n_iters=10,
|
||||
use_soar=False,
|
||||
),
|
||||
IVFIndexParam(
|
||||
metric_type=MetricType.L2,
|
||||
quantize_type=QuantizeType.INT8,
|
||||
n_list=200,
|
||||
n_iters=20,
|
||||
use_soar=True,
|
||||
),
|
||||
IVFIndexParam(
|
||||
metric_type=MetricType.COSINE,
|
||||
quantize_type=QuantizeType.FP16,
|
||||
n_list=150,
|
||||
n_iters=15,
|
||||
use_soar=False,
|
||||
),
|
||||
],
|
||||
DataType.VECTOR_FP16: [
|
||||
HnswIndexParam(),
|
||||
FlatIndexParam(),
|
||||
# IVFIndexParam(),
|
||||
],
|
||||
DataType.VECTOR_INT8: [
|
||||
HnswIndexParam(),
|
||||
FlatIndexParam(),
|
||||
# IVFIndexParam(),
|
||||
],
|
||||
DataType.SPARSE_VECTOR_FP32: [
|
||||
HnswIndexParam(),
|
||||
FlatIndexParam(),
|
||||
HnswIndexParam(
|
||||
metric_type=MetricType.IP,
|
||||
m=16,
|
||||
ef_construction=100,
|
||||
quantize_type=QuantizeType.FP16,
|
||||
),
|
||||
],
|
||||
DataType.SPARSE_VECTOR_FP16: [
|
||||
HnswIndexParam(),
|
||||
FlatIndexParam(),
|
||||
HnswIndexParam(
|
||||
metric_type=MetricType.IP,
|
||||
m=16,
|
||||
ef_construction=100,
|
||||
),
|
||||
],
|
||||
}
|
||||
|
||||
VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS = [
|
||||
(data_type, param)
|
||||
for data_type, params in VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP.items()
|
||||
for param in params
|
||||
]
|
||||
|
||||
INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP = {
|
||||
DataType.VECTOR_FP32: [
|
||||
InvertIndexParam(),
|
||||
],
|
||||
DataType.VECTOR_FP16: [
|
||||
InvertIndexParam(),
|
||||
],
|
||||
DataType.VECTOR_INT8: [
|
||||
InvertIndexParam(),
|
||||
],
|
||||
DataType.SPARSE_VECTOR_FP32: [
|
||||
HnswIndexParam(metric_type=MetricType.L2),
|
||||
FlatIndexParam(metric_type=MetricType.COSINE),
|
||||
IVFIndexParam(),
|
||||
InvertIndexParam(),
|
||||
],
|
||||
DataType.SPARSE_VECTOR_FP16: [
|
||||
HnswIndexParam(metric_type=MetricType.L2),
|
||||
FlatIndexParam(metric_type=MetricType.COSINE),
|
||||
IVFIndexParam(),
|
||||
InvertIndexParam(),
|
||||
],
|
||||
}
|
||||
|
||||
INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS = [
|
||||
(data_type, param)
|
||||
for data_type, params in INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP.items()
|
||||
for param in params
|
||||
]
|
||||
|
||||
COLLECTION_NAME_MAX_LENGTH = 64
|
||||
|
||||
COLLECTION_NAME_VALID_LIST = [
|
||||
"col",
|
||||
"C0llECTION",
|
||||
"Collection1",
|
||||
"collection_2",
|
||||
"123collection-",
|
||||
"a" * COLLECTION_NAME_MAX_LENGTH,
|
||||
]
|
||||
|
||||
COLLECTION_NAME_INVALID_LIST = [
|
||||
"l",
|
||||
"1C",
|
||||
"",
|
||||
" ",
|
||||
None,
|
||||
"abcdefghijklmnopqrstuvwxzy123456abcdefghijklmnopqrstuvwxzy1234561",
|
||||
"test/",
|
||||
"!@#$%^&*()test",
|
||||
]
|
||||
|
||||
FIELD_NAME_VALID_LIST = [
|
||||
"1",
|
||||
"12",
|
||||
"col",
|
||||
"ID",
|
||||
"name1",
|
||||
"Weigt_12-",
|
||||
"123age",
|
||||
"name_with_underscores",
|
||||
"123numeric_start",
|
||||
"name-with-dashes",
|
||||
]
|
||||
|
||||
FIELD_NAME_INVALID_LIST = [
|
||||
"",
|
||||
" ",
|
||||
None,
|
||||
"abcdefghijklmnopqrstuvwxzy1234561",
|
||||
"test/",
|
||||
"!@#$%^&*()test",
|
||||
"name@with#special$chars",
|
||||
"name with spaces",
|
||||
]
|
||||
|
||||
FIELD_LIST_MAX_LENGTH = 1024
|
||||
VECTOR_LIST_MAX_LENGTH = 5
|
||||
DENSE_VECTOR_MAX_DIMENSION = 20000
|
||||
SPARSE_VECTOR_MAX_DIMENSION = 4096
|
||||
|
||||
FIELD_VECTOR_LIST_DIMENSION_VALID_LIST = [
|
||||
# field_list_len, vector_list_len, dimension
|
||||
(1, 1, 1),
|
||||
(2, 2, 512),
|
||||
(512, 3, 1024),
|
||||
(1024, 4, 20000),
|
||||
]
|
||||
|
||||
FIELD_VECTOR_LIST_DIMENSION_INVALID_LIST = [
|
||||
# field_list_len, vector_list_len, dimension
|
||||
(1, 1, 0),
|
||||
(1, 1, -1),
|
||||
(1, 1, "1"),
|
||||
(1, 1, 20001),
|
||||
]
|
||||
|
||||
|
||||
INCOMPATIBLE_CONSTRUCTOR_ERROR_MSG = "incompatible constructor arguments"
|
||||
SCHEMA_VALIDATE_ERROR_MSG = "schema validate failed"
|
||||
CREATE_READ_ONLY_ERROR_MSG = "Unable to create collection with read-only mode"
|
||||
INCOMPATIBLE_FUNCTION_ERROR_MSG = "incompatible function arguments"
|
||||
INVALID_PATH_ERROR_MSG = "path validate failed"
|
||||
INDEX_NON_EXISTENT_COLUMN_ERROR_MSG = "not found in schema"
|
||||
ACCESS_DESTROYED_COLLECTION_ERROR_MSG = "is already destroyed"
|
||||
COLLECTION_PATH_NOT_EXIST_ERROR_MSG = "not exist"
|
||||
NOT_SUPPORT_ADD_COLUMN_ERROR_MSG = "Only support basic numeric data type"
|
||||
NOT_EXIST_COLUMN_TO_DROP_ERROR_MSG = "Column not exists"
|
||||
@@ -0,0 +1,126 @@
|
||||
from zvec import (
|
||||
CollectionOption,
|
||||
IndexOption,
|
||||
OptimizeOption,
|
||||
InvertIndexParam,
|
||||
HnswIndexParam,
|
||||
IVFIndexParam,
|
||||
FlatIndexParam,
|
||||
DataType,
|
||||
IndexType,
|
||||
QuantizeType,
|
||||
)
|
||||
|
||||
SUPPORT_SCALAR_DATA_TYPES = [
|
||||
DataType.BOOL,
|
||||
DataType.FLOAT,
|
||||
DataType.DOUBLE,
|
||||
DataType.INT32,
|
||||
DataType.INT64,
|
||||
DataType.UINT32,
|
||||
DataType.UINT64,
|
||||
DataType.STRING,
|
||||
DataType.ARRAY_BOOL,
|
||||
DataType.ARRAY_FLOAT,
|
||||
DataType.ARRAY_DOUBLE,
|
||||
DataType.ARRAY_INT32,
|
||||
DataType.ARRAY_INT64,
|
||||
DataType.ARRAY_UINT32,
|
||||
DataType.ARRAY_UINT64,
|
||||
DataType.ARRAY_STRING,
|
||||
]
|
||||
|
||||
DEFAULT_SCALAR_FIELD_NAME = {
|
||||
DataType.BOOL: "bool_field",
|
||||
DataType.FLOAT: "float_field",
|
||||
DataType.DOUBLE: "double_field",
|
||||
DataType.INT32: "int32_field",
|
||||
DataType.INT64: "int64_field",
|
||||
DataType.UINT32: "uint32_field",
|
||||
DataType.UINT64: "uint64_field",
|
||||
DataType.STRING: "string_field",
|
||||
DataType.ARRAY_BOOL: "array_bool_field",
|
||||
DataType.ARRAY_FLOAT: "array_float_field",
|
||||
DataType.ARRAY_DOUBLE: "array_double_field",
|
||||
DataType.ARRAY_INT32: "array_int32_field",
|
||||
DataType.ARRAY_INT64: "array_int64_field",
|
||||
DataType.ARRAY_UINT32: "array_uint32_field",
|
||||
DataType.ARRAY_UINT64: "array_uint64_field",
|
||||
DataType.ARRAY_STRING: "array_string_field",
|
||||
}
|
||||
|
||||
SUPPORT_SCALAR_INDEX_TYPES = [
|
||||
IndexType.INVERT,
|
||||
]
|
||||
|
||||
SUPPORT_VECTOR_DATA_TYPES = [
|
||||
DataType.VECTOR_FP16,
|
||||
DataType.VECTOR_FP32,
|
||||
DataType.VECTOR_INT8,
|
||||
DataType.SPARSE_VECTOR_FP32,
|
||||
DataType.SPARSE_VECTOR_FP16,
|
||||
]
|
||||
|
||||
SUPPORT_VECTOR_INDEX_TYPES = [
|
||||
IndexType.FLAT,
|
||||
IndexType.HNSW,
|
||||
IndexType.IVF,
|
||||
]
|
||||
|
||||
DEFAULT_VECTOR_FIELD_NAME = {
|
||||
DataType.VECTOR_FP16: "vector_fp16_field",
|
||||
DataType.VECTOR_FP32: "vector_fp32_field",
|
||||
DataType.VECTOR_INT8: "vector_int8_field",
|
||||
DataType.SPARSE_VECTOR_FP32: "sparse_vector_fp32_field",
|
||||
DataType.SPARSE_VECTOR_FP16: "sparse_vector_fp16_field",
|
||||
}
|
||||
|
||||
DEFAULT_VECTOR_DIMENSION = 128
|
||||
|
||||
SUPPORT_VECTOR_DATA_TYPE_INDEX_MAP = {
|
||||
DataType.VECTOR_FP16: [IndexType.FLAT, IndexType.HNSW, IndexType.IVF],
|
||||
DataType.VECTOR_FP32: [IndexType.FLAT, IndexType.HNSW, IndexType.IVF],
|
||||
DataType.VECTOR_INT8: [IndexType.FLAT, IndexType.HNSW],
|
||||
DataType.SPARSE_VECTOR_FP32: [IndexType.FLAT, IndexType.HNSW],
|
||||
DataType.SPARSE_VECTOR_FP16: [IndexType.FLAT, IndexType.HNSW],
|
||||
}
|
||||
|
||||
SUPPORT_VECTOR_DATA_TYPE_INDEX_MAP_PARAMS = [
|
||||
(data_type, index_type)
|
||||
for data_type, index_types in SUPPORT_VECTOR_DATA_TYPE_INDEX_MAP.items()
|
||||
for index_type in index_types
|
||||
]
|
||||
|
||||
DEFAULT_INDEX_PARAMS = {
|
||||
IndexType.FLAT: FlatIndexParam(),
|
||||
IndexType.HNSW: HnswIndexParam(),
|
||||
IndexType.IVF: IVFIndexParam(),
|
||||
IndexType.INVERT: InvertIndexParam(),
|
||||
}
|
||||
|
||||
SUPPORT_VECTOR_DATA_TYPE_QUANT_MAP = {
|
||||
DataType.VECTOR_FP32: [QuantizeType.FP16, QuantizeType.INT8, QuantizeType.INT4],
|
||||
DataType.SPARSE_VECTOR_FP32: [QuantizeType.FP16],
|
||||
}
|
||||
|
||||
SUPPORT_ADD_COLUMN_DATA_TYPE = [
|
||||
DataType.INT32,
|
||||
DataType.UINT32,
|
||||
DataType.INT64,
|
||||
DataType.UINT64,
|
||||
DataType.FLOAT,
|
||||
DataType.DOUBLE,
|
||||
]
|
||||
|
||||
NOT_SUPPORT_ADD_COLUMN_DATA_TYPE = [
|
||||
DataType.BOOL,
|
||||
DataType.STRING,
|
||||
DataType.ARRAY_BOOL,
|
||||
DataType.ARRAY_INT32,
|
||||
DataType.ARRAY_INT64,
|
||||
DataType.ARRAY_UINT32,
|
||||
DataType.ARRAY_UINT64,
|
||||
DataType.ARRAY_FLOAT,
|
||||
DataType.ARRAY_DOUBLE,
|
||||
DataType.ARRAY_STRING,
|
||||
]
|
||||
@@ -0,0 +1,429 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import pytest
|
||||
import threading
|
||||
import numpy as np
|
||||
import zvec
|
||||
|
||||
from zvec import (
|
||||
CollectionOption,
|
||||
InvertIndexParam,
|
||||
HnswIndexParam,
|
||||
Collection,
|
||||
Doc,
|
||||
DataType,
|
||||
FieldSchema,
|
||||
VectorSchema,
|
||||
)
|
||||
|
||||
|
||||
class TestCollectionConcurrency:
|
||||
@pytest.fixture(scope="function")
|
||||
def test_collection(self, tmp_path_factory):
|
||||
"""Fixture to create a test collection"""
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name",
|
||||
DataType.STRING,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(),
|
||||
),
|
||||
FieldSchema("weight", DataType.FLOAT, nullable=True),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
),
|
||||
VectorSchema(
|
||||
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
collection_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=str(collection_path),
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert coll is not None, "Failed to create and open collection"
|
||||
|
||||
yield coll
|
||||
|
||||
# Clean up
|
||||
if hasattr(coll, "destroy") and coll is not None:
|
||||
try:
|
||||
coll.destroy()
|
||||
except Exception as e:
|
||||
print(f"Warning: failed to destroy collection: {e}")
|
||||
|
||||
def test_concurrent_read_write(self, test_collection: Collection):
|
||||
results = []
|
||||
|
||||
def insert_docs(thread_id):
|
||||
try:
|
||||
docs = [
|
||||
Doc(
|
||||
id=f"{thread_id}_{i}",
|
||||
fields={
|
||||
"id": int(f"{thread_id}{i}"),
|
||||
"name": f"thread_{thread_id}_doc_{i}",
|
||||
"weight": float(i),
|
||||
},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: float(i), 2: float(i * 2)},
|
||||
},
|
||||
)
|
||||
for i in range(5)
|
||||
]
|
||||
|
||||
result = test_collection.insert(docs)
|
||||
results.append((thread_id, "insert", len(result)))
|
||||
except Exception as e:
|
||||
results.append((thread_id, "insert_exception", str(e)))
|
||||
|
||||
def query_docs(thread_id):
|
||||
try:
|
||||
result = test_collection.query(filter="id > 0", topk=10)
|
||||
results.append((thread_id, "query", len(result)))
|
||||
except Exception as e:
|
||||
results.append((thread_id, "query_exception", str(e)))
|
||||
|
||||
# Create threads for concurrent operations
|
||||
threads = []
|
||||
|
||||
# Start insert threads
|
||||
for i in range(3):
|
||||
thread = threading.Thread(target=insert_docs, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Start query threads
|
||||
for i in range(3):
|
||||
thread = threading.Thread(target=query_docs, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Wait for all threads to complete
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
# Analyze results
|
||||
insert_results = [r for r in results if r[1] == "insert"]
|
||||
query_results = [r for r in results if r[1] == "query"]
|
||||
|
||||
logging.info(
|
||||
f"Concurrent read/write results - Inserts: {len(insert_results)}, Queries: {len(query_results)}"
|
||||
)
|
||||
|
||||
# At least some operations should succeed
|
||||
assert len(insert_results) + len(query_results) > 0
|
||||
|
||||
def test_concurrent_query(self, test_collection: Collection):
|
||||
# First insert some data
|
||||
docs = [
|
||||
Doc(
|
||||
id=f"{i}",
|
||||
fields={"id": i, "name": f"test_{i}", "weight": float(i)},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: float(i), 2: float(i * 2)},
|
||||
},
|
||||
)
|
||||
for i in range(20)
|
||||
]
|
||||
|
||||
insert_result = test_collection.insert(docs)
|
||||
assert len(insert_result) == 20
|
||||
|
||||
results = []
|
||||
|
||||
def query_operation(thread_id):
|
||||
"""Perform query operation from a thread"""
|
||||
try:
|
||||
result = test_collection.query(filter=f"id > {thread_id}", topk=5)
|
||||
results.append((thread_id, "query", len(result)))
|
||||
except Exception as e:
|
||||
results.append((thread_id, "query_exception", str(e)))
|
||||
|
||||
# Create multiple threads for concurrent queries
|
||||
threads = []
|
||||
for i in range(5):
|
||||
thread = threading.Thread(target=query_operation, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Wait for all threads to complete
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
# Analyze results
|
||||
query_results = [r for r in results if r[1] == "query"]
|
||||
logging.info(f"Concurrent query results - Queries: {len(query_results)}")
|
||||
|
||||
# All query operations should succeed
|
||||
assert len(query_results) == 5
|
||||
|
||||
def test_concurrent_modifications(self, test_collection: Collection):
|
||||
# First insert some data
|
||||
docs = [
|
||||
Doc(
|
||||
id=f"{i}",
|
||||
fields={"id": i, "name": f"test_{i}", "weight": float(i)},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: float(i), 2: float(i * 2)},
|
||||
},
|
||||
)
|
||||
for i in range(10)
|
||||
]
|
||||
|
||||
insert_result = test_collection.insert(docs)
|
||||
assert len(insert_result) == 10
|
||||
|
||||
results = []
|
||||
|
||||
def update_operation(thread_id):
|
||||
"""Perform update operation from a thread"""
|
||||
try:
|
||||
# Each thread updates different documents
|
||||
update_docs = [
|
||||
Doc(
|
||||
id=f"{i}",
|
||||
fields={
|
||||
"id": i,
|
||||
"name": f"updated_by_thread_{thread_id}",
|
||||
"weight": float(i + thread_id),
|
||||
},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: float(i) + 0.5, 2: float(i * 2) + 0.5},
|
||||
},
|
||||
)
|
||||
for i in range(thread_id * 2, thread_id * 2 + 2)
|
||||
]
|
||||
|
||||
result = test_collection.update(update_docs)
|
||||
results.append((thread_id, "update", len(result)))
|
||||
except Exception as e:
|
||||
results.append((thread_id, "update_exception", str(e)))
|
||||
|
||||
def delete_operation(thread_id):
|
||||
"""Perform delete operation from a thread"""
|
||||
try:
|
||||
# Each thread deletes different documents
|
||||
delete_ids = [f"{thread_id * 2 + 2}", f"{thread_id * 2 + 3}"]
|
||||
result = test_collection.delete(delete_ids)
|
||||
results.append((thread_id, "delete", len(result)))
|
||||
except Exception as e:
|
||||
results.append((thread_id, "delete_exception", str(e)))
|
||||
|
||||
# Create threads for concurrent operations
|
||||
threads = []
|
||||
|
||||
# Start update threads
|
||||
for i in range(3):
|
||||
thread = threading.Thread(target=update_operation, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Start delete threads
|
||||
for i in range(2):
|
||||
thread = threading.Thread(target=delete_operation, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Wait for all threads to complete
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
# Analyze results
|
||||
update_results = [r for r in results if r[1] == "update"]
|
||||
delete_results = [r for r in results if r[1] == "delete"]
|
||||
|
||||
logging.info(
|
||||
f"Concurrent modification results - Updates: {len(update_results)}, Deletes: {len(delete_results)}"
|
||||
)
|
||||
|
||||
# At least some operations should succeed
|
||||
assert len(update_results) + len(delete_results) > 0
|
||||
|
||||
def test_read_write_locking(self, test_collection: Collection):
|
||||
# Perform operations that should be thread-safe
|
||||
docs = [
|
||||
Doc(
|
||||
id=f"{i}",
|
||||
fields={"id": i, "name": f"test_{i}", "weight": float(i)},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: float(i), 2: float(i * 2)},
|
||||
},
|
||||
)
|
||||
for i in range(5)
|
||||
]
|
||||
|
||||
# Insert data
|
||||
insert_result = test_collection.insert(docs)
|
||||
assert len(insert_result) == 5
|
||||
|
||||
# Concurrent operations should not cause data corruption
|
||||
results = []
|
||||
|
||||
def mixed_operation(thread_id):
|
||||
"""Perform mixed operations from a thread"""
|
||||
try:
|
||||
# Mix of read and write operations
|
||||
if thread_id % 2 == 0:
|
||||
# Read operation
|
||||
result = test_collection.fetch([f"{thread_id % 5}"])
|
||||
results.append((thread_id, "read", len(result)))
|
||||
else:
|
||||
# Write operation
|
||||
doc = Doc(
|
||||
id=f"{thread_id % 5}",
|
||||
fields={
|
||||
"id": thread_id % 5,
|
||||
"name": f"mixed_op_{thread_id}",
|
||||
"weight": float(thread_id),
|
||||
},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: float(thread_id), 2: float(thread_id * 2)},
|
||||
},
|
||||
)
|
||||
result = test_collection.upsert(doc)
|
||||
results.append((thread_id, "write", len(result)))
|
||||
except Exception as e:
|
||||
results.append((thread_id, "exception", str(e)))
|
||||
|
||||
# Create multiple threads
|
||||
threads = []
|
||||
for i in range(10):
|
||||
thread = threading.Thread(target=mixed_operation, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Wait for all threads to complete
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
# Verify that the collection is still in a consistent state
|
||||
final_result = test_collection.query()
|
||||
assert len(final_result) >= 0 # Should not crash or return corrupted data
|
||||
|
||||
def test_race_condition_detection(self, test_collection: Collection):
|
||||
# Insert initial data
|
||||
docs = [
|
||||
Doc(
|
||||
id=f"{i}",
|
||||
fields={"id": i, "name": f"initial_{i}", "weight": float(i)},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: float(i), 2: float(i * 2)},
|
||||
},
|
||||
)
|
||||
for i in range(10)
|
||||
]
|
||||
|
||||
insert_result = test_collection.insert(docs)
|
||||
assert len(insert_result) == 10
|
||||
|
||||
# Perform many rapid concurrent operations
|
||||
operation_count = 100
|
||||
results = []
|
||||
|
||||
def rapid_operation(op_id):
|
||||
"""Perform rapid operations"""
|
||||
try:
|
||||
# Alternate between different types of operations
|
||||
if op_id % 4 == 0:
|
||||
# Insert
|
||||
doc = Doc(
|
||||
id=f"rapid_{op_id}",
|
||||
fields={
|
||||
"id": op_id,
|
||||
"name": f"rapid_{op_id}",
|
||||
"weight": float(op_id),
|
||||
},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: float(op_id), 2: float(op_id * 2)},
|
||||
},
|
||||
)
|
||||
result = test_collection.insert(doc)
|
||||
results.append(("insert", len(result)))
|
||||
elif op_id % 4 == 1:
|
||||
# Update
|
||||
doc = Doc(
|
||||
id=f"{op_id % 10}",
|
||||
fields={
|
||||
"id": op_id % 10,
|
||||
"name": f"rapid_update_{op_id}",
|
||||
"weight": float(op_id),
|
||||
},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: float(op_id), 2: float(op_id * 2)},
|
||||
},
|
||||
)
|
||||
result = test_collection.update(doc)
|
||||
results.append(("update", len(result)))
|
||||
elif op_id % 4 == 2:
|
||||
# Query
|
||||
result = test_collection.query(filter=f"id > {op_id % 5}", topk=3)
|
||||
results.append(("query", len(result)))
|
||||
else:
|
||||
# Fetch
|
||||
result = test_collection.fetch([f"{op_id % 10}"])
|
||||
results.append(("fetch", len(result)))
|
||||
except Exception as e:
|
||||
results.append(("exception", str(e)))
|
||||
|
||||
# Create many threads for rapid concurrent operations
|
||||
threads = []
|
||||
for i in range(operation_count):
|
||||
thread = threading.Thread(target=rapid_operation, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Wait for all threads to complete
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
# Verify collection is still functional
|
||||
final_query = test_collection.query()
|
||||
assert len(final_query) >= 0 # Should not be corrupted
|
||||
|
||||
logging.info(
|
||||
f"Rapid concurrent operations completed - Total operations: {len(results)}"
|
||||
)
|
||||
@@ -0,0 +1,792 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
import threading
|
||||
import os
|
||||
|
||||
from distance_helper import *
|
||||
from fixture_helper import *
|
||||
from doc_helper import *
|
||||
from params_helper import *
|
||||
|
||||
|
||||
def check_collection_info(
|
||||
coll: Collection, schema: CollectionSchema, option: CollectionOption, path: str
|
||||
):
|
||||
assert coll is not None, "Failed to create and open collection"
|
||||
assert coll.path == path
|
||||
assert coll.schema.name == schema.name
|
||||
assert list(coll.schema.fields) == list(schema.fields)
|
||||
assert list(coll.schema.vectors) == list(schema.vectors)
|
||||
assert coll.option.read_only == option.read_only
|
||||
assert coll.option.enable_mmap == option.enable_mmap
|
||||
|
||||
|
||||
def check_collection_basic(coll: Collection, optimize: bool = False):
|
||||
schema = coll.schema
|
||||
|
||||
docs = [generate_doc(i, schema) for i in range(10)]
|
||||
|
||||
results = coll.insert(docs=docs)
|
||||
assert len(results) == len(docs)
|
||||
for result in results:
|
||||
assert result.ok()
|
||||
|
||||
assert coll.stats.doc_count == len(docs)
|
||||
|
||||
def check_fetch_query():
|
||||
results = coll.fetch([str(i) for i in range(len(docs))])
|
||||
assert len(results) == len(docs)
|
||||
for i in range(len(docs)):
|
||||
assert str(i) in results
|
||||
|
||||
results = coll.query()
|
||||
assert len(results) == len(docs)
|
||||
|
||||
check_fetch_query()
|
||||
|
||||
if optimize:
|
||||
coll.optimize()
|
||||
check_fetch_query()
|
||||
|
||||
|
||||
def check_collection_full(coll: Collection):
|
||||
test_doc = generate_doc(1, coll.schema)
|
||||
|
||||
insert_result = coll.insert(test_doc)
|
||||
assert insert_result.ok()
|
||||
|
||||
stats = coll.stats
|
||||
assert stats.doc_count == 1
|
||||
|
||||
fetched_docs = coll.fetch(ids=["1"])
|
||||
assert len(fetched_docs) == 1
|
||||
assert "1" in fetched_docs
|
||||
assert fetched_docs["1"] is not None
|
||||
assert is_doc_equal(fetched_docs["1"], test_doc, coll.schema)
|
||||
|
||||
query_result = coll.query()
|
||||
assert len(query_result) == 1
|
||||
|
||||
updated_doc = Doc(
|
||||
id="1",
|
||||
fields={"int32_field": 1},
|
||||
vectors={"vector_fp32_field": [0.2] * 128},
|
||||
)
|
||||
update_result = coll.update(updated_doc)
|
||||
assert update_result.ok()
|
||||
|
||||
upserted_doc = generate_doc(1, coll.schema)
|
||||
upsert_result = coll.upsert(upserted_doc)
|
||||
assert upsert_result.ok()
|
||||
|
||||
# 8. Delete document
|
||||
delete_result = coll.delete("1")
|
||||
assert delete_result.ok()
|
||||
|
||||
# Verify document was deleted
|
||||
stats = coll.stats
|
||||
assert stats.doc_count == 0
|
||||
|
||||
|
||||
valid_collection_options = [
|
||||
# (read_only, enable_mmap)
|
||||
(False, True),
|
||||
(False, False),
|
||||
]
|
||||
invalid_collection_options = [
|
||||
# (read_only, enable_mmap)
|
||||
(True, True),
|
||||
(True, False),
|
||||
]
|
||||
duplicate_names_test = [
|
||||
("field1", "field1", "vector1", "vector2"),
|
||||
("field1", "field2", "vector1", "vector1"),
|
||||
(
|
||||
"shared_name1",
|
||||
"shared_name2",
|
||||
"shared_name1",
|
||||
"shared_name2",
|
||||
),
|
||||
]
|
||||
long_names = [
|
||||
"a" * 100, # 100 characters
|
||||
"b" * 200, # 200 characters
|
||||
]
|
||||
|
||||
valid_path_list = [
|
||||
"/tmp/nonexistent/directory/test_collection",
|
||||
"test/collection/with/slashes",
|
||||
]
|
||||
invalid_path_list = [
|
||||
"invalid:path",
|
||||
"",
|
||||
"test_collection_with_spaces ",
|
||||
"test@#$%collection",
|
||||
]
|
||||
|
||||
|
||||
class TestCreateAndOpen:
|
||||
@pytest.mark.parametrize("collection_name", COLLECTION_NAME_VALID_LIST)
|
||||
def test_valid_collection_name(
|
||||
self,
|
||||
collection_temp_dir,
|
||||
collection_name,
|
||||
collection_option,
|
||||
sample_field_list,
|
||||
sample_vector_list,
|
||||
):
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name=collection_name,
|
||||
fields=sample_field_list,
|
||||
vectors=sample_vector_list,
|
||||
)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
check_collection_info(
|
||||
coll, collection_schema, collection_option, collection_temp_dir
|
||||
)
|
||||
check_collection_basic(coll)
|
||||
|
||||
coll.destroy()
|
||||
|
||||
@pytest.mark.parametrize("collection_name", COLLECTION_NAME_INVALID_LIST)
|
||||
def test_invalid_collection_name(
|
||||
self,
|
||||
collection_temp_dir,
|
||||
collection_name,
|
||||
collection_option,
|
||||
sample_field_list,
|
||||
sample_vector_list,
|
||||
):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name=collection_name,
|
||||
fields=sample_field_list,
|
||||
vectors=sample_vector_list,
|
||||
)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
|
||||
@pytest.mark.parametrize("name_prefix", FIELD_NAME_VALID_LIST)
|
||||
def test_valid_field_vector_name(
|
||||
self,
|
||||
collection_temp_dir,
|
||||
collection_option,
|
||||
name_prefix,
|
||||
sample_field_list,
|
||||
sample_vector_list,
|
||||
):
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=sample_field_list,
|
||||
vectors=sample_vector_list,
|
||||
)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
check_collection_info(
|
||||
coll, collection_schema, collection_option, collection_temp_dir
|
||||
)
|
||||
check_collection_basic(coll)
|
||||
|
||||
coll.destroy()
|
||||
|
||||
@pytest.mark.parametrize("field_name", FIELD_NAME_INVALID_LIST)
|
||||
def test_invalid_field_name(
|
||||
self, collection_temp_dir, collection_option, field_name
|
||||
):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
field_list = [FieldSchema(field_name, DataType.STRING)]
|
||||
vector_list = [
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
]
|
||||
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="collection_name", fields=field_list, vectors=vector_list
|
||||
)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
|
||||
@pytest.mark.parametrize("vector_name", FIELD_NAME_INVALID_LIST)
|
||||
def test_invalid_vector_name(
|
||||
self, collection_temp_dir, collection_option, vector_name
|
||||
):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
field_list = [
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
)
|
||||
]
|
||||
vector_list = [
|
||||
VectorSchema(vector_name, DataType.VECTOR_FP32, dimension=128)
|
||||
]
|
||||
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="collection_name", fields=field_list, vectors=vector_list
|
||||
)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field_list_len,vector_list_len,dimension",
|
||||
FIELD_VECTOR_LIST_DIMENSION_VALID_LIST,
|
||||
)
|
||||
def test_valid_field_vector_size_dimension(
|
||||
self,
|
||||
collection_temp_dir,
|
||||
collection_option,
|
||||
field_list_len,
|
||||
vector_list_len,
|
||||
dimension,
|
||||
):
|
||||
field_list = []
|
||||
vector_list = []
|
||||
for i in range(0, field_list_len):
|
||||
field_list.append(
|
||||
FieldSchema("id_" + str(i), DataType.INT64, nullable=True)
|
||||
)
|
||||
|
||||
for i in range(0, vector_list_len):
|
||||
vector_list.append(
|
||||
VectorSchema(
|
||||
"dense_vector_" + str(i),
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=dimension,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
)
|
||||
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_dense_vector_list", fields=field_list, vectors=vector_list
|
||||
)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
check_collection_info(
|
||||
coll, collection_schema, collection_option, collection_temp_dir
|
||||
)
|
||||
check_collection_basic(coll)
|
||||
|
||||
coll.destroy()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field_list_len,vector_list_len,dimension",
|
||||
FIELD_VECTOR_LIST_DIMENSION_INVALID_LIST,
|
||||
)
|
||||
def test_invalid_field_vector_size_dimension(
|
||||
self,
|
||||
collection_temp_dir,
|
||||
collection_option,
|
||||
vector_list_len,
|
||||
field_list_len,
|
||||
dimension,
|
||||
):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
field_list = []
|
||||
vector_list = []
|
||||
for i in range(0, field_list_len):
|
||||
field_list.append(
|
||||
FieldSchema(
|
||||
"id_" + str(i),
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
)
|
||||
)
|
||||
|
||||
for i in range(0, vector_list_len):
|
||||
vector_list.append(
|
||||
VectorSchema(
|
||||
"dense_vector_" + str(i),
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=dimension,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
)
|
||||
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_dense_vector_list", fields=field_list, vectors=vector_list
|
||||
)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
|
||||
def test_valid_single_vector_field_construction(
|
||||
self, collection_temp_dir, collection_option
|
||||
):
|
||||
field = FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=True,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
)
|
||||
|
||||
vector = VectorSchema(
|
||||
"dense_vector",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_single_dense_vector_non_list",
|
||||
fields=field,
|
||||
vectors=vector, # Non-list form
|
||||
)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
check_collection_info(
|
||||
coll, collection_schema, collection_option, collection_temp_dir
|
||||
)
|
||||
check_collection_basic(coll)
|
||||
coll.destroy()
|
||||
|
||||
def test_collection_concurrent_create(
|
||||
self, collection_temp_dir, basic_schema, collection_option
|
||||
):
|
||||
results = []
|
||||
errors = []
|
||||
lock = threading.Lock()
|
||||
|
||||
# Function to be executed by each thread
|
||||
def create_collection_thread(thread_id):
|
||||
try:
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=basic_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
with lock:
|
||||
results.append((thread_id, coll))
|
||||
except Exception as e:
|
||||
with lock:
|
||||
errors.append((thread_id, str(e)))
|
||||
|
||||
threads = []
|
||||
for i in range(5):
|
||||
thread = threading.Thread(target=create_collection_thread, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
assert len(results) == 1, (
|
||||
f"Expected exactly one successful creation, but got {len(results)}"
|
||||
)
|
||||
assert len(errors) == 4, (
|
||||
f"Expected exactly four failures, but got {len(errors)}"
|
||||
)
|
||||
|
||||
successful_thread_id, successful_collection = results[0]
|
||||
assert successful_collection is not None, (
|
||||
"Successful creation should return a valid collection"
|
||||
)
|
||||
assert successful_collection.path == collection_temp_dir, (
|
||||
"Collection path mismatch"
|
||||
)
|
||||
|
||||
def test_create_open_loop(
|
||||
self, collection_temp_dir, collection_option, full_schema
|
||||
):
|
||||
for cycle in range(10):
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=full_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
assert coll is not None, (
|
||||
f"Failed to create and open collection in cycle {cycle}"
|
||||
)
|
||||
assert coll.path == collection_temp_dir, (
|
||||
f"Collection path mismatch in cycle {cycle}"
|
||||
)
|
||||
|
||||
del coll
|
||||
|
||||
reopened_coll = zvec.open(
|
||||
path=collection_temp_dir, option=collection_option
|
||||
)
|
||||
assert reopened_coll is not None, (
|
||||
f"Failed to reopen collection in cycle {cycle}"
|
||||
)
|
||||
assert reopened_coll.path == collection_temp_dir, (
|
||||
f"Reopened collection path mismatch in cycle {cycle}"
|
||||
)
|
||||
|
||||
check_collection_full(reopened_coll)
|
||||
|
||||
reopened_coll.destroy()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data_type, index_param", VALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS
|
||||
)
|
||||
def test_valid_vector_index_params(
|
||||
self,
|
||||
data_type,
|
||||
index_param,
|
||||
single_vector_schema_with_index_param,
|
||||
collection_temp_dir,
|
||||
collection_option,
|
||||
):
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=single_vector_schema_with_index_param,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
check_collection_info(
|
||||
coll,
|
||||
single_vector_schema_with_index_param,
|
||||
collection_option,
|
||||
collection_temp_dir,
|
||||
)
|
||||
|
||||
check_collection_basic(coll, True)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"data_type, index_param", INVALID_VECTOR_DATA_TYPE_INDEX_PARAM_MAP_PARAMS
|
||||
)
|
||||
def test_invalid_vector_index_params(
|
||||
self,
|
||||
data_type,
|
||||
index_param,
|
||||
single_vector_schema_with_index_param,
|
||||
collection_temp_dir,
|
||||
collection_option,
|
||||
):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=single_vector_schema_with_index_param,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
|
||||
def test_open_concurrent_same_path(self, tmp_path_factory, collection_option):
|
||||
"""Test concurrent opening of the same collection path.
|
||||
|
||||
- Multi-threading concurrency: 5 threads simultaneously open the same collection
|
||||
- Result verification: Verify that only one can open successfully, others must fail
|
||||
"""
|
||||
# Create a temporary directory and path for the collection
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "concurrent_open_test_collection"
|
||||
|
||||
# First, create a collection that we'll try to open concurrently
|
||||
field_list = [
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name", DataType.STRING, nullable=False, index_param=InvertIndexParam()
|
||||
),
|
||||
]
|
||||
|
||||
vector_list = [
|
||||
VectorSchema(
|
||||
"dense_vector",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
]
|
||||
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="concurrent_open_test_collection",
|
||||
fields=field_list,
|
||||
vectors=vector_list,
|
||||
)
|
||||
|
||||
# Create the collection first
|
||||
coll = zvec.create_and_open(
|
||||
path=str(collection_path),
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
# Close the collection so we can test opening it
|
||||
if hasattr(coll, "close") and coll is not None:
|
||||
coll.close()
|
||||
|
||||
# Shared variables to collect results from threads
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
# Lock for thread-safe operations
|
||||
lock = threading.Lock()
|
||||
# Clean up the created collection reference
|
||||
del coll
|
||||
|
||||
# Function to be executed by each thread
|
||||
def open_collection_thread(thread_id):
|
||||
try:
|
||||
reopened_coll = zvec.open(
|
||||
path=str(collection_path), option=collection_option
|
||||
)
|
||||
with lock:
|
||||
results.append((thread_id, reopened_coll))
|
||||
# Clean up the collection if opened successfully
|
||||
if hasattr(reopened_coll, "close") and reopened_coll is not None:
|
||||
reopened_coll.close()
|
||||
except Exception as e:
|
||||
with lock:
|
||||
errors.append((thread_id, str(e)))
|
||||
|
||||
# Create and start 5 threads
|
||||
threads = []
|
||||
for i in range(5):
|
||||
thread = threading.Thread(target=open_collection_thread, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Wait for all threads to complete
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
# Verify results:
|
||||
# 1. Only one open should succeed (exactly one collection in results)
|
||||
# 2. Others should fail (4 errors in errors)
|
||||
assert len(results) == 1, (
|
||||
f"Expected exactly one successful open, but got {len(results)}"
|
||||
)
|
||||
assert len(errors) == 4, (
|
||||
f"Expected exactly four failures, but got {len(errors)}"
|
||||
)
|
||||
|
||||
# Additional verification: check that the successful open has a valid collection
|
||||
successful_thread_id, successful_collection = results[0]
|
||||
assert successful_collection is not None, (
|
||||
"Successful open should return a valid collection"
|
||||
)
|
||||
assert successful_collection.path == str(collection_path), (
|
||||
"Collection path mismatch"
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("read_only,enable_mmap", valid_collection_options)
|
||||
def test_valid_option(
|
||||
self, collection_temp_dir, basic_schema, read_only, enable_mmap
|
||||
):
|
||||
option = CollectionOption(read_only=read_only, enable_mmap=enable_mmap)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=basic_schema,
|
||||
option=option,
|
||||
)
|
||||
|
||||
check_collection_info(coll, basic_schema, option, collection_temp_dir)
|
||||
check_collection_basic(coll)
|
||||
|
||||
coll.destroy()
|
||||
|
||||
def test_valid_none_option(self, collection_temp_dir, basic_schema):
|
||||
zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=basic_schema,
|
||||
option=None,
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize("read_only,enable_mmap", invalid_collection_options)
|
||||
def test_invalid_option(
|
||||
self, collection_temp_dir, basic_schema, read_only, enable_mmap
|
||||
):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=basic_schema,
|
||||
option=CollectionOption(read_only=read_only, enable_mmap=enable_mmap),
|
||||
)
|
||||
|
||||
assert CREATE_READ_ONLY_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"field_name1,field_name2,vector_name1,vector_name2",
|
||||
duplicate_names_test,
|
||||
)
|
||||
def test_duplicate_field_names(
|
||||
self,
|
||||
collection_temp_dir,
|
||||
collection_option,
|
||||
field_name1,
|
||||
field_name2,
|
||||
vector_name1,
|
||||
vector_name2,
|
||||
):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
field_name1,
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
field_name2,
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
vector_name1,
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
),
|
||||
VectorSchema(
|
||||
vector_name2,
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
|
||||
@pytest.mark.parametrize("long_name", long_names)
|
||||
def test_invalid_long_field_names(
|
||||
self, collection_option, collection_temp_dir, long_name
|
||||
):
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name=long_name,
|
||||
fields=[
|
||||
FieldSchema(
|
||||
long_name + "_field",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
long_name + "_vector",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
|
||||
def test_invalid_empty_fields_and_vectors(
|
||||
self, collection_temp_dir, collection_option
|
||||
):
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[], # Empty fields
|
||||
vectors=[], # Empty vectors
|
||||
)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
coll = zvec.create_and_open(
|
||||
path=collection_temp_dir,
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert SCHEMA_VALIDATE_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
|
||||
@pytest.mark.parametrize("valid_path", valid_path_list)
|
||||
def test_valid_path(self, basic_schema, collection_option, valid_path):
|
||||
if os.path.exists(valid_path):
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(valid_path)
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=valid_path, schema=basic_schema, option=collection_option
|
||||
)
|
||||
|
||||
check_collection_info(coll, basic_schema, collection_option, valid_path)
|
||||
|
||||
coll.destroy()
|
||||
|
||||
@pytest.mark.parametrize("invalid_path", invalid_path_list)
|
||||
def test_invalid_path(self, basic_schema, collection_option, invalid_path):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
coll = zvec.create_and_open(
|
||||
path=invalid_path, schema=basic_schema, option=collection_option
|
||||
)
|
||||
|
||||
assert INVALID_PATH_ERROR_MSG in str(exc_info.value), str(exc_info.value)
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,328 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import logging
|
||||
import pytest
|
||||
import numpy as np
|
||||
import zvec
|
||||
|
||||
from zvec import (
|
||||
CollectionOption,
|
||||
InvertIndexParam,
|
||||
HnswIndexParam,
|
||||
DataType,
|
||||
Collection,
|
||||
Doc,
|
||||
FieldSchema,
|
||||
VectorSchema,
|
||||
VectorQuery,
|
||||
)
|
||||
|
||||
|
||||
class TestCollectionExceptionHandling:
|
||||
@pytest.fixture(scope="function")
|
||||
def test_collection(self, tmp_path_factory):
|
||||
"""Fixture to create a test collection"""
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name",
|
||||
DataType.STRING,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(),
|
||||
),
|
||||
FieldSchema("weight", DataType.FLOAT, nullable=True),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
),
|
||||
VectorSchema(
|
||||
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
collection_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=str(collection_path),
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert coll is not None, "Failed to create and open collection"
|
||||
|
||||
yield coll
|
||||
|
||||
# Clean up
|
||||
if hasattr(coll, "destroy") and coll is not None:
|
||||
try:
|
||||
coll.destroy()
|
||||
except Exception as e:
|
||||
print(f"Warning: failed to destroy collection: {e}")
|
||||
|
||||
def test_create_and_open_missing_path(self, tmp_path_factory):
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name",
|
||||
DataType.STRING,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(),
|
||||
),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
collection_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
coll = zvec.create_and_open(
|
||||
schema=collection_schema, option=collection_option
|
||||
)
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing path parameter"
|
||||
)
|
||||
|
||||
def test_create_and_open_missing_schema(self, tmp_path_factory):
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
collection_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
coll = zvec.create_and_open(
|
||||
path=str(collection_path), option=collection_option
|
||||
)
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing schema parameter"
|
||||
)
|
||||
|
||||
def test_open_missing_path(self):
|
||||
collection_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
coll = zvec.open(option=collection_option)
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing path parameter"
|
||||
)
|
||||
|
||||
def test_insert_missing_docs(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
result = test_collection.insert()
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing docs parameter"
|
||||
)
|
||||
|
||||
def test_update_missing_docs(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
result = test_collection.update()
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing docs parameter"
|
||||
)
|
||||
|
||||
def test_upsert_missing_docs(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
result = test_collection.upsert()
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing docs parameter"
|
||||
)
|
||||
|
||||
def test_delete_missing_ids(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
result = test_collection.delete()
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing ids parameter"
|
||||
)
|
||||
|
||||
def test_fetch_missing_ids(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
result = test_collection.fetch()
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing ids parameter"
|
||||
)
|
||||
|
||||
def test_query_missing_vectorquery_field_name(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
result = test_collection.query(vectors=[VectorQuery()])
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing VectorQuery field_name parameter"
|
||||
)
|
||||
|
||||
def test_add_column_missing_field_schema(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
test_collection.add_column()
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing field_schema parameter"
|
||||
)
|
||||
|
||||
def test_alter_column_missing_old_name(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
test_collection.alter_column(new_name="new_name")
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing old_name parameter"
|
||||
)
|
||||
|
||||
def test_alter_column_missing_new_name(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
test_collection.alter_column(old_name="old_name")
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing new_name parameter"
|
||||
)
|
||||
|
||||
def test_drop_column_missing_field_name(self, test_collection: Collection):
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
test_collection.drop_column()
|
||||
assert exc_info.value is not None, (
|
||||
"Expected exception for missing field_name parameter"
|
||||
)
|
||||
|
||||
def test_invalid_parameter_types(self, test_collection: Collection):
|
||||
# This test depends on specific implementation details
|
||||
# Generally, we would expect TypeErrors or similar exceptions
|
||||
pass
|
||||
|
||||
def test_missing_required_parameters(self, test_collection: Collection):
|
||||
# This test depends on specific implementation details
|
||||
# Generally, we would expect TypeErrors or similar exceptions
|
||||
pass
|
||||
|
||||
def test_empty_collection_operations(self, tmp_path_factory):
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="empty_test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name",
|
||||
DataType.STRING,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(),
|
||||
),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
collection_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "empty_test_collection"
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=str(collection_path),
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert coll is not None, "Failed to create and open collection"
|
||||
|
||||
# Test fetch on empty collection
|
||||
result = coll.fetch(["1"])
|
||||
assert len(result) >= 0 # May be empty or have special handling
|
||||
|
||||
# Test query on empty collection
|
||||
result = coll.query()
|
||||
assert len(result) == 0
|
||||
|
||||
# Test update on empty collection
|
||||
doc = Doc(
|
||||
id="1",
|
||||
fields={"id": 1, "name": "test"},
|
||||
vectors={"dense": np.random.random(128).tolist()},
|
||||
)
|
||||
|
||||
result = coll.update(doc)
|
||||
# Should handle gracefully, possibly with NOT_FOUND status
|
||||
|
||||
# Clean up
|
||||
if hasattr(coll, "destroy") and coll is not None:
|
||||
try:
|
||||
coll.destroy()
|
||||
except Exception as e:
|
||||
print(f"Warning: failed to destroy collection: {e}")
|
||||
|
||||
def test_resource_management(self, test_collection: Collection):
|
||||
doc = Doc(
|
||||
id="1",
|
||||
fields={"id": 1, "name": "test", "weight": 80.5},
|
||||
vectors={
|
||||
"dense": np.random.random(128).tolist(),
|
||||
"sparse": {1: 1.0, 2: 2.0},
|
||||
},
|
||||
)
|
||||
|
||||
# Insert
|
||||
result = test_collection.insert(doc)
|
||||
assert result.ok()
|
||||
|
||||
# Fetch
|
||||
result = test_collection.fetch(["1"])
|
||||
assert len(result) == 1
|
||||
|
||||
# Query
|
||||
result = test_collection.query()
|
||||
assert len(result) >= 0
|
||||
|
||||
# Update
|
||||
result = test_collection.update(doc)
|
||||
assert result.ok()
|
||||
|
||||
# Delete
|
||||
result = test_collection.delete("1")
|
||||
assert result.ok()
|
||||
|
||||
def test_exception_resource_cleanup(self, test_collection: Collection):
|
||||
# This test would need to simulate exception conditions
|
||||
# which is difficult without specific failure injection points
|
||||
pass
|
||||
@@ -0,0 +1,960 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
import threading
|
||||
import numpy as np
|
||||
|
||||
from fixture_helper import *
|
||||
|
||||
COLLECTION_OPTION_TEST_CASES_VALID = [
|
||||
# (read_only, enable_mmap, description)
|
||||
(False, True, "Read-write with mmap enabled"),
|
||||
(False, False, "Read-write with mmap disabled"),
|
||||
(True, True, "Read-only with mmap enabled"),
|
||||
(True, False, "Read-only with mmap disabled"),
|
||||
]
|
||||
|
||||
# Test data for invalid paths
|
||||
INVALID_PATH_LIST = [
|
||||
"/nonexistent/directory/test_collection",
|
||||
"invalid:path",
|
||||
"", # Empty path
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def collection_schema():
|
||||
return zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name", DataType.STRING, nullable=False, index_param=InvertIndexParam()
|
||||
),
|
||||
FieldSchema(
|
||||
"weight", DataType.FLOAT, nullable=False, index_param=InvertIndexParam()
|
||||
),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
),
|
||||
VectorSchema(
|
||||
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def single_doc():
|
||||
id = 0
|
||||
return Doc(
|
||||
id=f"{id}",
|
||||
fields={"id": id, "name": "test"},
|
||||
vectors={
|
||||
"dense": [id + 0.1] * 128,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="function")
|
||||
def test_collection(
|
||||
tmp_path_factory, collection_schema, collection_option
|
||||
) -> Generator[Any, Any, Collection]:
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
coll = zvec.create_and_open(
|
||||
path=str(collection_path), schema=collection_schema, option=collection_option
|
||||
)
|
||||
|
||||
assert coll is not None, "Failed to create and open collection"
|
||||
assert coll.path == str(collection_path)
|
||||
assert coll.schema.name == collection_schema.name
|
||||
assert list(coll.schema.fields) == list(collection_schema.fields)
|
||||
assert list(coll.schema.vectors) == list(collection_schema.vectors)
|
||||
assert coll.option.read_only == collection_option.read_only
|
||||
assert coll.option.enable_mmap == collection_option.enable_mmap
|
||||
|
||||
try:
|
||||
yield coll
|
||||
finally:
|
||||
if hasattr(coll, "destroy") and coll is not None:
|
||||
try:
|
||||
coll.destroy()
|
||||
except Exception as e:
|
||||
print(f"Warning: failed to destroy collection: {e}")
|
||||
|
||||
|
||||
class TestCollectionOpen:
|
||||
def test_open_basic_functionality(
|
||||
self, tmp_path_factory, collection_schema, collection_option
|
||||
):
|
||||
import sys
|
||||
import time
|
||||
import os
|
||||
|
||||
# Create unique temp directory
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
# Ensure the path exists
|
||||
collection_path_str = str(collection_path)
|
||||
print(f"DEBUG: Collection path: {collection_path_str}")
|
||||
print(f"DEBUG: Temp directory exists: {temp_dir.exists()}")
|
||||
|
||||
# Create and open collection first
|
||||
created_coll = zvec.create_and_open(
|
||||
path=collection_path_str, schema=collection_schema, option=collection_option
|
||||
)
|
||||
|
||||
assert created_coll is not None, (
|
||||
f"Failed to create collection, returned None instead of valid Collection object. Path: {collection_path_str}"
|
||||
)
|
||||
assert created_coll.path == collection_path_str, (
|
||||
f"Collection path mismatch. Expected: {collection_path_str}, Actual: {created_coll.path}"
|
||||
)
|
||||
assert created_coll.schema.name == "test_collection", (
|
||||
f"Collection schema name mismatch. Expected: test_collection, Actual: {created_coll.schema.name}"
|
||||
)
|
||||
|
||||
# Insert multiple documents to verify persistence
|
||||
docs = []
|
||||
for i in range(3):
|
||||
doc = Doc(
|
||||
id=f"{i}",
|
||||
fields={"id": i, "name": f"test_{i}", "weight": float(i * 10)},
|
||||
vectors={
|
||||
"dense": [float(j + i) for j in range(128)],
|
||||
"sparse": {j: float(j + i) for j in range(5)},
|
||||
},
|
||||
)
|
||||
docs.append(doc)
|
||||
|
||||
result = created_coll.insert(docs)
|
||||
assert len(result) == 3, f"Expected 3 insertion results, but got {len(result)}"
|
||||
for i, res in enumerate(result):
|
||||
assert res.ok(), (
|
||||
f"Insertion result {i} is not OK. Status code: {res.code()}, Message: {res.message()}"
|
||||
)
|
||||
|
||||
# Verify documents were inserted using fetch interface
|
||||
fetched_docs_after_insert = created_coll.fetch(["0", "1", "2"])
|
||||
assert len(fetched_docs_after_insert) == 3, (
|
||||
f"Expected 3 fetched documents after insertion, but got {len(fetched_docs_after_insert)}"
|
||||
)
|
||||
assert "0" in fetched_docs_after_insert, (
|
||||
"Document with ID '0' not found in fetched results after insertion"
|
||||
)
|
||||
assert "1" in fetched_docs_after_insert, (
|
||||
"Document with ID '1' not found in fetched results after insertion"
|
||||
)
|
||||
assert "2" in fetched_docs_after_insert, (
|
||||
"Document with ID '2' not found in fetched results after insertion"
|
||||
)
|
||||
|
||||
# Verify fetched document content after insertion
|
||||
for i in range(3):
|
||||
doc = fetched_docs_after_insert[f"{i}"]
|
||||
assert doc is not None, (
|
||||
f"Fetched document with ID '{i}' is None after insertion"
|
||||
)
|
||||
assert doc.id == f"{i}", (
|
||||
f"Document ID mismatch for document '{i}' after insertion. Expected: {i}, Actual: {doc.id}"
|
||||
)
|
||||
assert doc.field("id") == i, (
|
||||
f"Document id field mismatch for document '{i}' after insertion. Expected: {i}, Actual: {doc.field('id')}"
|
||||
)
|
||||
assert doc.field("name") == f"test_{i}", (
|
||||
f"Document name field mismatch for document '{i}' after insertion. Expected: test_{i}, Actual: {doc.field('name')}"
|
||||
)
|
||||
assert doc.field("weight") == float(i * 10), (
|
||||
f"Document weight field mismatch for document '{i}' after insertion. Expected: {float(i * 10)}, Actual: {doc.field('weight')}"
|
||||
)
|
||||
|
||||
# Verify vector access after insertion
|
||||
assert doc.vector("dense") is not None, (
|
||||
f"Document {i} should have dense vector after insertion"
|
||||
)
|
||||
assert doc.vector("sparse") is not None, (
|
||||
f"Document {i} should have sparse vector after insertion"
|
||||
)
|
||||
|
||||
# Verify vector types after insertion
|
||||
assert isinstance(doc.vector("dense"), list), (
|
||||
f"Document {i} dense vector should be dict after insertion, got {type(doc.vector('dense'))}"
|
||||
)
|
||||
assert isinstance(doc.vector("sparse"), dict), (
|
||||
f"Document {i} sparse vector should be dict after insertion, got {type(doc.vector('sparse'))}"
|
||||
)
|
||||
|
||||
# Verify documents were inserted using stats
|
||||
stats = created_coll.stats
|
||||
assert stats is not None, "Collection stats should not be None"
|
||||
assert stats.doc_count == 3, (
|
||||
f"Document count mismatch after insertion. Expected: 3, Actual: {stats.doc_count}"
|
||||
)
|
||||
|
||||
# Store the collection path before cleanup
|
||||
collection_path = created_coll.path
|
||||
|
||||
# Clean up the created collection reference
|
||||
del created_coll
|
||||
|
||||
# Wait and verify the path still exists
|
||||
print(f"DEBUG: Collection path after destroy: {collection_path}")
|
||||
print(f"DEBUG: Path exists after destroy: {os.path.exists(collection_path)}")
|
||||
|
||||
# Now open the existing collection
|
||||
try:
|
||||
print(f"DEBUG: Path exists before open: {os.path.exists(collection_path)}")
|
||||
|
||||
# List contents of parent directory for debugging
|
||||
parent_dir = os.path.dirname(collection_path)
|
||||
if os.path.exists(parent_dir):
|
||||
print(f"DEBUG: Parent directory contents: {os.listdir(parent_dir)}")
|
||||
|
||||
opened_coll = zvec.open(path=collection_path, option=collection_option)
|
||||
|
||||
assert opened_coll is not None, (
|
||||
f"Failed to open existing collection at path: {collection_path}. Returned None instead of valid Collection object"
|
||||
)
|
||||
assert opened_coll.path == collection_path, (
|
||||
f"Opened collection path mismatch. Expected: {collection_path}, Actual: {opened_coll.path}"
|
||||
)
|
||||
assert opened_coll.schema.name == "test_collection", (
|
||||
f"Opened collection schema name mismatch. Expected: test_collection, Actual: {opened_coll.schema.name}"
|
||||
)
|
||||
|
||||
# Check reference count of opened collection
|
||||
opened_ref_count = sys.getrefcount(opened_coll)
|
||||
print(f"DEBUG: Reference count of opened collection: {opened_ref_count}")
|
||||
|
||||
# Verify data persistence
|
||||
# Verify data persistence using fetch interface
|
||||
fetched_docs = opened_coll.fetch(["0", "1", "2"])
|
||||
assert len(fetched_docs) == 3, (
|
||||
f"Expected 3 fetched documents after reopening, but got {len(fetched_docs)}"
|
||||
)
|
||||
assert "0" in fetched_docs, (
|
||||
"Document with ID '0' not found in fetched results after reopening"
|
||||
)
|
||||
assert "1" in fetched_docs, (
|
||||
"Document with ID '1' not found in fetched results after reopening"
|
||||
)
|
||||
assert "2" in fetched_docs, (
|
||||
"Document with ID '2' not found in fetched results after reopening"
|
||||
)
|
||||
|
||||
# Verify fetched document content after reopening collection
|
||||
for i in range(3):
|
||||
doc = fetched_docs[f"{i}"]
|
||||
assert doc is not None, (
|
||||
f"Fetched document with ID '{i}' is None after reopening collection"
|
||||
)
|
||||
assert doc.id == f"{i}", (
|
||||
f"Document ID mismatch for document '{i}' after reopening. Expected: {i}, Actual: {doc.id}"
|
||||
)
|
||||
assert doc.field("id") == i, (
|
||||
f"Document id field mismatch for document '{i}' after reopening. Expected: {i}, Actual: {doc.field('id')}"
|
||||
)
|
||||
assert doc.field("name") == f"test_{i}", (
|
||||
f"Document name field mismatch for document '{i}' after reopening. Expected: test_{i}, Actual: {doc.field('name')}"
|
||||
)
|
||||
assert doc.field("weight") == float(i * 10), (
|
||||
f"Document weight field mismatch for document '{i}' after reopening. Expected: {float(i * 10)}, Actual: {doc.field('weight')}"
|
||||
)
|
||||
|
||||
# Verify vector access after reopening
|
||||
assert doc.vector("dense") is not None, (
|
||||
f"Document {i} should have dense vector after reopening"
|
||||
)
|
||||
assert doc.vector("sparse") is not None, (
|
||||
f"Document {i} should have sparse vector after reopening"
|
||||
)
|
||||
|
||||
# Verify vector types after reopening
|
||||
assert isinstance(doc.vector("dense"), list), (
|
||||
f"Document {i} dense vector should be dict after reopening, got {type(doc.vector('dense'))}"
|
||||
)
|
||||
assert isinstance(doc.vector("sparse"), dict), (
|
||||
f"Document {i} sparse vector should be dict after reopening, got {type(doc.vector('sparse'))}"
|
||||
)
|
||||
|
||||
# Verify score attribute exists
|
||||
assert hasattr(doc, "score"), (
|
||||
f"Document {i} should have a score attribute after reopening"
|
||||
)
|
||||
assert isinstance(doc.score, (int, float)), (
|
||||
f"Document {i} score should be numeric after reopening, got {type(doc.score)}"
|
||||
)
|
||||
# For fetch operations, score is typically 0.0
|
||||
assert doc.score == 0.0, (
|
||||
f"Document {i} score should be 0.0 for fetch operation after reopening, but got {doc.score}"
|
||||
)
|
||||
|
||||
# Test query functionality
|
||||
query_result = opened_coll.query(include_vector=True)
|
||||
assert len(query_result) == 3, (
|
||||
f"Expected 3 query results, but got {len(query_result)}"
|
||||
)
|
||||
|
||||
# Verify query results have proper structure and content with detailed validation
|
||||
returned_doc_ids = set()
|
||||
for doc in query_result:
|
||||
# Verify basic document structure
|
||||
assert doc.id is not None, f"Query result document should have an ID"
|
||||
assert doc.id in ["0", "1", "2"], (
|
||||
f"Query result document ID should be one of ['0', '1', '2'], but got {doc.id}"
|
||||
)
|
||||
returned_doc_ids.add(doc.id)
|
||||
|
||||
# Verify field access
|
||||
assert doc.field("id") is not None, (
|
||||
f"Document {doc.id} should have id field"
|
||||
)
|
||||
assert doc.field("name") is not None, (
|
||||
f"Document {doc.id} should have name field"
|
||||
)
|
||||
assert doc.field("weight") is not None, (
|
||||
f"Document {doc.id} should have weight field"
|
||||
)
|
||||
|
||||
# Verify field values
|
||||
expected_id = int(doc.id)
|
||||
assert doc.field("id") == expected_id, (
|
||||
f"Document {doc.id} id field mismatch. Expected: {expected_id}, Actual: {doc.field('id')}"
|
||||
)
|
||||
assert doc.field("name") == f"test_{expected_id}", (
|
||||
f"Document {doc.id} name field mismatch. Expected: test_{expected_id}, Actual: {doc.field('name')}"
|
||||
)
|
||||
assert doc.field("weight") == float(expected_id * 10), (
|
||||
f"Document {doc.id} weight field mismatch. Expected: {float(expected_id * 10)}, Actual: {doc.field('weight')}"
|
||||
)
|
||||
|
||||
# Verify vector access
|
||||
assert doc.vector("dense") is not None, (
|
||||
f"Document {doc.id} should have dense vector"
|
||||
)
|
||||
assert doc.vector("sparse") is not None, (
|
||||
f"Document {doc.id} should have sparse vector"
|
||||
)
|
||||
|
||||
# Verify vector types
|
||||
assert isinstance(doc.vector("dense"), list), (
|
||||
f"Document {doc.id} dense vector should be list, got {type(doc.vector('dense'))}"
|
||||
)
|
||||
assert isinstance(doc.vector("sparse"), dict), (
|
||||
f"Document {doc.id} sparse vector should be dict, got {type(doc.vector('sparse'))}"
|
||||
)
|
||||
|
||||
# Verify score attribute exists
|
||||
assert hasattr(doc, "score"), (
|
||||
f"Document {doc.id} should have a score attribute"
|
||||
)
|
||||
assert isinstance(doc.score, (int, float)), (
|
||||
f"Document {doc.id} score should be numeric, got {type(doc.score)}"
|
||||
)
|
||||
|
||||
# Verify all expected documents are returned
|
||||
expected_doc_ids = {"0", "1", "2"}
|
||||
assert returned_doc_ids == expected_doc_ids, (
|
||||
f"Query should return all expected documents. Expected: {expected_doc_ids}, Actual: {returned_doc_ids}"
|
||||
)
|
||||
|
||||
# === Enhanced validation based on test_collection_dql_operations.py ===
|
||||
|
||||
# Verify vector field names accessibility for all documents
|
||||
for doc in query_result:
|
||||
vector_names = doc.vector_names()
|
||||
expected_vector_names = {"dense", "sparse"}
|
||||
assert set(vector_names) == expected_vector_names, (
|
||||
f"Document {doc.id} vector names mismatch. Expected: {expected_vector_names}, Actual: {set(vector_names)}"
|
||||
)
|
||||
|
||||
# Verify all vector fields can be accessed
|
||||
for vector_name in expected_vector_names:
|
||||
vector_data = doc.vector(vector_name)
|
||||
assert vector_data is not None, (
|
||||
f"Document {doc.id} should have accessible vector '{vector_name}'"
|
||||
)
|
||||
if vector_name == "dense":
|
||||
assert isinstance(vector_data, list), (
|
||||
f"Document {doc.id} vector '{vector_name}' should be list, got {type(vector_data)}"
|
||||
)
|
||||
else:
|
||||
assert isinstance(vector_data, dict), (
|
||||
f"Document {doc.id} vector '{vector_name}' should be dict, got {type(vector_data)}"
|
||||
)
|
||||
|
||||
# Test query with filter
|
||||
filtered_result = opened_coll.query(filter="id >= 1", include_vector=True)
|
||||
assert len(filtered_result) == 2, (
|
||||
f"Expected 2 filtered query results (id >= 1), but got {len(filtered_result)}"
|
||||
)
|
||||
|
||||
# Verify filtered query results
|
||||
filtered_doc_ids = set()
|
||||
for doc in filtered_result:
|
||||
assert doc.id is not None, (
|
||||
f"Filtered query result document should have an ID"
|
||||
)
|
||||
assert doc.id in ["1", "2"], (
|
||||
f"Filtered query result document ID should be one of ['1', '2'], but got {doc.id}"
|
||||
)
|
||||
filtered_doc_ids.add(doc.id)
|
||||
|
||||
# Verify filter condition is satisfied
|
||||
doc_id = int(doc.id)
|
||||
assert doc_id >= 1, (
|
||||
f"Document {doc.id} should satisfy filter condition id >= 1"
|
||||
)
|
||||
|
||||
# Verify document structure
|
||||
assert doc.field("id") is not None, (
|
||||
f"Document {doc.id} should have id field"
|
||||
)
|
||||
assert doc.field("name") is not None, (
|
||||
f"Document {doc.id} should have name field"
|
||||
)
|
||||
assert doc.field("weight") is not None, (
|
||||
f"Document {doc.id} should have weight field"
|
||||
)
|
||||
|
||||
# Verify field values
|
||||
assert doc.field("id") == doc_id, (
|
||||
f"Document {doc.id} id field mismatch. Expected: {doc_id}, Actual: {doc.field('id')}"
|
||||
)
|
||||
assert doc.field("name") == f"test_{doc_id}", (
|
||||
f"Document {doc.id} name field mismatch. Expected: test_{doc_id}, Actual: {doc.field('name')}"
|
||||
)
|
||||
assert doc.field("weight") == float(doc_id * 10), (
|
||||
f"Document {doc.id} weight field mismatch. Expected: {float(doc_id * 10)}, Actual: {doc.field('weight')}"
|
||||
)
|
||||
|
||||
# Verify vector access
|
||||
assert doc.vector("dense") is not None, (
|
||||
f"Document {doc.id} should have dense vector"
|
||||
)
|
||||
assert doc.vector("sparse") is not None, (
|
||||
f"Document {doc.id} should have sparse vector"
|
||||
)
|
||||
|
||||
# Verify score attribute exists
|
||||
assert hasattr(doc, "score"), (
|
||||
f"Document {doc.id} should have a score attribute"
|
||||
)
|
||||
assert isinstance(doc.score, (int, float)), (
|
||||
f"Document {doc.id} score should be numeric, got {type(doc.score)}"
|
||||
)
|
||||
|
||||
# Verify filtered documents
|
||||
expected_filtered_ids = {"1", "2"}
|
||||
assert filtered_doc_ids == expected_filtered_ids, (
|
||||
f"Filtered query should return expected documents. Expected: {expected_filtered_ids}, Actual: {filtered_doc_ids}"
|
||||
)
|
||||
|
||||
# Test vector query functionality for dense vectors
|
||||
query_vector_dense = [0.1] * 128
|
||||
vector_query_result = opened_coll.query(
|
||||
VectorQuery(field_name="dense", vector=query_vector_dense)
|
||||
)
|
||||
assert len(vector_query_result) > 0, (
|
||||
f"Expected at least 1 vector query result, but got {len(vector_query_result)}"
|
||||
)
|
||||
|
||||
# Verify vector query results structure
|
||||
for doc in vector_query_result[:3]: # Check first 3 results
|
||||
assert doc.id is not None, (
|
||||
f"Vector query result document should have an ID"
|
||||
)
|
||||
assert doc.id in ["0", "1", "2"], (
|
||||
f"Vector query result document ID should be one of ['0', '1', '2'], but got {doc.id}"
|
||||
)
|
||||
|
||||
# Verify document structure
|
||||
assert doc.field("id") is not None, (
|
||||
f"Document {doc.id} should have id field"
|
||||
)
|
||||
assert doc.field("name") is not None, (
|
||||
f"Document {doc.id} should have name field"
|
||||
)
|
||||
assert doc.field("weight") is not None, (
|
||||
f"Document {doc.id} should have weight field"
|
||||
)
|
||||
|
||||
# Verify vector access
|
||||
assert doc.vector("dense") is not None, (
|
||||
f"Document {doc.id} should have dense vector"
|
||||
)
|
||||
assert doc.vector("sparse") is not None, (
|
||||
f"Document {doc.id} should have sparse vector"
|
||||
)
|
||||
|
||||
# Verify score attribute exists and is numeric
|
||||
assert hasattr(doc, "score"), (
|
||||
f"Document {doc.id} should have a score attribute"
|
||||
)
|
||||
assert isinstance(doc.score, (int, float)), (
|
||||
f"Document {doc.id} score should be numeric, got {type(doc.score)}"
|
||||
)
|
||||
|
||||
# For dense vector queries, score should typically be non-negative (depending on metric)
|
||||
# Note: This may vary based on the metric type used
|
||||
assert doc.score >= 0 or doc.score < 0, (
|
||||
f"Document {doc.id} score should be a valid number"
|
||||
)
|
||||
|
||||
# Test vector query functionality for sparse vectors
|
||||
query_vector_sparse = {1: 1.0, 2: 2.0, 3: 3.0}
|
||||
sparse_vector_query_result = opened_coll.query(
|
||||
VectorQuery(field_name="sparse", vector=query_vector_sparse)
|
||||
)
|
||||
assert len(sparse_vector_query_result) > 0, (
|
||||
f"Expected at least 1 sparse vector query result, but got {len(sparse_vector_query_result)}"
|
||||
)
|
||||
|
||||
# Verify sparse vector query results structure
|
||||
for doc in sparse_vector_query_result[:3]: # Check first 3 results
|
||||
assert doc.id is not None, (
|
||||
f"Sparse vector query result document should have an ID"
|
||||
)
|
||||
assert doc.id in ["0", "1", "2"], (
|
||||
f"Sparse vector query result document ID should be one of ['0', '1', '2'], but got {doc.id}"
|
||||
)
|
||||
|
||||
# Verify document structure
|
||||
assert doc.field("id") is not None, (
|
||||
f"Document {doc.id} should have id field"
|
||||
)
|
||||
assert doc.field("name") is not None, (
|
||||
f"Document {doc.id} should have name field"
|
||||
)
|
||||
assert doc.field("weight") is not None, (
|
||||
f"Document {doc.id} should have weight field"
|
||||
)
|
||||
|
||||
# Verify vector access
|
||||
assert doc.vector("dense") is not None, (
|
||||
f"Document {doc.id} should have dense vector"
|
||||
)
|
||||
assert doc.vector("sparse") is not None, (
|
||||
f"Document {doc.id} should have sparse vector"
|
||||
)
|
||||
|
||||
# Verify score attribute exists and is numeric
|
||||
assert hasattr(doc, "score"), (
|
||||
f"Document {doc.id} should have a score attribute"
|
||||
)
|
||||
assert isinstance(doc.score, (int, float)), (
|
||||
f"Document {doc.id} score should be numeric, got {type(doc.score)}"
|
||||
)
|
||||
|
||||
# Clean up
|
||||
if hasattr(opened_coll, "destroy") and opened_coll is not None:
|
||||
opened_coll.destroy()
|
||||
print("DEBUG: Opened collection destroyed successfully")
|
||||
|
||||
except Exception as e:
|
||||
logging.error("Exception occurred: [{}]".format(e))
|
||||
raise e
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"read_only,enable_mmap,description", COLLECTION_OPTION_TEST_CASES_VALID
|
||||
)
|
||||
@pytest.mark.parametrize("createAndopen_enable_mmap", [True, False])
|
||||
def test_open_with_different_collection_options_valid(
|
||||
self,
|
||||
tmp_path_factory,
|
||||
createAndopen_enable_mmap,
|
||||
read_only,
|
||||
enable_mmap,
|
||||
description,
|
||||
collection_schema,
|
||||
):
|
||||
# Create collection with initial option
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
initial_option = CollectionOption(
|
||||
read_only=False, enable_mmap=createAndopen_enable_mmap
|
||||
)
|
||||
|
||||
# Create and open collection first
|
||||
created_coll = zvec.create_and_open(
|
||||
path=str(collection_path), schema=collection_schema, option=initial_option
|
||||
)
|
||||
|
||||
assert created_coll is not None, "Failed to create collection"
|
||||
|
||||
# Clean up the created collection reference
|
||||
del created_coll
|
||||
|
||||
# Now open with different options
|
||||
collection_option = CollectionOption(
|
||||
read_only=read_only, enable_mmap=enable_mmap
|
||||
)
|
||||
|
||||
try:
|
||||
opened_coll = zvec.open(path=str(collection_path), option=collection_option)
|
||||
|
||||
assert opened_coll is not None, (
|
||||
f"Failed to open collection with option: {description}. Returned None instead of valid Collection object. Path: {collection_path}"
|
||||
)
|
||||
assert opened_coll.path == str(collection_path), (
|
||||
f"Opened collection path mismatch. Expected: {collection_path}, Actual: {opened_coll.path}"
|
||||
)
|
||||
assert opened_coll.schema.name == collection_schema.name, (
|
||||
f"Opened collection schema name mismatch. Expected: {collection_schema.name}, Actual: {opened_coll.schema.name}"
|
||||
)
|
||||
assert opened_coll.option.read_only == read_only, (
|
||||
f"Opened collection read_only option mismatch. Expected: {read_only}, Actual: {opened_coll.option.read_only}"
|
||||
)
|
||||
assert opened_coll.option.enable_mmap == createAndopen_enable_mmap, (
|
||||
f"Opened collection mmap option mismatch. Expected: {createAndopen_enable_mmap}, Actual: {opened_coll.option.enable_mmap}"
|
||||
)
|
||||
|
||||
# Clean up
|
||||
if (
|
||||
hasattr(opened_coll, "destroy")
|
||||
and opened_coll is not None
|
||||
and read_only == False
|
||||
):
|
||||
opened_coll.destroy()
|
||||
|
||||
except Exception as e:
|
||||
logging.error("Exception occurred: [{}]".format(e))
|
||||
pytest.fail(f"Failed to open collection with different options: {e}")
|
||||
|
||||
def test_open_with_none_option(self, tmp_path_factory, collection_schema):
|
||||
# Create collection
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
initial_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
# Create and open collection first
|
||||
created_coll = zvec.create_and_open(
|
||||
path=str(collection_path), schema=collection_schema, option=initial_option
|
||||
)
|
||||
|
||||
assert created_coll is not None, (
|
||||
f"Failed to create collection. Returned None instead of valid Collection object. Path: {collection_path}"
|
||||
)
|
||||
|
||||
# Clean up the created collection reference
|
||||
del created_coll
|
||||
|
||||
# Now open with None option
|
||||
with pytest.raises(Exception) as exc_info:
|
||||
zvec.open(path=str(collection_path), option=None)
|
||||
|
||||
assert "incompatible function arguments" in str(exc_info.value), (
|
||||
f"Expected 'incompatible function arguments' error, but got: {exc_info.value}"
|
||||
)
|
||||
|
||||
def test_reopen_collection(self, tmp_path_factory):
|
||||
# Prepare schema
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name",
|
||||
DataType.STRING,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(),
|
||||
),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
collection_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
# Create collection
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
# Create and open collection
|
||||
coll1 = zvec.create_and_open(
|
||||
path=str(collection_path),
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert coll1 is not None, "Failed to create and open collection"
|
||||
|
||||
# Insert some data
|
||||
doc = Doc(
|
||||
id="1",
|
||||
fields={"id": 1, "name": "test"},
|
||||
vectors={"dense": np.random.random(128).tolist()},
|
||||
)
|
||||
|
||||
result = coll1.insert(doc)
|
||||
assert result.ok()
|
||||
|
||||
# Close the first collection (delete reference)
|
||||
del coll1
|
||||
|
||||
# Reopen the collection
|
||||
coll2 = zvec.open(path=str(collection_path), option=collection_option)
|
||||
|
||||
assert coll2 is not None, "Failed to reopen collection"
|
||||
assert coll2.path == str(collection_path)
|
||||
assert coll2.schema.name == collection_schema.name
|
||||
|
||||
# Verify data is still there
|
||||
fetched_docs = coll2.fetch(["1"])
|
||||
assert "1" in fetched_docs
|
||||
fetched_doc = fetched_docs["1"]
|
||||
assert fetched_doc.id == "1"
|
||||
assert fetched_doc.field("name") == "test"
|
||||
|
||||
# Clean up
|
||||
if hasattr(coll2, "destroy") and coll2 is not None:
|
||||
try:
|
||||
coll2.destroy()
|
||||
except Exception as e:
|
||||
print(f"Warning: failed to destroy collection: {e}")
|
||||
|
||||
def test_open_concurrent_same_path(self, tmp_path_factory):
|
||||
# First create a collection
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name",
|
||||
DataType.STRING,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(),
|
||||
),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
collection_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
# Create collection path
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
# First create the collection
|
||||
created_coll = zvec.create_and_open(
|
||||
path=str(collection_path),
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert created_coll is not None, "Failed to create collection"
|
||||
|
||||
# Close the collection so we can test concurrent opening
|
||||
if hasattr(created_coll, "close") and created_coll is not None:
|
||||
created_coll.close()
|
||||
|
||||
# Shared variables to collect results from threads
|
||||
results = []
|
||||
errors = []
|
||||
|
||||
# Lock for thread-safe operations
|
||||
lock = threading.Lock()
|
||||
# Clean up the created collection reference
|
||||
del created_coll
|
||||
|
||||
# Function to be executed by each thread
|
||||
def open_collection_thread(thread_id):
|
||||
try:
|
||||
coll = zvec.open(path=str(collection_path), option=collection_option)
|
||||
with lock:
|
||||
results.append((thread_id, coll))
|
||||
# Close the collection if opened successfully
|
||||
if hasattr(coll, "close") and coll is not None:
|
||||
coll.close()
|
||||
except Exception as e:
|
||||
with lock:
|
||||
errors.append((thread_id, str(e)))
|
||||
|
||||
# Create 5 threads to call open concurrently
|
||||
threads = []
|
||||
for i in range(5):
|
||||
thread = threading.Thread(target=open_collection_thread, args=(i,))
|
||||
threads.append(thread)
|
||||
thread.start()
|
||||
|
||||
# Wait for all threads to complete
|
||||
for thread in threads:
|
||||
thread.join()
|
||||
|
||||
# Verify concurrency safety: only one should succeed, others should fail
|
||||
assert len(results) == 1, (
|
||||
f"Expected exactly one successful open, but got {len(results)}"
|
||||
)
|
||||
assert len(errors) == 4, (
|
||||
f"Expected exactly four failures, but got {len(errors)}"
|
||||
)
|
||||
|
||||
# Additional verification: check that the successful open has a valid collection
|
||||
successful_thread_id, successful_collection = results[0]
|
||||
assert successful_collection is not None, (
|
||||
"Successful open should return a valid collection"
|
||||
)
|
||||
assert successful_collection.path == str(collection_path), (
|
||||
"Collection path mismatch"
|
||||
)
|
||||
|
||||
# Clean up the successfully opened collection
|
||||
if (
|
||||
hasattr(successful_collection, "destroy")
|
||||
and successful_collection is not None
|
||||
):
|
||||
try:
|
||||
successful_collection.destroy()
|
||||
except Exception as e:
|
||||
print(f"Warning: failed to destroy collection: {e}")
|
||||
|
||||
def test_open_with_corrupted_files(self, tmp_path_factory):
|
||||
# First create a collection
|
||||
collection_schema = zvec.CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name",
|
||||
DataType.STRING,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(),
|
||||
),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
collection_option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
|
||||
# Create collection path
|
||||
temp_dir = tmp_path_factory.mktemp("zvec")
|
||||
collection_path = temp_dir / "test_collection"
|
||||
|
||||
# First create the collection
|
||||
created_coll = zvec.create_and_open(
|
||||
path=str(collection_path),
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert created_coll is not None, "Failed to create collection"
|
||||
|
||||
# Close the collection so we can manipulate its files
|
||||
if hasattr(created_coll, "close") and created_coll is not None:
|
||||
created_coll.close()
|
||||
|
||||
# Test case 1: Delete some files in the collection directory (simulate partial corruption)
|
||||
import os
|
||||
import shutil
|
||||
import random
|
||||
|
||||
# Get the collection directory path
|
||||
collection_dir = str(collection_path)
|
||||
|
||||
# List all files in the collection directory
|
||||
files_in_dir = []
|
||||
for root, dirs, files in os.walk(collection_dir):
|
||||
for file in files:
|
||||
files_in_dir.append(os.path.join(root, file))
|
||||
|
||||
# Randomly delete approximately half of the files to simulate partial corruption
|
||||
if files_in_dir:
|
||||
# Shuffle the list to randomly select files
|
||||
random.shuffle(files_in_dir)
|
||||
files_to_delete = files_in_dir[: len(files_in_dir) // 2]
|
||||
for file_path in files_to_delete:
|
||||
try:
|
||||
os.remove(file_path)
|
||||
except Exception as e:
|
||||
pass # Ignore errors during deletion
|
||||
|
||||
# Try to open the collection with missing files - should raise an exception
|
||||
with pytest.raises(Exception):
|
||||
zvec.open(path=str(collection_path), option=collection_option)
|
||||
|
||||
# Test case 2: Delete all files in the collection directory (simulate complete corruption)
|
||||
# Recreate the collection
|
||||
recreated_coll = zvec.create_and_open(
|
||||
path=str(collection_path) + "_all",
|
||||
schema=collection_schema,
|
||||
option=collection_option,
|
||||
)
|
||||
|
||||
assert recreated_coll is not None, "Failed to recreate collection"
|
||||
|
||||
# Close the collection so we can manipulate its files
|
||||
if hasattr(recreated_coll, "close") and recreated_coll is not None:
|
||||
recreated_coll.close()
|
||||
|
||||
# Delete all files in the collection directory
|
||||
try:
|
||||
shutil.rmtree(collection_dir)
|
||||
os.makedirs(collection_dir) # Recreate empty directory
|
||||
except Exception as e:
|
||||
pass # Ignore errors during deletion
|
||||
|
||||
# Try to open the collection with missing files - should raise an exception
|
||||
with pytest.raises(Exception):
|
||||
zvec.open(path=str(collection_path), option=collection_option)
|
||||
@@ -0,0 +1,307 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import logging
|
||||
import pytest
|
||||
import tempfile
|
||||
import os
|
||||
import sys
|
||||
import subprocess
|
||||
|
||||
import zvec
|
||||
import zvec
|
||||
from zvec import LogType, LogLevel
|
||||
|
||||
# Error messages
|
||||
INITIALIZATION_ERROR_MSG = "initialization failed"
|
||||
RUNTIME_ERROR_MSG = "RuntimeError"
|
||||
VALUE_ERROR_MSG = "ValueError"
|
||||
TYPE_ERROR_MSG = "TypeError"
|
||||
|
||||
|
||||
# ==================== helper ====================
|
||||
def run_in_subprocess(func):
|
||||
def wrapper(*args, **kwargs):
|
||||
if os.getenv("RUNNING_IN_SUBPROCESS"):
|
||||
return func(*args, **kwargs)
|
||||
|
||||
env = os.environ.copy()
|
||||
env["RUNNING_IN_SUBPROCESS"] = "1"
|
||||
env["PYTEST_CURRENT_TEST"] = func.__name__
|
||||
|
||||
import inspect
|
||||
|
||||
filepath = inspect.getfile(func)
|
||||
qualname = func.__qualname__.replace(".", "::")
|
||||
test_id = f"{filepath}::{qualname}"
|
||||
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
env["PYTHONPATH"] = project_root + ":" + env.get("PYTHONPATH", "")
|
||||
|
||||
cmd = [sys.executable, "-m", "pytest", "-v", "-s", test_id]
|
||||
|
||||
result = subprocess.run(cmd, env=env, capture_output=True, text=True)
|
||||
if result.returncode != 0:
|
||||
pytest.fail(
|
||||
f"Subprocess test {func.__name__} failed with code {result.returncode}\n"
|
||||
f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}"
|
||||
)
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
# ==================== Fixtures ====================
|
||||
@pytest.fixture(scope="function")
|
||||
def temp_log_dir(tmp_path_factory):
|
||||
return tmp_path_factory.mktemp("logs")
|
||||
|
||||
|
||||
# ==================== Tests ====================
|
||||
class TestDbConfigInitialization:
|
||||
@run_in_subprocess
|
||||
def test_init_default(self):
|
||||
# default config
|
||||
# log_type: Optional[LogType] = LogType.CONSOLE,
|
||||
# log_level: Optional[LogLevel] = LogLevel.WARN,
|
||||
# log_dir: Optional[str] = "./logs",
|
||||
# log_basename: Optional[str] = "zvec.log",
|
||||
# log_file_size: Optional[int] = 2048,
|
||||
# log_overdue_days: Optional[int] = 7,
|
||||
zvec.init()
|
||||
|
||||
@run_in_subprocess
|
||||
def test_init_file_logger(self):
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
zvec.init(
|
||||
log_level=LogLevel.DEBUG,
|
||||
log_type=LogType.FILE,
|
||||
)
|
||||
# assert logdir exist
|
||||
log_dir = Path("./logs")
|
||||
assert log_dir.exists()
|
||||
|
||||
# validate write log
|
||||
col = zvec.create_and_open(
|
||||
"/tmp/test/1",
|
||||
zvec.CollectionSchema(
|
||||
name="test",
|
||||
vectors=zvec.VectorSchema(
|
||||
dimension=4,
|
||||
data_type=zvec.DataType.VECTOR_FP32,
|
||||
name="image",
|
||||
),
|
||||
),
|
||||
)
|
||||
col.insert(docs=[zvec.Doc(id="1", vectors={"image": [1.0, 2.0, 3.0, 4.0]})])
|
||||
assert any(log_dir.glob("zvec.log.*"))
|
||||
|
||||
# clear
|
||||
col.destroy()
|
||||
shutil.rmtree(log_dir, ignore_errors=True)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_init_with_mixed_config(self):
|
||||
zvec.init(
|
||||
memory_limit_mb=128,
|
||||
log_type=LogType.FILE,
|
||||
query_threads=1,
|
||||
log_level=LogLevel.WARN,
|
||||
)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_repeated_initialization_runtime_error(self):
|
||||
zvec.init()
|
||||
with pytest.raises(RuntimeError):
|
||||
zvec.init()
|
||||
|
||||
|
||||
class TestDbConfigMemoryLimitValidation:
|
||||
@run_in_subprocess
|
||||
def test_memory_limit_min_valid(self):
|
||||
# MIN_MEMORY_LIMIT_BYTES is 100M
|
||||
with pytest.raises(RuntimeError):
|
||||
zvec.init(memory_limit_mb=99)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_memory_limit_invalid_value(self):
|
||||
# memory_limit_mb must >= 0 and must be int and if None, set default value
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(memory_limit_mb=0)
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(memory_limit_mb=-1)
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(memory_limit_mb="512")
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(memory_limit_mb=512.5)
|
||||
|
||||
|
||||
class TestDbConfigThreadValidation:
|
||||
@run_in_subprocess
|
||||
def test_query_threads(self):
|
||||
zvec.init(query_threads=1)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_query_threads_invalid(self):
|
||||
# query_threads must >= 0 and must be int and if None, set default value
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(query_threads=0)
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(query_threads=-1)
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(query_threads="value")
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(query_threads=512.5)
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(query_threads="512")
|
||||
|
||||
@run_in_subprocess
|
||||
def test_optimize_threads(self):
|
||||
zvec.init(optimize_threads=1)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_optimize_threads_invalid(self):
|
||||
# optimize_threads must >= 0 and must be int and if None, set default value
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(optimize_threads=0)
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(optimize_threads=-1)
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(optimize_threads="value")
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(optimize_threads=512.5)
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(optimize_threads="512")
|
||||
|
||||
|
||||
class TestDbConfigRatioValidation:
|
||||
@run_in_subprocess
|
||||
def test_init_invert_to_forward_scan_ratio(self):
|
||||
# must be in [0,1]
|
||||
zvec.init(invert_to_forward_scan_ratio=0.8)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_init_invert_to_forward_scan_ratio_invalid(self):
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(invert_to_forward_scan_ratio=1.1)
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(invert_to_forward_scan_ratio=-0.1)
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(invert_to_forward_scan_ratio="0.8")
|
||||
|
||||
@run_in_subprocess
|
||||
def test_init_brute_force_by_keys_ratio(self):
|
||||
zvec.init(brute_force_by_keys_ratio=0.8)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_init_brute_force_by_keys_ratio_invalid(self):
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(brute_force_by_keys_ratio=1.1)
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(brute_force_by_keys_ratio=-0.1)
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(brute_force_by_keys_ratio="0.8")
|
||||
|
||||
|
||||
class TestDbConfigLogValidation:
|
||||
@run_in_subprocess
|
||||
def test_log_type_valid(self):
|
||||
zvec.init(log_type=LogType.CONSOLE)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_log_type_invalid(self):
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_type="FILE")
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_type="")
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_type="invalid")
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_type=123)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_log_level_valid(self):
|
||||
zvec.init(log_level=LogLevel.ERROR)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_log_level_invalid(self):
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_level="WARN")
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_level="")
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_level="invalid")
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_level=123)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_init_file_logger(self):
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
|
||||
temp_dir = tempfile.mkdtemp(prefix="log_test_")
|
||||
abs_temp_dir = os.path.abspath(temp_dir)
|
||||
|
||||
zvec.init(
|
||||
log_level=LogLevel.DEBUG,
|
||||
log_type=LogType.FILE,
|
||||
log_dir=abs_temp_dir,
|
||||
log_basename="test",
|
||||
)
|
||||
|
||||
# assert logdir exist
|
||||
log_dir = Path(abs_temp_dir)
|
||||
assert log_dir.exists()
|
||||
|
||||
# validate write log
|
||||
col = zvec.create_and_open(
|
||||
"/tmp/test/1",
|
||||
zvec.CollectionSchema(
|
||||
name="test",
|
||||
vectors=zvec.VectorSchema(
|
||||
dimension=4,
|
||||
data_type=zvec.DataType.VECTOR_FP32,
|
||||
name="image",
|
||||
),
|
||||
),
|
||||
)
|
||||
col.insert(docs=[zvec.Doc(id="1", vectors={"image": [1.0, 2.0, 3.0, 4.0]})])
|
||||
assert any(log_dir.glob("test.*"))
|
||||
|
||||
# clear
|
||||
col.destroy()
|
||||
shutil.rmtree(log_dir, ignore_errors=True)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_log_file_size_invalid(self):
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_type=LogType.FILE, log_file_size="df")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(log_type=LogType.FILE, log_file_size=0)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(log_type=LogType.FILE, log_file_size=-1)
|
||||
|
||||
@run_in_subprocess
|
||||
def test_log_overdue_days_invalid(self):
|
||||
with pytest.raises(TypeError):
|
||||
zvec.init(log_type=LogType.FILE, log_overdue_days="df")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(log_type=LogType.FILE, log_overdue_days=0)
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
zvec.init(log_type=LogType.FILE, log_overdue_days=-1)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,584 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
from _zvec import _Doc
|
||||
from zvec.model.convert import convert_to_py_doc, convert_to_cpp_doc
|
||||
from zvec import Doc, CollectionSchema, DataType, FieldSchema, VectorSchema
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Convert Cpp Doc Test Case
|
||||
# ----------------------------
|
||||
class TestConvertCppDoc:
|
||||
def test_default(self):
|
||||
doc = Doc(id="1")
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=FieldSchema("name", DataType.STRING),
|
||||
)
|
||||
|
||||
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
assert cpp_doc is not None
|
||||
assert cpp_doc.pk() == doc.id
|
||||
|
||||
def test_with_field_notin_schema(self):
|
||||
doc = Doc(id="1", fields={"name": "Tom"})
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema("id", DataType.UINT64),
|
||||
FieldSchema("salary", DataType.UINT32),
|
||||
FieldSchema("age", DataType.INT32),
|
||||
FieldSchema("create_at", DataType.INT64),
|
||||
FieldSchema("author", DataType.STRING),
|
||||
FieldSchema("weight", DataType.FLOAT),
|
||||
],
|
||||
)
|
||||
with pytest.raises(ValueError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
def test_with_scalar_fields(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema("id", DataType.UINT64),
|
||||
FieldSchema("salary", DataType.UINT32),
|
||||
FieldSchema("age", DataType.INT32),
|
||||
FieldSchema("create_at", DataType.INT64),
|
||||
FieldSchema("author", DataType.STRING),
|
||||
FieldSchema("weight", DataType.FLOAT),
|
||||
FieldSchema("bmi", DataType.DOUBLE),
|
||||
FieldSchema("is_male", DataType.BOOL),
|
||||
],
|
||||
)
|
||||
doc = Doc(
|
||||
id="1",
|
||||
fields={
|
||||
"id": 1,
|
||||
"salary": 1000,
|
||||
"age": 18,
|
||||
"create_at": 1640995200,
|
||||
"bmi": 80.0 / 200.0,
|
||||
"author": "Tom",
|
||||
"weight": 80.0,
|
||||
"is_male": True,
|
||||
},
|
||||
)
|
||||
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
assert cpp_doc is not None
|
||||
assert cpp_doc.pk() == doc.id
|
||||
assert cpp_doc.get_any("id", DataType.UINT64) == 1
|
||||
assert cpp_doc.get_any("salary", DataType.UINT32) == 1000
|
||||
assert cpp_doc.get_any("age", DataType.INT32) == 18
|
||||
assert cpp_doc.get_any("create_at", DataType.INT64) == 1640995200
|
||||
assert cpp_doc.get_any("author", DataType.STRING) == "Tom"
|
||||
assert math.isclose(
|
||||
cpp_doc.get_any("weight", DataType.FLOAT), 80.0, rel_tol=1e-6
|
||||
)
|
||||
assert math.isclose(
|
||||
cpp_doc.get_any("bmi", DataType.DOUBLE), 80.0 / 200.0, rel_tol=1e-6
|
||||
)
|
||||
assert cpp_doc.get_any("is_male", DataType.BOOL) == True
|
||||
|
||||
def test_with_array_fields(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema("tags", DataType.ARRAY_STRING),
|
||||
FieldSchema("ids", DataType.ARRAY_UINT64),
|
||||
FieldSchema("marks", DataType.ARRAY_UINT32),
|
||||
FieldSchema("x", DataType.ARRAY_INT32),
|
||||
FieldSchema("y", DataType.ARRAY_INT64),
|
||||
FieldSchema("scores", DataType.ARRAY_FLOAT),
|
||||
FieldSchema("ratios", DataType.ARRAY_DOUBLE),
|
||||
FieldSchema("results", DataType.ARRAY_BOOL),
|
||||
],
|
||||
)
|
||||
|
||||
doc = Doc(
|
||||
id="1",
|
||||
fields={
|
||||
"tags": ["tag1", "tag2", "tag3"],
|
||||
"ids": [111111111111, 222222222222, 333333333333],
|
||||
"marks": [100, 200, 300],
|
||||
"x": [1, 2, 3],
|
||||
"y": [100, 200, 300],
|
||||
"scores": [1.1, 2.2, 3.3],
|
||||
"ratios": [0.1, 0.2, 0.3],
|
||||
"results": [True, False, True],
|
||||
},
|
||||
)
|
||||
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
assert cpp_doc is not None
|
||||
assert cpp_doc.pk() == doc.id
|
||||
assert cpp_doc.get_any("tags", DataType.ARRAY_STRING) == doc.field("tags")
|
||||
assert cpp_doc.get_any("ids", DataType.ARRAY_UINT64) == doc.field("ids")
|
||||
assert cpp_doc.get_any("marks", DataType.ARRAY_UINT32) == doc.field("marks")
|
||||
assert cpp_doc.get_any("x", DataType.ARRAY_INT32) == doc.field("x")
|
||||
assert cpp_doc.get_any("y", DataType.ARRAY_INT64) == doc.field("y")
|
||||
scores = cpp_doc.get_any("scores", DataType.ARRAY_FLOAT)
|
||||
for i in range(len(doc.field("scores"))):
|
||||
assert math.isclose(scores[i], doc.field("scores")[i], rel_tol=1e-1)
|
||||
ratios = cpp_doc.get_any("ratios", DataType.ARRAY_DOUBLE)
|
||||
for i in range(len(doc.field("ratios"))):
|
||||
assert math.isclose(ratios[i], doc.field("ratios")[i], rel_tol=1e-1)
|
||||
results = cpp_doc.get_any("results", DataType.ARRAY_BOOL)
|
||||
for i in range(len(doc.field("results"))):
|
||||
assert results[i] == doc.field("results")[i]
|
||||
|
||||
def test_with_dense_vector_fields(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
name="embedding",
|
||||
data_type=DataType.VECTOR_FP16,
|
||||
dimension=4,
|
||||
),
|
||||
VectorSchema(
|
||||
name="image",
|
||||
data_type=DataType.VECTOR_FP32,
|
||||
dimension=8,
|
||||
),
|
||||
VectorSchema(
|
||||
name="text",
|
||||
data_type=DataType.VECTOR_INT8,
|
||||
dimension=32,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
doc = Doc(
|
||||
id="1",
|
||||
vectors={
|
||||
"embedding": [1.1] * 4,
|
||||
"image": [2.2] * 8,
|
||||
"text": [4] * 32,
|
||||
},
|
||||
)
|
||||
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
assert cpp_doc is not None
|
||||
assert cpp_doc.pk() == doc.id
|
||||
|
||||
embedding_vector = cpp_doc.get_any("embedding", DataType.VECTOR_FP16)
|
||||
assert len(embedding_vector) == 4
|
||||
for i in range(4):
|
||||
assert math.isclose(
|
||||
embedding_vector[i], doc.vector("embedding")[i], rel_tol=1e-1
|
||||
)
|
||||
|
||||
image_vector = cpp_doc.get_any("image", DataType.VECTOR_FP32)
|
||||
assert len(image_vector) == 8
|
||||
for i in range(8):
|
||||
assert math.isclose(image_vector[i], doc.vector("image")[i], rel_tol=1e-1)
|
||||
|
||||
text_vector = cpp_doc.get_any("text", DataType.VECTOR_INT8)
|
||||
assert len(text_vector) == 32
|
||||
for i in range(32):
|
||||
assert text_vector[i] == doc.vectors["text"][i]
|
||||
|
||||
def test_with_sparse_vector_fields(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
name="author",
|
||||
data_type=DataType.SPARSE_VECTOR_FP32,
|
||||
),
|
||||
VectorSchema(
|
||||
name="content",
|
||||
data_type=DataType.SPARSE_VECTOR_FP16,
|
||||
),
|
||||
],
|
||||
)
|
||||
doc = Doc(
|
||||
id="1",
|
||||
vectors={
|
||||
"author": {1: 1.1, 2: 2.2, 3: 3.3},
|
||||
"content": {4: 4.4, 5: 5.5, 6: 6.6},
|
||||
},
|
||||
)
|
||||
|
||||
cpp_doc = convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
assert cpp_doc is not None
|
||||
assert cpp_doc.pk() == doc.id
|
||||
|
||||
author_vector = cpp_doc.get_any("author", DataType.SPARSE_VECTOR_FP32)
|
||||
assert isinstance(author_vector, dict)
|
||||
for key, value in doc.vector("author").items():
|
||||
assert math.isclose(author_vector[key], value, rel_tol=1e-1)
|
||||
|
||||
content_vector = cpp_doc.get_any("content", DataType.SPARSE_VECTOR_FP16)
|
||||
assert isinstance(content_vector, dict)
|
||||
for key, value in doc.vector("content").items():
|
||||
assert math.isclose(content_vector[key], value, rel_tol=1e-1)
|
||||
|
||||
def test_with_scalar_fields_error_datatype(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema("id", DataType.UINT64),
|
||||
FieldSchema("salary", DataType.UINT32),
|
||||
FieldSchema("age", DataType.INT32),
|
||||
FieldSchema("create_at", DataType.INT64),
|
||||
FieldSchema("author", DataType.STRING),
|
||||
FieldSchema("weight", DataType.FLOAT),
|
||||
FieldSchema("bmi", DataType.DOUBLE),
|
||||
FieldSchema("is_male", DataType.BOOL),
|
||||
],
|
||||
)
|
||||
doc = Doc(
|
||||
id="1",
|
||||
fields={
|
||||
"id": "1",
|
||||
},
|
||||
)
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"salary": "1000"})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"age": "18"})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"create_at": "2021-01-01"})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"author": 1})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"weight": "80.5"})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"bmi": "25.0"})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"is_male": "true"})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
def test_with_array_fields_error_datatype(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema("tags", DataType.ARRAY_STRING),
|
||||
FieldSchema("ids", DataType.ARRAY_UINT64),
|
||||
FieldSchema("marks", DataType.ARRAY_UINT32),
|
||||
FieldSchema("x", DataType.ARRAY_INT32),
|
||||
FieldSchema("y", DataType.ARRAY_INT64),
|
||||
FieldSchema("scores", DataType.ARRAY_FLOAT),
|
||||
FieldSchema("ratios", DataType.ARRAY_DOUBLE),
|
||||
FieldSchema("results", DataType.ARRAY_BOOL),
|
||||
],
|
||||
)
|
||||
|
||||
doc = Doc(id="1", fields={"tags": [1, 2, 3]})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"ids": ["1", "2", "3"]})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"marks": [1.1, 2.2, 3.3]})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"x": [1.1, 2.2, 3.3]})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"y": [1.1, 2.2, 3.3]})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"scores": ["1", "2", "3"]})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"ratios": ["1", "2", "3"]})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", fields={"results": ["1", "2", "3"]})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
def test_with_vector_fields_error_datatype(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
name="embedding",
|
||||
data_type=DataType.VECTOR_FP16,
|
||||
dimension=4,
|
||||
),
|
||||
VectorSchema(
|
||||
name="image",
|
||||
data_type=DataType.VECTOR_FP32,
|
||||
dimension=8,
|
||||
),
|
||||
VectorSchema(
|
||||
name="text",
|
||||
data_type=DataType.VECTOR_INT8,
|
||||
dimension=32,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
doc = Doc(id="1", vectors={"image": ["1.1"] * 4})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", vectors={"text": ["1"] * 4})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(id="1", vectors={"embedding": ["1"] * 4})
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
def test_with_sparse_vector_error_datatype(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
name="author",
|
||||
data_type=DataType.SPARSE_VECTOR_FP32,
|
||||
),
|
||||
VectorSchema(
|
||||
name="content",
|
||||
data_type=DataType.SPARSE_VECTOR_FP16,
|
||||
),
|
||||
],
|
||||
)
|
||||
doc = Doc(
|
||||
id="1",
|
||||
vectors={
|
||||
"author": {"1": 1.1, "2": 2.2, "3": 3.3},
|
||||
},
|
||||
)
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(
|
||||
id="1",
|
||||
vectors={
|
||||
"content": {"1": 1.1, "2": 2.2, "3": 3.3},
|
||||
},
|
||||
)
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
doc = Doc(
|
||||
id="1",
|
||||
vectors={
|
||||
"author": {1: "1", 2: "2", 3: "3"},
|
||||
},
|
||||
)
|
||||
with pytest.raises(TypeError):
|
||||
convert_to_cpp_doc(doc, collection_schema=schema)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Convert Py Doc Test Case
|
||||
# ----------------------------
|
||||
class TestConvertPyDoc:
|
||||
def test_default(self):
|
||||
doc = _Doc()
|
||||
doc.set_pk("1")
|
||||
doc.set_score(1.0)
|
||||
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=FieldSchema("name", DataType.STRING),
|
||||
)
|
||||
|
||||
py_doc = convert_to_py_doc(doc, schema)
|
||||
assert py_doc.id == "1"
|
||||
assert py_doc.score == 1.0
|
||||
|
||||
def test_with_scalar_fields(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema("id", DataType.UINT64),
|
||||
FieldSchema("salary", DataType.UINT32),
|
||||
FieldSchema("age", DataType.INT32),
|
||||
FieldSchema("create_at", DataType.INT64),
|
||||
FieldSchema("author", DataType.STRING),
|
||||
FieldSchema("weight", DataType.FLOAT),
|
||||
FieldSchema("bmi", DataType.DOUBLE),
|
||||
FieldSchema("is_male", DataType.BOOL),
|
||||
],
|
||||
)
|
||||
doc = _Doc()
|
||||
doc.set_pk("1")
|
||||
doc.set_any("id", schema.field("id")._get_object(), 1111111111111111)
|
||||
doc.set_any("salary", schema.field("salary")._get_object(), 1000)
|
||||
doc.set_any("age", schema.field("age")._get_object(), 18)
|
||||
doc.set_any("create_at", schema.field("create_at")._get_object(), 1640995200)
|
||||
doc.set_any("author", schema.field("author")._get_object(), "Tom")
|
||||
doc.set_any("weight", schema.field("weight")._get_object(), 80.0)
|
||||
doc.set_any("bmi", schema.field("bmi")._get_object(), 80.0 / 200.0)
|
||||
doc.set_any("is_male", schema.field("is_male")._get_object(), True)
|
||||
|
||||
py_doc = convert_to_py_doc(doc, schema)
|
||||
assert py_doc.id == "1"
|
||||
assert py_doc.field("id") == 1111111111111111
|
||||
assert py_doc.field("salary") == 1000
|
||||
assert py_doc.field("age") == 18
|
||||
assert py_doc.field("create_at") == 1640995200
|
||||
assert py_doc.field("author") == "Tom"
|
||||
assert py_doc.field("weight") == 80.0
|
||||
assert py_doc.field("bmi") == 80.0 / 200.0
|
||||
assert py_doc.field("is_male") == True
|
||||
|
||||
def test_with_array_fields(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema("tags", DataType.ARRAY_STRING),
|
||||
FieldSchema("ids", DataType.ARRAY_UINT64),
|
||||
FieldSchema("marks", DataType.ARRAY_UINT32),
|
||||
FieldSchema("x", DataType.ARRAY_INT32),
|
||||
FieldSchema("y", DataType.ARRAY_INT64),
|
||||
FieldSchema("scores", DataType.ARRAY_FLOAT),
|
||||
FieldSchema("ratios", DataType.ARRAY_DOUBLE),
|
||||
FieldSchema("results", DataType.ARRAY_BOOL),
|
||||
],
|
||||
)
|
||||
|
||||
doc = _Doc()
|
||||
doc.set_pk("1")
|
||||
doc.set_any(
|
||||
"tags", schema.field("tags")._get_object(), ["tag1", "tag2", "tag3"]
|
||||
)
|
||||
doc.set_any(
|
||||
"ids",
|
||||
schema.field("ids")._get_object(),
|
||||
[111111111111, 222222222222, 3333333333333],
|
||||
)
|
||||
doc.set_any("marks", schema.field("marks")._get_object(), [1000, 2000, 3000])
|
||||
doc.set_any("x", schema.field("x")._get_object(), [1, 2, 3])
|
||||
doc.set_any("y", schema.field("y")._get_object(), [100, 200, 300])
|
||||
doc.set_any("scores", schema.field("scores")._get_object(), [0.1, 0.2, 0.3])
|
||||
doc.set_any("ratios", schema.field("ratios")._get_object(), [0.1, 0.2, 0.3])
|
||||
doc.set_any(
|
||||
"results", schema.field("results")._get_object(), [True, False, True]
|
||||
)
|
||||
|
||||
py_doc = convert_to_py_doc(doc, schema)
|
||||
assert py_doc.field("tags") == ["tag1", "tag2", "tag3"]
|
||||
assert py_doc.field("ids") == [111111111111, 222222222222, 3333333333333]
|
||||
assert py_doc.field("marks") == [1000, 2000, 3000]
|
||||
assert py_doc.field("x") == [1, 2, 3]
|
||||
assert py_doc.field("y") == [100, 200, 300]
|
||||
|
||||
scores = doc.get_any("scores", DataType.ARRAY_FLOAT)
|
||||
for i in range(len(scores)):
|
||||
assert math.isclose(scores[i], py_doc.field("scores")[i], rel_tol=1e-1)
|
||||
ratios = doc.get_any("ratios", DataType.ARRAY_DOUBLE)
|
||||
for i in range(len(ratios)):
|
||||
assert math.isclose(ratios[i], py_doc.field("ratios")[i], rel_tol=1e-1)
|
||||
results = doc.get_any("results", DataType.ARRAY_BOOL)
|
||||
for i in range(len(results)):
|
||||
assert results[i] == py_doc.field("results")[i]
|
||||
|
||||
def test_with_dense_vector_fields(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
name="embedding",
|
||||
data_type=DataType.VECTOR_FP16,
|
||||
dimension=4,
|
||||
),
|
||||
VectorSchema(
|
||||
name="image",
|
||||
data_type=DataType.VECTOR_FP32,
|
||||
dimension=8,
|
||||
),
|
||||
VectorSchema(
|
||||
name="text",
|
||||
data_type=DataType.VECTOR_INT8,
|
||||
dimension=32,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
doc = _Doc()
|
||||
doc.set_pk("1")
|
||||
doc.set_any("embedding", schema.vector("embedding")._get_object(), [1.1] * 4)
|
||||
doc.set_any("image", schema.vector("image")._get_object(), [2.2] * 8)
|
||||
doc.set_any("text", schema.vector("text")._get_object(), [4] * 32)
|
||||
|
||||
py_doc = convert_to_py_doc(doc, schema)
|
||||
assert py_doc.id == "1"
|
||||
|
||||
embedding_vector = py_doc.vector("embedding")
|
||||
assert len(embedding_vector) == 4
|
||||
for i in range(4):
|
||||
assert math.isclose(
|
||||
py_doc.vector("embedding")[i], embedding_vector[i], rel_tol=1e-1
|
||||
)
|
||||
|
||||
image_vector = py_doc.vector("image")
|
||||
assert len(image_vector) == 8
|
||||
for i in range(8):
|
||||
assert math.isclose(
|
||||
py_doc.vector("image")[i], image_vector[i], rel_tol=1e-1
|
||||
)
|
||||
|
||||
text_vector = py_doc.vector("text")
|
||||
assert len(text_vector) == 32
|
||||
for i in range(32):
|
||||
assert py_doc.vector("text")[i] == text_vector[i]
|
||||
|
||||
def test_with_sparse_vector_fields(self):
|
||||
schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
name="author",
|
||||
data_type=DataType.SPARSE_VECTOR_FP32,
|
||||
),
|
||||
VectorSchema(
|
||||
name="content",
|
||||
data_type=DataType.SPARSE_VECTOR_FP16,
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
doc = _Doc()
|
||||
doc.set_pk("1")
|
||||
doc.set_any(
|
||||
"author", schema.vector("author")._get_object(), {1: 1.1, 2: 2.2, 3: 3.3}
|
||||
)
|
||||
doc.set_any(
|
||||
"content", schema.vector("content")._get_object(), {4: 4.4, 5: 5.5, 6: 6.6}
|
||||
)
|
||||
|
||||
py_doc = convert_to_py_doc(doc, schema)
|
||||
assert py_doc.id == "1"
|
||||
|
||||
author_vector = py_doc.vector("author")
|
||||
assert isinstance(author_vector, dict)
|
||||
for key, value in doc.get_any("author", DataType.SPARSE_VECTOR_FP32).items():
|
||||
assert math.isclose(author_vector[key], value, rel_tol=1e-1)
|
||||
|
||||
content_vector = py_doc.vector("content")
|
||||
assert isinstance(content_vector, dict)
|
||||
for key, value in doc.get_any("content", DataType.SPARSE_VECTOR_FP16).items():
|
||||
assert math.isclose(content_vector[key], value, rel_tol=1e-1)
|
||||
@@ -0,0 +1,269 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import pytest
|
||||
|
||||
|
||||
from _zvec import _Doc
|
||||
from zvec import FieldSchema, VectorSchema, Doc, DataType
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# PyDoc Test Case
|
||||
# ----------------------------
|
||||
class TestPyDoc:
|
||||
def test_default(self):
|
||||
Doc(id="1")
|
||||
|
||||
def test_with_single_vector(self):
|
||||
doc = Doc(id="1", vectors={"dense": [1, 2, 3]})
|
||||
assert doc is not None
|
||||
assert doc.id == "1"
|
||||
assert doc.vector("dense") == [1, 2, 3]
|
||||
|
||||
def test_with_hybrid_vectors(self):
|
||||
doc = Doc(
|
||||
id="1", vectors={"dense": [1, 2, 3], "sparse": {1: 1.0, 2: 2.0, 3: 3.0}}
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.id == "1"
|
||||
assert doc.vector("dense") == [1, 2, 3]
|
||||
assert doc.vector("sparse") == {1: 1.0, 2: 2.0, 3: 3.0}
|
||||
|
||||
def test_with_multi_vectors(self):
|
||||
doc = Doc(
|
||||
id="1",
|
||||
vectors={
|
||||
"image": [1, 2, 3],
|
||||
"description": [4, 5, 6],
|
||||
"keys": {1: 1.0, 2: 2.0, 3: 3.0},
|
||||
},
|
||||
fields={"author": "Tom", "age": 19, "is_male": True, "weight": 60.5},
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.id == "1"
|
||||
assert doc.vector("image") == [1, 2, 3]
|
||||
assert doc.vector("description") == [4, 5, 6]
|
||||
assert doc.vector("keys") == {1: 1.0, 2: 2.0, 3: 3.0}
|
||||
assert doc.field("author") == "Tom"
|
||||
assert doc.field("age") == 19
|
||||
assert doc.field("is_male") == True
|
||||
assert doc.field("weight") == 60.5
|
||||
|
||||
def test_with_numpy_array(self):
|
||||
import numpy as np
|
||||
|
||||
doc = Doc._from_tuple(
|
||||
(
|
||||
"1",
|
||||
0.0,
|
||||
None,
|
||||
{
|
||||
"image": np.array([1, 2, 3]),
|
||||
"description": np.random.random(512),
|
||||
"keys": {1: 1.0, 2: 2.0, 3: 3.0},
|
||||
},
|
||||
)
|
||||
)
|
||||
assert doc is not None
|
||||
assert doc.id == "1"
|
||||
assert doc.vector("image") == [1, 2, 3]
|
||||
assert doc.vector("keys") == {1: 1.0, 2: 2.0, 3: 3.0}
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# CppDoc Test Case
|
||||
# ----------------------------
|
||||
class TestCppDoc:
|
||||
def test_default(self):
|
||||
doc = _Doc()
|
||||
assert doc is not None
|
||||
|
||||
def test_doc_set_pk(self):
|
||||
doc = _Doc()
|
||||
doc.set_pk("1")
|
||||
assert doc.pk() == "1"
|
||||
|
||||
def test_doc_set_score(self):
|
||||
doc = _Doc()
|
||||
doc.set_score(0.9)
|
||||
assert math.isclose(doc.score(), 0.9, rel_tol=1e-6)
|
||||
|
||||
def test_doc_get_null_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("author", DataType.STRING, nullable=True)
|
||||
doc.set_any("author", schema._get_object(), None)
|
||||
assert doc.has_field("author")
|
||||
assert doc.get_any("author", schema.data_type) is None
|
||||
|
||||
def test_doc_get_set_has_null_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("author", DataType.STRING, nullable=False)
|
||||
with pytest.raises(ValueError):
|
||||
doc.set_any("author", schema._get_object(), None)
|
||||
|
||||
def test_doc_get_set_has_string_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("author", DataType.STRING)
|
||||
doc.set_any("author", schema._get_object(), "Tom")
|
||||
assert doc.has_field("author")
|
||||
assert doc.get_any("author", DataType.STRING) == "Tom"
|
||||
|
||||
def test_doc_get_set_has_bool_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("is_male", DataType.BOOL)
|
||||
doc.set_any("is_male", schema._get_object(), True)
|
||||
assert doc.has_field("is_male")
|
||||
assert doc.get_any("is_male", DataType.BOOL) == True
|
||||
|
||||
def test_doc_get_set_has_int32_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("age", DataType.INT32)
|
||||
doc.set_any("age", schema._get_object(), 19)
|
||||
assert doc.has_field("age")
|
||||
assert doc.get_any("age", DataType.INT32) == 19
|
||||
|
||||
def test_doc_get_set_has_int64_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("id", DataType.INT64)
|
||||
doc.set_any("id", schema._get_object(), 1111111111111111111)
|
||||
assert doc.has_field("id")
|
||||
assert doc.get_any("id", DataType.INT64) == 1111111111111111111
|
||||
|
||||
def test_doc_get_set_has_float_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("weight", DataType.FLOAT)
|
||||
doc.set_any("weight", schema._get_object(), 60.5)
|
||||
assert doc.has_field("weight")
|
||||
assert math.isclose(doc.get_any("weight", DataType.FLOAT), 60.5, rel_tol=1e-6)
|
||||
|
||||
def test_doc_get_set_has_double_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("height", DataType.DOUBLE)
|
||||
doc.set_any("height", schema._get_object(), 1.77777777777)
|
||||
assert doc.has_field("height")
|
||||
assert math.isclose(
|
||||
doc.get_any("height", DataType.DOUBLE), 1.7777777777, rel_tol=1e-9
|
||||
)
|
||||
|
||||
def test_doc_get_set_has_uint32_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("id", DataType.UINT32)
|
||||
doc.set_any("id", schema._get_object(), 4294967295)
|
||||
assert doc.has_field("id")
|
||||
assert doc.get_any("id", DataType.UINT32) == 4294967295
|
||||
|
||||
def test_doc_get_set_has_uint64_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("id", DataType.UINT64)
|
||||
doc.set_any("id", schema._get_object(), 18446744073709551615)
|
||||
assert doc.has_field("id")
|
||||
assert doc.get_any("id", DataType.UINT64) == 18446744073709551615
|
||||
|
||||
def test_doc_get_set_has_array_string_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("tags", DataType.ARRAY_STRING)
|
||||
doc.set_any("tags", schema._get_object(), ["tag1", "tag2", "tag3"])
|
||||
assert doc.has_field("tags")
|
||||
assert doc.get_any("tags", DataType.ARRAY_STRING) == ["tag1", "tag2", "tag3"]
|
||||
|
||||
def test_doc_get_set_has_array_int32_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("ids", DataType.ARRAY_INT32)
|
||||
doc.set_any("ids", schema._get_object(), [1, 2, 3])
|
||||
assert doc.has_field("ids")
|
||||
assert doc.get_any("ids", DataType.ARRAY_INT32) == [1, 2, 3]
|
||||
|
||||
def test_doc_get_set_has_array_int64_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("ids", DataType.ARRAY_INT64)
|
||||
doc.set_any("ids", schema._get_object(), [1, 2, 3])
|
||||
assert doc.has_field("ids")
|
||||
assert doc.get_any("ids", DataType.ARRAY_INT64) == [1, 2, 3]
|
||||
|
||||
def test_doc_get_set_has_array_float_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("weights", DataType.ARRAY_FLOAT)
|
||||
doc.set_any("weights", schema._get_object(), [1.0, 2.0, 3.0])
|
||||
assert doc.has_field("weights")
|
||||
assert doc.get_any("weights", DataType.ARRAY_FLOAT) == [1.0, 2.0, 3.0]
|
||||
|
||||
def test_doc_get_set_has_array_double_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("heights", DataType.ARRAY_DOUBLE)
|
||||
doc.set_any("heights", schema._get_object(), [1.0, 2.0, 3.0])
|
||||
assert doc.has_field("heights")
|
||||
assert doc.get_any("heights", DataType.ARRAY_DOUBLE) == [1.0, 2.0, 3.0]
|
||||
|
||||
def test_doc_get_set_has_array_bool_field(self):
|
||||
doc = _Doc()
|
||||
schema = FieldSchema("bools", DataType.ARRAY_BOOL)
|
||||
doc.set_any("bools", schema._get_object(), [True, False, True])
|
||||
assert doc.has_field("bools")
|
||||
assert doc.get_any("bools", DataType.ARRAY_BOOL) == [True, False, True]
|
||||
|
||||
def test_doc_get_set_has_vector_fp16(self):
|
||||
doc = _Doc()
|
||||
schema = VectorSchema("image", DataType.VECTOR_FP16)
|
||||
doc.set_any("image", schema._get_object(), [1.0, 2.0, 3.0])
|
||||
assert doc.has_field("image")
|
||||
image_vector = doc.get_any("image", DataType.VECTOR_FP16)
|
||||
assert image_vector is not None
|
||||
for i in range(len(image_vector)):
|
||||
assert math.isclose(image_vector[i], [1.0, 2.0, 3.0][i], rel_tol=1e-6)
|
||||
|
||||
def test_doc_get_set_has_vector_fp32(self):
|
||||
doc = _Doc()
|
||||
schema = VectorSchema("image", DataType.VECTOR_FP32)
|
||||
doc.set_any("image", schema._get_object(), [1.111111, 2.222222, 3.333333])
|
||||
assert doc.has_field("image")
|
||||
vector = doc.get_any("image", DataType.VECTOR_FP32)
|
||||
assert vector is not None
|
||||
for i in range(len(vector)):
|
||||
assert math.isclose(
|
||||
vector[i], [1.111111, 2.222222, 3.333333][i], rel_tol=1e-6
|
||||
)
|
||||
|
||||
def test_doc_get_set_has_vector_int8(self):
|
||||
doc = _Doc()
|
||||
schema = VectorSchema("image", DataType.VECTOR_INT8)
|
||||
doc.set_any("image", schema._get_object(), [1, 2, 3])
|
||||
assert doc.has_field("image")
|
||||
assert doc.get_any("image", DataType.VECTOR_INT8) == [1, 2, 3]
|
||||
|
||||
def test_doc_get_set_has_sparse_vector_fp32(self):
|
||||
doc = _Doc()
|
||||
sparse = {1: 1.111111, 2: 2.222222, 3: 3.333333}
|
||||
schema = VectorSchema("key", DataType.SPARSE_VECTOR_FP32)
|
||||
doc.set_any("key", schema._get_object(), sparse)
|
||||
assert doc.has_field("key")
|
||||
vector = doc.get_any("key", DataType.SPARSE_VECTOR_FP32)
|
||||
assert vector is not None
|
||||
assert isinstance(vector, dict)
|
||||
for key, value in sparse.items():
|
||||
assert math.isclose(vector[key], value, rel_tol=1e-6)
|
||||
|
||||
def test_doc_get_set_has_sparse_vector_fp16(self):
|
||||
doc = _Doc()
|
||||
sparse = {1: 1.1, 2: 2.2, 3: 3.3}
|
||||
schema = VectorSchema("key", DataType.SPARSE_VECTOR_FP16)
|
||||
doc.set_any("key", schema._get_object(), sparse)
|
||||
assert doc.has_field("key")
|
||||
vector = doc.get_any("key", DataType.SPARSE_VECTOR_FP16)
|
||||
assert vector is not None
|
||||
assert isinstance(vector, dict)
|
||||
for key, value in sparse.items():
|
||||
assert math.isclose(vector[key], value, rel_tol=1e-1)
|
||||
@@ -0,0 +1,131 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
|
||||
import pytest
|
||||
from zvec.extension import QwenEmbeddingFunction
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# QwenEmbeddingFunction Test Case
|
||||
# ----------------------------
|
||||
class TestQwenEmbeddingFunction:
|
||||
def test_init_with_api_key(self):
|
||||
# Test initialization with explicit API key
|
||||
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
|
||||
assert embedding_func.dimension == 128
|
||||
assert embedding_func.model == "text-embedding-v4"
|
||||
assert embedding_func._api_key == "test_key"
|
||||
|
||||
@patch.dict(os.environ, {"DASHSCOPE_API_KEY": "env_key"})
|
||||
def test_init_with_env_api_key(self):
|
||||
# Test initialization with API key from environment
|
||||
embedding_func = QwenEmbeddingFunction(dimension=128)
|
||||
assert embedding_func._api_key == "env_key"
|
||||
|
||||
def test_init_without_api_key(self):
|
||||
# Test initialization without API key raises ValueError
|
||||
with pytest.raises(ValueError, match="DashScope API key is required"):
|
||||
QwenEmbeddingFunction(dimension=128)
|
||||
|
||||
@patch.dict(os.environ, {"DASHSCOPE_API_KEY": ""})
|
||||
def test_init_with_empty_env_api_key(self):
|
||||
# Test initialization with empty API key from environment
|
||||
with pytest.raises(ValueError, match="DashScope API key is required"):
|
||||
QwenEmbeddingFunction(dimension=128)
|
||||
|
||||
def test_model_property(self):
|
||||
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
|
||||
assert embedding_func.model == "text-embedding-v4"
|
||||
|
||||
embedding_func = QwenEmbeddingFunction(
|
||||
dimension=128, model="custom-model", api_key="test_key"
|
||||
)
|
||||
assert embedding_func.model == "custom-model"
|
||||
|
||||
@patch("zvec.extension.embedding.require_module")
|
||||
def test_embed_with_empty_text(self, mock_require_module):
|
||||
# Test embed method with empty text raises ValueError
|
||||
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="Input text cannot be empty or whitespace only"
|
||||
):
|
||||
embedding_func.embed("")
|
||||
|
||||
with pytest.raises(TypeError):
|
||||
embedding_func.embed(None)
|
||||
|
||||
@patch("zvec.extension.embedding.require_module")
|
||||
def test_embed_success(self, mock_require_module):
|
||||
# Test successful embedding
|
||||
mock_dashscope = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = HTTPStatus.OK
|
||||
mock_response.output = {"embeddings": [{"embedding": [0.1, 0.2, 0.3]}]}
|
||||
mock_dashscope.TextEmbedding.call.return_value = mock_response
|
||||
mock_require_module.return_value = mock_dashscope
|
||||
|
||||
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
|
||||
result = embedding_func.embed("test text")
|
||||
|
||||
assert result == [0.1, 0.2, 0.3]
|
||||
mock_dashscope.TextEmbedding.call.assert_called_once_with(
|
||||
model="text-embedding-v4",
|
||||
input="test text",
|
||||
dimension=128,
|
||||
output_type="dense",
|
||||
)
|
||||
|
||||
@patch("zvec.extension.embedding.require_module")
|
||||
def test_embed_http_error(self, mock_require_module):
|
||||
# Test embedding with HTTP error
|
||||
mock_dashscope = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = HTTPStatus.BAD_REQUEST
|
||||
mock_response.message = "Bad Request"
|
||||
mock_dashscope.TextEmbedding.call.return_value = mock_response
|
||||
mock_require_module.return_value = mock_dashscope
|
||||
|
||||
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
embedding_func.embed("test text")
|
||||
|
||||
@patch("zvec.extension.embedding.require_module")
|
||||
def test_embed_invalid_response(self, mock_require_module):
|
||||
# Test embedding with invalid response (wrong number of embeddings)
|
||||
mock_dashscope = MagicMock()
|
||||
mock_response = MagicMock()
|
||||
mock_response.status_code = HTTPStatus.OK
|
||||
mock_response.output.embeddings = []
|
||||
mock_dashscope.TextEmbedding.call.return_value = mock_response
|
||||
mock_require_module.return_value = mock_dashscope
|
||||
|
||||
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="test_key")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
embedding_func.embed("test text")
|
||||
|
||||
@pytest.mark.skip(reason="Qwen Embedding is not available in CI")
|
||||
def test_embed(self):
|
||||
# Test embedding with invalid dimension
|
||||
embedding_func = QwenEmbeddingFunction(dimension=128, api_key="xxx")
|
||||
dense = embedding_func("test text")
|
||||
assert len(dense) == 128
|
||||
@@ -0,0 +1,415 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from zvec import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
FlatIndexParam,
|
||||
HnswIndexParam,
|
||||
IndexOption,
|
||||
InvertIndexParam,
|
||||
IVFIndexParam,
|
||||
OptimizeOption,
|
||||
HnswQueryParam,
|
||||
IVFQueryParam,
|
||||
VectorQuery,
|
||||
IndexType,
|
||||
MetricType,
|
||||
QuantizeType,
|
||||
DataType,
|
||||
VectorSchema,
|
||||
)
|
||||
|
||||
from _zvec.param import _VectorQuery
|
||||
|
||||
# ----------------------------
|
||||
# Invert Index Param Test Case
|
||||
# ----------------------------
|
||||
|
||||
|
||||
class TestInvertIndexParam:
|
||||
def test_default(self):
|
||||
param = InvertIndexParam()
|
||||
assert param.enable_range_optimization is False
|
||||
assert param.enable_extended_wildcard is False
|
||||
assert param.type == IndexType.INVERT
|
||||
|
||||
def test_custom(self):
|
||||
param = InvertIndexParam(
|
||||
enable_range_optimization=True, enable_extended_wildcard=True
|
||||
)
|
||||
assert param.enable_range_optimization is True
|
||||
assert param.enable_extended_wildcard is True
|
||||
|
||||
def test_readonly(self):
|
||||
param = InvertIndexParam()
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
param.enable_range_optimization = False
|
||||
param.enable_extended_wildcard = False
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Hnsw Index Param Test Case
|
||||
# ----------------------------
|
||||
|
||||
|
||||
class TestHnswIndexParam:
|
||||
def test_default(self):
|
||||
param = HnswIndexParam()
|
||||
assert param.metric_type == MetricType.IP
|
||||
assert param.m == 100
|
||||
assert param.ef_construction == 500
|
||||
assert param.quantize_type == QuantizeType.UNDEFINED
|
||||
assert param.type == IndexType.HNSW
|
||||
|
||||
def test_custom(self):
|
||||
param = HnswIndexParam(
|
||||
metric_type=MetricType.L2,
|
||||
m=10,
|
||||
ef_construction=1000,
|
||||
quantize_type=QuantizeType.FP16,
|
||||
)
|
||||
assert param.metric_type == MetricType.L2
|
||||
assert param.m == 10
|
||||
assert param.ef_construction == 1000
|
||||
assert param.quantize_type == QuantizeType.FP16
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"attr", ["metric_type", "m", "ef_construction", "quantize_type"]
|
||||
)
|
||||
def test_readonly_attributes(self, attr):
|
||||
param = HnswIndexParam()
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
setattr(param, attr, getattr(param, attr))
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Flat Index Param Test Case
|
||||
# ----------------------------
|
||||
class TestFlatIndexParam:
|
||||
def test_default(self):
|
||||
param = FlatIndexParam()
|
||||
assert param.type == IndexType.FLAT
|
||||
assert param.quantize_type == QuantizeType.UNDEFINED
|
||||
assert param.metric_type == MetricType.IP
|
||||
|
||||
def test_custom(self):
|
||||
param = FlatIndexParam(
|
||||
metric_type=MetricType.L2, quantize_type=QuantizeType.INT8
|
||||
)
|
||||
assert param.metric_type == MetricType.L2
|
||||
assert param.quantize_type == QuantizeType.INT8
|
||||
|
||||
@pytest.mark.parametrize("attr", ["metric_type", "quantize_type"])
|
||||
def test_readonly_attributes(self, attr):
|
||||
param = FlatIndexParam()
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
setattr(param, attr, getattr(param, attr))
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Ivf Index Param Test Case
|
||||
# ----------------------------
|
||||
class TestIVFIndexParam:
|
||||
def test_default(self):
|
||||
param = IVFIndexParam()
|
||||
assert param.metric_type == MetricType.IP
|
||||
assert param.n_list == 0
|
||||
assert param.quantize_type == QuantizeType.UNDEFINED
|
||||
assert param.type == IndexType.IVF
|
||||
|
||||
def test_custom(self):
|
||||
param = IVFIndexParam(
|
||||
metric_type=MetricType.L2, n_list=1000, quantize_type=QuantizeType.FP16
|
||||
)
|
||||
assert param.metric_type == MetricType.L2
|
||||
assert param.n_list == 1000
|
||||
assert param.quantize_type == QuantizeType.FP16
|
||||
assert param.type == IndexType.IVF
|
||||
|
||||
@pytest.mark.parametrize("attr", ["metric_type", "n_list", "quantize_type"])
|
||||
def test_readonly_attributes(self, attr):
|
||||
param = IVFIndexParam()
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
setattr(param, attr, getattr(param, attr))
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# CollectionOption Test Case
|
||||
# ----------------------------
|
||||
class TestCollectionOption:
|
||||
def test_default(self):
|
||||
option = CollectionOption()
|
||||
assert option is not None
|
||||
assert option.read_only == False
|
||||
assert option.enable_mmap == True
|
||||
|
||||
def test_custom(self):
|
||||
option = CollectionOption(read_only=True, enable_mmap=False)
|
||||
assert option.read_only == True
|
||||
assert option.enable_mmap == False
|
||||
|
||||
option = CollectionOption(read_only=False, enable_mmap=True)
|
||||
assert option.read_only == False
|
||||
assert option.enable_mmap == True
|
||||
|
||||
@pytest.mark.parametrize("attr", ["read_only", "enable_mmap"])
|
||||
def test_readonly_attributes(self, attr):
|
||||
param = CollectionOption()
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
setattr(param, attr, getattr(param, attr))
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# IndexOption Test Case
|
||||
# ----------------------------
|
||||
class TestIndexOption:
|
||||
def test_default(self):
|
||||
option = IndexOption()
|
||||
assert option is not None
|
||||
assert option.concurrency == 0
|
||||
|
||||
def test_custom(self):
|
||||
option = IndexOption(concurrency=10)
|
||||
assert option.concurrency == 10
|
||||
|
||||
@pytest.mark.parametrize("attr", ["concurrency"])
|
||||
def test_readonly_attributes(self, attr):
|
||||
param = IndexOption()
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
setattr(param, attr, getattr(param, attr))
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# AddColumnOption Test Case
|
||||
# ----------------------------
|
||||
class TestAddColumnOption:
|
||||
def test_default(self):
|
||||
option = AddColumnOption()
|
||||
assert option is not None
|
||||
assert option.concurrency == 0
|
||||
|
||||
def test_custom(self):
|
||||
option = AddColumnOption(concurrency=10)
|
||||
assert option.concurrency == 10
|
||||
|
||||
@pytest.mark.parametrize("attr", ["concurrency"])
|
||||
def test_readonly_attributes(self, attr):
|
||||
param = AddColumnOption()
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
setattr(param, attr, getattr(param, attr))
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# AlterColumnOption Test Case
|
||||
# ----------------------------
|
||||
class TestAlterColumnOption:
|
||||
def test_default(self):
|
||||
option = AlterColumnOption()
|
||||
assert option is not None
|
||||
assert option.concurrency == 0
|
||||
|
||||
def test_custom(self):
|
||||
option = AlterColumnOption(concurrency=10)
|
||||
assert option.concurrency == 10
|
||||
|
||||
@pytest.mark.parametrize("attr", ["concurrency"])
|
||||
def test_readonly_attributes(self, attr):
|
||||
param = AlterColumnOption()
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
setattr(param, attr, getattr(param, attr))
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# OptimizeOption Test Case
|
||||
# ----------------------------
|
||||
class TestOptimizeOption:
|
||||
def test_default(self):
|
||||
option = OptimizeOption()
|
||||
assert option is not None
|
||||
assert option.concurrency == 0
|
||||
|
||||
def test_custom(self):
|
||||
option = OptimizeOption(concurrency=10)
|
||||
assert option.concurrency == 10
|
||||
|
||||
@pytest.mark.parametrize("attr", ["concurrency"])
|
||||
def test_readonly_attributes(self, attr):
|
||||
param = OptimizeOption()
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
setattr(param, attr, getattr(param, attr))
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# HnswQueryParam Test Case
|
||||
# ----------------------------
|
||||
class TestHnswQueryParam:
|
||||
def test_default(self):
|
||||
param = HnswQueryParam()
|
||||
assert param is not None
|
||||
assert param.ef == 300
|
||||
assert param.is_using_refiner == False
|
||||
assert param.radius == 0
|
||||
assert param.is_linear == False
|
||||
|
||||
def test_custom(self):
|
||||
param = HnswQueryParam(ef=10, is_using_refiner=True, radius=30, is_linear=True)
|
||||
assert param.ef == 10
|
||||
assert param.is_using_refiner == True
|
||||
assert param.radius == 30
|
||||
assert param.is_linear == True
|
||||
|
||||
def test_readonly_attributes(self):
|
||||
param = HnswQueryParam()
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
param.ef = 10
|
||||
param.is_using_refiner = True
|
||||
param.radius = 30
|
||||
param.is_linear = True
|
||||
|
||||
|
||||
# # ----------------------------
|
||||
# # IVFQueryParam Test Case
|
||||
# # ----------------------------
|
||||
# class TestIVFQueryParam:
|
||||
# def test_default(self):
|
||||
# param = IVFQueryParam()
|
||||
# assert param is not None
|
||||
# assert param.nprobe == 10
|
||||
# assert param.is_using_refiner == False
|
||||
# assert param.radius == 0
|
||||
# assert param.is_linear == False
|
||||
# assert param.scale_factor == 10
|
||||
#
|
||||
# def test_custom(self):
|
||||
# param = IVFQueryParam(
|
||||
# nprobe=20,
|
||||
# is_using_refiner=True,
|
||||
# radius=30,
|
||||
# is_linear=True,
|
||||
# scale_factor=40
|
||||
# )
|
||||
# assert param.nprobe == 20
|
||||
# assert param.is_using_refiner == True
|
||||
# assert param.radius == 30
|
||||
# assert param.is_linear == True
|
||||
# assert param.scale_factor == 40
|
||||
|
||||
|
||||
class TestVectorQuery:
|
||||
def test_init_with_valid_id(self):
|
||||
vq = VectorQuery(field_name="embedding", id="doc123")
|
||||
assert vq.field_name == "embedding"
|
||||
assert vq.id == "doc123"
|
||||
assert vq.vector is None
|
||||
assert vq.param is None
|
||||
|
||||
def test_init_with_valid_vector(self):
|
||||
vec = [0.1, 0.2, 0.3]
|
||||
param = HnswQueryParam(ef=300)
|
||||
vq = VectorQuery(field_name="embedding", vector=vec, param=param)
|
||||
assert vq.field_name == "embedding"
|
||||
assert vq.vector == vec
|
||||
assert vq.param == param
|
||||
|
||||
def test_init_both_id_and_vector_raises_error(self):
|
||||
with pytest.raises(ValueError):
|
||||
VectorQuery(field_name="embedding", id="doc123", vector=[0.1])._validate()
|
||||
|
||||
def test_init_without_field_name_raises_error(self):
|
||||
with pytest.raises(ValueError):
|
||||
VectorQuery(field_name=None)._validate()
|
||||
|
||||
def test_has_id_returns_true_when_id_set(self):
|
||||
vq = VectorQuery(field_name="embedding", id="doc123")
|
||||
assert vq.has_id()
|
||||
|
||||
def test_has_id_returns_false_when_no_id(self):
|
||||
vq = VectorQuery(field_name="embedding", vector=[0.1])
|
||||
assert not vq.has_id()
|
||||
|
||||
def test_has_vector_returns_true_with_non_empty_vector(self):
|
||||
vq = VectorQuery(field_name="embedding", vector=[0.1])
|
||||
assert vq.has_vector()
|
||||
|
||||
def test_validate_fails_on_both_id_and_vector(self):
|
||||
vq = VectorQuery(field_name="test", id="doc123", vector=[0.1])
|
||||
with pytest.raises(ValueError):
|
||||
vq._validate()
|
||||
@@ -0,0 +1,297 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict, Union
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import math
|
||||
from _zvec.param import _VectorQuery
|
||||
|
||||
import pytest
|
||||
from zvec.executor.query_executor import (
|
||||
MultiVectorQueryExecutor,
|
||||
NoVectorQueryExecutor,
|
||||
QueryContext,
|
||||
QueryExecutor,
|
||||
QueryExecutorFactory,
|
||||
SingleVectorQueryExecutor,
|
||||
VectorQuery,
|
||||
)
|
||||
from zvec import RrfReRanker, HnswQueryParam, CollectionSchema, VectorSchema, DataType
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Mock Vector Schema
|
||||
# ----------------------------
|
||||
class MockVectorSchema(VectorSchema):
|
||||
def __init__(self, name="test_vector"):
|
||||
self._name = name
|
||||
|
||||
@property
|
||||
def name(self):
|
||||
return self._name
|
||||
|
||||
def _get_object(self):
|
||||
return MagicMock()
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Mock Collection Schema
|
||||
# ----------------------------
|
||||
class MockCollectionSchema(CollectionSchema):
|
||||
def __init__(self, vectors=Union[VectorSchema, Dict[str, VectorSchema]]):
|
||||
self._vectors = (
|
||||
[vectors] if not isinstance(vectors, Dict) else list(vectors.values())
|
||||
)
|
||||
|
||||
@property
|
||||
def vectors(self):
|
||||
return self._vectors
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# VectorQuery Test Case
|
||||
# ----------------------------
|
||||
class TestVectorQuery:
|
||||
def test_init(self):
|
||||
query = VectorQuery(field_name="test_field")
|
||||
assert query.field_name == "test_field"
|
||||
assert query.id is None
|
||||
assert query.vector is None
|
||||
assert query.param is None
|
||||
|
||||
param = HnswQueryParam()
|
||||
query = VectorQuery(
|
||||
field_name="test_field", id="test_id", vector=[1, 2, 3], param=param
|
||||
)
|
||||
assert query.field_name == "test_field"
|
||||
assert query.id == "test_id"
|
||||
assert query.vector == [1, 2, 3]
|
||||
assert query.param == param
|
||||
|
||||
def test_has_id(self):
|
||||
query = VectorQuery(field_name="test_field")
|
||||
assert not query.has_id()
|
||||
|
||||
query = VectorQuery(field_name="test_field", id="test_id")
|
||||
assert query.has_id()
|
||||
|
||||
def test_has_vector(self):
|
||||
query = VectorQuery(field_name="test_field")
|
||||
assert not query.has_vector()
|
||||
|
||||
query = VectorQuery(field_name="test_field", vector=[])
|
||||
assert not query.has_vector()
|
||||
|
||||
query = VectorQuery(field_name="test_field", vector=[1, 2, 3])
|
||||
assert query.has_vector()
|
||||
|
||||
def test_validate_dense_fp16_convert(self):
|
||||
v = _VectorQuery()
|
||||
schema = VectorSchema(name="test", data_type=DataType.VECTOR_FP16)
|
||||
vec = np.array([1.1, 2.1, 3.1], dtype=np.float16)
|
||||
v.set_vector(schema._get_object(), vec)
|
||||
ret = v.get_vector(schema._get_object())
|
||||
assert np.array_equal(vec, ret)
|
||||
|
||||
def test_validate_dense_fp32_convert(self):
|
||||
v = _VectorQuery()
|
||||
schema = VectorSchema(name="test", data_type=DataType.VECTOR_FP32)
|
||||
vec = np.array([1.1, 2.1, 3.1], dtype=np.float32)
|
||||
v.set_vector(schema._get_object(), vec)
|
||||
ret = v.get_vector(schema._get_object())
|
||||
assert np.array_equal(vec, ret)
|
||||
|
||||
def test_validate_dense_fp64_convert(self):
|
||||
v = _VectorQuery()
|
||||
schema = VectorSchema(name="test", data_type=DataType.VECTOR_FP64)
|
||||
vec = np.array([1.1, 2.1, 3.1], dtype=np.float64)
|
||||
v.set_vector(schema._get_object(), vec)
|
||||
ret = v.get_vector(schema._get_object())
|
||||
assert np.array_equal(vec, ret)
|
||||
|
||||
def test_validate_dense_int8_convert(self):
|
||||
v = _VectorQuery()
|
||||
schema = VectorSchema(name="test", data_type=DataType.VECTOR_INT8)
|
||||
vec = np.array([1, 2, 3], dtype=np.int8)
|
||||
v.set_vector(schema._get_object(), vec)
|
||||
ret = v.get_vector(schema._get_object())
|
||||
assert np.array_equal(vec, ret)
|
||||
|
||||
def test_validate_sparse_fp32_convert(self):
|
||||
v = _VectorQuery()
|
||||
schema = VectorSchema(name="test", data_type=DataType.SPARSE_VECTOR_FP32)
|
||||
vec = {1: 1.1, 2: 2.2, 3: 3.3}
|
||||
v.set_vector(schema._get_object(), vec)
|
||||
ret = v.get_vector(schema._get_object())
|
||||
for k in vec.keys():
|
||||
assert math.isclose(vec[k], ret[k], abs_tol=1e-6)
|
||||
|
||||
def test_validate_sparse_fp16_convert(self):
|
||||
v = _VectorQuery()
|
||||
schema = VectorSchema(name="test", data_type=DataType.SPARSE_VECTOR_FP16)
|
||||
vec = {1: 1.1, 2: 2.2, 3: 3.3}
|
||||
v.set_vector(schema._get_object(), vec)
|
||||
ret = v.get_vector(schema._get_object())
|
||||
for k in vec.keys():
|
||||
assert math.isclose(np.float16(vec[k]), ret[k], abs_tol=1e-6)
|
||||
|
||||
|
||||
class TestQueryContext:
|
||||
def test_init(self):
|
||||
ctx = QueryContext(topk=10)
|
||||
assert ctx.topk == 10
|
||||
assert ctx.queries == []
|
||||
assert ctx.filter is None
|
||||
assert ctx.reranker is None
|
||||
assert ctx.output_fields is None
|
||||
assert ctx.include_vector is False
|
||||
assert ctx.core_vectors == []
|
||||
|
||||
def test_properties(self):
|
||||
queries = [VectorQuery(field_name="test")]
|
||||
reranker = RrfReRanker()
|
||||
output_fields = ["field1", "field2"]
|
||||
|
||||
ctx = QueryContext(
|
||||
topk=5,
|
||||
filter="test_filter",
|
||||
include_vector=True,
|
||||
queries=queries,
|
||||
output_fields=output_fields,
|
||||
reranker=reranker,
|
||||
)
|
||||
|
||||
assert ctx.topk == 5
|
||||
assert ctx.queries == queries
|
||||
assert ctx.filter == "test_filter"
|
||||
assert ctx.reranker == reranker
|
||||
assert ctx.output_fields == output_fields
|
||||
assert ctx.include_vector is True
|
||||
|
||||
def test_core_vectors_setter(self):
|
||||
ctx = QueryContext(topk=10)
|
||||
core_vectors = [MagicMock()]
|
||||
ctx.core_vectors = core_vectors
|
||||
assert ctx.core_vectors == core_vectors
|
||||
|
||||
|
||||
class TestNoVectorQueryExecutor:
|
||||
def test_init(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = NoVectorQueryExecutor(schema)
|
||||
assert isinstance(executor, QueryExecutor)
|
||||
|
||||
def test_do_validate_with_queries(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = NoVectorQueryExecutor(schema)
|
||||
ctx = QueryContext(topk=10, queries=[VectorQuery(field_name="test")])
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="Collection does not support query with vector or id"
|
||||
):
|
||||
executor._do_validate(ctx)
|
||||
|
||||
def test_do_validate_without_queries(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = NoVectorQueryExecutor(schema)
|
||||
ctx = QueryContext(topk=10)
|
||||
|
||||
executor._do_validate(ctx)
|
||||
|
||||
def test_do_build(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = NoVectorQueryExecutor(schema)
|
||||
ctx = QueryContext(topk=5, filter="test_filter")
|
||||
|
||||
result = executor._do_build(ctx, MagicMock())
|
||||
assert len(result) == 1
|
||||
assert result[0].topk == 5
|
||||
assert result[0].filter == "test_filter"
|
||||
|
||||
|
||||
class TestSingleVectorQueryExecutor:
|
||||
def test_init(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = SingleVectorQueryExecutor(schema)
|
||||
assert isinstance(executor, NoVectorQueryExecutor)
|
||||
|
||||
def test_do_validate_multiple_queries(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = SingleVectorQueryExecutor(schema)
|
||||
queries = [VectorQuery(field_name="test1"), VectorQuery(field_name="test2")]
|
||||
ctx = QueryContext(topk=10, queries=queries)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError,
|
||||
match="Collection has only one vector field, cannot query with multiple vectors",
|
||||
):
|
||||
executor._do_validate(ctx)
|
||||
|
||||
def test_do_build_without_queries(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = SingleVectorQueryExecutor(schema)
|
||||
ctx = QueryContext(topk=5)
|
||||
|
||||
result = executor._do_build(ctx, MagicMock())
|
||||
assert len(result) == 1
|
||||
assert result[0].topk == 5
|
||||
|
||||
|
||||
class TestMultiVectorQueryExecutor:
|
||||
def test_init(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = MultiVectorQueryExecutor(schema)
|
||||
assert isinstance(executor, SingleVectorQueryExecutor)
|
||||
|
||||
def test_do_validate_multiple_queries_without_reranker(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = MultiVectorQueryExecutor(schema)
|
||||
queries = [VectorQuery(field_name="test1"), VectorQuery(field_name="test2")]
|
||||
ctx = QueryContext(topk=10, queries=queries)
|
||||
|
||||
with pytest.raises(
|
||||
ValueError, match="Reranker is required for multi-vector query"
|
||||
):
|
||||
executor._do_validate(ctx)
|
||||
|
||||
def test_do_validate_multiple_queries_with_reranker(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = MultiVectorQueryExecutor(schema)
|
||||
queries = [VectorQuery(field_name="test1"), VectorQuery(field_name="test2")]
|
||||
reranker = RrfReRanker()
|
||||
ctx = QueryContext(topk=10, queries=queries, reranker=reranker)
|
||||
|
||||
executor._do_validate(ctx)
|
||||
|
||||
|
||||
class TestQueryExecutorFactory:
|
||||
def test_create_no_vectors(self):
|
||||
schema = MockCollectionSchema()
|
||||
executor = QueryExecutorFactory.create(schema)
|
||||
assert isinstance(executor, NoVectorQueryExecutor)
|
||||
|
||||
def test_create_single_vector(self):
|
||||
schema = MockCollectionSchema(vectors=MockVectorSchema())
|
||||
executor = QueryExecutorFactory.create(schema)
|
||||
assert isinstance(executor, SingleVectorQueryExecutor)
|
||||
|
||||
def test_create_multiple_vectors(self):
|
||||
schema = MockCollectionSchema(
|
||||
vectors={"test1": MockVectorSchema(), "test2": MockVectorSchema()}
|
||||
)
|
||||
executor = QueryExecutorFactory.create(schema)
|
||||
assert isinstance(executor, MultiVectorQueryExecutor)
|
||||
@@ -0,0 +1,184 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import patch
|
||||
import pytest
|
||||
import math
|
||||
|
||||
from zvec import RrfReRanker, WeightedReRanker, Doc, MetricType
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# RrfRanker Test Case
|
||||
# ----------------------------
|
||||
class TestRrfReRanker:
|
||||
def test_init(self):
|
||||
reranker = RrfReRanker(
|
||||
query="test", topn=5, rerank_field="content", rank_constant=100
|
||||
)
|
||||
assert reranker.query == "test"
|
||||
assert reranker.topn == 5
|
||||
assert reranker.rerank_field == "content"
|
||||
assert reranker.rank_constant == 100
|
||||
|
||||
def test_rrf_score(self):
|
||||
reranker = RrfReRanker(query="test", rank_constant=60)
|
||||
# 根据公式 1.0 / (k + rank + 1),其中k=60
|
||||
assert reranker._rrf_score(0) == 1.0 / (60 + 0 + 1)
|
||||
assert reranker._rrf_score(1) == 1.0 / (60 + 1 + 1)
|
||||
assert reranker._rrf_score(10) == 1.0 / (60 + 10 + 1)
|
||||
|
||||
def test_rerank(self):
|
||||
reranker = RrfReRanker(query="test", topn=3)
|
||||
|
||||
doc1 = Doc(id="1", score=0.8)
|
||||
doc2 = Doc(id="2", score=0.7)
|
||||
doc3 = Doc(id="3", score=0.9)
|
||||
doc4 = Doc(id="4", score=0.6)
|
||||
|
||||
query_results = {"vector1": [doc1, doc2, doc3], "vector2": [doc3, doc1, doc4]}
|
||||
|
||||
results = reranker.rerank(query_results)
|
||||
|
||||
assert len(results) <= reranker.topn
|
||||
|
||||
for doc in results:
|
||||
assert hasattr(doc, "score")
|
||||
|
||||
scores = [doc.score for doc in results]
|
||||
assert scores == sorted(scores, reverse=True)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# WeightedRanker Test Case
|
||||
# ----------------------------
|
||||
class TestWeightedReRanker:
|
||||
def test_init(self):
|
||||
weights = {"vector1": 0.7, "vector2": 0.3}
|
||||
reranker = WeightedReRanker(
|
||||
query="test",
|
||||
topn=5,
|
||||
rerank_field="content",
|
||||
metric=MetricType.L2,
|
||||
weights=weights,
|
||||
)
|
||||
assert reranker.query == "test"
|
||||
assert reranker.topn == 5
|
||||
assert reranker.rerank_field == "content"
|
||||
assert reranker.metric == MetricType.L2
|
||||
assert reranker.weights == weights
|
||||
|
||||
def test_normalize_score(self):
|
||||
reranker = WeightedReRanker(query="test")
|
||||
|
||||
score = reranker._normalize_score(1.0, MetricType.L2)
|
||||
expected = 1.0 - 2 * math.atan(1.0) / math.pi
|
||||
assert score == expected
|
||||
|
||||
score = reranker._normalize_score(1.0, MetricType.IP)
|
||||
expected = 0.5 + math.atan(1.0) / math.pi
|
||||
assert score == expected
|
||||
|
||||
score = reranker._normalize_score(1.0, MetricType.COSINE)
|
||||
expected = 1.0 - 1.0 / 2.0
|
||||
assert score == expected
|
||||
|
||||
with pytest.raises(ValueError, match="Unsupported metric type"):
|
||||
reranker._normalize_score(1.0, "unsupported_metric")
|
||||
|
||||
def test_rerank(self):
|
||||
weights = {"vector1": 0.7, "vector2": 0.3}
|
||||
reranker = WeightedReRanker(
|
||||
query="test", topn=3, weights=weights, metric=MetricType.L2
|
||||
)
|
||||
|
||||
doc1 = Doc(id="1", score=0.8)
|
||||
doc2 = Doc(id="2", score=0.7)
|
||||
doc3 = Doc(id="3", score=0.9)
|
||||
|
||||
query_results = {"vector1": [doc1, doc2], "vector2": [doc2, doc3]}
|
||||
|
||||
results = reranker.rerank(query_results)
|
||||
|
||||
assert len(results) <= reranker.topn
|
||||
|
||||
for doc in results:
|
||||
assert hasattr(doc, "score")
|
||||
|
||||
scores = [doc.score for doc in results]
|
||||
assert scores == sorted(scores, reverse=True)
|
||||
|
||||
|
||||
# # ----------------------------
|
||||
# # QwenReRanker Test Case
|
||||
# # ----------------------------
|
||||
# class TestQwenReRanker:
|
||||
# def test_init_without_query(self):
|
||||
# with pytest.raises(ValueError):
|
||||
# QwenReRanker()
|
||||
#
|
||||
# def test_init_without_api_key(self):
|
||||
# with patch.dict(os.environ, {"DASHSCOPE_API_KEY": ""}):
|
||||
# with pytest.raises(ValueError, match="DashScope API key is required"):
|
||||
# QwenReRanker(query="test")
|
||||
#
|
||||
# @patch.dict(os.environ, {"DASHSCOPE_API_KEY": "test_key"})
|
||||
# def test_init_with_env_api_key(self):
|
||||
# reranker = QwenReRanker(query="test")
|
||||
# assert reranker.query == "test"
|
||||
# assert reranker._api_key == "test_key"
|
||||
#
|
||||
# def test_model_property(self):
|
||||
# reranker = QwenReRanker(query="test", api_key="test_key")
|
||||
# assert reranker.model == "gte-rerank-v2"
|
||||
#
|
||||
# reranker = QwenReRanker(query="test", model="custom-model", api_key="test_key")
|
||||
# assert reranker.model == "custom-model"
|
||||
#
|
||||
# def test_rerank_empty_results(self):
|
||||
# reranker = QwenReRanker(query="test", api_key="test_key")
|
||||
# results = reranker.rerank({})
|
||||
# assert results == []
|
||||
#
|
||||
# def test_rerank_no_documents(self):
|
||||
# reranker = QwenReRanker(query="test", api_key="test_key")
|
||||
# query_results = {"vector1": [Doc(id="1")]}
|
||||
# with pytest.raises(ValueError, match="No documents to rerank"):
|
||||
# reranker.rerank(query_results)
|
||||
#
|
||||
# @pytest.mark.skip(reason="Qwen ReRanker is not available in CI")
|
||||
# def test_rerank_success(self):
|
||||
# reranker = QwenReRanker(
|
||||
# topn=3,
|
||||
# query="test",
|
||||
# api_key="*",
|
||||
# rerank_field="content",
|
||||
# )
|
||||
# query_results = {
|
||||
# "vector1": [
|
||||
# Doc(id="1", fields={"content": "This is a test document."}),
|
||||
# Doc(id="2", fields={"content": "Another test document."}),
|
||||
# Doc(id="3", fields={"content": "Yet another test document."}),
|
||||
# Doc(id="4", fields={"content": "One more test document."}),
|
||||
# ],
|
||||
# "vector2": [
|
||||
# Doc(id="5", fields={"content": "This is a test document2."}),
|
||||
# Doc(id="6", fields={"content": "Another test document2."}),
|
||||
# Doc(id="7", fields={"content": "Yet another test document2."}),
|
||||
# Doc(id="8", fields={"content": "One more test document2."}),
|
||||
# ],
|
||||
# }
|
||||
# results = reranker.rerank(query_results)
|
||||
# assert len(results) == 3
|
||||
@@ -0,0 +1,246 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from zvec import (
|
||||
CollectionSchema,
|
||||
CollectionStats,
|
||||
FieldSchema,
|
||||
VectorSchema,
|
||||
HnswIndexParam,
|
||||
InvertIndexParam,
|
||||
DataType,
|
||||
IndexType,
|
||||
MetricType,
|
||||
)
|
||||
|
||||
# ----------------------------
|
||||
# FieldSchema Test Case
|
||||
# ----------------------------
|
||||
|
||||
|
||||
class TestFieldSchema:
|
||||
def test_default(self):
|
||||
field = FieldSchema("field", data_type=DataType.FLOAT)
|
||||
assert field.name == "field"
|
||||
assert field.data_type == DataType.FLOAT
|
||||
assert field.nullable is False
|
||||
assert field.index_param is None
|
||||
|
||||
def test_custom(self):
|
||||
field_1 = FieldSchema(
|
||||
name="float",
|
||||
data_type=DataType.FLOAT,
|
||||
nullable=True,
|
||||
index_param=InvertIndexParam(),
|
||||
)
|
||||
assert field_1.name == "float"
|
||||
assert field_1.data_type == DataType.FLOAT
|
||||
assert field_1.nullable is True
|
||||
assert field_1.index_param.enable_range_optimization is False
|
||||
|
||||
field_2 = FieldSchema(
|
||||
name="str",
|
||||
data_type=DataType.STRING,
|
||||
nullable=True,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
)
|
||||
assert field_2.name == "str"
|
||||
assert field_2.data_type == DataType.STRING
|
||||
assert field_2.nullable is True
|
||||
assert field_2.index_param.enable_range_optimization is True
|
||||
|
||||
def test_readonly(self):
|
||||
field = FieldSchema(
|
||||
name="float",
|
||||
data_type=DataType.FLOAT,
|
||||
nullable=True,
|
||||
index_param=InvertIndexParam(),
|
||||
)
|
||||
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
field.index_param = InvertIndexParam(enable_range_optimization=True)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# VectorSchema Test Case
|
||||
# ----------------------------
|
||||
class TestVectorSchema:
|
||||
def test_default(self):
|
||||
field = VectorSchema("vector", data_type=DataType.VECTOR_FP32, dimension=128)
|
||||
assert field.name == "vector"
|
||||
assert field.data_type == DataType.VECTOR_FP32
|
||||
assert field.dimension == 128
|
||||
assert field.index_param is not None
|
||||
assert field.index_param.type == IndexType.FLAT
|
||||
assert field.index_param.metric_type == MetricType.IP
|
||||
|
||||
def test_custom(self):
|
||||
field = VectorSchema(
|
||||
name="vector",
|
||||
data_type=DataType.VECTOR_INT8,
|
||||
dimension=512,
|
||||
index_param=HnswIndexParam(
|
||||
metric_type=MetricType.COSINE, m=15, ef_construction=300
|
||||
),
|
||||
)
|
||||
assert field.name == "vector"
|
||||
assert field.data_type == DataType.VECTOR_INT8
|
||||
assert field.index_param.metric_type == MetricType.COSINE
|
||||
assert field.index_param.m == 15
|
||||
assert field.index_param.ef_construction == 300
|
||||
|
||||
def test_readonly(self):
|
||||
field = VectorSchema(
|
||||
name="vector",
|
||||
dimension=128,
|
||||
data_type=DataType.VECTOR_INT8,
|
||||
)
|
||||
|
||||
import sys
|
||||
|
||||
if sys.version_info >= (3, 11):
|
||||
match_pattern = r"(can't set attribute|has no setter|readonly attribute)"
|
||||
else:
|
||||
match_pattern = r"can't set attribute"
|
||||
with pytest.raises(AttributeError, match=match_pattern):
|
||||
field.dimension = 4
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# CollectionSchema Test Case
|
||||
# ----------------------------
|
||||
class TestCollectionSchema:
|
||||
def test_collection_schema_with_single_field(self):
|
||||
collection_schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=FieldSchema(
|
||||
name="id",
|
||||
data_type=DataType.INT64,
|
||||
index_param=InvertIndexParam(),
|
||||
nullable=False,
|
||||
),
|
||||
vectors=VectorSchema(
|
||||
name="vector",
|
||||
data_type=DataType.VECTOR_INT8,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
),
|
||||
)
|
||||
|
||||
assert collection_schema is not None
|
||||
assert collection_schema.name == "test_collection"
|
||||
assert len(collection_schema.fields) == 1
|
||||
assert len(collection_schema.vectors) == 1
|
||||
|
||||
field = collection_schema.field("id")
|
||||
assert field is not None
|
||||
assert field.name == "id"
|
||||
assert field.data_type == DataType.INT64
|
||||
assert not field.nullable
|
||||
assert field.index_param.type == IndexType.INVERT
|
||||
assert not field.index_param.enable_range_optimization
|
||||
|
||||
vector = collection_schema.vector("vector")
|
||||
assert vector is not None
|
||||
assert vector.name == "vector"
|
||||
assert vector.data_type == DataType.VECTOR_INT8
|
||||
assert vector.dimension == 128
|
||||
assert vector.index_param.type == IndexType.HNSW
|
||||
assert vector.index_param.m == 100
|
||||
assert vector.index_param.ef_construction == 500
|
||||
assert vector.index_param.metric_type == MetricType.IP
|
||||
|
||||
def test_collection_schema_with_multi_fields(self):
|
||||
collection_schema = CollectionSchema(
|
||||
name="test_collection",
|
||||
fields=[
|
||||
FieldSchema(
|
||||
"id",
|
||||
DataType.INT64,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(enable_range_optimization=True),
|
||||
),
|
||||
FieldSchema(
|
||||
"name",
|
||||
DataType.STRING,
|
||||
nullable=False,
|
||||
index_param=InvertIndexParam(),
|
||||
),
|
||||
FieldSchema(
|
||||
"weight",
|
||||
DataType.INT32,
|
||||
nullable=True,
|
||||
),
|
||||
],
|
||||
vectors=[
|
||||
VectorSchema(
|
||||
"dense",
|
||||
DataType.VECTOR_FP32,
|
||||
dimension=128,
|
||||
index_param=HnswIndexParam(),
|
||||
),
|
||||
VectorSchema(
|
||||
"sparse", DataType.SPARSE_VECTOR_FP32, index_param=HnswIndexParam()
|
||||
),
|
||||
],
|
||||
)
|
||||
assert collection_schema is not None
|
||||
assert collection_schema.name == "test_collection"
|
||||
assert len(collection_schema.fields) == 3
|
||||
assert len(collection_schema.vectors) == 2
|
||||
|
||||
field_id = collection_schema.field("id")
|
||||
assert field_id is not None
|
||||
assert field_id.name == "id"
|
||||
assert field_id.data_type == DataType.INT64
|
||||
assert not field_id.nullable
|
||||
assert field_id.index_param.type == IndexType.INVERT
|
||||
|
||||
dense = collection_schema.vector("dense")
|
||||
assert dense is not None
|
||||
assert dense.name == "dense"
|
||||
assert dense.data_type == DataType.VECTOR_FP32
|
||||
assert dense.dimension == 128
|
||||
assert dense.index_param.type == IndexType.HNSW
|
||||
|
||||
sparse = collection_schema.vector("sparse")
|
||||
assert sparse is not None
|
||||
assert sparse.name == "sparse"
|
||||
assert sparse.data_type == DataType.SPARSE_VECTOR_FP32
|
||||
assert sparse.dimension == 0
|
||||
assert sparse.index_param.type == IndexType.HNSW
|
||||
|
||||
assert str(collection_schema) is not None
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# CollectionStats Test Case
|
||||
# ----------------------------
|
||||
class TestCollectionStats:
|
||||
"""
|
||||
The constructor of CollectionStats is not provided.
|
||||
It can only be obtained through collection.stats()
|
||||
"""
|
||||
|
||||
def test_collection_stats(self):
|
||||
stats = CollectionStats()
|
||||
assert stats is not None
|
||||
@@ -0,0 +1,141 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from zvec import (
|
||||
DataType,
|
||||
IndexType,
|
||||
MetricType,
|
||||
QuantizeType,
|
||||
Status,
|
||||
StatusCode,
|
||||
)
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Enum Test Case
|
||||
# ----------------------------
|
||||
@pytest.mark.parametrize(
|
||||
"member, name",
|
||||
[
|
||||
(DataType.FLOAT, "FLOAT"),
|
||||
(IndexType.HNSW, "HNSW"),
|
||||
(MetricType.COSINE, "COSINE"),
|
||||
(QuantizeType.INT8, "INT8"),
|
||||
(StatusCode.OK, "OK"),
|
||||
],
|
||||
)
|
||||
def test_enum_names(member, name):
|
||||
assert member.name == name
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"member, value",
|
||||
[
|
||||
(DataType.FLOAT, 8),
|
||||
(IndexType.HNSW, 1),
|
||||
(MetricType.COSINE, 3),
|
||||
(QuantizeType.INT8, 2),
|
||||
(StatusCode.OK, 0),
|
||||
],
|
||||
)
|
||||
def test_enum_values(member, value):
|
||||
assert member.value == value
|
||||
|
||||
|
||||
@pytest.mark.parametrize("member", ["L2", "IP", "COSINE"])
|
||||
def test_metric_type_has_member(member):
|
||||
assert member in MetricType.__members__
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"member",
|
||||
[
|
||||
"STRING",
|
||||
"BOOL",
|
||||
"INT32",
|
||||
"INT64",
|
||||
"FLOAT",
|
||||
"DOUBLE",
|
||||
"UINT32",
|
||||
"UINT64",
|
||||
"VECTOR_FP16",
|
||||
"VECTOR_FP32",
|
||||
"VECTOR_FP64",
|
||||
"VECTOR_INT8",
|
||||
"SPARSE_VECTOR_FP32",
|
||||
"SPARSE_VECTOR_FP16",
|
||||
"ARRAY_STRING",
|
||||
"ARRAY_INT32",
|
||||
"ARRAY_INT64",
|
||||
"ARRAY_FLOAT",
|
||||
"ARRAY_DOUBLE",
|
||||
"ARRAY_BOOL",
|
||||
"ARRAY_UINT32",
|
||||
"ARRAY_UINT64",
|
||||
],
|
||||
)
|
||||
def test_data_type_has_member(member):
|
||||
assert member in DataType.__members__
|
||||
|
||||
|
||||
@pytest.mark.parametrize("member", ["HNSW", "IVF", "FLAT", "INVERT"])
|
||||
def test_index_type_has_member(member):
|
||||
assert member in IndexType.__members__
|
||||
|
||||
|
||||
@pytest.mark.parametrize("member", ["FP16", "INT8", "INT4", "UNDEFINED"])
|
||||
def test_quantize_type_has_member(member):
|
||||
assert member in QuantizeType.__members__
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"member",
|
||||
[
|
||||
"OK",
|
||||
"UNKNOWN",
|
||||
"NOT_FOUND",
|
||||
"ALREADY_EXISTS",
|
||||
"INVALID_ARGUMENT",
|
||||
"PERMISSION_DENIED",
|
||||
"FAILED_PRECONDITION",
|
||||
"RESOURCE_EXHAUSTED",
|
||||
"UNAVAILABLE",
|
||||
"INTERNAL_ERROR",
|
||||
"NOT_SUPPORTED",
|
||||
],
|
||||
)
|
||||
def test_status_code_has_member(member):
|
||||
assert member in StatusCode.__members__
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# Status Test Case
|
||||
# ----------------------------
|
||||
class TestStatus:
|
||||
def test_status_code(self):
|
||||
status = Status(StatusCode.OK)
|
||||
assert status.code() == StatusCode.OK
|
||||
|
||||
def test_status_message(self):
|
||||
status = Status(StatusCode.OK, "OK")
|
||||
assert status.message() == "OK"
|
||||
|
||||
status = Status(StatusCode.NOT_FOUND, "Not Found")
|
||||
assert status.message() == "Not Found"
|
||||
|
||||
def test_status_ok(self):
|
||||
status = Status(StatusCode.OK)
|
||||
assert status.ok()
|
||||
@@ -0,0 +1,94 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from zvec import require_module
|
||||
|
||||
|
||||
# ----------------------------
|
||||
# require_module func Test Case
|
||||
# ----------------------------
|
||||
def test_require_module_success():
|
||||
module = require_module("os")
|
||||
assert module is not None
|
||||
assert hasattr(module, "path")
|
||||
|
||||
|
||||
def test_require_module_with_submodule_success():
|
||||
module = require_module("os.path")
|
||||
assert module is not None
|
||||
assert hasattr(module, "join")
|
||||
|
||||
|
||||
def test_require_module_import_error():
|
||||
with pytest.raises(ImportError) as exc_info:
|
||||
require_module("nonexistent_module")
|
||||
|
||||
exception_msg = str(exc_info.value)
|
||||
assert "Required package 'nonexistent_module' is not installed." in exception_msg
|
||||
|
||||
|
||||
def test_require_module_with_mitigation_import_error():
|
||||
with pytest.raises(ImportError) as exc_info:
|
||||
require_module("nonexistent_module.submodule", mitigation="custom_package")
|
||||
|
||||
exception_msg = str(exc_info.value)
|
||||
assert "Required package 'custom_package' is not installed." in exception_msg
|
||||
assert (
|
||||
"Module 'nonexistent_module.submodule' is part of 'nonexistent_module'"
|
||||
in exception_msg
|
||||
)
|
||||
assert "please pip install 'custom_package'." in exception_msg
|
||||
|
||||
|
||||
def test_require_module_submodule_import_error():
|
||||
with pytest.raises(ImportError) as exc_info:
|
||||
require_module("os.nonexistent_submodule")
|
||||
|
||||
exception_msg = str(exc_info.value)
|
||||
assert (
|
||||
"Required package 'os.nonexistent_submodule' is not installed." in exception_msg
|
||||
)
|
||||
assert "Module 'os.nonexistent_submodule' is part of 'os'" in exception_msg
|
||||
assert "please pip install 'os'." in exception_msg
|
||||
|
||||
|
||||
@patch("importlib.import_module")
|
||||
def test_require_module_wraps_original_exception(mock_import_module):
|
||||
original_exception = ImportError("Original error")
|
||||
mock_import_module.side_effect = original_exception
|
||||
|
||||
with pytest.raises(ImportError) as exc_info:
|
||||
require_module("some_module")
|
||||
|
||||
assert exc_info.value.__cause__ is original_exception
|
||||
|
||||
|
||||
@patch("importlib.import_module")
|
||||
def test_require_module_calls_importlib(mock_import_module):
|
||||
mock_module = MagicMock()
|
||||
mock_import_module.return_value = mock_module
|
||||
|
||||
result = require_module("test_module")
|
||||
|
||||
mock_import_module.assert_called_once_with("test_module")
|
||||
assert result is mock_module
|
||||
|
||||
|
||||
def test_require_module_with_openai():
|
||||
with pytest.raises(ImportError) as exc_info:
|
||||
require_module("openai")
|
||||
@@ -0,0 +1,133 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from importlib.metadata import PackageNotFoundError
|
||||
|
||||
|
||||
# ==============================
|
||||
# Public API — grouped by category
|
||||
# ==============================
|
||||
|
||||
from . import model as model
|
||||
|
||||
# —— Extensions & typing ——
|
||||
from .extension import DenseEmbeddingFunction, ReRanker, RrfReRanker, WeightedReRanker
|
||||
from .model import param as param
|
||||
from .model import schema as schema
|
||||
|
||||
# —— Core data structures ——
|
||||
from .model.collection import Collection
|
||||
from .model.doc import Doc
|
||||
|
||||
# —— Query & index parameters ——
|
||||
from .model.param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
FlatIndexParam,
|
||||
HnswIndexParam,
|
||||
HnswQueryParam,
|
||||
IndexOption,
|
||||
InvertIndexParam,
|
||||
IVFIndexParam,
|
||||
IVFQueryParam,
|
||||
OptimizeOption,
|
||||
)
|
||||
from .model.param.vector_query import VectorQuery
|
||||
|
||||
# —— Schema & field definitions ——
|
||||
from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
|
||||
|
||||
# —— tools ——
|
||||
from .tool import require_module
|
||||
from .typing import (
|
||||
DataType,
|
||||
IndexType,
|
||||
MetricType,
|
||||
QuantizeType,
|
||||
Status,
|
||||
StatusCode,
|
||||
)
|
||||
from .typing.enum import LogLevel, LogType
|
||||
|
||||
# —— lifecycle ——
|
||||
from .zvec import create_and_open, init, open
|
||||
|
||||
# ==============================
|
||||
# Public interface declaration
|
||||
# ==============================
|
||||
__all__ = [
|
||||
# Zvec functions
|
||||
"create_and_open",
|
||||
"init",
|
||||
"open",
|
||||
# Core classes
|
||||
"Collection",
|
||||
"Doc",
|
||||
# Schema
|
||||
"CollectionSchema",
|
||||
"FieldSchema",
|
||||
"VectorSchema",
|
||||
"CollectionStats",
|
||||
# Parameters
|
||||
"VectorQuery",
|
||||
"InvertIndexParam",
|
||||
"HnswIndexParam",
|
||||
"FlatIndexParam",
|
||||
"IVFIndexParam",
|
||||
"CollectionOption",
|
||||
"IndexOption",
|
||||
"OptimizeOption",
|
||||
"AddColumnOption",
|
||||
"AlterColumnOption",
|
||||
"HnswQueryParam",
|
||||
"IVFQueryParam",
|
||||
# Extensions
|
||||
"ReRanker",
|
||||
"DenseEmbeddingFunction",
|
||||
"RrfReRanker",
|
||||
"WeightedReRanker",
|
||||
# Typing
|
||||
"DataType",
|
||||
"MetricType",
|
||||
"QuantizeType",
|
||||
"IndexType",
|
||||
"LogLevel",
|
||||
"LogType",
|
||||
"Status",
|
||||
"StatusCode",
|
||||
# Tools
|
||||
"require_module",
|
||||
]
|
||||
|
||||
# ==============================
|
||||
# Version handling
|
||||
# ==============================
|
||||
__version__: str
|
||||
|
||||
try:
|
||||
from importlib.metadata import version
|
||||
except ImportError:
|
||||
from importlib_metadata import version # Python < 3.8
|
||||
|
||||
try:
|
||||
__version__ = version("zvec")
|
||||
except Exception:
|
||||
__version__ = "unknown"
|
||||
@@ -0,0 +1,172 @@
|
||||
"""
|
||||
Zvec core module
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
|
||||
from . import typing
|
||||
from .extension import ReRanker, RrfReRanker, WeightedReRanker
|
||||
from .extension.embedding import DenseEmbeddingFunction
|
||||
from .model import param, schema
|
||||
from .model.collection import Collection
|
||||
from .model.doc import Doc
|
||||
from .model.param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
FlatIndexParam,
|
||||
HnswIndexParam,
|
||||
HnswQueryParam,
|
||||
IndexOption,
|
||||
InvertIndexParam,
|
||||
IVFIndexParam,
|
||||
IVFQueryParam,
|
||||
OptimizeOption,
|
||||
)
|
||||
from .model.param.vector_query import VectorQuery
|
||||
from .model.schema import CollectionSchema, CollectionStats, FieldSchema, VectorSchema
|
||||
from .tool import require_module
|
||||
from .typing import (
|
||||
DataType,
|
||||
IndexType,
|
||||
MetricType,
|
||||
QuantizeType,
|
||||
Status,
|
||||
StatusCode,
|
||||
)
|
||||
from .typing.enum import LogLevel, LogType
|
||||
from .zvec import create_and_open, init, open
|
||||
|
||||
__all__: list = [
|
||||
"AddColumnOption",
|
||||
"AlterColumnOption",
|
||||
"Collection",
|
||||
"CollectionOption",
|
||||
"CollectionSchema",
|
||||
"CollectionStats",
|
||||
"DataType",
|
||||
"DenseEmbeddingFunction",
|
||||
"DenseEmbeddingFunction",
|
||||
"Doc",
|
||||
"FieldSchema",
|
||||
"FlatIndexParam",
|
||||
"HnswIndexParam",
|
||||
"HnswQueryParam",
|
||||
"IVFIndexParam",
|
||||
"IVFQueryParam",
|
||||
"IndexOption",
|
||||
"IndexType",
|
||||
"InvertIndexParam",
|
||||
"LogLevel",
|
||||
"LogType",
|
||||
"MetricType",
|
||||
"OptimizeOption",
|
||||
"QuantizeType",
|
||||
"ReRanker",
|
||||
"ReRanker",
|
||||
"RrfReRanker",
|
||||
"Status",
|
||||
"StatusCode",
|
||||
"VectorQuery",
|
||||
"VectorSchema",
|
||||
"WeightedReRanker",
|
||||
"create_and_open",
|
||||
"init",
|
||||
"open",
|
||||
"require_module",
|
||||
]
|
||||
|
||||
class _Collection:
|
||||
@staticmethod
|
||||
def CreateAndOpen(
|
||||
arg0: str, arg1: schema._CollectionSchema, arg2: param.CollectionOption
|
||||
) -> _Collection: ...
|
||||
@staticmethod
|
||||
def Open(arg0: str, arg1: param.CollectionOption) -> _Collection: ...
|
||||
def AddColumn(
|
||||
self,
|
||||
arg0: str,
|
||||
arg1: schema._FieldSchema,
|
||||
arg2: str,
|
||||
arg3: param.AddColumnOption,
|
||||
) -> None: ...
|
||||
def AlterColumn(
|
||||
self,
|
||||
arg0: str,
|
||||
arg1: str,
|
||||
arg2: schema._FieldSchema,
|
||||
arg3: param.AlterColumnOption,
|
||||
) -> None: ...
|
||||
def CreateIndex(
|
||||
self, arg0: str, arg1: param.IndexParam, arg2: param.IndexOption
|
||||
) -> None: ...
|
||||
def Delete(self, arg0: collections.abc.Sequence[str]) -> list[typing.Status]: ...
|
||||
def DeleteByFilter(self, arg0: str) -> None: ...
|
||||
def Destroy(self) -> None: ...
|
||||
def DropColumn(self, arg0: str) -> None: ...
|
||||
def DropIndex(self, arg0: str) -> None: ...
|
||||
def Fetch(self, arg0: collections.abc.Sequence[str]) -> dict[str, _Doc]: ...
|
||||
def Flush(self) -> None: ...
|
||||
def GroupByQuery(self, arg0: ...) -> list[...]: ...
|
||||
def Insert(self, arg0: collections.abc.Sequence[_Doc]) -> list[typing.Status]: ...
|
||||
def Optimize(self, arg0: param.OptimizeOption) -> None: ...
|
||||
def Options(self) -> param.CollectionOption: ...
|
||||
def Path(self) -> str: ...
|
||||
def Query(self, arg0: param._VectorQuery) -> list[_Doc]: ...
|
||||
def Schema(self) -> schema._CollectionSchema: ...
|
||||
def Stats(self) -> schema.CollectionStats: ...
|
||||
def Update(self, arg0: collections.abc.Sequence[_Doc]) -> list[typing.Status]: ...
|
||||
def Upsert(self, arg0: collections.abc.Sequence[_Doc]) -> list[typing.Status]: ...
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
|
||||
class _Doc:
|
||||
def __getstate__(self) -> bytes: ...
|
||||
def __init__(self) -> None: ...
|
||||
def __setstate__(self, arg0: bytes) -> None: ...
|
||||
def field_names(self) -> list[str]: ...
|
||||
def get_any(self, arg0: str, arg1: typing.DataType) -> typing.Any: ...
|
||||
def has_field(self, arg0: str) -> bool: ...
|
||||
def pk(self) -> str: ...
|
||||
def score(self) -> float: ...
|
||||
def set_any(self, arg0: str, arg1: typing.DataType, arg2: typing.Any) -> bool: ...
|
||||
def set_pk(self, arg0: str) -> None: ...
|
||||
def set_score(self, arg0: typing.SupportsFloat) -> None: ...
|
||||
|
||||
class _DocOp:
|
||||
"""
|
||||
Members:
|
||||
|
||||
INSERT
|
||||
|
||||
UPDATE
|
||||
|
||||
DELETE
|
||||
|
||||
UPSERT
|
||||
"""
|
||||
|
||||
DELETE: typing.ClassVar[_DocOp] # value = <_DocOp.DELETE: 3>
|
||||
INSERT: typing.ClassVar[_DocOp] # value = <_DocOp.INSERT: 0>
|
||||
UPDATE: typing.ClassVar[_DocOp] # value = <_DocOp.UPDATE: 2>
|
||||
UPSERT: typing.ClassVar[_DocOp] # value = <_DocOp.UPSERT: 1>
|
||||
__members__: typing.ClassVar[
|
||||
dict[str, _DocOp]
|
||||
] # value = {'INSERT': <_DocOp.INSERT: 0>, 'UPDATE': <_DocOp.UPDATE: 2>, 'DELETE': <_DocOp.DELETE: 3>, 'UPSERT': <_DocOp.UPSERT: 1>}
|
||||
|
||||
def __eq__(self, other: typing.Any) -> bool: ...
|
||||
def __getstate__(self) -> int: ...
|
||||
def __hash__(self) -> int: ...
|
||||
def __index__(self) -> int: ...
|
||||
def __init__(self, value: typing.SupportsInt) -> None: ...
|
||||
def __int__(self) -> int: ...
|
||||
def __ne__(self, other: typing.Any) -> bool: ...
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, state: typing.SupportsInt) -> None: ...
|
||||
def __str__(self) -> str: ...
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
@property
|
||||
def value(self) -> int: ...
|
||||
@@ -0,0 +1,18 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from .constants import DenseVectorType, SparseVectorType, VectorType
|
||||
|
||||
__all__ = ["DenseVectorType", "SparseVectorType", "VectorType"]
|
||||
@@ -0,0 +1,22 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Union
|
||||
|
||||
import numpy as np
|
||||
|
||||
DenseVectorType = Union[list[float], list[int], np.ndarray]
|
||||
SparseVectorType = dict[int, float]
|
||||
VectorType = Optional[Union[DenseVectorType, SparseVectorType]]
|
||||
@@ -0,0 +1,26 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from .query_executor import (
|
||||
QueryContext,
|
||||
QueryExecutor,
|
||||
QueryExecutorFactory,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"QueryContext",
|
||||
"QueryExecutor",
|
||||
"QueryExecutorFactory",
|
||||
]
|
||||
@@ -0,0 +1,307 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Optional, Union, final
|
||||
|
||||
import numpy as np
|
||||
from _zvec import _Collection
|
||||
from _zvec.param import _VectorQuery
|
||||
|
||||
from ..extension import ReRanker, RrfReRanker, WeightedReRanker
|
||||
from ..model.convert import convert_to_py_doc
|
||||
from ..model.doc import Doc
|
||||
from ..model.param.vector_query import VectorQuery
|
||||
from ..model.schema import CollectionSchema
|
||||
from ..typing import DataType
|
||||
|
||||
__all__ = [
|
||||
"QueryContext",
|
||||
"QueryExecutor",
|
||||
"QueryExecutorFactory",
|
||||
]
|
||||
|
||||
DTYPE_MAP = {
|
||||
DataType.VECTOR_FP16.value: np.float16,
|
||||
DataType.VECTOR_FP32.value: np.float32,
|
||||
DataType.VECTOR_FP64.value: np.float64,
|
||||
DataType.VECTOR_INT8.value: np.int8,
|
||||
}
|
||||
|
||||
|
||||
def convert_to_numpy(vec: Union[list, np.ndarray], dtype: np.dtype) -> np.ndarray:
|
||||
if isinstance(vec, np.ndarray):
|
||||
if vec.dtype == dtype and vec.ndim == 1:
|
||||
return vec
|
||||
return np.asarray(vec, dtype=dtype).flatten()
|
||||
|
||||
try:
|
||||
arr = np.asarray(vec, dtype=dtype)
|
||||
if arr.ndim != 1:
|
||||
arr = arr.flatten()
|
||||
return arr
|
||||
except (ValueError, TypeError) as e:
|
||||
raise TypeError(
|
||||
f"Cannot convert input to 1D numpy array with dtype={dtype}: {type(vec)}"
|
||||
) from e
|
||||
|
||||
|
||||
class QueryContext:
|
||||
def __init__(
|
||||
self,
|
||||
topk: int,
|
||||
filter: Optional[str] = None,
|
||||
include_vector: bool = False,
|
||||
queries: Optional[list[VectorQuery]] = None,
|
||||
output_fields: Optional[list[str]] = None,
|
||||
reranker: Optional[ReRanker] = None,
|
||||
):
|
||||
# query param
|
||||
self._filter = filter
|
||||
self._queries = queries or []
|
||||
self._topk = topk
|
||||
self._include_vector = include_vector
|
||||
self._output_fields = output_fields
|
||||
|
||||
# reranker
|
||||
self._reranker = reranker
|
||||
|
||||
# core vectors
|
||||
self._core_vectors = []
|
||||
|
||||
@property
|
||||
def topk(self):
|
||||
return self._topk
|
||||
|
||||
@property
|
||||
def queries(self):
|
||||
return self._queries
|
||||
|
||||
@property
|
||||
def filter(self):
|
||||
return self._filter
|
||||
|
||||
@property
|
||||
def reranker(self):
|
||||
return self._reranker
|
||||
|
||||
@property
|
||||
def output_fields(self):
|
||||
return self._output_fields
|
||||
|
||||
@property
|
||||
def include_vector(self):
|
||||
return self._include_vector
|
||||
|
||||
@property
|
||||
def core_vectors(self):
|
||||
return self._core_vectors
|
||||
|
||||
@core_vectors.setter
|
||||
def core_vectors(self, core_vectors: list[_VectorQuery]):
|
||||
self._core_vectors = core_vectors
|
||||
|
||||
|
||||
class QueryExecutor(ABC):
|
||||
def __init__(self, schema: CollectionSchema):
|
||||
self._schema = schema
|
||||
self._concurrency = max(1, int(os.getenv("ZVEC_QUERY_CONCURRENCY", "1")))
|
||||
|
||||
@abstractmethod
|
||||
def _do_validate(self, ctx: QueryContext) -> None:
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def _do_build(
|
||||
self, ctx: QueryContext, collection: _Collection
|
||||
) -> list[_VectorQuery]:
|
||||
pass
|
||||
|
||||
def _do_build_query_wo_vector(self, ctx: QueryContext) -> _VectorQuery:
|
||||
core_vector = _VectorQuery()
|
||||
core_vector.topk = ctx.topk
|
||||
core_vector.include_vector = ctx.include_vector
|
||||
if ctx.filter:
|
||||
core_vector.filter = ctx.filter
|
||||
if ctx.output_fields:
|
||||
core_vector.output_fields = ctx.output_fields
|
||||
return core_vector
|
||||
|
||||
def _do_build_query_with_vector(
|
||||
self, ctx: QueryContext, query: VectorQuery, collection: _Collection
|
||||
) -> _VectorQuery:
|
||||
core_vector = self._do_build_query_wo_vector(ctx)
|
||||
core_vector.field_name = query.field_name
|
||||
if query.param:
|
||||
core_vector.query_params = query.param
|
||||
|
||||
vector_schema = (
|
||||
self._schema.vector(query.field_name) if query else self._schema.vectors[0]
|
||||
)
|
||||
|
||||
if vector_schema is None:
|
||||
raise ValueError("No vector field found")
|
||||
|
||||
# set output_fields
|
||||
core_vector.output_fields = ctx.output_fields
|
||||
|
||||
# set vector
|
||||
if query.has_vector():
|
||||
vec_data = query.vector
|
||||
else:
|
||||
fetched = collection.Fetch([query.id])
|
||||
doc = next(iter(fetched.values()))
|
||||
if not doc:
|
||||
return core_vector
|
||||
vec_data = doc.get_any(vector_schema.name, vector_schema.data_type)
|
||||
|
||||
target_dtype = DTYPE_MAP.get(vector_schema.data_type.value)
|
||||
core_vector.set_vector(
|
||||
vector_schema._get_object(),
|
||||
convert_to_numpy(vec_data, target_dtype) if target_dtype else vec_data,
|
||||
)
|
||||
return core_vector
|
||||
|
||||
def _do_execute(
|
||||
self, vectors: list[_VectorQuery], collection: _Collection
|
||||
) -> dict[str, list[Doc]]:
|
||||
query_cnt = len(vectors)
|
||||
if query_cnt == 0:
|
||||
raise ValueError("No query to execute")
|
||||
|
||||
if len(vectors) == 1 or self._concurrency == 1:
|
||||
results = {}
|
||||
for query in vectors:
|
||||
docs = collection.Query(query)
|
||||
results[query.field_name] = [
|
||||
convert_to_py_doc(doc, self._schema) for doc in docs
|
||||
]
|
||||
return results
|
||||
|
||||
results = {}
|
||||
with ThreadPoolExecutor(max_workers=self._concurrency) as executor:
|
||||
future_to_query = {
|
||||
executor.submit(collection.Query, query): query.field_name
|
||||
for query in vectors
|
||||
}
|
||||
|
||||
for future in as_completed(future_to_query):
|
||||
field_name = future_to_query[future]
|
||||
try:
|
||||
docs = future.result()
|
||||
results[field_name] = [
|
||||
convert_to_py_doc(doc, self._schema) for doc in docs
|
||||
]
|
||||
except Exception as e:
|
||||
raise e
|
||||
return results
|
||||
|
||||
def _do_merge_rerank_results(
|
||||
self, ctx: QueryContext, docs_map: dict[str, list[Doc]]
|
||||
) -> list[Doc]:
|
||||
query_result_cnt = len(docs_map) if docs_map else 0
|
||||
if query_result_cnt == 0:
|
||||
raise ValueError("Query results is none and dost not to rerank")
|
||||
if query_result_cnt == 1:
|
||||
if not ctx.reranker or isinstance(
|
||||
ctx.reranker, (RrfReRanker, WeightedReRanker)
|
||||
):
|
||||
return next(iter(docs_map.values()))
|
||||
return ctx.reranker.rerank(docs_map)
|
||||
return ctx.reranker.rerank(docs_map)
|
||||
|
||||
@final
|
||||
def execute(self, ctx: QueryContext, collection: _Collection) -> list[Doc]:
|
||||
# 1. validate query
|
||||
self._do_validate(ctx)
|
||||
# 2. build query vector
|
||||
query_vectors = self._do_build(ctx, collection)
|
||||
if not query_vectors:
|
||||
raise ValueError("No query to execute")
|
||||
# 3. execute query
|
||||
docs = self._do_execute(query_vectors, collection)
|
||||
# 4. merge and rerank result
|
||||
return self._do_merge_rerank_results(ctx, docs)
|
||||
|
||||
|
||||
class NoVectorQueryExecutor(QueryExecutor):
|
||||
def __init__(self, schema: CollectionSchema):
|
||||
super().__init__(schema)
|
||||
|
||||
def _do_validate(self, ctx: QueryContext) -> None:
|
||||
if len(ctx.queries) > 0:
|
||||
raise ValueError("Collection does not support query with vector or id")
|
||||
|
||||
def _do_build(
|
||||
self, ctx: QueryContext, _collection: _Collection
|
||||
) -> list[_VectorQuery]:
|
||||
return [self._do_build_query_wo_vector(ctx)]
|
||||
|
||||
|
||||
class SingleVectorQueryExecutor(NoVectorQueryExecutor):
|
||||
def __init__(self, schema: CollectionSchema) -> None:
|
||||
super().__init__(schema)
|
||||
|
||||
def _do_validate(self, ctx: QueryContext) -> None:
|
||||
if len(ctx.queries) > 1:
|
||||
raise ValueError(
|
||||
"Collection has only one vector field, cannot query with multiple vectors"
|
||||
)
|
||||
for query in ctx.queries:
|
||||
query._validate()
|
||||
|
||||
def _do_build(
|
||||
self, ctx: QueryContext, collection: _Collection
|
||||
) -> list[_VectorQuery]:
|
||||
if len(ctx.queries) == 0:
|
||||
return [self._do_build_query_wo_vector(ctx)]
|
||||
vectors = []
|
||||
for query in ctx.queries:
|
||||
vectors.append(self._do_build_query_with_vector(ctx, query, collection))
|
||||
return vectors
|
||||
|
||||
|
||||
class MultiVectorQueryExecutor(SingleVectorQueryExecutor):
|
||||
def __init__(self, schema: CollectionSchema) -> None:
|
||||
super().__init__(schema)
|
||||
|
||||
def _do_validate(self, ctx: QueryContext) -> None:
|
||||
if len(ctx.queries) > 1 and ctx.reranker is None:
|
||||
raise ValueError("Reranker is required for multi-vector query")
|
||||
seen_fields = set()
|
||||
for query in ctx.queries:
|
||||
query._validate()
|
||||
field = query.field_name
|
||||
if field in seen_fields:
|
||||
raise ValueError(f"Query field name '{field}' appears more than once")
|
||||
seen_fields.add(field)
|
||||
|
||||
def _do_execute(
|
||||
self, vectors: list[_VectorQuery], collection: _Collection
|
||||
) -> dict[str, list[Doc]]:
|
||||
return super()._do_execute(vectors, collection)
|
||||
|
||||
|
||||
class QueryExecutorFactory:
|
||||
@staticmethod
|
||||
def create(schema: CollectionSchema) -> QueryExecutor:
|
||||
vectors = schema.vectors
|
||||
if len(vectors) == 0:
|
||||
return NoVectorQueryExecutor(schema)
|
||||
if len(vectors) == 1:
|
||||
return SingleVectorQueryExecutor(schema)
|
||||
return MultiVectorQueryExecutor(schema)
|
||||
@@ -0,0 +1,26 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from .embedding import DenseEmbeddingFunction, QwenEmbeddingFunction
|
||||
from .rerank import QwenReRanker, ReRanker, RrfReRanker, WeightedReRanker
|
||||
|
||||
__all__ = [
|
||||
"DenseEmbeddingFunction",
|
||||
"QwenEmbeddingFunction",
|
||||
"QwenReRanker",
|
||||
"ReRanker",
|
||||
"RrfReRanker",
|
||||
"WeightedReRanker",
|
||||
]
|
||||
@@ -0,0 +1,188 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from functools import lru_cache
|
||||
from http import HTTPStatus
|
||||
from typing import Optional, Union
|
||||
|
||||
from ..tool import require_module
|
||||
from ..typing import DataType
|
||||
|
||||
|
||||
class DenseEmbeddingFunction(ABC):
|
||||
"""Abstract base class for dense vector embedding functions.
|
||||
|
||||
Dense embedding functions map text to fixed-length real-valued vectors.
|
||||
Subclasses must implement the ``embed()`` method.
|
||||
|
||||
Args:
|
||||
dimension (int): Dimensionality of the output embedding vector.
|
||||
data_type (DataType, optional): Numeric type of the embedding.
|
||||
Defaults to ``DataType.VECTOR_FP32``.
|
||||
|
||||
Note:
|
||||
This class is callable: ``embedding_func("text")`` is equivalent to
|
||||
``embedding_func.embed("text")``.
|
||||
"""
|
||||
|
||||
def __init__(self, dimension: int, data_type: DataType = DataType.VECTOR_FP32):
|
||||
self._dimension = dimension
|
||||
self._data_type = data_type
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
"""int: The expected dimensionality of the embedding vector."""
|
||||
return self._dimension
|
||||
|
||||
@property
|
||||
def data_type(self) -> DataType:
|
||||
"""DataType: The numeric data type of the embedding (e.g., VECTOR_FP32)."""
|
||||
return self._data_type
|
||||
|
||||
@abstractmethod
|
||||
def embed(self, text: str) -> list[Union[int, float]]:
|
||||
"""Generate a dense embedding vector for the input text.
|
||||
|
||||
Args:
|
||||
text (str): Input text to embed.
|
||||
|
||||
Returns:
|
||||
list[Union[int, float]]: A list of numbers representing the embedding.
|
||||
Length must equal ``self.dimension``.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def __call__(self, text: str) -> list[Union[int, float]]:
|
||||
return self.embed(text)
|
||||
|
||||
|
||||
class SparseEmbeddingFunction(ABC):
|
||||
"""Abstract base class for sparse vector embedding functions.
|
||||
|
||||
Sparse embedding functions map text to a dictionary of {index: weight},
|
||||
where only non-zero dimensions are stored.
|
||||
|
||||
Note:
|
||||
Subclasses must implement the ``embed()`` method.
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def embed(self, text: str) -> dict[int, float]:
|
||||
"""Generate a sparse embedding for the input text.
|
||||
Args:
|
||||
text (str): Input text to embed.
|
||||
|
||||
Returns:
|
||||
dict[int, float]: Mapping from dimension index to non-zero weight.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class QwenEmbeddingFunction(DenseEmbeddingFunction):
|
||||
"""Dense embedding function using Qwen (DashScope) Text Embedding API.
|
||||
|
||||
This implementation uses the DashScope service to generate embeddings
|
||||
via Qwen's text embedding models (e.g., ``text-embedding-v4``).
|
||||
|
||||
Args:
|
||||
dimension (int): Desired embedding dimension (e.g., 1024).
|
||||
model (str, optional): DashScope embedding model name.
|
||||
Defaults to ``"text-embedding-v4"``.
|
||||
api_key (Optional[str], optional): DashScope API key. If not provided,
|
||||
reads from ``DASHSCOPE_API_KEY`` environment variable.
|
||||
|
||||
Raises:
|
||||
ValueError: If API key is missing or input text is invalid.
|
||||
|
||||
Note:
|
||||
Requires the ``dashscope`` Python package.
|
||||
Embedding results are cached using ``functools.lru_cache`` (maxsize=10).
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dimension: int,
|
||||
model: str = "text-embedding-v4",
|
||||
api_key: Optional[str] = None,
|
||||
):
|
||||
super().__init__(dimension, DataType.VECTOR_FP32)
|
||||
self._model = model
|
||||
self._api_key = api_key or os.environ.get("DASHSCOPE_API_KEY")
|
||||
if not self._api_key:
|
||||
raise ValueError("DashScope API key is required")
|
||||
|
||||
@property
|
||||
def model(self) -> str:
|
||||
"""str: The DashScope embedding model name in use."""
|
||||
return self._model
|
||||
|
||||
def _connection(self):
|
||||
dashscope = require_module("dashscope")
|
||||
dashscope.api_key = self._api_key
|
||||
return dashscope
|
||||
|
||||
@lru_cache(maxsize=10)
|
||||
def embed(self, text: str) -> list[Union[int, float]]:
|
||||
"""
|
||||
Generate embedding for a given text using Qwen (via DashScope).
|
||||
|
||||
Args:
|
||||
text (str): Input text to embed. Must be non-empty and valid string.
|
||||
|
||||
Returns:
|
||||
list[Union[int, float]]: The dense embedding vector.
|
||||
|
||||
Raises:
|
||||
ValueError: If input is invalid or API response is malformed.
|
||||
RuntimeError: If network or internal error occurs during API call.
|
||||
"""
|
||||
if not isinstance(text, str):
|
||||
raise TypeError(f"Expected 'text' to be str, got {type(text).__name__}")
|
||||
|
||||
text = text.strip()
|
||||
if not text:
|
||||
raise ValueError("Input text cannot be empty or whitespace only")
|
||||
|
||||
resp = self._connection().TextEmbedding.call(
|
||||
model=self.model, input=text, dimension=self.dimension, output_type="dense"
|
||||
)
|
||||
|
||||
if resp.status_code != HTTPStatus.OK:
|
||||
error_msg = getattr(resp, "message", "Unknown error")
|
||||
error_detail = f"Status={resp.status_code}, Message={error_msg}"
|
||||
raise ValueError(f"QwenEmbedding failed: {error_detail}")
|
||||
|
||||
output = getattr(resp, "output", None)
|
||||
if not isinstance(output, dict):
|
||||
raise ValueError("Invalid response: missing or malformed 'output' field")
|
||||
|
||||
embeddings = output.get("embeddings")
|
||||
if not isinstance(embeddings, list):
|
||||
raise ValueError(
|
||||
"Invalid response: 'embeddings' field is missing or not a list"
|
||||
)
|
||||
|
||||
if len(embeddings) != 1:
|
||||
raise ValueError(
|
||||
f"Expected 1 embedding, got {len(embeddings)}. Response: {resp}"
|
||||
)
|
||||
|
||||
first_emb = embeddings[0]
|
||||
if not isinstance(first_emb, dict):
|
||||
raise ValueError("Invalid response: embedding item is not a dictionary")
|
||||
|
||||
return list(first_emb.get("embedding"))
|
||||
@@ -0,0 +1,343 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import heapq
|
||||
import math
|
||||
import os
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict
|
||||
from http import HTTPStatus
|
||||
from typing import Optional
|
||||
|
||||
from ..model.doc import Doc
|
||||
from ..tool import require_module
|
||||
from ..typing import MetricType
|
||||
|
||||
|
||||
class ReRanker(ABC):
|
||||
"""Abstract base class for re-ranking search results.
|
||||
|
||||
Re-rankers refine the output of one or more vector queries by applying
|
||||
a secondary scoring strategy. They are used in the ``query()`` method of
|
||||
``Collection`` via the ``reranker`` parameter.
|
||||
|
||||
Args:
|
||||
query (Optional[str], optional): Query text used for re-ranking.
|
||||
Required for LLM-based re-rankers. Defaults to None.
|
||||
topn (int, optional): Number of top documents to return after re-ranking.
|
||||
Defaults to 10.
|
||||
rerank_field (Optional[str], optional): Field name used as input for
|
||||
re-ranking (e.g., document title or body). Defaults to None.
|
||||
|
||||
Note:
|
||||
Subclasses must implement the ``rerank()`` method.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
query: Optional[str] = None,
|
||||
topn: int = 10,
|
||||
rerank_field: Optional[str] = None,
|
||||
):
|
||||
self._query = query
|
||||
self._topn = topn
|
||||
self._rerank_field = rerank_field
|
||||
|
||||
@property
|
||||
def topn(self) -> int:
|
||||
"""int: Number of top documents to return after re-ranking."""
|
||||
return self._topn
|
||||
|
||||
@property
|
||||
def query(self) -> str:
|
||||
"""str: Query text used for re-ranking."""
|
||||
return self._query
|
||||
|
||||
@property
|
||||
def rerank_field(self) -> Optional[str]:
|
||||
"""Optional[str]: Field name used as re-ranking input."""
|
||||
return self._rerank_field
|
||||
|
||||
@abstractmethod
|
||||
def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
|
||||
"""Re-rank documents from one or more vector queries.
|
||||
|
||||
Args:
|
||||
query_results (dict[str, list[Doc]]): Mapping from vector field name
|
||||
to list of retrieved documents (sorted by relevance).
|
||||
|
||||
Returns:
|
||||
list[Doc]: Re-ranked list of documents (length ≤ ``topn``),
|
||||
with updated ``score`` fields.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
class RrfReRanker(ReRanker):
|
||||
"""Re-ranker using Reciprocal Rank Fusion (RRF).
|
||||
|
||||
RRF combines results from multiple queries without requiring relevance scores.
|
||||
It assigns higher weight to documents that appear early in multiple result lists.
|
||||
|
||||
The RRF score for a document at rank ``r`` is: ``1 / (k + r + 1)``,
|
||||
where ``k`` is the rank constant.
|
||||
|
||||
Args:
|
||||
query (Optional[str], optional): Ignored by RRF. Defaults to None.
|
||||
topn (int, optional): Number of top documents to return. Defaults to 10.
|
||||
rerank_field (Optional[str], optional): Ignored by RRF. Defaults to None.
|
||||
rank_constant (int, optional): Smoothing constant ``k`` in RRF formula.
|
||||
Larger values reduce the impact of early ranks. Defaults to 60.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
query: Optional[str] = None,
|
||||
topn: int = 10,
|
||||
rerank_field: Optional[str] = None,
|
||||
rank_constant: int = 60,
|
||||
):
|
||||
super().__init__(query, topn, rerank_field)
|
||||
self._rank_constant = rank_constant
|
||||
|
||||
@property
|
||||
def rank_constant(self) -> int:
|
||||
return self._rank_constant
|
||||
|
||||
def _rrf_score(self, rank: int):
|
||||
return 1.0 / (self._rank_constant + rank + 1)
|
||||
|
||||
def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
|
||||
"""Apply Reciprocal Rank Fusion to combine multiple query results.
|
||||
|
||||
Args:
|
||||
query_results (dict[str, list[Doc]]): Results from one or more vector queries.
|
||||
|
||||
Returns:
|
||||
list[Doc]: Re-ranked documents with RRF scores in the ``score`` field.
|
||||
"""
|
||||
rrf_scores: dict[str, float] = defaultdict(float)
|
||||
id_to_doc: dict[str, Doc] = {}
|
||||
|
||||
for _, query_result in query_results.items():
|
||||
for rank, doc in enumerate(query_result):
|
||||
doc_id = doc.id
|
||||
rrf_score = self._rrf_score(rank)
|
||||
rrf_scores[doc_id] += rrf_score
|
||||
if doc_id not in id_to_doc:
|
||||
id_to_doc[doc_id] = doc
|
||||
|
||||
top_docs = heapq.nlargest(self.topn, rrf_scores.items(), key=lambda x: x[1])
|
||||
results = []
|
||||
for doc_id, rrf_score in top_docs:
|
||||
doc = id_to_doc[doc_id]
|
||||
new_doc = doc._replace(score=rrf_score)
|
||||
results.append(new_doc)
|
||||
return results
|
||||
|
||||
|
||||
class WeightedReRanker(ReRanker):
|
||||
"""Re-ranker that combines scores from multiple vector fields using weights.
|
||||
|
||||
Each vector field's relevance score is normalized based on its metric type,
|
||||
then scaled by a user-provided weight. Final scores are summed across fields.
|
||||
|
||||
Args:
|
||||
query (Optional[str], optional): Ignored. Defaults to None.
|
||||
topn (int, optional): Number of top documents to return. Defaults to 10.
|
||||
rerank_field (Optional[str], optional): Ignored. Defaults to None.
|
||||
metric (MetricType, optional): Distance metric used for score normalization.
|
||||
Defaults to ``MetricType.L2``.
|
||||
weights (Optional[dict[str, float]], optional): Weight per vector field.
|
||||
Fields not listed use weight 1.0. Defaults to None.
|
||||
|
||||
Note:
|
||||
Supported metrics: L2, IP, COSINE. Scores are normalized to [0, 1].
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
query: Optional[str] = None,
|
||||
topn: int = 10,
|
||||
rerank_field: Optional[str] = None,
|
||||
metric: MetricType = MetricType.L2,
|
||||
weights: Optional[dict[str, float]] = None,
|
||||
):
|
||||
super().__init__(query, topn, rerank_field)
|
||||
self._weights = weights
|
||||
self._metric = metric
|
||||
|
||||
@property
|
||||
def weights(self) -> dict[str, float]:
|
||||
"""dict[str, float]: Weight mapping for vector fields."""
|
||||
return self._weights
|
||||
|
||||
@property
|
||||
def metric(self) -> MetricType:
|
||||
"""MetricType: Distance metric used for score normalization."""
|
||||
return self._metric
|
||||
|
||||
def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
|
||||
"""Combine scores from multiple vector fields using weighted sum.
|
||||
|
||||
Args:
|
||||
query_results (dict[str, list[Doc]]): Results per vector field.
|
||||
|
||||
Returns:
|
||||
list[Doc]: Re-ranked documents with combined scores in ``score`` field.
|
||||
"""
|
||||
weighted_scores: dict[str, float] = defaultdict(float)
|
||||
id_to_doc: dict[str, Doc] = {}
|
||||
|
||||
for vector_name, query_result in query_results.items():
|
||||
for _, doc in enumerate(query_result):
|
||||
doc_id = doc.id
|
||||
weighted_score = self._normalize_score(
|
||||
doc.score, self.metric
|
||||
) * self.weights.get(vector_name, 1.0)
|
||||
weighted_scores[doc_id] += weighted_score
|
||||
if doc_id not in id_to_doc:
|
||||
id_to_doc[doc_id] = doc
|
||||
|
||||
top_docs = heapq.nlargest(
|
||||
self.topn, weighted_scores.items(), key=lambda x: x[1]
|
||||
)
|
||||
results = []
|
||||
for doc_id, weighted_score in top_docs:
|
||||
doc = id_to_doc[doc_id]
|
||||
new_doc = doc._replace(score=weighted_score)
|
||||
results.append(new_doc)
|
||||
return results
|
||||
|
||||
def _normalize_score(self, score: float, metric: MetricType) -> float:
|
||||
if metric == MetricType.L2:
|
||||
return 1.0 - 2 * math.atan(score) / math.pi
|
||||
if metric == MetricType.IP:
|
||||
return 0.5 + math.atan(score) / math.pi
|
||||
if metric == MetricType.COSINE:
|
||||
return 1.0 - score / 2.0
|
||||
raise ValueError("Unsupported metric type")
|
||||
|
||||
|
||||
class QwenReRanker(ReRanker):
|
||||
"""Re-ranker using Qwen (DashScope) LLM-based re-ranking API.
|
||||
|
||||
This re-ranker sends documents to the DashScope TextReRank service for
|
||||
cross-encoder style re-ranking based on semantic relevance to the query.
|
||||
|
||||
Args:
|
||||
query (str): Query text for semantic re-ranking. **Required**.
|
||||
topn (int, optional): Number of top documents to return. Defaults to 10.
|
||||
rerank_field (str): Field name containing document text for re-ranking.
|
||||
**Required**.
|
||||
model (str, optional): DashScope re-ranking model name.
|
||||
Defaults to ``"gte-rerank-v2"``.
|
||||
api_key (Optional[str], optional): DashScope API key. If not provided,
|
||||
reads from ``DASHSCOPE_API_KEY`` environment variable.
|
||||
|
||||
Raises:
|
||||
ValueError: If ``query`` is missing, ``rerank_field`` is missing,
|
||||
or API key is not provided.
|
||||
|
||||
Note:
|
||||
Requires the ``dashscope`` Python package.
|
||||
Documents without content in ``rerank_field`` are skipped.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
query: Optional[str] = None,
|
||||
topn: int = 10,
|
||||
rerank_field: Optional[str] = None,
|
||||
model: str = "gte-rerank-v2",
|
||||
api_key: Optional[str] = None,
|
||||
):
|
||||
super().__init__(query, topn, rerank_field)
|
||||
if not query:
|
||||
raise ValueError("Query is required for reranking")
|
||||
self._model = model
|
||||
self._api_key = api_key or os.environ.get("DASHSCOPE_API_KEY")
|
||||
if not self._api_key:
|
||||
raise ValueError("DashScope API key is required")
|
||||
|
||||
@property
|
||||
def model(self) -> str:
|
||||
"""str: DashScope re-ranking model name."""
|
||||
return self._model
|
||||
|
||||
def _connection(self):
|
||||
dashscope = require_module("dashscope")
|
||||
dashscope.api_key = self._api_key
|
||||
return dashscope
|
||||
|
||||
def rerank(self, query_results: dict[str, list[Doc]]) -> list[Doc]:
|
||||
"""Re-rank documents using Qwen's TextReRank API.
|
||||
|
||||
Args:
|
||||
query_results (dict[str, list[Doc]]): Results from vector search.
|
||||
|
||||
Returns:
|
||||
list[Doc]: Re-ranked documents with relevance scores from Qwen.
|
||||
|
||||
Raises:
|
||||
ValueError: If API call fails or no valid documents are found.
|
||||
"""
|
||||
if not query_results:
|
||||
return []
|
||||
|
||||
id_to_doc: dict[str, Doc] = {}
|
||||
doc_ids = []
|
||||
contents = []
|
||||
|
||||
for _, query_result in query_results.items():
|
||||
for doc in query_result:
|
||||
doc_id = doc.id
|
||||
if doc_id in id_to_doc:
|
||||
continue
|
||||
|
||||
field_value = doc.field(self.rerank_field)
|
||||
rank_content = str(field_value).strip() if field_value else ""
|
||||
if not rank_content:
|
||||
continue
|
||||
|
||||
id_to_doc[doc_id] = doc
|
||||
doc_ids.append(doc_id)
|
||||
contents.append(rank_content)
|
||||
|
||||
if not contents:
|
||||
raise ValueError("No documents to rerank")
|
||||
|
||||
resp = self._connection().TextReRank.call(
|
||||
model=self.model,
|
||||
query=self.query,
|
||||
documents=list(contents),
|
||||
top_n=self.topn,
|
||||
return_documents=False,
|
||||
)
|
||||
|
||||
if resp.status_code != HTTPStatus.OK:
|
||||
raise ValueError(
|
||||
f"QwenReranker failed with status {resp.status_code}: {resp.message}"
|
||||
)
|
||||
|
||||
results = []
|
||||
for item in resp.output.results:
|
||||
idx = item.index
|
||||
doc_id = doc_ids[idx]
|
||||
doc = id_to_doc[doc_id]
|
||||
new_doc = doc._replace(score=item.relevance_score)
|
||||
results.append(new_doc)
|
||||
|
||||
return results
|
||||
@@ -0,0 +1,22 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from .collection import Collection
|
||||
from .doc import Doc
|
||||
from .param.vector_query import VectorQuery
|
||||
from .schema.collection_schema import CollectionSchema
|
||||
from .schema.field_schema import FieldSchema
|
||||
|
||||
__all__ = ["Collection", "CollectionSchema", "Doc", "FieldSchema", "VectorQuery"]
|
||||
@@ -0,0 +1,381 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional, Union
|
||||
|
||||
from _zvec import _Collection
|
||||
|
||||
from ..executor import QueryContext, QueryExecutorFactory
|
||||
from ..extension import ReRanker
|
||||
from ..typing import Status
|
||||
from .convert import convert_to_cpp_doc, convert_to_py_doc
|
||||
from .doc import Doc
|
||||
from .param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
FlatIndexParam,
|
||||
HnswIndexParam,
|
||||
IndexOption,
|
||||
InvertIndexParam,
|
||||
IVFIndexParam,
|
||||
OptimizeOption,
|
||||
)
|
||||
from .param.vector_query import VectorQuery
|
||||
from .schema import CollectionSchema, CollectionStats, FieldSchema
|
||||
|
||||
__all__ = ["Collection"]
|
||||
|
||||
_VECTOR_INDEX_TYPES = (HnswIndexParam, IVFIndexParam, FlatIndexParam)
|
||||
|
||||
|
||||
class Collection:
|
||||
"""Represents an opened collection in Zvec.
|
||||
|
||||
A `Collection` provides methods for data definition (DDL), data manipulation (DML),
|
||||
and querying (DQL). It is obtained via `create_and_open()` or `open()`.
|
||||
|
||||
This class is not meant to be instantiated directly; use factory functions instead.
|
||||
"""
|
||||
|
||||
def __init__(self, obj: _Collection):
|
||||
self._obj = obj
|
||||
self._schema = None
|
||||
self._querier = None
|
||||
|
||||
@classmethod
|
||||
def _from_core(cls, core_collection: _Collection) -> Collection:
|
||||
if not core_collection:
|
||||
raise ValueError("Collection is None")
|
||||
inst = cls.__new__(cls)
|
||||
inst._obj = core_collection
|
||||
schema = CollectionSchema._from_core(core_collection.Schema())
|
||||
inst._schema = schema
|
||||
inst._querier = QueryExecutorFactory.create(schema)
|
||||
return inst
|
||||
|
||||
@property
|
||||
def path(self) -> str:
|
||||
"""str: The filesystem path of the collection."""
|
||||
return self._obj.Path()
|
||||
|
||||
@property
|
||||
def option(self) -> CollectionOption:
|
||||
"""CollectionOption: The options used to open the collection."""
|
||||
return self._obj.Options()
|
||||
|
||||
@property
|
||||
def schema(self) -> CollectionSchema:
|
||||
"""CollectionSchema: The schema defining the structure of the collection."""
|
||||
return self._schema
|
||||
|
||||
@property
|
||||
def stats(self) -> CollectionStats:
|
||||
"""CollectionStats: Runtime statistics about the collection (e.g., doc count, size)."""
|
||||
return self._obj.Stats()
|
||||
|
||||
# ========== Collection DDL Methods ==========
|
||||
def destroy(self) -> None:
|
||||
"""Permanently delete the collection from disk.
|
||||
|
||||
Warning:
|
||||
This operation is irreversible. All data will be lost.
|
||||
"""
|
||||
self._obj.Destroy()
|
||||
|
||||
def flush(self) -> None:
|
||||
"""Force all pending writes to disk.
|
||||
|
||||
Ensures durability of recent inserts/updates.
|
||||
"""
|
||||
self._obj.Flush()
|
||||
|
||||
# ========== Index DDL Methods ==========
|
||||
def create_index(
|
||||
self,
|
||||
field_name: str,
|
||||
index_param: Union[
|
||||
HnswIndexParam, IVFIndexParam, FlatIndexParam, InvertIndexParam
|
||||
],
|
||||
option: IndexOption = IndexOption(),
|
||||
) -> None:
|
||||
"""Create an index on a field.
|
||||
|
||||
Vector index types (HNSW, IVF, FLAT) can only be applied to vector fields.
|
||||
Inverted index (`InvertIndexParam`) is for scalar fields.
|
||||
|
||||
Args:
|
||||
field_name (str): Name of the field to index.
|
||||
index_param (Union[HnswIndexParam, IVFIndexParam, FlatIndexParam, InvertIndexParam]):
|
||||
Index configuration.
|
||||
option (Optional[IndexOption], optional): Index creation options.
|
||||
Defaults to ``IndexOption()``.
|
||||
|
||||
Raises:
|
||||
ValueError: If a vector index is applied to a non-vector field.
|
||||
"""
|
||||
if index_param in _VECTOR_INDEX_TYPES and not self.schema.vector(field_name):
|
||||
supported_types = ", ".join(cls.__name__ for cls in _VECTOR_INDEX_TYPES)
|
||||
raise ValueError(
|
||||
f"Cannot apply vector index to non-vector field '{field_name}'. "
|
||||
f"The field must be of vector type to use index types like {supported_types}."
|
||||
)
|
||||
self._obj.CreateIndex(field_name, index_param, option)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
|
||||
def drop_index(self, field_name: str) -> None:
|
||||
"""Remove the index from a field.
|
||||
|
||||
Args:
|
||||
field_name (str): Name of the indexed field.
|
||||
"""
|
||||
self._obj.DropIndex(field_name)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
|
||||
def optimize(self, option: OptimizeOption = OptimizeOption()) -> None:
|
||||
"""Optimize the collection (e.g., merge segments, rebuild index).
|
||||
|
||||
Args:
|
||||
option (Optional[OptimizeOption], optional): Optimization options.
|
||||
Defaults to ``OptimizeOption()``.
|
||||
"""
|
||||
self._obj.Optimize(option)
|
||||
|
||||
# ========== COLUMN DDL Methods ==========
|
||||
def add_column(
|
||||
self,
|
||||
field_schema: FieldSchema,
|
||||
expression: str = "",
|
||||
option: AddColumnOption = AddColumnOption(),
|
||||
) -> None:
|
||||
"""Add a new column to the collection.
|
||||
|
||||
The column is populated using the provided expression (e.g., SQL-like formula).
|
||||
|
||||
Args:
|
||||
field_schema (FieldSchema): Schema definition for the new column.
|
||||
expression (str): Expression to compute values for existing documents.
|
||||
option (Optional[AddColumnOption], optional): Options for the operation.
|
||||
Defaults to ``AddColumnOption()``.
|
||||
"""
|
||||
self._obj.AddColumn(
|
||||
field_schema.name, field_schema._get_object(), expression, option
|
||||
)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
|
||||
def drop_column(self, field_name: str) -> None:
|
||||
"""Remove a column from the collection.
|
||||
|
||||
Args:
|
||||
field_name (str): Name of the column to drop.
|
||||
"""
|
||||
self._obj.DropColumn(field_name)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
|
||||
def alter_column(
|
||||
self,
|
||||
old_name: str,
|
||||
new_name: Optional[str] = None,
|
||||
field_schema: Optional[FieldSchema] = None,
|
||||
option: AlterColumnOption = AlterColumnOption(),
|
||||
) -> None:
|
||||
"""Rename a column, update its schema.
|
||||
|
||||
This method supports three atomic operations:
|
||||
1. Rename only (when `field_schema` is None).
|
||||
2. Modify schema only (when `new_name` is None or empty string).
|
||||
|
||||
Args:
|
||||
old_name (str): The current name of the column to be altered.
|
||||
new_name (Optional[str]): The new name for the column.
|
||||
- If provided and non-empty, the column will be renamed.
|
||||
- If `None` or empty string, no rename occurs.
|
||||
field_schema (Optional[FieldSchema]): The new schema definition.
|
||||
- If provided, the column's type, dimension, or other properties will be updated.
|
||||
- If `None`, only renaming (if requested) is performed.
|
||||
option (AlterColumnOption, optional): Options controlling the alteration behavior.
|
||||
Defaults to ``AlterColumnOption()``.
|
||||
|
||||
**Limitation**: This operation **only supports scalar numeric columns**. such as:
|
||||
- `DOUBLE`, `FLOAT`,
|
||||
- `INT32`, `INT64`, `UINT32`, `UINT64`
|
||||
|
||||
Note:
|
||||
- Schema modification may trigger data migration or index rebuild.
|
||||
|
||||
Examples:
|
||||
>>> # Rename column only
|
||||
>>> results = collection.alter_column(old_name="id", new_name="doc_id")
|
||||
|
||||
>>> # Modify schema only
|
||||
>>> new_schema = FieldSchema(name="doc_id", dtype=DataType.INT64)
|
||||
>>> collection.alter_column("id", field_schema=new_schema)
|
||||
"""
|
||||
self._obj.AlterColumn(
|
||||
old_name,
|
||||
new_name or "",
|
||||
field_schema._get_object() if field_schema else None,
|
||||
option,
|
||||
)
|
||||
self._schema = CollectionSchema._from_core(self._obj.Schema())
|
||||
|
||||
# ========== Collection DDL Methods ==========
|
||||
def insert(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
|
||||
"""Insert new documents into the collection.
|
||||
|
||||
Documents must have unique IDs and conform to the schema.
|
||||
|
||||
Args:
|
||||
docs (Union[Doc, list[Doc]]): One or more documents to insert.
|
||||
|
||||
Returns:
|
||||
Union[Status, list[Status]]: If a single Doc was given, returns its Status;
|
||||
if a list was given, returns a list of Status objects.
|
||||
"""
|
||||
is_single = isinstance(docs, Doc)
|
||||
doc_list = [docs] if is_single else docs
|
||||
results = self._obj.Insert(
|
||||
[convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
|
||||
)
|
||||
return results[0] if is_single else results
|
||||
|
||||
def upsert(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
|
||||
"""Insert new documents or update existing ones by ID.
|
||||
|
||||
Args:
|
||||
docs (Union[Doc, list[Doc]]): Documents to upsert.
|
||||
|
||||
Returns:
|
||||
Union[Status, list[Status]]: If a single Doc was given, returns its Status;
|
||||
if a list was given, returns a list of Status objects.
|
||||
"""
|
||||
is_single = isinstance(docs, Doc)
|
||||
doc_list = [docs] if is_single else docs
|
||||
results = self._obj.Upsert(
|
||||
[convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
|
||||
)
|
||||
return results[0] if is_single else results
|
||||
|
||||
def update(self, docs: Union[Doc, list[Doc]]) -> Union[Status, list[Status]]:
|
||||
"""Update existing documents by ID.
|
||||
|
||||
Only specified fields are updated; others remain unchanged.
|
||||
|
||||
Args:
|
||||
docs (Union[Doc, list[Doc]]): Documents containing updated fields.
|
||||
|
||||
Returns:
|
||||
Union[Status, list[Status]]: If a single Doc was given, returns its Status;
|
||||
if a list was given, returns a list of Status objects.
|
||||
"""
|
||||
is_single = isinstance(docs, Doc)
|
||||
doc_list = [docs] if is_single else docs
|
||||
results = self._obj.Update(
|
||||
[convert_to_cpp_doc(doc, self.schema) for doc in doc_list]
|
||||
)
|
||||
return results[0] if is_single else results
|
||||
|
||||
def delete(self, ids: Union[str, list[str]]) -> Union[Status, list[Status]]:
|
||||
"""Delete documents by ID.
|
||||
|
||||
Args:
|
||||
ids (Union[str, list[str]]): One or more document IDs to delete.
|
||||
|
||||
Returns:
|
||||
Union[Status, list[Status]]: If a single id was given, returns its Status;
|
||||
if a list was given, returns a list of Status objects.
|
||||
"""
|
||||
is_single = isinstance(ids, str)
|
||||
id_list = [ids] if isinstance(ids, str) else ids
|
||||
results = self._obj.Delete(id_list)
|
||||
return results[0] if is_single else results
|
||||
|
||||
def delete_by_filter(self, filter: str) -> None:
|
||||
"""Delete documents matching a filter expression.
|
||||
|
||||
Args:
|
||||
filter (str): Boolean expression (e.g., ``"age > 30"``).
|
||||
"""
|
||||
self._obj.DeleteByFilter(filter)
|
||||
|
||||
# ========== Collection DQL-fetch Methods ==========
|
||||
def fetch(self, ids: Union[str, list[str]]) -> dict[str, Doc]:
|
||||
"""Retrieve documents by ID.
|
||||
|
||||
Args:
|
||||
ids (Union[str, list[str]]): Document IDs to fetch.
|
||||
|
||||
Returns:
|
||||
dict[str, Doc]: Mapping from ID to document. Missing IDs are omitted.
|
||||
"""
|
||||
ids = [ids] if isinstance(ids, str) else ids
|
||||
docs = self._obj.Fetch(ids)
|
||||
return {
|
||||
doc_id: py_doc
|
||||
for doc_id, core_doc in docs.items()
|
||||
if (py_doc := convert_to_py_doc(core_doc, self.schema)) is not None
|
||||
}
|
||||
|
||||
# ========== Collection DQL-Query Methods ==========
|
||||
|
||||
def query(
|
||||
self,
|
||||
vectors: Optional[Union[VectorQuery, list[VectorQuery]]] = None,
|
||||
*,
|
||||
topk: int = 10,
|
||||
filter: Optional[str] = None,
|
||||
include_vector: bool = False,
|
||||
output_fields: Optional[list[str]] = None,
|
||||
reranker: Optional[ReRanker] = None,
|
||||
) -> list[Doc]:
|
||||
"""Perform vector similarity search with optional filtering and re-ranking.
|
||||
|
||||
At least one `VectorQuery` must be provided.
|
||||
|
||||
Args:
|
||||
vectors (Optional[Union[VectorQuery, list[VectorQuery]]], optional):
|
||||
One or more vector queries. Defaults to None.
|
||||
topk (int, optional): Number of nearest neighbors to return.
|
||||
Defaults to 10.
|
||||
filter (Optional[str], optional): Boolean expression to pre-filter candidates.
|
||||
Defaults to None.
|
||||
include_vector (bool, optional): Whether to include vector data in results.
|
||||
Defaults to False.
|
||||
output_fields (Optional[list[str]], optional): Scalar fields to include.
|
||||
If None, all fields are returned. Defaults to None.
|
||||
reranker (Optional[ReRanker], optional): Re-ranker to refine results.
|
||||
Defaults to None.
|
||||
|
||||
Returns:
|
||||
list[Doc]: Top-k matching documents, sorted by relevance score.
|
||||
|
||||
Examples:
|
||||
>>> from zvec import VectorQuery
|
||||
>>> results = collection.query(
|
||||
... vectors=VectorQuery("embedding", vector=[0.1, 0.2]),
|
||||
... topk=5,
|
||||
... filter="category == 'tech'",
|
||||
... output_fields=["title", "url"]
|
||||
... )
|
||||
"""
|
||||
ctx = QueryContext(
|
||||
topk=topk,
|
||||
filter=filter,
|
||||
queries=[vectors] if isinstance(vectors, VectorQuery) else vectors,
|
||||
include_vector=include_vector,
|
||||
output_fields=output_fields,
|
||||
reranker=reranker,
|
||||
)
|
||||
return self._querier.execute(ctx, self._obj)
|
||||
@@ -0,0 +1,54 @@
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from _zvec import _Doc
|
||||
|
||||
from .doc import Doc
|
||||
from .schema import CollectionSchema
|
||||
|
||||
|
||||
def convert_to_cpp_doc(doc: Doc, collection_schema: CollectionSchema) -> _Doc:
|
||||
if not doc or not collection_schema:
|
||||
return None
|
||||
|
||||
_doc = _Doc()
|
||||
|
||||
# set pk
|
||||
_doc.set_pk(doc.id)
|
||||
|
||||
# set scalar fields
|
||||
for k, v in doc.fields.items():
|
||||
field_schema = collection_schema.field(k)
|
||||
if not field_schema:
|
||||
raise ValueError(
|
||||
f"schema validate failed: {k} not found in collection schema"
|
||||
)
|
||||
_doc.set_any(k, field_schema._get_object(), v)
|
||||
|
||||
# set vector fields
|
||||
for k, v in doc.vectors.items():
|
||||
vector_schema = collection_schema.vector(k)
|
||||
if not vector_schema:
|
||||
raise ValueError(
|
||||
f"schema validate failed: {k} not found in collection schema"
|
||||
)
|
||||
_doc.set_any(k, vector_schema._get_object(), v)
|
||||
return _doc
|
||||
|
||||
|
||||
def convert_to_py_doc(doc: _Doc, collection_schema: CollectionSchema) -> Doc:
|
||||
if not doc or not collection_schema:
|
||||
return None
|
||||
|
||||
data_tuple = doc.get_all(collection_schema._get_object())
|
||||
return Doc._from_tuple(data_tuple)
|
||||
@@ -0,0 +1,173 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Optional
|
||||
|
||||
from ..common import VectorType
|
||||
|
||||
__all__ = [
|
||||
"Doc",
|
||||
]
|
||||
|
||||
|
||||
class Doc:
|
||||
"""Represents a retrieved document with optional metadata, fields, and vectors.
|
||||
|
||||
This immutable data class encapsulates the result of a search or retrieval
|
||||
operation. It includes the document ID, relevance score (if applicable),
|
||||
scalar fields, and vector embeddings.
|
||||
|
||||
During initialization, any `numpy.ndarray` in `vectors` is automatically
|
||||
converted to a plain Python list for JSON serialization and immutability.
|
||||
|
||||
Attributes:
|
||||
id (str): Unique identifier of the document.
|
||||
score (Optional[float], optional): Relevance score from search.
|
||||
Defaults to None.
|
||||
vectors (Optional[dict[str, VectorType]], optional): Named vector
|
||||
embeddings associated with the document. Values are converted to
|
||||
lists if originally `np.ndarray`. Defaults to None.
|
||||
fields (Optional[dict[str, Any]], optional): Scalar metadata fields
|
||||
(e.g., title, timestamp). Defaults to None.
|
||||
|
||||
Examples:
|
||||
>>> import numpy as np
|
||||
>>> import zvec
|
||||
>>> doc = zvec.Doc(
|
||||
... id="doc1",
|
||||
... score=0.95,
|
||||
... vectors={"emb": np.array([0.1, 0.2, 0.3])},
|
||||
... fields={"title": "Hello World"}
|
||||
... )
|
||||
>>> print(doc.vector("emb"))
|
||||
[0.1, 0.2, 0.3]
|
||||
>>> print(doc.has_field("title"))
|
||||
True
|
||||
"""
|
||||
|
||||
__slots__ = ("id", "score", "vectors", "fields")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
id: str,
|
||||
score: Optional[float] = None,
|
||||
vectors: Optional[dict[str, VectorType]] = None,
|
||||
fields: Optional[dict[str, Any]] = None,
|
||||
):
|
||||
self.id = id
|
||||
self.score = score
|
||||
self.vectors = vectors or {}
|
||||
self.fields = fields or {}
|
||||
|
||||
def has_field(self, name: str) -> bool:
|
||||
"""Check if the document contains a scalar field with the given name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the field to check.
|
||||
|
||||
Returns:
|
||||
bool: True if the field exists, False otherwise.
|
||||
"""
|
||||
return name in self.fields
|
||||
|
||||
def has_vector(self, name: str) -> bool:
|
||||
"""Check if the document contains a vector with the given name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the vector to check.
|
||||
|
||||
Returns:
|
||||
bool: True if the vector exists, False otherwise.
|
||||
"""
|
||||
return name in self.vectors
|
||||
|
||||
def vector(self, name: str):
|
||||
"""Get a vector by name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the vector.
|
||||
|
||||
Returns:
|
||||
Any: The vector (as a list) if it exists, otherwise None.
|
||||
"""
|
||||
return self.vectors and self.vectors.get(name)
|
||||
|
||||
def field(self, name: str):
|
||||
"""Get a scalar field by name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the field.
|
||||
|
||||
Returns:
|
||||
Any: The field value if it exists, otherwise None.
|
||||
"""
|
||||
return self.fields and self.fields.get(name)
|
||||
|
||||
def vector_names(self) -> list[str]:
|
||||
"""Get the list of all vector names in this document.
|
||||
|
||||
Returns:
|
||||
list[str]: A list of vector field names. Empty if no vectors.
|
||||
"""
|
||||
return [] if not self.vectors else list(self.vectors.keys())
|
||||
|
||||
def field_names(self) -> list[str]:
|
||||
"""Get the list of all scalar field names in this document.
|
||||
|
||||
Returns:
|
||||
list[str]: A list of field names. Empty if no fields.
|
||||
"""
|
||||
return [] if not self.fields else list(self.fields.keys())
|
||||
|
||||
def __repr__(self) -> str:
|
||||
try:
|
||||
schema = {
|
||||
"id": self.id,
|
||||
"score": self.score,
|
||||
"fields": self.fields,
|
||||
"vectors": self.vectors,
|
||||
}
|
||||
return json.dumps(schema, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"<Doc error during repr: {e}>"
|
||||
|
||||
def _replace(self, **changes):
|
||||
new_tuple = (
|
||||
changes.get("id", self.id),
|
||||
changes.get("score", self.score),
|
||||
changes.get("fields", self.fields.copy() if self.fields else None),
|
||||
changes.get("vectors", self.vectors.copy() if self.vectors else None),
|
||||
)
|
||||
return type(self)._from_tuple(new_tuple)
|
||||
|
||||
@classmethod
|
||||
def _from_tuple(
|
||||
cls, data_tuple: tuple[str, float, dict[str, Any], dict[str, VectorType]]
|
||||
):
|
||||
obj = object.__new__(cls)
|
||||
obj.id = data_tuple[0]
|
||||
obj.score = data_tuple[1]
|
||||
obj.fields = data_tuple[2] or {}
|
||||
|
||||
vectors = data_tuple[3]
|
||||
if vectors is not None:
|
||||
obj.vectors = {
|
||||
name: (vec.tolist() if hasattr(vec, "tolist") else vec)
|
||||
for name, vec in vectors.items()
|
||||
}
|
||||
else:
|
||||
obj.vectors = {}
|
||||
return obj
|
||||
@@ -0,0 +1,42 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from _zvec.param import (
|
||||
AddColumnOption,
|
||||
AlterColumnOption,
|
||||
CollectionOption,
|
||||
FlatIndexParam,
|
||||
HnswIndexParam,
|
||||
HnswQueryParam,
|
||||
IndexOption,
|
||||
InvertIndexParam,
|
||||
IVFIndexParam,
|
||||
IVFQueryParam,
|
||||
OptimizeOption,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"AddColumnOption",
|
||||
"AlterColumnOption",
|
||||
"CollectionOption",
|
||||
"FlatIndexParam",
|
||||
"HnswIndexParam",
|
||||
"HnswQueryParam",
|
||||
"IVFIndexParam",
|
||||
"IVFQueryParam",
|
||||
"IndexOption",
|
||||
"InvertIndexParam",
|
||||
"OptimizeOption",
|
||||
]
|
||||
@@ -0,0 +1,692 @@
|
||||
"""
|
||||
This module contains the params of Zvec
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import typing
|
||||
|
||||
import _zvec.typing
|
||||
|
||||
__all__: list[str] = [
|
||||
"AddColumnOption",
|
||||
"AlterColumnOption",
|
||||
"CollectionOption",
|
||||
"FlatIndexParam",
|
||||
"HnswIndexParam",
|
||||
"HnswQueryParam",
|
||||
"IVFIndexParam",
|
||||
"IVFQueryParam",
|
||||
"IndexOption",
|
||||
"IndexParam",
|
||||
"InvertIndexParam",
|
||||
"OptimizeOption",
|
||||
"QueryParam",
|
||||
"SegmentOption",
|
||||
"VectorIndexParam",
|
||||
]
|
||||
|
||||
class AddColumnOption:
|
||||
"""
|
||||
|
||||
Options for adding a new column to a collection.
|
||||
|
||||
Attributes:
|
||||
concurrency (int): Number of threads to use when backfilling data
|
||||
for the new column. If 0, auto-detect is used. Default is 0.
|
||||
|
||||
Examples:
|
||||
>>> opt = AddColumnOption(concurrency=1)
|
||||
>>> print(opt.concurrency)
|
||||
1
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
|
||||
"""
|
||||
Constructs an AddColumnOption instance.
|
||||
|
||||
Args:
|
||||
concurrency (int, optional): Number of threads for data backfill.
|
||||
0 means auto-detect. Defaults to 0.
|
||||
"""
|
||||
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
@property
|
||||
def concurrency(self) -> int:
|
||||
"""
|
||||
int: Number of threads used when adding a column (0 = auto).
|
||||
"""
|
||||
|
||||
class AlterColumnOption:
|
||||
"""
|
||||
|
||||
Options for altering an existing column (e.g., changing index settings).
|
||||
|
||||
Attributes:
|
||||
concurrency (int): Number of threads to use during the alteration process.
|
||||
If 0, the system will choose an optimal value automatically.
|
||||
Default is 0.
|
||||
|
||||
Examples:
|
||||
>>> opt = AlterColumnOption(concurrency=1)
|
||||
>>> print(opt.concurrency)
|
||||
1
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
|
||||
"""
|
||||
Constructs an AlterColumnOption instance.
|
||||
|
||||
Args:
|
||||
concurrency (int, optional): Number of threads for column alteration.
|
||||
0 means auto-detect. Defaults to 0.
|
||||
"""
|
||||
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
@property
|
||||
def concurrency(self) -> int:
|
||||
"""
|
||||
int: Number of threads used when altering a column (0 = auto).
|
||||
"""
|
||||
|
||||
class CollectionOption:
|
||||
"""
|
||||
|
||||
Options for opening or creating a collection.
|
||||
|
||||
Attributes:
|
||||
read_only (bool): Whether the collection is opened in read-only mode.
|
||||
Default is False.
|
||||
enable_mmap (bool): Whether to use memory-mapped I/O for data files.
|
||||
Default is True.
|
||||
|
||||
Examples:
|
||||
>>> opt = CollectionOption(read_only=True, enable_mmap=False)
|
||||
>>> print(opt.read_only)
|
||||
True
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(self, read_only: bool = False, enable_mmap: bool = True) -> None:
|
||||
"""
|
||||
Constructs a CollectionOption instance.
|
||||
|
||||
Args:
|
||||
read_only (bool, optional): Open collection in read-only mode.
|
||||
Defaults to False.
|
||||
enable_mmap (bool, optional): Enable memory-mapped I/O.
|
||||
Defaults to True.
|
||||
"""
|
||||
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
@property
|
||||
def enable_mmap(self) -> bool: ...
|
||||
@property
|
||||
def read_only(self) -> bool: ...
|
||||
|
||||
class FlatIndexParam(VectorIndexParam):
|
||||
"""
|
||||
|
||||
Parameters for configuring a flat (brute-force) index.
|
||||
|
||||
A flat index performs exact nearest neighbor search by comparing the query
|
||||
vector against all vectors in the collection. It is simple, accurate, and
|
||||
suitable for small to medium datasets or as a baseline.
|
||||
|
||||
Attributes:
|
||||
metric_type (MetricType): Distance metric used for similarity computation.
|
||||
Default is ``MetricType.IP`` (inner product).
|
||||
quantize_type (QuantizeType): Optional quantization type for vector
|
||||
compression (e.g., FP16, INT8). Use ``QuantizeType.UNDEFINED`` to
|
||||
disable quantization. Default is ``QuantizeType.UNDEFINED``.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import MetricType, QuantizeType
|
||||
>>> params = FlatIndexParam(
|
||||
... metric_type=MetricType.L2,
|
||||
... quantize_type=QuantizeType.FP16
|
||||
... )
|
||||
>>> print(params)
|
||||
{'metric_type': 'L2', 'quantize_type': 'FP16'}
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(
|
||||
self,
|
||||
metric_type: _zvec.typing.MetricType = ...,
|
||||
quantize_type: _zvec.typing.QuantizeType = ...,
|
||||
) -> None:
|
||||
"""
|
||||
Constructs a FlatIndexParam instance.
|
||||
|
||||
Args:
|
||||
metric_type (MetricType, optional): Distance metric. Defaults to MetricType.IP.
|
||||
quantize_type (QuantizeType, optional): Vector quantization type.
|
||||
Defaults to QuantizeType.UNDEFINED (no quantization).
|
||||
"""
|
||||
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
def to_dict(self) -> dict:
|
||||
"""
|
||||
Convert to dictionary with all fields
|
||||
"""
|
||||
|
||||
class HnswIndexParam(VectorIndexParam):
|
||||
"""
|
||||
|
||||
Parameters for configuring an HNSW (Hierarchical Navigable Small World) index.
|
||||
|
||||
HNSW is a graph-based approximate nearest neighbor search index. This class
|
||||
encapsulates its construction hyperparameters.
|
||||
|
||||
Attributes:
|
||||
metric_type (MetricType): Distance metric used for similarity computation.
|
||||
Default is ``MetricType.IP`` (inner product).
|
||||
m (int): Number of bi-directional links created for every new element
|
||||
during construction. Higher values improve accuracy but increase
|
||||
memory usage and construction time. Default is 100.
|
||||
ef_construction (int): Size of the dynamic candidate list for nearest
|
||||
neighbors during index construction. Larger values yield better
|
||||
graph quality at the cost of slower build time. Default is 500.
|
||||
quantize_type (QuantizeType): Optional quantization type for vector
|
||||
compression (e.g., FP16, INT8). Default is `QuantizeType.UNDEFINED` to
|
||||
disable quantization.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import MetricType, QuantizeType
|
||||
>>> params = HnswIndexParam(
|
||||
... metric_type=MetricType.COSINE,
|
||||
... m=16,
|
||||
... ef_construction=200,
|
||||
... quantize_type=QuantizeType.INT8
|
||||
... )
|
||||
>>> print(params)
|
||||
{'metric_type': 'IP', 'm': 16, 'ef_construction': 200, 'quantize_type': 'INT8'}
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(
|
||||
self,
|
||||
metric_type: _zvec.typing.MetricType = ...,
|
||||
m: typing.SupportsInt = 100,
|
||||
ef_construction: typing.SupportsInt = 500,
|
||||
quantize_type: _zvec.typing.QuantizeType = ...,
|
||||
) -> None: ...
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
def to_dict(self) -> dict:
|
||||
"""
|
||||
Convert to dictionary with all fields
|
||||
"""
|
||||
|
||||
@property
|
||||
def ef_construction(self) -> int:
|
||||
"""
|
||||
int: Candidate list size during index construction.
|
||||
"""
|
||||
|
||||
@property
|
||||
def m(self) -> int:
|
||||
"""
|
||||
int: Maximum number of neighbors per node in upper layers.
|
||||
"""
|
||||
|
||||
class HnswQueryParam(QueryParam):
|
||||
"""
|
||||
|
||||
Query parameters for HNSW (Hierarchical Navigable Small World) index.
|
||||
|
||||
Controls the trade-off between search speed and accuracy via the `ef` parameter.
|
||||
|
||||
Attributes:
|
||||
type (IndexType): Always ``IndexType.HNSW``.
|
||||
ef (int): Size of the dynamic candidate list during search.
|
||||
Larger values improve recall but slow down search.
|
||||
Default is 300.
|
||||
radius (float): Search radius for range queries. Default is 0.0.
|
||||
is_linear (bool): Force linear search. Default is False.
|
||||
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
|
||||
|
||||
Examples:
|
||||
>>> params = HnswQueryParam(ef=300)
|
||||
>>> print(params.ef)
|
||||
300
|
||||
>>> print(params.to_dict() if hasattr(params, 'to_dict') else params)
|
||||
{"type":"HNSW", "ef":300}
|
||||
"""
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(
|
||||
self,
|
||||
ef: typing.SupportsInt = 300,
|
||||
radius: typing.SupportsFloat = 0.0,
|
||||
is_linear: bool = False,
|
||||
is_using_refiner: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Constructs an HnswQueryParam instance.
|
||||
|
||||
Args:
|
||||
ef (int, optional): Search-time candidate list size.
|
||||
Higher values improve accuracy. Defaults to 100.
|
||||
radius (float, optional): Search radius for range queries. Default is 0.0.
|
||||
is_linear (bool, optional): Force linear search. Default is False.
|
||||
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
|
||||
"""
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
@property
|
||||
def ef(self) -> int:
|
||||
"""
|
||||
int: Size of the dynamic candidate list during HNSW search.
|
||||
"""
|
||||
|
||||
class IVFIndexParam(VectorIndexParam):
|
||||
"""
|
||||
|
||||
Parameters for configuring an IVF (Inverted File Index) index.
|
||||
|
||||
IVF partitions the vector space into clusters (inverted lists). At query time,
|
||||
only a subset of clusters is searched, providing a trade-off between speed
|
||||
and accuracy.
|
||||
|
||||
Attributes:
|
||||
metric_type (MetricType): Distance metric used for similarity computation.
|
||||
Default is ``MetricType.IP`` (inner product).
|
||||
n_list (int): Number of clusters (inverted lists) to partition the dataset into.
|
||||
If set to 0, the system will auto-select a reasonable value based on data size.
|
||||
Default is 0 (auto).
|
||||
n_iters (int): Number of iterations for k-means clustering during index training.
|
||||
Higher values yield more stable centroids. Default is 10.
|
||||
use_soar (bool): Whether to enable SOAR (Scalable Optimized Adaptive Routing)
|
||||
for improved IVF search performance. Default is False.
|
||||
quantize_type (QuantizeType): Optional quantization type for vector
|
||||
compression (e.g., FP16, INT8). Default is ``QuantizeType.UNDEFINED``.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import MetricType, QuantizeType
|
||||
>>> params = IVFIndexParam(
|
||||
... metric_type=MetricType.COSINE,
|
||||
... n_list=100,
|
||||
... n_iters=15,
|
||||
... use_soar=True,
|
||||
... quantize_type=QuantizeType.INT8
|
||||
... )
|
||||
>>> print(params.n_list)
|
||||
100
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(
|
||||
self,
|
||||
metric_type: _zvec.typing.MetricType = ...,
|
||||
n_list: typing.SupportsInt = 0,
|
||||
n_iters: typing.SupportsInt = 10,
|
||||
use_soar: bool = False,
|
||||
quantize_type: _zvec.typing.QuantizeType = ...,
|
||||
) -> None:
|
||||
"""
|
||||
Constructs an IVFIndexParam instance.
|
||||
|
||||
Args:
|
||||
metric_type (MetricType, optional): Distance metric. Defaults to MetricType.IP.
|
||||
n_list (int, optional): Number of inverted lists (clusters). Set to 0 for auto.
|
||||
Defaults to 0.
|
||||
n_iters (int, optional): Number of k-means iterations during training.
|
||||
Defaults to 10.
|
||||
use_soar (bool, optional): Enable SOAR optimization. Defaults to False.
|
||||
quantize_type (QuantizeType, optional): Vector quantization type.
|
||||
Defaults to QuantizeType.UNDEFINED.
|
||||
"""
|
||||
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
def to_dict(self) -> dict:
|
||||
"""
|
||||
Convert to dictionary with all fields
|
||||
"""
|
||||
|
||||
@property
|
||||
def n_iters(self) -> int:
|
||||
"""
|
||||
int: Number of k-means iterations during training.
|
||||
"""
|
||||
|
||||
@property
|
||||
def n_list(self) -> int:
|
||||
"""
|
||||
int: Number of inverted lists (0 = auto).
|
||||
"""
|
||||
|
||||
@property
|
||||
def use_soar(self) -> bool:
|
||||
"""
|
||||
bool: Whether SOAR optimization is enabled.
|
||||
"""
|
||||
|
||||
class IVFQueryParam(QueryParam):
|
||||
"""
|
||||
|
||||
Query parameters for IVF (Inverted File Index) index.
|
||||
|
||||
Controls how many inverted lists (`nprobe`) to visit during search.
|
||||
|
||||
Attributes:
|
||||
type (IndexType): Always ``IndexType.IVF``.
|
||||
nprobe (int): Number of closest clusters (inverted lists) to search.
|
||||
Higher values improve recall but increase latency.
|
||||
Default is 10.
|
||||
radius (float): Search radius for range queries. Default is 0.0.
|
||||
is_linear (bool): Force linear search. Default is False.
|
||||
|
||||
Examples:
|
||||
>>> params = IVFQueryParam(nprobe=20)
|
||||
>>> print(params.nprobe)
|
||||
20
|
||||
"""
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(self, nprobe: typing.SupportsInt = 10) -> None:
|
||||
"""
|
||||
Constructs an IVFQueryParam instance.
|
||||
|
||||
Args:
|
||||
nprobe (int, optional): Number of inverted lists to probe during search.
|
||||
Higher values improve accuracy. Defaults to 10.
|
||||
"""
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
@property
|
||||
def nprobe(self) -> int:
|
||||
"""
|
||||
int: Number of inverted lists to search during IVF query.
|
||||
"""
|
||||
|
||||
class IndexOption:
|
||||
"""
|
||||
|
||||
Options for creating an index.
|
||||
|
||||
Attributes:
|
||||
concurrency (int): Number of threads to use during index creation.
|
||||
If 0, the system will choose an optimal value automatically.
|
||||
Default is 0.
|
||||
|
||||
Examples:
|
||||
>>> opt = IndexOption(concurrency=4)
|
||||
>>> print(opt.concurrency)
|
||||
4
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
|
||||
"""
|
||||
Constructs an IndexOption instance.
|
||||
|
||||
Args:
|
||||
concurrency (int, optional): Number of concurrent threads.
|
||||
0 means auto-detect. Defaults to 0.
|
||||
"""
|
||||
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
@property
|
||||
def concurrency(self) -> int:
|
||||
"""
|
||||
int: Number of threads used for index creation (0 = auto).
|
||||
"""
|
||||
|
||||
class IndexParam:
|
||||
"""
|
||||
|
||||
Base class for all index parameter configurations.
|
||||
|
||||
This abstract base class defines the common interface for index types.
|
||||
It should not be instantiated directly; use derived classes instead.
|
||||
|
||||
Attributes:
|
||||
type (IndexType): The type of the index (e.g., HNSW, FLAT, INVERT).
|
||||
"""
|
||||
|
||||
__hash__: typing.ClassVar[None] = None
|
||||
|
||||
def __eq__(self, arg0: typing.Any) -> bool: ...
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
def clone(self) -> IndexParam: ...
|
||||
def to_dict(self) -> dict:
|
||||
"""
|
||||
Convert to dictionary with all fields
|
||||
"""
|
||||
|
||||
@property
|
||||
def type(self) -> _zvec.typing.IndexType:
|
||||
"""
|
||||
IndexType: The type of the index.
|
||||
"""
|
||||
|
||||
class InvertIndexParam(IndexParam):
|
||||
"""
|
||||
|
||||
Parameters for configuring an invert index.
|
||||
|
||||
This class controls whether range query
|
||||
optimization is enabled for invert index structures.
|
||||
|
||||
Attributes:
|
||||
type (IndexType): Always `IndexType.INVERTED`.
|
||||
enable_range_optimization (bool): Whether range optimization is enabled.
|
||||
enable_extended_wildcard (bool): Whether extended wildcard (suffix and infix) search is enabled.
|
||||
|
||||
Examples:
|
||||
>>> params = InvertIndexParam(enable_range_optimization=True, enable_extended_wildcard=False)
|
||||
>>> print(params.enable_range_optimization)
|
||||
True
|
||||
>>> print(params.enable_extended_wildcard)
|
||||
False
|
||||
>>> config = params.to_dict()
|
||||
>>> print(config)
|
||||
{'enable_range_optimization': True, 'enable_extended_wildcard': False}
|
||||
"""
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(
|
||||
self,
|
||||
enable_range_optimization: bool = False,
|
||||
enable_extended_wildcard: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Constructs an InvertIndexParam instance.
|
||||
|
||||
Args:
|
||||
enable_range_optimization (bool, optional): If True, enables range query
|
||||
optimization for the invert index. Defaults to False.
|
||||
enable_extended_wildcard (bool, optional): If True, enables extended wildcard
|
||||
search including suffix and infix patterns. Defaults to False.
|
||||
"""
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
def to_dict(self) -> dict:
|
||||
"""
|
||||
Convert to dictionary with all fields
|
||||
"""
|
||||
@property
|
||||
def enable_extended_wildcard(self) -> bool:
|
||||
"""
|
||||
bool: Whether extended wildcard (suffix and infix) search is enabled.
|
||||
Note: Prefix search is always enabled regardless of this setting.
|
||||
"""
|
||||
@property
|
||||
def enable_range_optimization(self) -> bool:
|
||||
"""
|
||||
bool: Whether range optimization is enabled for this inverted index.
|
||||
"""
|
||||
|
||||
class OptimizeOption:
|
||||
"""
|
||||
|
||||
Options for optimizing a collection (e.g., merging segments).
|
||||
|
||||
Attributes:
|
||||
concurrency (int): Number of threads to use during optimization.
|
||||
If 0, the system will choose an optimal value automatically.
|
||||
Default is 0.
|
||||
|
||||
Examples:
|
||||
>>> opt = OptimizeOption(concurrency=2)
|
||||
>>> print(opt.concurrency)
|
||||
2
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(self, concurrency: typing.SupportsInt = 0) -> None:
|
||||
"""
|
||||
Constructs an OptimizeOption instance.
|
||||
|
||||
Args:
|
||||
concurrency (int, optional): Number of concurrent threads.
|
||||
0 means auto-detect. Defaults to 0.
|
||||
"""
|
||||
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
@property
|
||||
def concurrency(self) -> int:
|
||||
"""
|
||||
int: Number of threads used for optimization (0 = auto).
|
||||
"""
|
||||
|
||||
class QueryParam:
|
||||
"""
|
||||
|
||||
Base class for all query parameter configurations.
|
||||
|
||||
This abstract base class defines common query settings such as search radius
|
||||
and whether to force linear (brute-force) search. It should not be instantiated
|
||||
directly; use derived classes like `HnswQueryParam` or `IVFQueryParam`.
|
||||
|
||||
Attributes:
|
||||
type (IndexType): The index type this query is configured for.
|
||||
radius (float): Search radius for range queries. Used in combination with
|
||||
top-k to filter results. Default is 0.0 (disabled).
|
||||
is_linear (bool): If True, forces brute-force linear search instead of
|
||||
using the index. Useful for debugging or small datasets. Default is False.
|
||||
is_using_refiner (bool, optional): Whether to use refiner for the query. Default is False.
|
||||
"""
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
@property
|
||||
def is_linear(self) -> bool:
|
||||
"""
|
||||
bool: Whether to bypass the index and use brute-force linear search.
|
||||
"""
|
||||
@property
|
||||
def is_using_refiner(self) -> bool:
|
||||
"""
|
||||
bool: Whether to use refiner for the query.
|
||||
"""
|
||||
@property
|
||||
def radius(self) -> float:
|
||||
"""
|
||||
IndexType: The type of index this query targets.
|
||||
"""
|
||||
@property
|
||||
def type(self) -> _zvec.typing.IndexType:
|
||||
"""
|
||||
IndexType: The type of index this query targets.
|
||||
"""
|
||||
|
||||
class SegmentOption:
|
||||
"""
|
||||
|
||||
Options for segment-level operations.
|
||||
|
||||
Currently, this class mirrors CollectionOption and is used internally.
|
||||
It supports read-only mode, memory mapping, and buffer configuration.
|
||||
|
||||
Note:
|
||||
This class is primarily for internal use. Most users should use
|
||||
CollectionOption instead.
|
||||
|
||||
Examples:
|
||||
>>> opt = SegmentOption()
|
||||
>>> print(opt.enable_mmap)
|
||||
True
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(self) -> None:
|
||||
"""
|
||||
Constructs a SegmentOption with default settings.
|
||||
"""
|
||||
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
@property
|
||||
def enable_mmap(self) -> bool:
|
||||
"""
|
||||
bool: Whether memory-mapped I/O is enabled.
|
||||
"""
|
||||
|
||||
@property
|
||||
def max_buffer_size(self) -> int:
|
||||
"""
|
||||
int: Maximum buffer size in bytes (internal use).
|
||||
"""
|
||||
|
||||
@property
|
||||
def read_only(self) -> bool:
|
||||
"""
|
||||
bool: Whether the segment is read-only.
|
||||
"""
|
||||
|
||||
class VectorIndexParam(IndexParam):
|
||||
"""
|
||||
|
||||
Base class for vector index parameter configurations.
|
||||
|
||||
Encapsulates common settings for all vector index types.
|
||||
|
||||
Attributes:
|
||||
type (IndexType): The specific vector index type (e.g., HNSW, FLAT).
|
||||
metric_type (MetricType): Distance metric used for similarity search.
|
||||
quantize_type (QuantizeType): Optional vector quantization type.
|
||||
"""
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
def to_dict(self) -> dict:
|
||||
"""
|
||||
Convert to dictionary with all fields
|
||||
"""
|
||||
|
||||
@property
|
||||
def metric_type(self) -> _zvec.typing.MetricType:
|
||||
"""
|
||||
MetricType: Distance metric (e.g., IP, COSINE, L2).
|
||||
"""
|
||||
|
||||
@property
|
||||
def quantize_type(self) -> _zvec.typing.QuantizeType:
|
||||
"""
|
||||
QuantizeType: Vector quantization type (e.g., FP16, INT8).
|
||||
"""
|
||||
|
||||
class _VectorQuery:
|
||||
field_name: str
|
||||
filter: str
|
||||
include_vector: bool
|
||||
query_params: QueryParam
|
||||
|
||||
def __getstate__(self) -> tuple: ...
|
||||
def __init__(self) -> None: ...
|
||||
def __setstate__(self, arg0: tuple) -> None: ...
|
||||
def set_vector(self, arg0: ..., arg1: typing.Any) -> None: ...
|
||||
@property
|
||||
def output_fields(self) -> list[str] | None: ...
|
||||
@output_fields.setter
|
||||
def output_fields(self, arg0: collections.abc.Sequence[str] | None) -> None: ...
|
||||
@property
|
||||
def topk(self) -> int: ...
|
||||
@topk.setter
|
||||
def topk(self, arg0: typing.SupportsInt) -> None: ...
|
||||
@@ -0,0 +1,80 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional, Union
|
||||
|
||||
from ...common import VectorType
|
||||
from . import HnswQueryParam, IVFQueryParam
|
||||
|
||||
__all__ = ["VectorQuery"]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class VectorQuery:
|
||||
"""Represents a vector search query for a specific field in a collection.
|
||||
|
||||
A `VectorQuery` can be constructed using either a document ID (to look up
|
||||
its vector) or an explicit vector. It may optionally include index-specific
|
||||
query parameters to control search behavior (e.g., `ef` for HNSW, `nprobe` for IVF).
|
||||
|
||||
Exactly one of `id` or `vector` should be provided. If both are given,
|
||||
behavior is implementation-defined (typically `id` takes precedence).
|
||||
|
||||
Attributes:
|
||||
field_name (str): Name of the vector field to query.
|
||||
id (Optional[str], optional): Document ID to fetch vector from. Default is None.
|
||||
vector (VectorType, optional): Explicit query vector. Default is None.
|
||||
param (Optional[Union[HnswQueryParam, IVFQueryParam]], optional):
|
||||
Index-specific query parameters. Default is None.
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
>>> # Query by ID
|
||||
>>> q1 = zvec.VectorQuery(field_name="embedding", id="doc123")
|
||||
>>> # Query by vector
|
||||
>>> q2 = zvec.VectorQuery(
|
||||
... field_name="embedding",
|
||||
... vector=[0.1, 0.2, 0.3],
|
||||
... param=HnswQueryParam(ef=300)
|
||||
... )
|
||||
"""
|
||||
|
||||
field_name: str
|
||||
id: Optional[str] = None
|
||||
vector: VectorType = None
|
||||
param: Optional[Union[HnswQueryParam, IVFQueryParam]] = None
|
||||
|
||||
def has_id(self) -> bool:
|
||||
"""Check if the query is based on a document ID.
|
||||
|
||||
Returns:
|
||||
bool: True if `id` is set, False otherwise.
|
||||
"""
|
||||
return self.id is not None
|
||||
|
||||
def has_vector(self) -> bool:
|
||||
"""Check if the query contains an explicit vector.
|
||||
|
||||
Returns:
|
||||
bool: True if `vector` is non-empty, False otherwise.
|
||||
"""
|
||||
return self.vector is not None and len(self.vector) > 0
|
||||
|
||||
def _validate(self) -> None:
|
||||
if self.field_name is None:
|
||||
raise ValueError("Field name cannot be empty")
|
||||
if self.id and self.vector:
|
||||
raise ValueError("Cannot provide both id and vector")
|
||||
@@ -0,0 +1,21 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from _zvec.schema import CollectionStats
|
||||
|
||||
from .collection_schema import CollectionSchema
|
||||
from .field_schema import FieldSchema, VectorSchema
|
||||
|
||||
__all__ = ["CollectionSchema", "CollectionStats", "FieldSchema", "VectorSchema"]
|
||||
@@ -0,0 +1,109 @@
|
||||
"""
|
||||
This module contains the schema of Zvec
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections.abc
|
||||
import typing
|
||||
|
||||
import _zvec.param
|
||||
import _zvec.typing
|
||||
|
||||
from .collection_schema import CollectionSchema
|
||||
from .field_schema import FieldSchema, VectorSchema
|
||||
|
||||
__all__: list[str] = [
|
||||
"CollectionSchema",
|
||||
"CollectionStats",
|
||||
"FieldSchema",
|
||||
"VectorSchema",
|
||||
]
|
||||
|
||||
class CollectionStats:
|
||||
def __init__(self) -> None: ...
|
||||
def __repr__(self) -> str: ...
|
||||
@property
|
||||
def doc_count(self) -> int: ...
|
||||
@property
|
||||
def index_completeness(self) -> dict[str, float]: ...
|
||||
|
||||
class _CollectionSchema:
|
||||
__hash__: typing.ClassVar[None] = None
|
||||
|
||||
def __eq__(self, arg0: _CollectionSchema) -> bool: ...
|
||||
def __init__(
|
||||
self, name: str, fields: collections.abc.Sequence[_FieldSchema]
|
||||
) -> None:
|
||||
"""
|
||||
Construct with name and list of fields
|
||||
"""
|
||||
|
||||
def __ne__(self, arg0: _CollectionSchema) -> bool: ...
|
||||
def fields(self) -> list[_FieldSchema]:
|
||||
"""
|
||||
Return list of all field schemas.
|
||||
"""
|
||||
|
||||
def forward_fields(self) -> list[_FieldSchema]:
|
||||
"""
|
||||
Return list of forward-indexed fields.
|
||||
"""
|
||||
|
||||
def get_field(self, field_name: str) -> _FieldSchema:
|
||||
"""
|
||||
Get field by name (const pointer), returns None if not found.
|
||||
"""
|
||||
|
||||
def get_forward_field(self, field_name: str) -> _FieldSchema:
|
||||
"""
|
||||
Get forward field (used for filtering).
|
||||
"""
|
||||
|
||||
def get_vector_field(self, field_name: str) -> _FieldSchema:
|
||||
"""
|
||||
Get vector field by name.
|
||||
"""
|
||||
|
||||
def has_field(self, field_name: str) -> bool:
|
||||
"""
|
||||
Check if a field exists.
|
||||
"""
|
||||
|
||||
def vector_fields(self) -> list[_FieldSchema]:
|
||||
"""
|
||||
Return list of vector fields.
|
||||
"""
|
||||
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
|
||||
class _FieldSchema:
|
||||
__hash__: typing.ClassVar[None] = None
|
||||
|
||||
def __eq__(self, arg0: _FieldSchema) -> bool: ...
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
data_type: _zvec.typing.DataType,
|
||||
nullable: bool = False,
|
||||
dimension: typing.SupportsInt = 0,
|
||||
index_param: _zvec.param.IndexParam = None,
|
||||
) -> None: ...
|
||||
def __ne__(self, arg0: _FieldSchema) -> bool: ...
|
||||
@property
|
||||
def data_type(self) -> _zvec.typing.DataType: ...
|
||||
@property
|
||||
def dimension(self) -> int: ...
|
||||
@property
|
||||
def index_param(self) -> typing.Any: ...
|
||||
@property
|
||||
def index_type(self) -> _zvec.typing.IndexType: ...
|
||||
@property
|
||||
def is_dense_vector(self) -> bool: ...
|
||||
@property
|
||||
def is_sparse_vector(self) -> bool: ...
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
@property
|
||||
def nullable(self) -> bool: ...
|
||||
@@ -0,0 +1,215 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Optional, Union
|
||||
|
||||
from _zvec.schema import _CollectionSchema, _FieldSchema
|
||||
|
||||
from .field_schema import FieldSchema, VectorSchema
|
||||
|
||||
__all__ = [
|
||||
"CollectionSchema",
|
||||
]
|
||||
|
||||
|
||||
class CollectionSchema:
|
||||
"""Defines the structure of a collection in Zvec.
|
||||
|
||||
A collection schema specifies the name of the collection and its fields,
|
||||
including both scalar fields (e.g., int, string) and vector fields.
|
||||
Field names must be unique across both scalar and vector fields.
|
||||
|
||||
Args:
|
||||
name (str): Name of the collection.
|
||||
fields (Optional[Union[FieldSchema, list[FieldSchema]]], optional):
|
||||
One or more scalar field definitions. Defaults to None.
|
||||
vectors (Optional[Union[VectorSchema, list[VectorSchema]]], optional):
|
||||
One or more vector field definitions. Defaults to None.
|
||||
|
||||
Raises:
|
||||
TypeError: If `fields` or `vectors` are of unsupported types.
|
||||
ValueError: If any field or vector name is duplicated.
|
||||
|
||||
Examples:
|
||||
>>> from zvec import FieldSchema, VectorSchema, DataType, IndexType
|
||||
>>> id_field = FieldSchema("id", DataType.INT64, is_primary=True)
|
||||
>>> emb_field = VectorSchema("embedding", dim=128, data_type=DataType.VECTOR_FP32)
|
||||
>>> schema = CollectionSchema(
|
||||
... name="my_collection",
|
||||
... fields=id_field,
|
||||
... vectors=emb_field
|
||||
... )
|
||||
>>> print(schema.name)
|
||||
my_collection
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
fields: Optional[Union[FieldSchema, list[FieldSchema]]] = None,
|
||||
vectors: Optional[Union[VectorSchema, list[VectorSchema]]] = None,
|
||||
):
|
||||
if name is None or not isinstance(name, str):
|
||||
raise ValueError(
|
||||
f"schema validate failed: collection name must be str, got {type(name).__name__}"
|
||||
)
|
||||
|
||||
# handle fields
|
||||
_fields_name: list[str] = []
|
||||
_fields_list: list[_FieldSchema] = []
|
||||
|
||||
self._check_fields(fields, _fields_name, _fields_list)
|
||||
self._check_vectors(vectors, _fields_name, _fields_list)
|
||||
|
||||
# init
|
||||
self._cpp_obj = _CollectionSchema(
|
||||
name=name,
|
||||
fields=_fields_list,
|
||||
)
|
||||
|
||||
def _check_fields(
|
||||
self,
|
||||
fields: Optional[Union[FieldSchema, list[FieldSchema]]],
|
||||
_fields_name: list[str],
|
||||
_fields_list: list[_FieldSchema],
|
||||
) -> None:
|
||||
field_items = []
|
||||
|
||||
if isinstance(fields, FieldSchema):
|
||||
field_items = [fields]
|
||||
elif isinstance(fields, list):
|
||||
field_items = fields
|
||||
elif fields is None:
|
||||
field_items = []
|
||||
else:
|
||||
raise TypeError(
|
||||
f"schema validate failed: invalid 'fields' type, expected FieldSchema or list[FieldSchema], "
|
||||
f"got {type(fields).__name__}"
|
||||
)
|
||||
|
||||
for idx, field in enumerate(field_items):
|
||||
if not isinstance(field, FieldSchema):
|
||||
raise TypeError(
|
||||
f"schema validate failed: invalid field type in 'fields' list, expected FieldSchema, "
|
||||
f"got {type(field).__name__} at index {idx}"
|
||||
)
|
||||
|
||||
if field.name in _fields_name:
|
||||
raise ValueError(
|
||||
f"schema validate failed: duplicate field name '{field.name}': field names must be unique"
|
||||
)
|
||||
_fields_name.append(field.name)
|
||||
_fields_list.append(field._get_object())
|
||||
|
||||
def _check_vectors(
|
||||
self,
|
||||
vectors: Optional[Union[VectorSchema, list[VectorSchema]]],
|
||||
_fields_name: list[str],
|
||||
_fields_list: list[_FieldSchema],
|
||||
) -> None:
|
||||
# handle vector
|
||||
if isinstance(vectors, VectorSchema):
|
||||
vectors_items = [vectors]
|
||||
elif isinstance(vectors, list):
|
||||
vectors_items = vectors
|
||||
elif vectors is None:
|
||||
vectors_items = []
|
||||
else:
|
||||
raise TypeError(
|
||||
f"schema validate failed: invalid 'vectors' type, expected VectorSchema or list[VectorSchema], "
|
||||
f"got {type(vectors).__name__}"
|
||||
)
|
||||
|
||||
for idx, vector in enumerate(vectors_items):
|
||||
if not isinstance(vector, VectorSchema):
|
||||
raise TypeError(
|
||||
f"schema validate failed: invalid vector type in 'vectors' list, expected VectorSchema, "
|
||||
f"got {type(vector).__name__} at index {idx}"
|
||||
)
|
||||
|
||||
if vector.name in _fields_name:
|
||||
raise ValueError(
|
||||
f"schema validate failed: duplicate vector name '{vector.name}', vector names must be unique "
|
||||
f"(conflicts with existing field or vector)"
|
||||
)
|
||||
_fields_name.append(vector.name)
|
||||
_fields_list.append(vector._get_object())
|
||||
|
||||
@classmethod
|
||||
def _from_core(cls, core_collection_schema: _CollectionSchema):
|
||||
inst = cls.__new__(cls)
|
||||
if not core_collection_schema:
|
||||
raise ValueError("schema validate failed: schema is null")
|
||||
inst._cpp_obj = core_collection_schema
|
||||
return inst
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""str: The name of the collection."""
|
||||
return self._cpp_obj.name
|
||||
|
||||
def field(self, name: str) -> Optional[FieldSchema]:
|
||||
"""Retrieve a scalar field by name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the field.
|
||||
|
||||
Returns:
|
||||
Optional[FieldSchema]: The field if found, otherwise None.
|
||||
"""
|
||||
_field = self._cpp_obj.get_forward_field(name)
|
||||
return FieldSchema._from_core(_field) if _field else None
|
||||
|
||||
def vector(self, name: str) -> Optional[VectorSchema]:
|
||||
"""Retrieve a vector field by name.
|
||||
|
||||
Args:
|
||||
name (str): Name of the vector field.
|
||||
|
||||
Returns:
|
||||
Optional[VectorSchema]: The vector field if found, otherwise None.
|
||||
"""
|
||||
_field = self._cpp_obj.get_vector_field(name)
|
||||
return VectorSchema._from_core(_field) if _field else None
|
||||
|
||||
@property
|
||||
def fields(self) -> list[FieldSchema]:
|
||||
"""list[FieldSchema]: All scalar (non-vector) fields in the schema."""
|
||||
_fields = self._cpp_obj.forward_fields()
|
||||
return [FieldSchema._from_core(_field) for _field in _fields]
|
||||
|
||||
@property
|
||||
def vectors(self) -> list[VectorSchema]:
|
||||
"""list[VectorSchema]: All vector fields in the schema."""
|
||||
_vectors = self._cpp_obj.vector_fields()
|
||||
return [VectorSchema._from_core(_vector) for _vector in _vectors]
|
||||
|
||||
def _get_object(self) -> _CollectionSchema:
|
||||
return self._cpp_obj
|
||||
|
||||
def __repr__(self) -> str:
|
||||
try:
|
||||
schema = {
|
||||
"name": self.name,
|
||||
"fields": {field.name: field.__dict__() for field in self.fields},
|
||||
"vectors": {vector.name: vector.__dict__() for vector in self.vectors},
|
||||
}
|
||||
return json.dumps(schema, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"<CollectionSchema error during repr: {e}>"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__repr__()
|
||||
@@ -0,0 +1,300 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any, Optional, Union
|
||||
|
||||
from _zvec.schema import _FieldSchema
|
||||
|
||||
from zvec.model.param import (
|
||||
FlatIndexParam,
|
||||
HnswIndexParam,
|
||||
InvertIndexParam,
|
||||
IVFIndexParam,
|
||||
)
|
||||
from zvec.typing import DataType
|
||||
|
||||
__all__ = [
|
||||
"FieldSchema",
|
||||
"VectorSchema",
|
||||
]
|
||||
|
||||
SUPPORT_VECTOR_DATA_TYPE = [
|
||||
DataType.VECTOR_FP16,
|
||||
DataType.VECTOR_FP32,
|
||||
DataType.VECTOR_FP64,
|
||||
DataType.VECTOR_INT8,
|
||||
DataType.SPARSE_VECTOR_FP16,
|
||||
DataType.SPARSE_VECTOR_FP32,
|
||||
]
|
||||
|
||||
SUPPORT_SCALAR_DATA_TYPE = [
|
||||
DataType.INT32,
|
||||
DataType.INT64,
|
||||
DataType.UINT32,
|
||||
DataType.UINT64,
|
||||
DataType.FLOAT,
|
||||
DataType.DOUBLE,
|
||||
DataType.STRING,
|
||||
DataType.BOOL,
|
||||
DataType.ARRAY_INT32,
|
||||
DataType.ARRAY_INT64,
|
||||
DataType.ARRAY_UINT32,
|
||||
DataType.ARRAY_UINT64,
|
||||
DataType.ARRAY_FLOAT,
|
||||
DataType.ARRAY_DOUBLE,
|
||||
DataType.ARRAY_STRING,
|
||||
DataType.ARRAY_BOOL,
|
||||
]
|
||||
|
||||
|
||||
class FieldSchema:
|
||||
"""Represents a scalar (non-vector) field in a collection schema.
|
||||
|
||||
A `FieldSchema` defines the name, data type, nullability, and optional
|
||||
inverted index configuration for a regular field (e.g., ID, timestamp, category).
|
||||
|
||||
Args:
|
||||
name (str): Name of the field. Must be unique within the collection.
|
||||
data_type (DataType): Data type of the field (e.g., INT64, STRING).
|
||||
nullable (bool, optional): Whether the field can contain null values.
|
||||
Defaults to False.
|
||||
index_param (Optional[InvertIndexParam], optional): Inverted index
|
||||
parameters for this field. Only applicable to fields that support
|
||||
indexing (e.g., scalar fields used in filtering). Defaults to None.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import DataType
|
||||
>>> from zvec.model.param import InvertIndexParam
|
||||
>>> id_field = FieldSchema(
|
||||
... name="id",
|
||||
... data_type=DataType.INT64,
|
||||
... nullable=False,
|
||||
... index_param=InvertIndexParam(enable_range_optimization=True)
|
||||
... )
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
data_type: DataType,
|
||||
nullable: bool = False,
|
||||
index_param: Optional[InvertIndexParam] = None,
|
||||
):
|
||||
if name is None or not isinstance(name, str):
|
||||
raise ValueError(
|
||||
f"schema validate failed: field name must be str, got {type(name).__name__}"
|
||||
)
|
||||
|
||||
if data_type not in SUPPORT_SCALAR_DATA_TYPE:
|
||||
raise ValueError(
|
||||
f"schema validate failed: scalar_field's data_type must be one of "
|
||||
f"{', '.join(str(dt) for dt in SUPPORT_SCALAR_DATA_TYPE)}, "
|
||||
f"but field[{name}]'s data_type is {data_type}"
|
||||
)
|
||||
|
||||
self._cpp_obj = _FieldSchema(
|
||||
name=name,
|
||||
data_type=data_type,
|
||||
dimension=0,
|
||||
nullable=nullable,
|
||||
index_param=index_param,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _from_core(cls, core_field_schema: _FieldSchema):
|
||||
if core_field_schema is None:
|
||||
raise ValueError("schema validate failed: field schema is None")
|
||||
inst = cls.__new__(cls)
|
||||
inst._cpp_obj = core_field_schema
|
||||
return inst
|
||||
|
||||
def _get_object(self) -> _FieldSchema:
|
||||
return self._cpp_obj
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""str: The name of the field."""
|
||||
return self._cpp_obj.name
|
||||
|
||||
@property
|
||||
def data_type(self) -> DataType:
|
||||
"""DataType: The data type of the field (e.g., INT64, STRING)."""
|
||||
return self._cpp_obj.data_type
|
||||
|
||||
@property
|
||||
def nullable(self) -> bool:
|
||||
"""bool: Whether the field allows null values."""
|
||||
return self._cpp_obj.nullable
|
||||
|
||||
@property
|
||||
def index_param(self) -> Optional[InvertIndexParam]:
|
||||
"""Optional[InvertIndexParam]: Inverted index configuration, if any."""
|
||||
return self._cpp_obj.index_param
|
||||
|
||||
def __dict__(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"data_type": (
|
||||
self.data_type.name
|
||||
if hasattr(self.data_type, "name")
|
||||
else str(self.data_type)
|
||||
),
|
||||
"nullable": self.nullable,
|
||||
"index_param": (
|
||||
self.index_param.to_dict() if self.index_param is not None else None
|
||||
),
|
||||
}
|
||||
|
||||
def __repr__(self) -> str:
|
||||
try:
|
||||
schema = self.__dict__()
|
||||
return json.dumps(schema, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"<FieldSchema error during repr: {e}>"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__repr__()
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, FieldSchema):
|
||||
return False
|
||||
return self._cpp_obj == other._cpp_obj
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash((self.name, self.data_type, self.nullable))
|
||||
|
||||
|
||||
class VectorSchema:
|
||||
"""Represents a vector field in a collection schema.
|
||||
|
||||
A `VectorSchema` defines the name, data type, dimensionality, and index
|
||||
configuration for a vector field used in similarity search.
|
||||
|
||||
Args:
|
||||
name (str): Name of the vector field. Must be unique within the collection.
|
||||
data_type (DataType): Vector data type (e.g., VECTOR_FP32, VECTOR_INT8).
|
||||
dimension (int, optional): Dimensionality of the vector. Must be > 0 for dense vectors;
|
||||
may be `None` for sparse vectors.
|
||||
index_param (Union[HnswIndexParam, IVFIndexParam, FlatIndexParam], optional):
|
||||
Index configuration for this vector field. Defaults to
|
||||
``HnswIndexParam()``.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import DataType
|
||||
>>> from zvec.model.param import HnswIndexParam
|
||||
>>> emb_field = VectorSchema(
|
||||
... name="embedding",
|
||||
... data_type=DataType.VECTOR_FP32,
|
||||
... dimension=128,
|
||||
... index_param=HnswIndexParam(ef_construction=200, m=16)
|
||||
... )
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
name: str,
|
||||
data_type: DataType,
|
||||
dimension: Optional[int] = 0,
|
||||
index_param: Optional[
|
||||
Union[HnswIndexParam, FlatIndexParam, IVFIndexParam]
|
||||
] = None,
|
||||
):
|
||||
if name is None or not isinstance(name, str):
|
||||
raise ValueError(
|
||||
f"schema validate failed: field name must be str, got {type(name).__name__}"
|
||||
)
|
||||
|
||||
if not isinstance(dimension, int) or dimension < 0:
|
||||
raise ValueError("schema validate failed: vector's dimension must be >= 0")
|
||||
|
||||
if data_type not in SUPPORT_VECTOR_DATA_TYPE:
|
||||
raise ValueError(
|
||||
f"schema validate failed: vector's data_type must be one of "
|
||||
f"{', '.join(str(dt) for dt in SUPPORT_VECTOR_DATA_TYPE)}, "
|
||||
f"but field[{name}]'s data_type is {data_type}"
|
||||
)
|
||||
|
||||
if index_param is None:
|
||||
index_param = FlatIndexParam()
|
||||
|
||||
self._cpp_obj = _FieldSchema(
|
||||
name=name,
|
||||
data_type=data_type,
|
||||
dimension=dimension,
|
||||
nullable=False,
|
||||
index_param=index_param,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _from_core(cls, core_field_schema: _FieldSchema):
|
||||
inst = cls.__new__(cls)
|
||||
inst._cpp_obj = core_field_schema
|
||||
return inst
|
||||
|
||||
def _get_object(self) -> _FieldSchema:
|
||||
return self._cpp_obj
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
"""str: The name of the vector field."""
|
||||
return self._cpp_obj.name
|
||||
|
||||
@property
|
||||
def data_type(self) -> DataType:
|
||||
"""DataType: The vector data type (e.g., VECTOR_FP32)."""
|
||||
return self._cpp_obj.data_type
|
||||
|
||||
@property
|
||||
def dimension(self) -> int:
|
||||
"""int: The dimensionality of the vector."""
|
||||
return self._cpp_obj.dimension
|
||||
|
||||
@property
|
||||
def index_param(self) -> Union[HnswIndexParam, IVFIndexParam, FlatIndexParam]:
|
||||
"""Union[HnswIndexParam, IVFIndexParam, FlatIndexParam]: Index configuration for the vector."""
|
||||
return self._cpp_obj.index_param
|
||||
|
||||
def __dict__(self) -> dict[str, Any]:
|
||||
return {
|
||||
"name": self.name,
|
||||
"data_type": (
|
||||
self.data_type.name
|
||||
if hasattr(self.data_type, "name")
|
||||
else str(self.data_type)
|
||||
),
|
||||
"dimension": self.dimension,
|
||||
"index_param": (
|
||||
self.index_param.to_dict() if self.index_param is not None else None
|
||||
),
|
||||
}
|
||||
|
||||
def __repr__(self) -> str:
|
||||
try:
|
||||
schema = self.__dict__()
|
||||
return json.dumps(schema, indent=2, ensure_ascii=False)
|
||||
except Exception as e:
|
||||
return f"<FieldSchema error during repr: {e}>"
|
||||
|
||||
def __str__(self) -> str:
|
||||
return self.__repr__()
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if not isinstance(other, VectorSchema):
|
||||
return False
|
||||
return self._cpp_obj == other._cpp_obj
|
||||
|
||||
def __hash__(self) -> int:
|
||||
return hash((self.name, self.data_type, self.dimension))
|
||||
@@ -0,0 +1,18 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from .util import require_module
|
||||
|
||||
__all__ = ["require_module"]
|
||||
@@ -0,0 +1,63 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
from typing import Any, Optional
|
||||
|
||||
|
||||
def require_module(module: str, mitigation: Optional[str] = None) -> Any:
|
||||
"""Import a Python module and raise a user-friendly error if it is not available.
|
||||
|
||||
This utility helps provide actionable error messages when optional dependencies
|
||||
are missing. It attempts to import the given module and, on failure, suggests
|
||||
a `pip install` command based on either the module name or an optional
|
||||
mitigation package name.
|
||||
|
||||
Args:
|
||||
module (str): The full module name to import (e.g., ``"numpy"``, ``"pandas.io.parquet"``).
|
||||
mitigation (Optional[str], optional): The package name to suggest for installation
|
||||
if the import fails. If not provided, the top-level package of `module`
|
||||
will be used (e.g., ``"pandas"`` for ``"pandas.io.parquet"``).
|
||||
|
||||
Returns:
|
||||
Any: The imported module object.
|
||||
|
||||
Raises:
|
||||
ImportError: If the module cannot be imported, with a clear installation hint.
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
>>> np = zvec.require_module("numpy")
|
||||
>>> pq = zvec.require_module("pyarrow.parquet", mitigation="pyarrow")
|
||||
|
||||
Note:
|
||||
This function is intended for lazy-loading optional dependencies
|
||||
with helpful error messages, not for core dependencies.
|
||||
"""
|
||||
try:
|
||||
return importlib.import_module(module)
|
||||
except ImportError as e:
|
||||
package = mitigation or module
|
||||
msg = f"Required package '{package}' is not installed. "
|
||||
if "." in module:
|
||||
top_level = module.split(".")[0]
|
||||
msg += f"Module '{module}' is part of '{top_level}', "
|
||||
if mitigation:
|
||||
msg += f"please pip install '{mitigation}'."
|
||||
else:
|
||||
msg += f"please pip install '{top_level}'."
|
||||
else:
|
||||
msg += f"Please pip install '{package}."
|
||||
raise ImportError(msg) from e
|
||||
@@ -0,0 +1,32 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from _zvec.typing import (
|
||||
DataType,
|
||||
IndexType,
|
||||
MetricType,
|
||||
QuantizeType,
|
||||
Status,
|
||||
StatusCode,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"DataType",
|
||||
"IndexType",
|
||||
"MetricType",
|
||||
"QuantizeType",
|
||||
"Status",
|
||||
"StatusCode",
|
||||
]
|
||||
@@ -0,0 +1,404 @@
|
||||
"""
|
||||
This module contains the basic data types of Zvec
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
__all__: list[str] = [
|
||||
"DataType",
|
||||
"IndexType",
|
||||
"MetricType",
|
||||
"QuantizeType",
|
||||
"Status",
|
||||
"StatusCode",
|
||||
]
|
||||
|
||||
class DataType:
|
||||
"""
|
||||
|
||||
Enumeration of supported data types in Zvec.
|
||||
|
||||
Includes scalar types, dense/sparse vector types, and array types.
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
>>> print(zvec.DataType.FLOAT)
|
||||
DataType.FLOAT
|
||||
>>> print(zvec.DataType.VECTOR_FP32)
|
||||
DataType.VECTOR_FP32
|
||||
|
||||
|
||||
Members:
|
||||
|
||||
STRING
|
||||
|
||||
BOOL
|
||||
|
||||
INT32
|
||||
|
||||
INT64
|
||||
|
||||
FLOAT
|
||||
|
||||
DOUBLE
|
||||
|
||||
UINT32
|
||||
|
||||
UINT64
|
||||
|
||||
VECTOR_FP16
|
||||
|
||||
VECTOR_FP32
|
||||
|
||||
VECTOR_FP64
|
||||
|
||||
VECTOR_INT8
|
||||
|
||||
SPARSE_VECTOR_FP32
|
||||
|
||||
SPARSE_VECTOR_FP16
|
||||
|
||||
ARRAY_STRING
|
||||
|
||||
ARRAY_INT32
|
||||
|
||||
ARRAY_INT64
|
||||
|
||||
ARRAY_FLOAT
|
||||
|
||||
ARRAY_DOUBLE
|
||||
|
||||
ARRAY_BOOL
|
||||
|
||||
ARRAY_UINT32
|
||||
|
||||
ARRAY_UINT64
|
||||
"""
|
||||
|
||||
ARRAY_BOOL: typing.ClassVar[DataType] # value = <DataType.ARRAY_BOOL: 42>
|
||||
ARRAY_DOUBLE: typing.ClassVar[DataType] # value = <DataType.ARRAY_DOUBLE: 48>
|
||||
ARRAY_FLOAT: typing.ClassVar[DataType] # value = <DataType.ARRAY_FLOAT: 47>
|
||||
ARRAY_INT32: typing.ClassVar[DataType] # value = <DataType.ARRAY_INT32: 43>
|
||||
ARRAY_INT64: typing.ClassVar[DataType] # value = <DataType.ARRAY_INT64: 44>
|
||||
ARRAY_STRING: typing.ClassVar[DataType] # value = <DataType.ARRAY_STRING: 41>
|
||||
ARRAY_UINT32: typing.ClassVar[DataType] # value = <DataType.ARRAY_UINT32: 45>
|
||||
ARRAY_UINT64: typing.ClassVar[DataType] # value = <DataType.ARRAY_UINT64: 46>
|
||||
BOOL: typing.ClassVar[DataType] # value = <DataType.BOOL: 3>
|
||||
DOUBLE: typing.ClassVar[DataType] # value = <DataType.DOUBLE: 9>
|
||||
FLOAT: typing.ClassVar[DataType] # value = <DataType.FLOAT: 8>
|
||||
INT32: typing.ClassVar[DataType] # value = <DataType.INT32: 4>
|
||||
INT64: typing.ClassVar[DataType] # value = <DataType.INT64: 5>
|
||||
SPARSE_VECTOR_FP16: typing.ClassVar[
|
||||
DataType
|
||||
] # value = <DataType.SPARSE_VECTOR_FP16: 30>
|
||||
SPARSE_VECTOR_FP32: typing.ClassVar[
|
||||
DataType
|
||||
] # value = <DataType.SPARSE_VECTOR_FP32: 31>
|
||||
STRING: typing.ClassVar[DataType] # value = <DataType.STRING: 2>
|
||||
UINT32: typing.ClassVar[DataType] # value = <DataType.UINT32: 6>
|
||||
UINT64: typing.ClassVar[DataType] # value = <DataType.UINT64: 7>
|
||||
VECTOR_FP16: typing.ClassVar[DataType] # value = <DataType.VECTOR_FP16: 22>
|
||||
VECTOR_FP32: typing.ClassVar[DataType] # value = <DataType.VECTOR_FP32: 23>
|
||||
VECTOR_FP64: typing.ClassVar[DataType] # value = <DataType.VECTOR_FP64: 24>
|
||||
VECTOR_INT8: typing.ClassVar[DataType] # value = <DataType.VECTOR_INT8: 26>
|
||||
__members__: typing.ClassVar[
|
||||
dict[str, DataType]
|
||||
] # value = {'STRING': <DataType.STRING: 2>, 'BOOL': <DataType.BOOL: 3>, 'INT32': <DataType.INT32: 4>, 'INT64': <DataType.INT64: 5>, 'FLOAT': <DataType.FLOAT: 8>, 'DOUBLE': <DataType.DOUBLE: 9>, 'UINT32': <DataType.UINT32: 6>, 'UINT64': <DataType.UINT64: 7>, 'VECTOR_FP16': <DataType.VECTOR_FP16: 22>, 'VECTOR_FP32': <DataType.VECTOR_FP32: 23>, 'VECTOR_FP64': <DataType.VECTOR_FP64: 24>, 'VECTOR_INT8': <DataType.VECTOR_INT8: 26>, 'SPARSE_VECTOR_FP32': <DataType.SPARSE_VECTOR_FP32: 31>, 'SPARSE_VECTOR_FP16': <DataType.SPARSE_VECTOR_FP16: 30>, 'ARRAY_STRING': <DataType.ARRAY_STRING: 41>, 'ARRAY_INT32': <DataType.ARRAY_INT32: 43>, 'ARRAY_INT64': <DataType.ARRAY_INT64: 44>, 'ARRAY_FLOAT': <DataType.ARRAY_FLOAT: 47>, 'ARRAY_DOUBLE': <DataType.ARRAY_DOUBLE: 48>, 'ARRAY_BOOL': <DataType.ARRAY_BOOL: 42>, 'ARRAY_UINT32': <DataType.ARRAY_UINT32: 45>, 'ARRAY_UINT64': <DataType.ARRAY_UINT64: 46>}
|
||||
|
||||
def __eq__(self, other: typing.Any) -> bool: ...
|
||||
def __getstate__(self) -> int: ...
|
||||
def __hash__(self) -> int: ...
|
||||
def __index__(self) -> int: ...
|
||||
def __init__(self, value: typing.SupportsInt) -> None: ...
|
||||
def __int__(self) -> int: ...
|
||||
def __ne__(self, other: typing.Any) -> bool: ...
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, state: typing.SupportsInt) -> None: ...
|
||||
def __str__(self) -> str: ...
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
@property
|
||||
def value(self) -> int: ...
|
||||
|
||||
class IndexType:
|
||||
"""
|
||||
|
||||
Enumeration of supported index types in Zvec.
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
>>> print(zvec.IndexType.HNSW)
|
||||
IndexType.HNSW
|
||||
|
||||
|
||||
Members:
|
||||
|
||||
UNDEFINED
|
||||
|
||||
HNSW
|
||||
|
||||
IVF
|
||||
|
||||
FLAT
|
||||
|
||||
INVERT
|
||||
"""
|
||||
|
||||
FLAT: typing.ClassVar[IndexType] # value = <IndexType.FLAT: 4>
|
||||
HNSW: typing.ClassVar[IndexType] # value = <IndexType.HNSW: 1>
|
||||
INVERT: typing.ClassVar[IndexType] # value = <IndexType.INVERT: 10>
|
||||
IVF: typing.ClassVar[IndexType] # value = <IndexType.IVF: 3>
|
||||
UNDEFINED: typing.ClassVar[IndexType] # value = <IndexType.UNDEFINED: 0>
|
||||
__members__: typing.ClassVar[
|
||||
dict[str, IndexType]
|
||||
] # value = {'UNDEFINED': <IndexType.UNDEFINED: 0>, 'HNSW': <IndexType.HNSW: 1>, 'IVF': <IndexType.IVF: 3>, 'FLAT': <IndexType.FLAT: 4>, 'INVERT': <IndexType.INVERT: 10>}
|
||||
|
||||
def __eq__(self, other: typing.Any) -> bool: ...
|
||||
def __getstate__(self) -> int: ...
|
||||
def __hash__(self) -> int: ...
|
||||
def __index__(self) -> int: ...
|
||||
def __init__(self, value: typing.SupportsInt) -> None: ...
|
||||
def __int__(self) -> int: ...
|
||||
def __ne__(self, other: typing.Any) -> bool: ...
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, state: typing.SupportsInt) -> None: ...
|
||||
def __str__(self) -> str: ...
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
@property
|
||||
def value(self) -> int: ...
|
||||
|
||||
class MetricType:
|
||||
"""
|
||||
|
||||
Enumeration of supported distance/similarity metrics.
|
||||
|
||||
- COSINE: Cosine similarity.
|
||||
- IP: Inner product (dot product).
|
||||
- L2: Euclidean distance (L2 norm).
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
>>> print(zvec.MetricType.COSINE)
|
||||
MetricType.COSINE
|
||||
|
||||
|
||||
Members:
|
||||
|
||||
COSINE
|
||||
|
||||
IP
|
||||
|
||||
L2
|
||||
"""
|
||||
|
||||
COSINE: typing.ClassVar[MetricType] # value = <MetricType.COSINE: 3>
|
||||
IP: typing.ClassVar[MetricType] # value = <MetricType.IP: 2>
|
||||
L2: typing.ClassVar[MetricType] # value = <MetricType.L2: 1>
|
||||
__members__: typing.ClassVar[
|
||||
dict[str, MetricType]
|
||||
] # value = {'COSINE': <MetricType.COSINE: 3>, 'IP': <MetricType.IP: 2>, 'L2': <MetricType.L2: 1>}
|
||||
|
||||
def __eq__(self, other: typing.Any) -> bool: ...
|
||||
def __getstate__(self) -> int: ...
|
||||
def __hash__(self) -> int: ...
|
||||
def __index__(self) -> int: ...
|
||||
def __init__(self, value: typing.SupportsInt) -> None: ...
|
||||
def __int__(self) -> int: ...
|
||||
def __ne__(self, other: typing.Any) -> bool: ...
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, state: typing.SupportsInt) -> None: ...
|
||||
def __str__(self) -> str: ...
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
@property
|
||||
def value(self) -> int: ...
|
||||
|
||||
class QuantizeType:
|
||||
"""
|
||||
|
||||
Enumeration of supported quantization types for vector compression.
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
>>> print(zvec.QuantizeType.INT8)
|
||||
QuantizeType.INT8
|
||||
|
||||
|
||||
Members:
|
||||
|
||||
UNDEFINED
|
||||
|
||||
FP16
|
||||
|
||||
INT8
|
||||
|
||||
INT4
|
||||
"""
|
||||
|
||||
FP16: typing.ClassVar[QuantizeType] # value = <QuantizeType.FP16: 1>
|
||||
INT4: typing.ClassVar[QuantizeType] # value = <QuantizeType.INT4: 3>
|
||||
INT8: typing.ClassVar[QuantizeType] # value = <QuantizeType.INT8: 2>
|
||||
UNDEFINED: typing.ClassVar[QuantizeType] # value = <QuantizeType.UNDEFINED: 0>
|
||||
__members__: typing.ClassVar[
|
||||
dict[str, QuantizeType]
|
||||
] # value = {'UNDEFINED': <QuantizeType.UNDEFINED: 0>, 'FP16': <QuantizeType.FP16: 1>, 'INT8': <QuantizeType.INT8: 2>, 'INT4': <QuantizeType.INT4: 3>}
|
||||
|
||||
def __eq__(self, other: typing.Any) -> bool: ...
|
||||
def __getstate__(self) -> int: ...
|
||||
def __hash__(self) -> int: ...
|
||||
def __index__(self) -> int: ...
|
||||
def __init__(self, value: typing.SupportsInt) -> None: ...
|
||||
def __int__(self) -> int: ...
|
||||
def __ne__(self, other: typing.Any) -> bool: ...
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, state: typing.SupportsInt) -> None: ...
|
||||
def __str__(self) -> str: ...
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
@property
|
||||
def value(self) -> int: ...
|
||||
|
||||
class Status:
|
||||
"""
|
||||
|
||||
Represents the outcome of a Zvec operation.
|
||||
|
||||
A `Status` object is either OK (success) or carries an error code and message.
|
||||
|
||||
Examples:
|
||||
>>> from zvec.typing import Status, StatusCode
|
||||
>>> s = Status()
|
||||
>>> print(s.ok())
|
||||
True
|
||||
>>> s = Status(StatusCode.INVALID_ARGUMENT, "Field not found")
|
||||
>>> print(s.code() == StatusCode.INVALID_ARGUMENT)
|
||||
True
|
||||
>>> print(s.message())
|
||||
Field not found
|
||||
"""
|
||||
|
||||
__hash__: typing.ClassVar[None] = None
|
||||
|
||||
@staticmethod
|
||||
def AlreadyExists(message: str) -> Status: ...
|
||||
@staticmethod
|
||||
def InternalError(message: str) -> Status: ...
|
||||
@staticmethod
|
||||
def InvalidArgument(message: str) -> Status: ...
|
||||
@staticmethod
|
||||
def NotFound(message: str) -> Status: ...
|
||||
@staticmethod
|
||||
def OK() -> Status:
|
||||
"""
|
||||
Create an OK status.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def PermissionDenied(message: str) -> Status: ...
|
||||
def __eq__(self, arg0: Status) -> bool: ...
|
||||
@typing.overload
|
||||
def __init__(self) -> None: ...
|
||||
@typing.overload
|
||||
def __init__(self, code: StatusCode, message: str = "") -> None:
|
||||
"""
|
||||
Construct a status with the given code and optional message.
|
||||
|
||||
Args:
|
||||
code (StatusCode): The status code.
|
||||
message (str, optional): Error message. Defaults to empty string.
|
||||
"""
|
||||
|
||||
def __ne__(self, arg0: Status) -> bool: ...
|
||||
def __repr__(self) -> str: ...
|
||||
def code(self) -> StatusCode:
|
||||
"""
|
||||
StatusCode: Returns the status code.
|
||||
"""
|
||||
|
||||
def message(self) -> str:
|
||||
"""
|
||||
str: Returns the error message (may be empty).
|
||||
"""
|
||||
|
||||
def ok(self) -> bool:
|
||||
"""
|
||||
bool: Returns True if the status is OK.
|
||||
"""
|
||||
|
||||
class StatusCode:
|
||||
"""
|
||||
|
||||
Enumeration of possible status codes for Zvec operations.
|
||||
|
||||
Used by the `Status` class to indicate success or failure reason.
|
||||
|
||||
|
||||
Members:
|
||||
|
||||
OK
|
||||
|
||||
NOT_FOUND
|
||||
|
||||
ALREADY_EXISTS
|
||||
|
||||
INVALID_ARGUMENT
|
||||
|
||||
PERMISSION_DENIED
|
||||
|
||||
FAILED_PRECONDITION
|
||||
|
||||
RESOURCE_EXHAUSTED
|
||||
|
||||
UNAVAILABLE
|
||||
|
||||
INTERNAL_ERROR
|
||||
|
||||
NOT_SUPPORTED
|
||||
|
||||
UNKNOWN
|
||||
"""
|
||||
|
||||
ALREADY_EXISTS: typing.ClassVar[
|
||||
StatusCode
|
||||
] # value = <StatusCode.ALREADY_EXISTS: 2>
|
||||
FAILED_PRECONDITION: typing.ClassVar[
|
||||
StatusCode
|
||||
] # value = <StatusCode.FAILED_PRECONDITION: 5>
|
||||
INTERNAL_ERROR: typing.ClassVar[
|
||||
StatusCode
|
||||
] # value = <StatusCode.INTERNAL_ERROR: 8>
|
||||
INVALID_ARGUMENT: typing.ClassVar[
|
||||
StatusCode
|
||||
] # value = <StatusCode.INVALID_ARGUMENT: 3>
|
||||
NOT_FOUND: typing.ClassVar[StatusCode] # value = <StatusCode.NOT_FOUND: 1>
|
||||
NOT_SUPPORTED: typing.ClassVar[StatusCode] # value = <StatusCode.NOT_SUPPORTED: 9>
|
||||
OK: typing.ClassVar[StatusCode] # value = <StatusCode.OK: 0>
|
||||
PERMISSION_DENIED: typing.ClassVar[
|
||||
StatusCode
|
||||
] # value = <StatusCode.PERMISSION_DENIED: 4>
|
||||
RESOURCE_EXHAUSTED: typing.ClassVar[
|
||||
StatusCode
|
||||
] # value = <StatusCode.RESOURCE_EXHAUSTED: 6>
|
||||
UNAVAILABLE: typing.ClassVar[StatusCode] # value = <StatusCode.UNAVAILABLE: 7>
|
||||
UNKNOWN: typing.ClassVar[StatusCode] # value = <StatusCode.UNKNOWN: 10>
|
||||
__members__: typing.ClassVar[
|
||||
dict[str, StatusCode]
|
||||
] # value = {'OK': <StatusCode.OK: 0>, 'NOT_FOUND': <StatusCode.NOT_FOUND: 1>, 'ALREADY_EXISTS': <StatusCode.ALREADY_EXISTS: 2>, 'INVALID_ARGUMENT': <StatusCode.INVALID_ARGUMENT: 3>, 'PERMISSION_DENIED': <StatusCode.PERMISSION_DENIED: 4>, 'FAILED_PRECONDITION': <StatusCode.FAILED_PRECONDITION: 5>, 'RESOURCE_EXHAUSTED': <StatusCode.RESOURCE_EXHAUSTED: 6>, 'UNAVAILABLE': <StatusCode.UNAVAILABLE: 7>, 'INTERNAL_ERROR': <StatusCode.INTERNAL_ERROR: 8>, 'NOT_SUPPORTED': <StatusCode.NOT_SUPPORTED: 9>, 'UNKNOWN': <StatusCode.UNKNOWN: 10>}
|
||||
|
||||
def __eq__(self, other: typing.Any) -> bool: ...
|
||||
def __getstate__(self) -> int: ...
|
||||
def __hash__(self) -> int: ...
|
||||
def __index__(self) -> int: ...
|
||||
def __init__(self, value: typing.SupportsInt) -> None: ...
|
||||
def __int__(self) -> int: ...
|
||||
def __ne__(self, other: typing.Any) -> bool: ...
|
||||
def __repr__(self) -> str: ...
|
||||
def __setstate__(self, state: typing.SupportsInt) -> None: ...
|
||||
def __str__(self) -> str: ...
|
||||
@property
|
||||
def name(self) -> str: ...
|
||||
@property
|
||||
def value(self) -> int: ...
|
||||
@@ -0,0 +1,62 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from enum import IntEnum
|
||||
|
||||
__all__ = ["LogLevel", "LogType"]
|
||||
|
||||
|
||||
class LogLevel(IntEnum):
|
||||
"""Enumeration of logging severity levels, ordered from lowest to highest priority.
|
||||
|
||||
Used to control verbosity and filtering of log messages. Higher numeric values
|
||||
indicate more severe conditions.
|
||||
|
||||
Note:
|
||||
``WARNING`` is an alias for ``WARN`` to match Python's built-in :mod:`logging`
|
||||
module convention.
|
||||
|
||||
Attributes:
|
||||
DEBUG (int): Detailed information, typically of interest only when diagnosing problems.
|
||||
INFO (int): Confirmation that things are working as expected.
|
||||
WARN (int): An indication that something unexpected happened, or indicative of
|
||||
potential future problems. (Alias: ``WARNING``)
|
||||
WARNING (int): Same as ``WARN``.
|
||||
ERROR (int): Due to a more serious problem, the software has not been able
|
||||
to perform some function.
|
||||
FATAL (int): A serious error, indicating that the program itself may be unable
|
||||
to continue running.
|
||||
"""
|
||||
|
||||
DEBUG = 0
|
||||
INFO = 1
|
||||
WARN = 2
|
||||
WARNING = 2
|
||||
ERROR = 3
|
||||
FATAL = 4
|
||||
|
||||
|
||||
class LogType(IntEnum):
|
||||
"""Enumeration of supported log output destinations.
|
||||
|
||||
Specifies where log messages should be written.
|
||||
|
||||
Attributes:
|
||||
CONSOLE (int): Output logs to standard output/error (e.g., terminal or IDE console).
|
||||
FILE (int): Write logs to a persistent file on disk.
|
||||
"""
|
||||
|
||||
CONSOLE = 0
|
||||
FILE = 1
|
||||
@@ -0,0 +1,226 @@
|
||||
# Copyright 2025-present the zvec project
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from _zvec import Initialize, _Collection
|
||||
|
||||
from .model import Collection
|
||||
from .model.param import CollectionOption
|
||||
from .model.schema import CollectionSchema
|
||||
|
||||
__all__ = ["create_and_open", "init", "open"]
|
||||
|
||||
from .typing.enum import LogLevel, LogType
|
||||
|
||||
|
||||
def init(
|
||||
*,
|
||||
log_type: Optional[LogType] = LogType.CONSOLE,
|
||||
log_level: Optional[LogLevel] = LogLevel.WARN,
|
||||
log_dir: Optional[str] = "./logs",
|
||||
log_basename: Optional[str] = "zvec.log",
|
||||
log_file_size: Optional[int] = 2048,
|
||||
log_overdue_days: Optional[int] = 7,
|
||||
query_threads: Optional[int] = None,
|
||||
optimize_threads: Optional[int] = None,
|
||||
invert_to_forward_scan_ratio: Optional[float] = None,
|
||||
brute_force_by_keys_ratio: Optional[float] = None,
|
||||
memory_limit_mb: Optional[int] = None,
|
||||
) -> None:
|
||||
"""Initialize Zvec with configuration options.
|
||||
|
||||
This function must be called before any other operation.
|
||||
It can only be called once — subsequent calls raise a ``RuntimeError``.
|
||||
|
||||
Parameters set to ``None`` are **omitted** from the configuration and
|
||||
fall back to Zvec's internal defaults, which may be derived from the runtime
|
||||
environment (e.g., cgroup CPU/memory limits). Explicitly provided values
|
||||
always override defaults.
|
||||
|
||||
Args:
|
||||
log_type (Optional[LogType], optional): Logger destination.
|
||||
- ``LogType.CONSOLE`` (default if omitted or set to this)
|
||||
- ``LogType.FILE``
|
||||
- If ``None``, uses internal default (currently ``CONSOLE``).
|
||||
log_level (Optional[LogLevel], optional): Minimum log severity.
|
||||
Default: ``LogLevel.WARN``.
|
||||
Accepted values: ``DEBUG``, ``INFO``, ``WARN``, ``ERROR``, ``FATAL``.
|
||||
If ``None``, uses internal default (``WARN``).
|
||||
log_dir (Optional[str], optional):
|
||||
Directory for log files (only used when ``log_type=FILE``).
|
||||
Parent directories are **not** created automatically.
|
||||
Default: ``"./logs"``.
|
||||
If ``None``, internal default is used.
|
||||
log_basename (Optional[str], optional):
|
||||
Base name for rotated log files (e.g., ``zvec.log.1``, ``zvec.log.2``).
|
||||
Default: ``"zvec.log"``.
|
||||
log_file_size (Optional[int], optional):
|
||||
Max size per log file in **MB** before rotation.
|
||||
Default: ``2048`` MB (2 GB).
|
||||
log_overdue_days (Optional[int], optional):
|
||||
Days to retain rotated log files before deletion.
|
||||
Default: ``7`` days.
|
||||
query_threads (Optional[int], optional):
|
||||
Number of threads for query execution.
|
||||
If ``None`` (default), inferred from available CPU cores (via cgroup).
|
||||
Must be ≥ 1 if provided.
|
||||
optimize_threads (Optional[int], optional):
|
||||
Threads for background tasks (e.g., compaction, indexing).
|
||||
If ``None``, defaults to same as ``query_threads`` or CPU count.
|
||||
invert_to_forward_scan_ratio (Optional[float], optional):
|
||||
Threshold to switch from inverted index to full forward scan.
|
||||
Range: [0.0, 1.0]. Higher → more aggressive index skipping.
|
||||
Default: ``0.9`` (if omitted).
|
||||
brute_force_by_keys_ratio (Optional[float], optional):
|
||||
Threshold to use brute-force key lookup over index.
|
||||
Lower → prefer index; higher → prefer brute-force.
|
||||
Range: [0.0, 1.0]. Default: ``0.1``.
|
||||
memory_limit_mb (Optional[int], optional):
|
||||
Soft memory cap in MB. Zvec may throttle or fail operations
|
||||
approaching this limit.
|
||||
If ``None``, inferred from cgroup memory limit * 0.8 (e.g., in Docker).
|
||||
Must be > 0 if provided.
|
||||
|
||||
Raises:
|
||||
RuntimeError: If Zvec is already initialized.
|
||||
ValueError: On invalid values (e.g., negative thread count, log level out of range).
|
||||
TypeError: If a value has incorrect type (e.g., string for ``query_threads``).
|
||||
|
||||
Note:
|
||||
- All ``None`` arguments are **excluded** from the configuration payload,
|
||||
allowing the core library to apply environment-aware defaults.
|
||||
- This design ensures container-friendliness: in Kubernetes/Docker,
|
||||
omitting ``memory_limit_mb`` and thread counts lets Zvec auto-adapt.
|
||||
|
||||
Examples:
|
||||
Initialize with defaults (log to console, auto-detect resources):
|
||||
>>> import zvec
|
||||
>>> zvec.init()
|
||||
|
||||
Customize logging to file with rotation:
|
||||
>>> zvec.init(
|
||||
... log_type=LogType.FILE,
|
||||
... log_dir="/var/log/zvec",
|
||||
... log_file_size=1024,
|
||||
... log_overdue_days=30
|
||||
... )
|
||||
|
||||
Limit resources explicitly:
|
||||
>>> zvec.init(
|
||||
... memory_limit_mb=2048,
|
||||
... query_threads=4,
|
||||
... optimize_threads=2
|
||||
... )
|
||||
|
||||
Fine-tune query heuristics:
|
||||
>>> zvec.init(
|
||||
... invert_to_forward_scan_ratio=0.95,
|
||||
... brute_force_by_keys_ratio=0.05
|
||||
... )
|
||||
"""
|
||||
# Build config dict, skipping None values
|
||||
config_dict = {}
|
||||
if log_type is not None:
|
||||
if not isinstance(log_type, LogType):
|
||||
raise TypeError("log_type must be LogType")
|
||||
config_dict["log_type"] = log_type.name
|
||||
if log_level is not None:
|
||||
if not isinstance(log_level, LogLevel):
|
||||
raise TypeError("log_level must be LogLevel")
|
||||
config_dict["log_level"] = log_level.name
|
||||
if log_dir is not None:
|
||||
config_dict["log_dir"] = log_dir
|
||||
if log_basename is not None:
|
||||
config_dict["log_basename"] = log_basename
|
||||
if log_file_size is not None:
|
||||
config_dict["log_file_size"] = log_file_size
|
||||
if log_overdue_days is not None:
|
||||
config_dict["log_overdue_days"] = log_overdue_days
|
||||
if query_threads is not None:
|
||||
config_dict["query_threads"] = query_threads
|
||||
if optimize_threads is not None:
|
||||
config_dict["optimize_threads"] = optimize_threads
|
||||
if invert_to_forward_scan_ratio is not None:
|
||||
config_dict["invert_to_forward_scan_ratio"] = invert_to_forward_scan_ratio
|
||||
if brute_force_by_keys_ratio is not None:
|
||||
config_dict["brute_force_by_keys_ratio"] = brute_force_by_keys_ratio
|
||||
if memory_limit_mb is not None:
|
||||
config_dict["memory_limit_mb"] = memory_limit_mb
|
||||
|
||||
Initialize(config_dict)
|
||||
|
||||
|
||||
def create_and_open(
|
||||
path: str,
|
||||
schema: CollectionSchema,
|
||||
option: Optional[CollectionOption] = None,
|
||||
) -> Collection:
|
||||
"""Create a new collection and open it for use.
|
||||
|
||||
If a collection already exists at the given path, it may raise an error
|
||||
depending on the underlying implementation.
|
||||
|
||||
Args:
|
||||
path (str): Path or name of the collection to create.
|
||||
schema (CollectionSchema): Schema defining the structure of the collection.
|
||||
option (CollectionOption): Configuration options
|
||||
for opening the collection. Defaults to a default-constructed
|
||||
``CollectionOption()`` if not provided.
|
||||
|
||||
Returns:
|
||||
Collection: An opened collection instance ready for operations.
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
>>> schema = zvec.CollectionSchema(
|
||||
... name="my_collection",
|
||||
... fields=[zvec.FieldSchema("id", zvec.DataType.INT64, nullable=True)]
|
||||
... )
|
||||
>>> coll = create_and_open("./my_collection", schema)
|
||||
"""
|
||||
if not isinstance(path, str):
|
||||
raise TypeError("path must be a string")
|
||||
if not isinstance(schema, CollectionSchema):
|
||||
raise TypeError("schema must be a CollectionSchema")
|
||||
|
||||
option = option or CollectionOption()
|
||||
if not isinstance(option, CollectionOption):
|
||||
raise TypeError("option must be a CollectionOption")
|
||||
|
||||
_collection = _Collection.CreateAndOpen(path, schema._get_object(), option)
|
||||
return Collection._from_core(_collection)
|
||||
|
||||
|
||||
def open(path: str, option: CollectionOption = CollectionOption()) -> Collection:
|
||||
"""Open an existing collection from disk.
|
||||
|
||||
The collection must have been previously created with ``create_and_open``.
|
||||
|
||||
Args:
|
||||
path (str): Path or name of the existing collection.
|
||||
option (CollectionOption): Configuration options
|
||||
for opening the collection. Defaults to a default-constructed
|
||||
``CollectionOption()`` if not provided.
|
||||
|
||||
Returns:
|
||||
Collection: An opened collection instance.
|
||||
|
||||
Examples:
|
||||
>>> import zvec
|
||||
>>> coll = zvec.open("./my_collection")
|
||||
"""
|
||||
_collection = _Collection.Open(path, option)
|
||||
return Collection._from_core(_collection)
|
||||
Executable
+41
@@ -0,0 +1,41 @@
|
||||
#!/bin/bash
|
||||
|
||||
project_name=proxima-zvec
|
||||
gcov_tool=gcov
|
||||
zip_html=false
|
||||
output_name=html
|
||||
keep_info=false
|
||||
|
||||
script_dir=$(cd "$(dirname "$0")"; pwd)
|
||||
source_base=$(dirname "$script_dir")
|
||||
filter_list="'*/tests/*' '*/thirdparty/*' '*/deps/*' '*/proto/*' '*/external/*' '*/sqlengine/antlr/gen/*'"
|
||||
|
||||
while getopts t:p:o:zk option; do
|
||||
case "$option" in
|
||||
t)
|
||||
gcov_tool=$OPTARG;;
|
||||
p)
|
||||
project_name=$OPTARG;;
|
||||
o)
|
||||
output_name=$OPTARG;;
|
||||
z)
|
||||
zip_html=true;;
|
||||
k)
|
||||
keep_info=true;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Process sources
|
||||
lcov -c -b "$source_base" -d . -o $project_name.lcov.info --gcov-tool=$gcov_tool --no-external || exit 1
|
||||
eval $(echo lcov -r $project_name.lcov.info -o $project_name-filtered.lcov.info $filter_list) || exit 1
|
||||
|
||||
# Gather HTML files
|
||||
genhtml -t "$project_name" -o $output_name $project_name-filtered.lcov.info || exit 1
|
||||
if [ "$keep_info" = false ]; then
|
||||
rm -rf *.lcov.info
|
||||
fi
|
||||
|
||||
# Zip HTML files
|
||||
if $zip_html ; then
|
||||
zip -r $output_name.zip $output_name/
|
||||
fi
|
||||
@@ -0,0 +1,13 @@
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
|
||||
# Retrieve version from git repository
|
||||
git_version(ZVEC_VERSION ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
|
||||
# Add repository
|
||||
cc_directory(ailego)
|
||||
cc_directory(core)
|
||||
cc_directory(db)
|
||||
if(BUILD_PYTHON_BINDINGS)
|
||||
cc_directory(binding)
|
||||
endif()
|
||||
@@ -0,0 +1,28 @@
|
||||
include(${PROJECT_ROOT_DIR}/cmake/bazel.cmake)
|
||||
include(${PROJECT_ROOT_DIR}/cmake/option.cmake)
|
||||
|
||||
find_package(Threads REQUIRED)
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
find_library(LIB_RT NAMES rt)
|
||||
else()
|
||||
set(LIB_RT "")
|
||||
endif()
|
||||
|
||||
git_version(GIT_SRCS_VER ${CMAKE_CURRENT_SOURCE_DIR})
|
||||
file(GLOB_RECURSE ALL_SRCS *.cc *.c *.h)
|
||||
|
||||
set(EXTRA_LIBS ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
|
||||
|
||||
if(UNIX AND NOT APPLE)
|
||||
list(APPEND EXTRA_LIBS ${LIB_RT})
|
||||
endif()
|
||||
|
||||
cc_library(
|
||||
NAME zvec_ailego STATIC STRICT PACKED
|
||||
SRCS ${ALL_SRCS}
|
||||
LIBS ${EXTRA_LIBS}
|
||||
Arrow::arrow_static
|
||||
Arrow::parquet_static
|
||||
VERSION "${GIT_SRCS_VER}"
|
||||
)
|
||||
@@ -0,0 +1,77 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "binary_quantizer.h"
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <iostream>
|
||||
#include <numeric>
|
||||
#include <ailego/math/normalizer.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
//! Feed the training data
|
||||
bool BinaryQuantizer::feed(const float *vec, size_t dim) {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
data_.emplace_back(vec[i]);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Train the quantizer
|
||||
bool BinaryQuantizer::train(void) {
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Quantize data: encode the float input to uint32_t output
|
||||
void BinaryQuantizer::encode(const float *in, size_t dim, uint32_t *out) const {
|
||||
for (size_t i = 0; i < dim; i += 32) {
|
||||
size_t remain = i + 32 <= dim ? 32 : dim - i;
|
||||
uint32_t data = 0;
|
||||
uint32_t mask = 1;
|
||||
|
||||
for (size_t j = 0; j < remain; j++) {
|
||||
if (in[i + j] >= threshold_) {
|
||||
data |= mask;
|
||||
}
|
||||
|
||||
mask <<= 1;
|
||||
}
|
||||
|
||||
*out = data;
|
||||
out++;
|
||||
}
|
||||
}
|
||||
|
||||
//! De-quantize data: decode the input uint32_t to float output
|
||||
//! bit value 1 will be mapped to 1.0
|
||||
//! bit value 0 will be mapped to -1.0
|
||||
void BinaryQuantizer::decode(const uint32_t *in, size_t dim, float *out) const {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
uint8_t bit = (in[i >> 5] >> (i & 31)) & 0x01;
|
||||
|
||||
if (bit == 1) {
|
||||
out[i] = 1.0f;
|
||||
} else {
|
||||
out[i] = -1.0f;
|
||||
}
|
||||
|
||||
// std::cout << "dim: " << i << ", value: " << (size_t)bit << std::endl;
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,69 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <vector>
|
||||
#include <ailego/internal/platform.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! Binary Quantization Algorithm
|
||||
*/
|
||||
class BinaryQuantizer {
|
||||
public:
|
||||
//! Constructor
|
||||
BinaryQuantizer(void) {}
|
||||
|
||||
//! Feed the training data
|
||||
bool feed(const float *vec, size_t dim);
|
||||
|
||||
//! Train the quantizer
|
||||
bool train(void);
|
||||
|
||||
//! Quantize data: encode the float input to uint32_t output
|
||||
void encode(const float *in, size_t dim, uint32_t *out) const;
|
||||
|
||||
//! De-quantize data: decode the input uint32_t to float output
|
||||
void decode(const uint32_t *in, size_t dim, float *out) const;
|
||||
|
||||
//! Get encoded elements in type of uint32_t
|
||||
static size_t EncodedSizeInBinary32(size_t dim) {
|
||||
return (dim + 31) / 32;
|
||||
}
|
||||
|
||||
//! Set quantization threshold
|
||||
void set_threshold(float threshold) {
|
||||
threshold_ = threshold;
|
||||
}
|
||||
|
||||
//! Get quantization threshold
|
||||
float threshold(void) const {
|
||||
return threshold_;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Disable them
|
||||
BinaryQuantizer(const BinaryQuantizer &) = delete;
|
||||
BinaryQuantizer &operator=(const BinaryQuantizer &) = delete;
|
||||
|
||||
private:
|
||||
//! Members
|
||||
std::vector<float> data_{};
|
||||
float threshold_{0.0f};
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,466 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "integer_quantizer.h"
|
||||
#include <algorithm>
|
||||
#include <cmath>
|
||||
#include <cstring>
|
||||
#include <numeric>
|
||||
#include <stdexcept>
|
||||
#include <ailego/math/normalizer.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
//! Make smooth the distribution to eliminate zero in hist
|
||||
static inline void MakeSmooth(std::vector<float> &dist) {
|
||||
constexpr float epsilon = std::numeric_limits<float>::epsilon();
|
||||
|
||||
// L1 Normalize first
|
||||
float norm = 1.0f;
|
||||
Normalizer<float>::L1(dist.data(), dist.size(), &norm);
|
||||
|
||||
size_t zero_count = std::count_if(dist.begin(), dist.end(), [](float val) {
|
||||
return (std::abs(val) < std::numeric_limits<float>::epsilon());
|
||||
});
|
||||
size_t nonzero_count = dist.size() - zero_count;
|
||||
|
||||
// Double check
|
||||
if (nonzero_count == 0 || zero_count == 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
float y = epsilon * zero_count / static_cast<float>(nonzero_count);
|
||||
for (auto &it : dist) {
|
||||
if (std::abs(it) < epsilon) {
|
||||
it += epsilon;
|
||||
} else {
|
||||
it -= y;
|
||||
}
|
||||
} // end of for
|
||||
}
|
||||
|
||||
//! Compute the Entropy of distribution p/q by Kullback-Leibler Divergence
|
||||
static inline double ComputeKlDivergence(const std::vector<float> &p,
|
||||
const std::vector<float> &q) {
|
||||
if (p.size() != q.size() || p.size() == 0) {
|
||||
return std::numeric_limits<float>::max();
|
||||
}
|
||||
|
||||
double v = 0.0f;
|
||||
for (size_t i = 0; i != p.size(); ++i) {
|
||||
if (p[i] == 0 || q[i] == 0) {
|
||||
return std::numeric_limits<double>::max();
|
||||
}
|
||||
v += p[i] * std::log(static_cast<double>(p[i]) / static_cast<double>(q[i]));
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
//! Expand the quantization distribution to origin distribution in
|
||||
//! [-threshold, threshold]
|
||||
static inline void ExpandCandidateDistribution(
|
||||
const std::vector<uint32_t> &distribution,
|
||||
const std::vector<float> &quantized_distribution, size_t threshold,
|
||||
std::vector<float> *expand_distribution) {
|
||||
expand_distribution->resize(threshold * 2, 0);
|
||||
float merged_cnt = static_cast<float>(expand_distribution->size()) /
|
||||
quantized_distribution.size();
|
||||
size_t left_boundary = distribution.size() / 2 - threshold;
|
||||
|
||||
for (size_t i = 0; i < quantized_distribution.size(); ++i) {
|
||||
float start = i * merged_cnt;
|
||||
float end = start + merged_cnt;
|
||||
const size_t start_ceil = static_cast<size_t>(std::ceil(start));
|
||||
const size_t end_floor = static_cast<size_t>(std::floor(end));
|
||||
float left_ratio = static_cast<float>(start_ceil) - start;
|
||||
float right_ratio = end - static_cast<float>(end_floor);
|
||||
float nonzero_count = 0;
|
||||
|
||||
//! Count the non-zeros bins, if the histogram bin is partially included,
|
||||
//! non-zero bins is also partially counted
|
||||
if (left_ratio > 0 && left_boundary + start_ceil > 0) {
|
||||
if (distribution[left_boundary + start_ceil - 1] != 0) {
|
||||
nonzero_count += left_ratio;
|
||||
}
|
||||
}
|
||||
if (right_ratio > 0 && left_boundary + end_floor < distribution.size()) {
|
||||
if (distribution[left_boundary + end_floor] != 0) {
|
||||
nonzero_count += right_ratio;
|
||||
}
|
||||
}
|
||||
for (size_t j = start_ceil; j < end_floor; j++) {
|
||||
nonzero_count += distribution[left_boundary + j] != 0;
|
||||
}
|
||||
if (nonzero_count == 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
//! expand the quantized value
|
||||
float value = quantized_distribution[i] / nonzero_count;
|
||||
if (left_ratio > 0 && start_ceil > 0) {
|
||||
(*expand_distribution)[start_ceil - 1] += value * left_ratio;
|
||||
}
|
||||
if (right_ratio > 0 && end_floor < expand_distribution->size()) {
|
||||
(*expand_distribution)[end_floor] += value * right_ratio;
|
||||
}
|
||||
for (size_t j = start_ceil; j < end_floor; j++) {
|
||||
if (distribution[left_boundary + j] != 0) {
|
||||
(*expand_distribution)[j] = value;
|
||||
}
|
||||
} // end of for
|
||||
} // end of for
|
||||
}
|
||||
|
||||
/*! Compute quantization threshold bins
|
||||
* Implement Int8 Quantization Algorithm ref:
|
||||
* http://on-demand.gputechconf.com/gtc/2017/presentation/s7310-8-bit-inference-with-tensorrt.pdf
|
||||
*/
|
||||
static inline size_t ComputeThreshold(const std::vector<uint32_t> &hist,
|
||||
const size_t target_bins) {
|
||||
std::vector<float> P_distribution(hist.size());
|
||||
size_t zero_point_index = hist.size() / 2;
|
||||
|
||||
size_t start_bin = target_bins / 2;
|
||||
size_t end_bin = hist.size() / 2;
|
||||
size_t negative_outliers_count = 0;
|
||||
size_t positive_outliers_count = 0;
|
||||
double min_divergence = std::numeric_limits<double>::max();
|
||||
size_t target_threshold = end_bin;
|
||||
|
||||
for (size_t threshold = start_bin; threshold <= end_bin; ++threshold) {
|
||||
negative_outliers_count += hist[zero_point_index - threshold];
|
||||
positive_outliers_count += hist[zero_point_index + threshold - 1];
|
||||
}
|
||||
|
||||
//! for each zero-axised quantization range: [-threshold, threshold], search
|
||||
//! the best solution
|
||||
for (size_t threshold = start_bin; threshold <= end_bin; ++threshold) {
|
||||
P_distribution.resize(threshold * 2);
|
||||
auto p_hist = &hist[zero_point_index - threshold];
|
||||
for (size_t i = 0; i != P_distribution.size(); ++i) {
|
||||
P_distribution[i] = static_cast<float>(p_hist[i]);
|
||||
}
|
||||
|
||||
negative_outliers_count -= hist[zero_point_index - threshold];
|
||||
positive_outliers_count -= hist[zero_point_index + threshold - 1];
|
||||
P_distribution[0] += negative_outliers_count;
|
||||
P_distribution[P_distribution.size() - 1] += positive_outliers_count;
|
||||
|
||||
//! Quantize the bins in range [-threshold, threshold] to target_bins
|
||||
std::vector<float> Q_distribution(target_bins, 0);
|
||||
float merged_cnt = static_cast<float>(threshold * 2) / target_bins;
|
||||
size_t left_boundary = zero_point_index - threshold;
|
||||
for (size_t i = 0; i < target_bins; ++i) {
|
||||
float start = i * merged_cnt;
|
||||
float end = start + merged_cnt;
|
||||
const size_t start_ceil = static_cast<size_t>(std::ceil(start));
|
||||
const size_t end_floor = static_cast<size_t>(std::floor(end));
|
||||
if (left_boundary + start_ceil > 0) {
|
||||
Q_distribution[i] +=
|
||||
((float)start_ceil - start) * hist[left_boundary + start_ceil - 1];
|
||||
}
|
||||
if (left_boundary + end_floor < hist.size()) {
|
||||
Q_distribution[i] +=
|
||||
(end - (float)end_floor) * hist[left_boundary + end_floor];
|
||||
}
|
||||
|
||||
for (size_t j = start_ceil; j < end_floor; j++) {
|
||||
Q_distribution[i] += hist[left_boundary + j];
|
||||
}
|
||||
}
|
||||
std::vector<float> Q_expand_distribution;
|
||||
ExpandCandidateDistribution(hist, Q_distribution, threshold,
|
||||
&Q_expand_distribution);
|
||||
|
||||
//! Compute Kullback-Leibler Divergence, normalize the smooth the data
|
||||
//! first. Ref: http://hanj.cs.illinois.edu/cs412/bk3/KL-divergence.pdf
|
||||
MakeSmooth(P_distribution);
|
||||
MakeSmooth(Q_expand_distribution);
|
||||
double divergence =
|
||||
ComputeKlDivergence(P_distribution, Q_expand_distribution);
|
||||
|
||||
if (divergence < min_divergence) {
|
||||
min_divergence = divergence;
|
||||
target_threshold = threshold;
|
||||
}
|
||||
}
|
||||
return target_threshold;
|
||||
}
|
||||
|
||||
// Quantize the value in range
|
||||
template <int RANGE_MIN, int RANGE_MAX>
|
||||
static inline float QuantizeValue(float val, float scale, float bias) {
|
||||
val = (val + bias) * scale;
|
||||
|
||||
if (val > RANGE_MAX) {
|
||||
val = RANGE_MAX;
|
||||
} else if (val < RANGE_MIN) {
|
||||
val = RANGE_MIN;
|
||||
}
|
||||
return val;
|
||||
}
|
||||
|
||||
// Init the historgram params
|
||||
#define INIT_HISTOGRAM() \
|
||||
{ \
|
||||
if (histogram_bins_ == 0) { \
|
||||
size_t range = non_bias_ \
|
||||
? std::max(std::abs(MIN_VALUE), std::abs(MAX_VALUE)) \
|
||||
: (MAX_VALUE - MIN_VALUE); \
|
||||
histogram_bins_ = std::max<size_t>(4096u, range * 8); \
|
||||
} \
|
||||
histogram_.resize((histogram_bins_ + 1) >> 1 << 1); \
|
||||
if (non_bias_) { \
|
||||
bias_ = 0.0f; \
|
||||
auto val = std::max(std::abs(max_), std::abs(min_)); \
|
||||
left_boundary_ = -val; \
|
||||
hist_interval_ = (val * 2) / static_cast<float>(histogram_.size()); \
|
||||
} else { \
|
||||
bias_ = -static_cast<float>(min_ + (max_ - min_) * 0.5); \
|
||||
left_boundary_ = min_; \
|
||||
hist_interval_ = (max_ - min_) / static_cast<float>(histogram_.size()); \
|
||||
} \
|
||||
}
|
||||
|
||||
// Feed vector and update the historgram
|
||||
#define UPDATE_HISTOGRAM(vec, dim) \
|
||||
{ \
|
||||
if (max_ < min_) { \
|
||||
return false; \
|
||||
} \
|
||||
if (histogram_.size() == 0) { \
|
||||
INIT_HISTOGRAM() \
|
||||
} \
|
||||
for (size_t i = 0; i < dim; ++i) { \
|
||||
ssize_t index = 0; \
|
||||
if (hist_interval_ > 0.0) { \
|
||||
index = \
|
||||
static_cast<ssize_t>((vec[i] - left_boundary_) / hist_interval_); \
|
||||
} \
|
||||
if (index < 0) { \
|
||||
index = 0; \
|
||||
} else if ((size_t)index >= histogram_.size()) { \
|
||||
index = histogram_.size() - 1; \
|
||||
} \
|
||||
ailego_assert_with((size_t)index < histogram_.size(), "Invalid index"); \
|
||||
histogram_[index] += 1; \
|
||||
} \
|
||||
return true; \
|
||||
}
|
||||
|
||||
// Train the quantizer
|
||||
#define TRAIN_QUANTIZER() \
|
||||
{ \
|
||||
auto sum = std::accumulate(histogram_.begin(), histogram_.end(), 0); \
|
||||
if (sum == 0) { \
|
||||
return false; \
|
||||
} \
|
||||
size_t target_bins = \
|
||||
ailego_align(static_cast<size_t>(MAX_VALUE - MIN_VALUE), 2); \
|
||||
auto threshold_bins = ComputeThreshold(histogram_, target_bins); \
|
||||
auto threshold = \
|
||||
(static_cast<float>(threshold_bins) + 0.5f) * hist_interval_; \
|
||||
scale_ = target_bins / 2 / threshold; \
|
||||
if (!non_bias_) { \
|
||||
bias_ += (MAX_VALUE + MIN_VALUE) * 0.5f / scale_; \
|
||||
} \
|
||||
scale_reciprocal_ = 1 / scale_; \
|
||||
return true; \
|
||||
}
|
||||
|
||||
// Feed the INT16 quantizer
|
||||
bool EntropyInt16Quantizer::feed(const float *vec, size_t dim) {
|
||||
UPDATE_HISTOGRAM(vec, dim)
|
||||
}
|
||||
|
||||
// Train the INT16 quantizer
|
||||
bool EntropyInt16Quantizer::train(void) {
|
||||
TRAIN_QUANTIZER()
|
||||
}
|
||||
|
||||
// Encode to INT16
|
||||
void EntropyInt16Quantizer::encode(const float *in, size_t dim,
|
||||
int16_t *out) const {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = static_cast<int16_t>(
|
||||
std::round(QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_)));
|
||||
}
|
||||
}
|
||||
|
||||
// Decode from INT16
|
||||
void EntropyInt16Quantizer::decode(const int16_t *in, size_t dim,
|
||||
float *out) const {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = in[i] * this->scale_reciprocal() - this->bias();
|
||||
}
|
||||
}
|
||||
|
||||
// Feed the UINT16 quantizer
|
||||
bool EntropyUInt16Quantizer::feed(const float *vec, size_t dim) {
|
||||
UPDATE_HISTOGRAM(vec, dim)
|
||||
}
|
||||
|
||||
// Train the UINT16 quantizer
|
||||
bool EntropyUInt16Quantizer::train(void) {
|
||||
TRAIN_QUANTIZER()
|
||||
}
|
||||
|
||||
// Encode to UINT16
|
||||
void EntropyUInt16Quantizer::encode(const float *in, size_t dim,
|
||||
uint16_t *out) const {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = static_cast_from_float_to_uint16(
|
||||
std::round(QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_)));
|
||||
}
|
||||
}
|
||||
|
||||
// Decode from INT16
|
||||
void EntropyUInt16Quantizer::decode(const uint16_t *in, size_t dim,
|
||||
float *out) const {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = in[i] * this->scale_reciprocal() - this->bias();
|
||||
}
|
||||
}
|
||||
|
||||
// Feed the INT8 quantizer
|
||||
bool EntropyInt8Quantizer::feed(const float *vec, size_t dim) {
|
||||
UPDATE_HISTOGRAM(vec, dim)
|
||||
}
|
||||
|
||||
// Train the INT8 quantizer
|
||||
bool EntropyInt8Quantizer::train(void) {
|
||||
TRAIN_QUANTIZER()
|
||||
}
|
||||
|
||||
// Encode to INT8
|
||||
void EntropyInt8Quantizer::encode(const float *in, size_t dim,
|
||||
int8_t *out) const {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = static_cast<int8_t>(
|
||||
std::round(QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_)));
|
||||
}
|
||||
}
|
||||
|
||||
// Decode from INT8
|
||||
void EntropyInt8Quantizer::decode(const int8_t *in, size_t dim,
|
||||
float *out) const {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = in[i] * this->scale_reciprocal() - this->bias();
|
||||
}
|
||||
}
|
||||
|
||||
// Feed the UINT8 quantizer
|
||||
bool EntropyUInt8Quantizer::feed(const float *vec, size_t dim) {
|
||||
UPDATE_HISTOGRAM(vec, dim)
|
||||
}
|
||||
|
||||
// Train the UINT8 quantizer
|
||||
bool EntropyUInt8Quantizer::train(void) {
|
||||
TRAIN_QUANTIZER()
|
||||
}
|
||||
|
||||
// Encode to INT8
|
||||
void EntropyUInt8Quantizer::encode(const float *in, size_t dim,
|
||||
uint8_t *out) const {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = static_cast_from_float_to_uint8(
|
||||
std::round(QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_)));
|
||||
}
|
||||
}
|
||||
|
||||
// Decode from UINT8
|
||||
void EntropyUInt8Quantizer::decode(const uint8_t *in, size_t dim,
|
||||
float *out) const {
|
||||
for (size_t i = 0; i < dim; ++i) {
|
||||
out[i] = in[i] * this->scale_reciprocal() - this->bias();
|
||||
}
|
||||
}
|
||||
|
||||
// Feed the INT4 quantizer
|
||||
bool EntropyInt4Quantizer::feed(const float *vec, size_t dim) {
|
||||
UPDATE_HISTOGRAM(vec, dim)
|
||||
}
|
||||
|
||||
// Train the INT4 quantizer
|
||||
bool EntropyInt4Quantizer::train(void) {
|
||||
TRAIN_QUANTIZER()
|
||||
}
|
||||
|
||||
// Encode to INT4
|
||||
void EntropyInt4Quantizer::encode(const float *in, size_t dim,
|
||||
uint8_t *out) const {
|
||||
ailego_assert_with(dim % 2 == 0, "Dimension must be aligned with 2");
|
||||
|
||||
for (size_t i = 0; i < dim; i += 2) {
|
||||
float lo = QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_);
|
||||
float hi = QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i + 1], scale_, bias_);
|
||||
out[i / 2] = (static_cast_from_float_to_uint8(std::round(hi)) << 4) |
|
||||
(static_cast_from_float_to_uint8(std::round(lo)) & 0xF);
|
||||
}
|
||||
}
|
||||
|
||||
// Decode from INT4
|
||||
void EntropyInt4Quantizer::decode(const uint8_t *in, size_t dim,
|
||||
float *out) const {
|
||||
ailego_assert_with(dim % 2 == 0, "Dimension must be aligned with 2");
|
||||
|
||||
size_t size = dim / 2;
|
||||
for (size_t i = 0; i < size; i += 1) {
|
||||
uint8_t v = in[i];
|
||||
int8_t lo = (static_cast<int8_t>(v << 4) >> 4);
|
||||
int8_t hi = (static_cast<int8_t>(v & 0xf0) >> 4);
|
||||
out[2 * i] = lo * this->scale_reciprocal() - this->bias();
|
||||
out[2 * i + 1] = hi * this->scale_reciprocal() - this->bias();
|
||||
}
|
||||
}
|
||||
|
||||
// Feed the UINT4 quantizer
|
||||
bool EntropyUInt4Quantizer::feed(const float *vec, size_t dim) {
|
||||
UPDATE_HISTOGRAM(vec, dim)
|
||||
}
|
||||
|
||||
// Train the UINT4 quantizer
|
||||
bool EntropyUInt4Quantizer::train(void) {
|
||||
TRAIN_QUANTIZER()
|
||||
}
|
||||
|
||||
// Encode to INT4
|
||||
void EntropyUInt4Quantizer::encode(const float *in, size_t dim,
|
||||
uint8_t *out) const {
|
||||
ailego_assert_with(dim % 2 == 0, "Dimension must be aligned with 2");
|
||||
|
||||
for (size_t i = 0; i < dim; i += 2) {
|
||||
float lo = QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i], scale_, bias_);
|
||||
float hi = QuantizeValue<MIN_VALUE, MAX_VALUE>(in[i + 1], scale_, bias_);
|
||||
out[i / 2] = (static_cast_from_float_to_uint8(std::round(hi)) << 4) |
|
||||
(static_cast_from_float_to_uint8(std::round(lo)) & 0xF);
|
||||
}
|
||||
}
|
||||
|
||||
// Decode from INT4
|
||||
void EntropyUInt4Quantizer::decode(const uint8_t *in, size_t dim,
|
||||
float *out) const {
|
||||
ailego_assert_with(dim % 2 == 0, "Dimension must be aligned with 2");
|
||||
|
||||
size_t size = dim / 2;
|
||||
for (size_t i = 0; i < size; i += 1) {
|
||||
uint8_t v = in[i];
|
||||
out[2 * i] = (v & 0xf) * this->scale_reciprocal() - this->bias();
|
||||
out[2 * i + 1] = (v >> 4) * this->scale_reciprocal() - this->bias();
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,238 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <limits>
|
||||
#include <vector>
|
||||
#include <ailego/internal/platform.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! Entropy-based Integer Quantization Algorithm
|
||||
*/
|
||||
template <typename T, int RANGE_MIN, int RANGE_MAX>
|
||||
class EntropyIntegerQuantizer {
|
||||
public:
|
||||
//! Primitive Built-in Types to store the quantized data
|
||||
using ValueType = typename std::remove_cv<T>::type;
|
||||
|
||||
//! Constants
|
||||
constexpr static int MIN_VALUE = RANGE_MIN;
|
||||
constexpr static int MAX_VALUE = RANGE_MAX;
|
||||
|
||||
// Check supporting type
|
||||
static_assert(std::is_integral<T>::value, "ValueType must be integral");
|
||||
|
||||
// Check template values
|
||||
static_assert(RANGE_MIN < RANGE_MAX, "Invalid value range");
|
||||
|
||||
//! Constructor
|
||||
EntropyIntegerQuantizer(void) {}
|
||||
|
||||
//! Set histogram bins in train
|
||||
void set_histogram_bins(size_t bins) {
|
||||
if (bins > (RANGE_MAX - RANGE_MIN)) {
|
||||
histogram_bins_ = bins;
|
||||
}
|
||||
}
|
||||
|
||||
//! Set quantization params scale
|
||||
void set_scale(float val) {
|
||||
if (val > 0.0f) {
|
||||
scale_ = val;
|
||||
scale_reciprocal_ = 1 / scale_;
|
||||
}
|
||||
}
|
||||
|
||||
//! Set quantization params bias
|
||||
void set_bias(float val) {
|
||||
bias_ = val;
|
||||
}
|
||||
|
||||
//! Set quantization params max
|
||||
void set_max(float val) {
|
||||
max_ = val;
|
||||
}
|
||||
|
||||
//! Set quantization params min
|
||||
void set_min(float val) {
|
||||
min_ = val;
|
||||
}
|
||||
|
||||
//! Set quantization params non bias
|
||||
void set_non_bias(bool val) {
|
||||
non_bias_ = val;
|
||||
}
|
||||
|
||||
//! Get histogram bins in train
|
||||
size_t histogram_bins(void) const {
|
||||
return histogram_bins_;
|
||||
}
|
||||
|
||||
//! Get quantization params scale
|
||||
float scale(void) const {
|
||||
return scale_;
|
||||
}
|
||||
|
||||
//! Get quantization params bias
|
||||
float bias(void) const {
|
||||
return bias_;
|
||||
}
|
||||
|
||||
//! Get quantization params max
|
||||
float max(void) const {
|
||||
return max_;
|
||||
}
|
||||
|
||||
//! Get quantization params min
|
||||
float min(void) const {
|
||||
return min_;
|
||||
}
|
||||
|
||||
//! Get quantization params non bias
|
||||
bool non_bias(void) const {
|
||||
return non_bias_;
|
||||
}
|
||||
|
||||
//! Retrieve the scale reciprocal for decoding
|
||||
float scale_reciprocal(void) const {
|
||||
return scale_reciprocal_;
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Disable them
|
||||
EntropyIntegerQuantizer(const EntropyIntegerQuantizer &) = delete;
|
||||
EntropyIntegerQuantizer &operator=(const EntropyIntegerQuantizer &) = delete;
|
||||
|
||||
//! Members
|
||||
size_t histogram_bins_{0};
|
||||
float hist_interval_{1.0f};
|
||||
float max_{std::numeric_limits<float>::min()};
|
||||
float min_{std::numeric_limits<float>::max()};
|
||||
float bias_{0.0f};
|
||||
float scale_{0.0f};
|
||||
float scale_reciprocal_{0.0f};
|
||||
float left_boundary_{0.0f};
|
||||
bool non_bias_{false};
|
||||
std::vector<uint32_t> histogram_{};
|
||||
};
|
||||
|
||||
/*! INT16 Quantizer
|
||||
*/
|
||||
class EntropyInt16Quantizer
|
||||
: public EntropyIntegerQuantizer<int16_t, -32767, 32767> {
|
||||
public:
|
||||
//! Feed the training data
|
||||
bool feed(const float *vec, size_t dim);
|
||||
|
||||
//! Train the quantizer
|
||||
bool train(void);
|
||||
|
||||
//! Encode float vector to int16
|
||||
void encode(const float *in, size_t dim, ValueType *out) const;
|
||||
|
||||
//! Decode to float vector from int16
|
||||
void decode(const ValueType *in, size_t dim, float *out) const;
|
||||
};
|
||||
|
||||
/*! UINT16 Quantizer
|
||||
*/
|
||||
class EntropyUInt16Quantizer
|
||||
: public EntropyIntegerQuantizer<uint16_t, 0, 65535> {
|
||||
public:
|
||||
//! Feed the training data
|
||||
bool feed(const float *vec, size_t dim);
|
||||
|
||||
//! Train the quantizer
|
||||
bool train(void);
|
||||
|
||||
//! Encode float vector to uint16
|
||||
void encode(const float *in, size_t dim, ValueType *out) const;
|
||||
|
||||
//! Decode to float vector from uint16
|
||||
void decode(const ValueType *in, size_t dim, float *out) const;
|
||||
};
|
||||
|
||||
/*! INT8 Quantizer
|
||||
*/
|
||||
class EntropyInt8Quantizer : public EntropyIntegerQuantizer<int8_t, -127, 127> {
|
||||
public:
|
||||
//! Feed the training data
|
||||
bool feed(const float *vec, size_t dim);
|
||||
|
||||
//! Train the quantizer
|
||||
bool train(void);
|
||||
|
||||
//! Encode float vector to int8
|
||||
void encode(const float *in, size_t dim, ValueType *out) const;
|
||||
|
||||
//! Decode to float vector from int8
|
||||
void decode(const ValueType *in, size_t dim, float *out) const;
|
||||
};
|
||||
|
||||
/*! UINT8 Quantizer
|
||||
*/
|
||||
class EntropyUInt8Quantizer : public EntropyIntegerQuantizer<uint8_t, 0, 255> {
|
||||
public:
|
||||
//! Feed the training data
|
||||
bool feed(const float *vec, size_t dim);
|
||||
|
||||
//! Train the quantizer
|
||||
bool train(void);
|
||||
|
||||
//! Encode float vector to uint8
|
||||
void encode(const float *in, size_t dim, ValueType *out) const;
|
||||
|
||||
//! Decode to float vector from uint8
|
||||
void decode(const ValueType *in, size_t dim, float *out) const;
|
||||
};
|
||||
|
||||
/*! INT4 Quantizer
|
||||
*/
|
||||
class EntropyInt4Quantizer : public EntropyIntegerQuantizer<uint8_t, -8, 7> {
|
||||
public:
|
||||
//! Feed the training data
|
||||
bool feed(const float *vec, size_t dim);
|
||||
|
||||
//! Train the quantizer
|
||||
bool train(void);
|
||||
|
||||
//! Encode float vector to int4
|
||||
void encode(const float *in, size_t dim, ValueType *out) const;
|
||||
|
||||
//! Decode to float vector from int4
|
||||
void decode(const ValueType *in, size_t dim, float *out) const;
|
||||
};
|
||||
|
||||
/*! UINT4 Quantizer
|
||||
*/
|
||||
class EntropyUInt4Quantizer : public EntropyIntegerQuantizer<uint8_t, 0, 15> {
|
||||
public:
|
||||
//! Feed the training data
|
||||
bool feed(const float *vec, size_t dim);
|
||||
|
||||
//! Train the quantizer
|
||||
bool train(void);
|
||||
|
||||
//! Encode float vector to uint4
|
||||
void encode(const float *in, size_t dim, ValueType *out) const;
|
||||
|
||||
//! Decode to float vector from uint4
|
||||
void decode(const ValueType *in, size_t dim, float *out) const;
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,363 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <random>
|
||||
#include <ailego/parallel/lock.h>
|
||||
#include <ailego/parallel/thread_pool.h>
|
||||
#include <ailego/utility/type_helper.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! Random Centroids Generator
|
||||
*/
|
||||
template <typename T, typename TPool>
|
||||
struct RandomCentroidsGenerator {
|
||||
//! Type of values
|
||||
using OwnerType = typename std::decay<T>::type;
|
||||
using ContainerType = typename OwnerType::ContainerType;
|
||||
using ContextType = typename OwnerType::ContextType;
|
||||
using ThreadPoolType = TPool;
|
||||
|
||||
//! constexpr variables
|
||||
constexpr static size_t BatchCount = OwnerType::BatchCount;
|
||||
|
||||
//! Generate centroids
|
||||
void operator()(OwnerType *owner, ThreadPoolType &) const {
|
||||
const auto &matrix = owner->feature_matrix();
|
||||
const auto &cache = owner->feature_cache();
|
||||
auto *centroids = owner->mutable_centroids();
|
||||
|
||||
ContainerType rows(cache.dimension());
|
||||
size_t m = matrix.count();
|
||||
size_t n = m + cache.count();
|
||||
size_t k = owner->k_value();
|
||||
std::mt19937 mt((std::random_device())());
|
||||
|
||||
rows.resize(BatchCount);
|
||||
centroids->reset(cache.dimension());
|
||||
centroids->reserve(k);
|
||||
|
||||
for (size_t i = 0; k > 0 && i < n; ++i) {
|
||||
if (mt() % (n - i) >= k) {
|
||||
continue;
|
||||
}
|
||||
// Selected a feature
|
||||
if (i < m) {
|
||||
ContextType::MatrixReverseTranspose(matrix[i / BatchCount * BatchCount],
|
||||
matrix.dimension(), rows.data());
|
||||
centroids->append(rows[i & (BatchCount - 1u)], matrix.dimension());
|
||||
} else {
|
||||
centroids->append(cache[i - m], cache.dimension());
|
||||
}
|
||||
--k;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/*! Lloyd's algorithm cluster
|
||||
*/
|
||||
template <typename T, typename TPool, typename TContext, typename TContainer>
|
||||
class LloydCluster {
|
||||
public:
|
||||
//! constexpr variables
|
||||
constexpr static size_t BatchCount = TContext::BatchCount;
|
||||
|
||||
//! Type of values
|
||||
using ThreadPoolType = TPool;
|
||||
using ContainerType = TContainer;
|
||||
using ContextType = TContext;
|
||||
using ValueType = typename TContext::ValueType;
|
||||
using StoreType = typename TContext::StoreType;
|
||||
|
||||
//! Constructor
|
||||
LloydCluster(size_t k, size_t dim)
|
||||
: k_value_(k),
|
||||
feature_cache_(dim),
|
||||
feature_matrix_(dim),
|
||||
centroids_matrix_(dim),
|
||||
centroids_(dim) {}
|
||||
|
||||
//! Constructor
|
||||
LloydCluster(size_t k, size_t dim, bool spherical)
|
||||
: k_value_(k),
|
||||
feature_cache_(dim),
|
||||
feature_matrix_(dim),
|
||||
centroids_matrix_(dim),
|
||||
centroids_(dim),
|
||||
spherical_{spherical} {}
|
||||
|
||||
//! Constructor
|
||||
LloydCluster(void) {}
|
||||
|
||||
//! Destructor
|
||||
~LloydCluster(void) {}
|
||||
|
||||
//! Append a feature
|
||||
void append(const StoreType *arr, size_t dim) {
|
||||
feature_cache_.append(arr, dim);
|
||||
|
||||
if (feature_cache_.count() == BatchCount) {
|
||||
size_t pos = feature_matrix_.count();
|
||||
feature_matrix_.resize(pos + BatchCount);
|
||||
ContextType::MatrixTranspose(feature_cache_.data(), dim,
|
||||
feature_matrix_[pos]);
|
||||
feature_cache_.clear();
|
||||
}
|
||||
}
|
||||
|
||||
//! Reset cluster
|
||||
void reset(size_t k, size_t dim) {
|
||||
k_value_ = k;
|
||||
feature_cache_.reset(dim);
|
||||
feature_matrix_.reset(dim);
|
||||
centroids_.reset(dim);
|
||||
centroids_matrix_.reset(dim);
|
||||
context_.clear();
|
||||
}
|
||||
|
||||
//! Reset cluster
|
||||
void reset(size_t k, size_t dim, bool spherical) {
|
||||
k_value_ = k;
|
||||
feature_cache_.reset(dim);
|
||||
feature_matrix_.reset(dim);
|
||||
centroids_.reset(dim);
|
||||
centroids_matrix_.reset(dim);
|
||||
context_.clear();
|
||||
spherical_ = spherical;
|
||||
}
|
||||
|
||||
//! Initialize centroids
|
||||
template <typename G = RandomCentroidsGenerator<LloydCluster, ThreadPoolType>>
|
||||
void init_centroids(ThreadPoolType &pool, const G &g = G()) {
|
||||
g(this, pool);
|
||||
}
|
||||
|
||||
//! Cluster one time
|
||||
template <typename ThreadPoolType>
|
||||
bool cluster_once(ThreadPoolType &pool, double *cost) {
|
||||
if (centroids_.empty()) {
|
||||
RandomCentroidsGenerator<LloydCluster, ThreadPoolType> g;
|
||||
this->init_centroids(pool, g);
|
||||
}
|
||||
if (centroids_.count() != k_value_) {
|
||||
return false;
|
||||
}
|
||||
context_.reset(centroids_.count(), centroids_.dimension());
|
||||
|
||||
size_t count = centroids_.count() / BatchCount * BatchCount;
|
||||
centroids_matrix_.resize(count);
|
||||
for (size_t i = 0; i != count; i += BatchCount) {
|
||||
ContextType::MatrixTranspose(centroids_[i], centroids_.dimension(),
|
||||
centroids_matrix_[i]);
|
||||
}
|
||||
size_t remain = static_cast<uint32_t>(centroids_.count() - count);
|
||||
if (remain > 0) {
|
||||
centroids_matrix_.append(centroids_[count], centroids_.dimension(),
|
||||
remain);
|
||||
}
|
||||
|
||||
// Using thread pool
|
||||
auto group = pool.make_group();
|
||||
if (!feature_matrix_.empty()) {
|
||||
size_t n = feature_matrix_.count() / BatchCount;
|
||||
size_t c = std::max<size_t>(n / pool.count() / 2u, 1u);
|
||||
size_t m = n / c * c;
|
||||
|
||||
for (size_t i = 0; i != m; i += c) {
|
||||
group->submit(Closure::New(this, &LloydCluster::cluster_matrix_features,
|
||||
i, i + c));
|
||||
}
|
||||
for (size_t i = m; i != n; i += 1) {
|
||||
group->submit(Closure::New(this, &LloydCluster::cluster_matrix_features,
|
||||
i, i + 1));
|
||||
}
|
||||
}
|
||||
if (!feature_cache_.empty()) {
|
||||
group->submit(Closure::New(this, &LloydCluster::cluster_cache_features));
|
||||
}
|
||||
group->wait_finish();
|
||||
|
||||
*cost = 0.0;
|
||||
for (size_t i = 0, n = centroids_.count(); i != n; ++i) {
|
||||
const auto &item = context_[i];
|
||||
item.centroid(centroids_[i], centroids_.dimension());
|
||||
*cost += item.cost();
|
||||
}
|
||||
|
||||
if (spherical_) {
|
||||
for (size_t i = 0, n = centroids_.count(); i != n; ++i) {
|
||||
float norm;
|
||||
ContextType::Norm2(centroids_[i], centroids_.dimension(), &norm);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Retrieve the controids
|
||||
ContainerType *mutable_centroids(void) {
|
||||
return ¢roids_;
|
||||
}
|
||||
|
||||
//! Retrieve the controids
|
||||
const ContainerType ¢roids(void) const {
|
||||
return centroids_;
|
||||
}
|
||||
|
||||
//! Retrieve the K value
|
||||
size_t k_value(void) const {
|
||||
return k_value_;
|
||||
}
|
||||
|
||||
//! Retrieve context
|
||||
const ContextType &context(void) const {
|
||||
return context_;
|
||||
}
|
||||
|
||||
//! Retrieve the feature cache
|
||||
const ContainerType &feature_cache(void) const {
|
||||
return feature_cache_;
|
||||
}
|
||||
|
||||
//! Retrieve the feature matrix
|
||||
const ContainerType &feature_matrix(void) const {
|
||||
return feature_matrix_;
|
||||
}
|
||||
|
||||
//! Reserve the feature matrix
|
||||
void feature_matrix_reserve(size_t count) {
|
||||
feature_matrix_.reserve(count);
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Cluster the cache features
|
||||
void cluster_cache_features(void) {
|
||||
float scores[BatchCount];
|
||||
|
||||
for (size_t i = 0, n = feature_cache_.count(); i != n; ++i) {
|
||||
size_t count = centroids_matrix_.count() / BatchCount * BatchCount;
|
||||
const StoreType *feature = feature_cache_[i];
|
||||
float nearest_score = std::numeric_limits<float>::max();
|
||||
size_t nearest_index = 0;
|
||||
|
||||
for (size_t j = 0; j != count; j += BatchCount) {
|
||||
ContextType::template BatchDistance<1>(centroids_matrix_[j], feature,
|
||||
centroids_matrix_.dimension(),
|
||||
scores);
|
||||
|
||||
for (size_t k = 0; k < BatchCount; ++k) {
|
||||
if (scores[k] < nearest_score) {
|
||||
nearest_score = scores[k];
|
||||
nearest_index = j + k;
|
||||
}
|
||||
}
|
||||
} // end of for
|
||||
|
||||
for (size_t j = count, total = centroids_matrix_.count(); j != total;
|
||||
++j) {
|
||||
ContextType::Distance(centroids_matrix_[j], feature,
|
||||
centroids_matrix_.dimension(), scores);
|
||||
|
||||
if (scores[0] < nearest_score) {
|
||||
nearest_score = scores[0];
|
||||
nearest_index = j;
|
||||
}
|
||||
}
|
||||
context_[nearest_index].append(feature, feature_cache_.dimension(),
|
||||
nearest_score);
|
||||
} // end of for
|
||||
}
|
||||
|
||||
//! Cluster the matrix features
|
||||
void cluster_matrix_features(size_t first, size_t last) {
|
||||
std::vector<float> scores(BatchCount * BatchCount, 0);
|
||||
ContainerType rows(centroids_matrix_.dimension());
|
||||
|
||||
auto comp = [](float i, float j) {
|
||||
if (std::isnan(i)) return false;
|
||||
if (std::isnan(j)) return true;
|
||||
|
||||
return i < j;
|
||||
};
|
||||
|
||||
float nearest_scores[BatchCount];
|
||||
size_t nearest_indexes[BatchCount];
|
||||
|
||||
rows.resize(BatchCount);
|
||||
for (size_t i = first * BatchCount; i != last * BatchCount;
|
||||
i += BatchCount) {
|
||||
size_t count = centroids_matrix_.count() / BatchCount * BatchCount;
|
||||
const StoreType *block = feature_matrix_[i];
|
||||
|
||||
std::fill(nearest_indexes, nearest_indexes + BatchCount, 0);
|
||||
std::fill(nearest_scores, nearest_scores + BatchCount,
|
||||
std::numeric_limits<float>::max());
|
||||
|
||||
for (size_t j = 0; j != count; j += BatchCount) {
|
||||
ContextType::template BatchDistance<BatchCount>(
|
||||
centroids_matrix_[j], block, centroids_matrix_.dimension(),
|
||||
&scores[0]);
|
||||
|
||||
for (size_t k = 0; k < BatchCount; ++k) {
|
||||
const float *start = &scores[k * BatchCount];
|
||||
const float *result =
|
||||
std::min_element(start, start + BatchCount, comp);
|
||||
if (*result < nearest_scores[k]) {
|
||||
nearest_scores[k] = *result;
|
||||
nearest_indexes[k] = j + (result - start);
|
||||
}
|
||||
}
|
||||
} // end of for
|
||||
|
||||
for (size_t j = count, total = centroids_matrix_.count(); j != total;
|
||||
++j) {
|
||||
ContextType::template BatchDistance<1>(block, centroids_matrix_[j],
|
||||
centroids_matrix_.dimension(),
|
||||
&scores[0]);
|
||||
|
||||
for (size_t k = 0; k < BatchCount; ++k) {
|
||||
float score = scores[k];
|
||||
if (score < nearest_scores[k]) {
|
||||
nearest_scores[k] = score;
|
||||
nearest_indexes[k] = j;
|
||||
}
|
||||
}
|
||||
} // end of for
|
||||
|
||||
ContextType::MatrixReverseTranspose(block, feature_matrix_.dimension(),
|
||||
rows.data());
|
||||
for (size_t k = 0; k < BatchCount; ++k) {
|
||||
context_[nearest_indexes[k]].append(
|
||||
rows[k], feature_matrix_.dimension(), nearest_scores[k]);
|
||||
}
|
||||
} // end of for
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
size_t k_value_{0u};
|
||||
ContainerType feature_cache_{};
|
||||
ContainerType feature_matrix_{};
|
||||
ContainerType centroids_matrix_{};
|
||||
ContainerType centroids_{};
|
||||
ContextType context_{};
|
||||
bool spherical_{false};
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,579 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "buffer_manager.h"
|
||||
#include <atomic>
|
||||
#include <mutex>
|
||||
#include <ailego/logger/logger.h>
|
||||
#include <ailego/pattern/defer.h>
|
||||
#include <arrow/io/api.h>
|
||||
#include <parquet/arrow/reader.h>
|
||||
|
||||
|
||||
namespace zvec {
|
||||
|
||||
|
||||
namespace ailego {
|
||||
|
||||
|
||||
namespace {
|
||||
|
||||
|
||||
struct IDHash {
|
||||
size_t operator()(const BufferID &buffer_id) const {
|
||||
size_t hash = std::hash<int>{}(static_cast<int>(buffer_id.type));
|
||||
hash = hash ^ (std::hash<uint64_t>{}(buffer_id.file_id));
|
||||
if (buffer_id.type == BufferID::TYPE::PARQUET) {
|
||||
hash = hash * 31 + std::hash<int>{}(buffer_id.parquet().column);
|
||||
hash = hash * 31 + std::hash<int>{}(buffer_id.parquet().row_group);
|
||||
} else if (buffer_id.type == BufferID::TYPE::VECTOR) {
|
||||
hash = hash * 31 + std::hash<uint32_t>{}(buffer_id.vector().offset);
|
||||
}
|
||||
return hash;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
struct IDEqual {
|
||||
bool operator()(const BufferID &a, const BufferID &b) const {
|
||||
if (a.type != b.type) {
|
||||
return false;
|
||||
}
|
||||
if (a.file_name != b.file_name) {
|
||||
return false;
|
||||
}
|
||||
if (a.file_id != b.file_id) {
|
||||
return false;
|
||||
}
|
||||
if (a.mtime != b.mtime) {
|
||||
return false;
|
||||
}
|
||||
if (a.type == BufferID::TYPE::PARQUET) {
|
||||
return a.parquet().column == b.parquet().column &&
|
||||
a.parquet().row_group == b.parquet().row_group;
|
||||
} else if (a.type == BufferID::TYPE::VECTOR) {
|
||||
return a.vector().offset == b.vector().offset;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
} // namespace
|
||||
|
||||
|
||||
struct BufferManager::BufferContext {
|
||||
BufferContext(const BufferID &id, BufferPool *p) : id(id), pool(p) {};
|
||||
BufferContext(const BufferContext &) = delete;
|
||||
BufferContext(BufferContext &&) = delete;
|
||||
BufferContext &operator=(const BufferContext &) = delete;
|
||||
BufferContext &operator=(BufferContext &&) = delete;
|
||||
|
||||
|
||||
~BufferContext() {
|
||||
if (vector) {
|
||||
free(vector);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
typedef std::unique_ptr<BufferManager::BufferContext> Pointer;
|
||||
|
||||
|
||||
enum State : uint32_t {
|
||||
IDLE = 0, // Empty and not held by any users, not in LRU
|
||||
RESERVED = 1, // Pinned by a user but no data yet, not in LRU
|
||||
IN_USE = 2, // Pinned by a user and data is present, not in LRU
|
||||
CACHED = 3, // Data is present but not held by any users, in LRU
|
||||
ERROR = 4 // Something went wrong, not in LRU
|
||||
};
|
||||
|
||||
|
||||
// Identifier for the buffer
|
||||
BufferID id;
|
||||
|
||||
// Current state
|
||||
State state{IDLE};
|
||||
|
||||
// The size of the buffer
|
||||
uint32_t size{0};
|
||||
|
||||
// Handle of the file backing this buffer
|
||||
File file;
|
||||
|
||||
// The number of external references to this buffer (via pin/unpin)
|
||||
std::atomic<uint32_t> refs_buf{0};
|
||||
|
||||
// The number of external references to this context (via BufferHandle)
|
||||
std::atomic<uint32_t> refs_context{0};
|
||||
|
||||
BufferPool *pool{nullptr};
|
||||
|
||||
// A shared pointer to the buffers allocated for arrow parquet data
|
||||
std::shared_ptr<arrow::ChunkedArray> arrow{nullptr};
|
||||
|
||||
// Guard original arrow buffers to prevent premature deletion
|
||||
std::vector<std::shared_ptr<arrow::Buffer>> arrow_refs{};
|
||||
|
||||
// A pointer to the buffer allocated for vector data
|
||||
void *vector{nullptr};
|
||||
|
||||
// Doubly linked LRU list
|
||||
BufferContext *next{nullptr};
|
||||
BufferContext *prev{nullptr};
|
||||
|
||||
|
||||
// Return a string representation of the status
|
||||
const std::string status_string() const;
|
||||
|
||||
// Populate the buffer with parquet data
|
||||
arrow::Status read_arrow_parquet();
|
||||
|
||||
// Populate the buffer with vector data
|
||||
bool read_vector();
|
||||
};
|
||||
|
||||
|
||||
const std::string BufferManager::BufferContext::status_string() const {
|
||||
std::string msg{id.to_string() + ": "};
|
||||
switch (state) {
|
||||
case State::IDLE: {
|
||||
msg += "Idle";
|
||||
break;
|
||||
}
|
||||
case State::RESERVED: {
|
||||
msg += "Reserved";
|
||||
break;
|
||||
}
|
||||
case State::IN_USE: {
|
||||
msg += "In use";
|
||||
break;
|
||||
}
|
||||
case State::CACHED: {
|
||||
msg += "Cached";
|
||||
break;
|
||||
}
|
||||
case State::ERROR: {
|
||||
msg += "Error";
|
||||
break;
|
||||
}
|
||||
}
|
||||
return msg;
|
||||
}
|
||||
|
||||
|
||||
arrow::Status BufferManager::BufferContext::read_arrow_parquet() {
|
||||
// TODO: file handler and memory pool can be optimized
|
||||
arrow::MemoryPool *mem_pool = arrow::default_memory_pool();
|
||||
|
||||
// Open file
|
||||
std::shared_ptr<arrow::io::RandomAccessFile> input;
|
||||
const auto &file_name = id.file_name;
|
||||
ARROW_ASSIGN_OR_RAISE(input, arrow::io::ReadableFile::Open(file_name));
|
||||
|
||||
// Open reader
|
||||
std::unique_ptr<parquet::arrow::FileReader> reader;
|
||||
ARROW_ASSIGN_OR_RAISE(reader, parquet::arrow::OpenFile(input, mem_pool));
|
||||
|
||||
// Perform read
|
||||
int row_group = id.parquet().row_group;
|
||||
int column = id.parquet().column;
|
||||
auto s = reader->RowGroup(row_group)->Column(column)->Read(&arrow);
|
||||
if (!s.ok()) {
|
||||
LOG_ERROR("Failed to read parquet file[%s]", file_name.c_str());
|
||||
arrow = nullptr;
|
||||
return s;
|
||||
}
|
||||
|
||||
// Compute the memory usage and hijack Arrow's buffers with our implementation
|
||||
for (auto &array : arrow->chunks()) {
|
||||
auto &buffers = array->data()->buffers;
|
||||
for (size_t buf_idx = 0; buf_idx < buffers.size(); ++buf_idx) {
|
||||
if (buffers[buf_idx] == nullptr) {
|
||||
continue;
|
||||
}
|
||||
// Keep references to original buffers to prevent premature deletion
|
||||
arrow_refs.emplace_back(buffers[buf_idx]);
|
||||
size += buffers[buf_idx]->capacity();
|
||||
// Create hijacked buffer with custom deleter that notifies us when Arrow
|
||||
// is finished with the buffer
|
||||
std::shared_ptr<arrow::Buffer> hijacked_buffer(
|
||||
buffers[buf_idx].get(), BufferManager::ArrowBufferDeleter(this));
|
||||
buffers[buf_idx] = hijacked_buffer;
|
||||
}
|
||||
}
|
||||
|
||||
return arrow::Status::OK();
|
||||
}
|
||||
|
||||
|
||||
bool BufferManager::BufferContext::read_vector() {
|
||||
const auto &file_name = id.file_name;
|
||||
if (!file.is_valid()) {
|
||||
if (!File::IsExist(file_name)) {
|
||||
LOG_ERROR("File[%s] does not exist", file_name.c_str());
|
||||
return false;
|
||||
}
|
||||
if (!File::IsRegular(file_name)) {
|
||||
LOG_ERROR("[%s] is not a regular file", file_name.c_str());
|
||||
return false;
|
||||
}
|
||||
if (!file.open(file_name.c_str(), true, false)) {
|
||||
LOG_ERROR("Failed to open file[%s]", file_name.c_str());
|
||||
return false;
|
||||
}
|
||||
}
|
||||
AILEGO_DEFER([this] { file.close(); });
|
||||
uint32_t len = id.vector().length;
|
||||
auto ret = posix_memalign((void **)&vector, 64, len); // 64-byte alignment
|
||||
if (ret != 0 || vector == nullptr) {
|
||||
LOG_ERROR("Failed to allocate buffer for file[%s]", file_name.c_str());
|
||||
return false;
|
||||
}
|
||||
uint32_t offset = id.vector().offset;
|
||||
if (file.read(offset, vector, len) != len) {
|
||||
LOG_ERROR("Failed to read file[%s]", file_name.c_str());
|
||||
free(vector);
|
||||
vector = nullptr;
|
||||
return false;
|
||||
}
|
||||
size = len;
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
// Thread-safe buffer pool implementation.
|
||||
//
|
||||
// BufferContext states:
|
||||
// 1. Must exist in the lookup (hash) table.
|
||||
// 2. LRU list presence:
|
||||
// - In LRU: holds memory but not pinned by any users
|
||||
// - Not in LRU: either holds memory pinned by users, or doesn't hold memory
|
||||
// 3. External references: when an external user acquires a context and pins the
|
||||
// memory, that context is removed from LRU list; when they unpins the
|
||||
// memory, that context is moved to LRU list if it was the last reference.
|
||||
//
|
||||
// Any operation on the hash table is protected by mutex_table_.
|
||||
// Any change to context state and LRU list is protected by mutex_context_.
|
||||
//
|
||||
class BufferManager::BufferPool {
|
||||
public:
|
||||
explicit BufferPool(uint64_t limit) : limit_(limit) {
|
||||
sentinel_.next = &sentinel_;
|
||||
sentinel_.prev = &sentinel_;
|
||||
}
|
||||
|
||||
|
||||
BufferContext *acquire_locked(BufferID &id) {
|
||||
std::lock_guard<std::mutex> lock(mutex_context_);
|
||||
if (auto iter = table_.find(id); iter != table_.end()) {
|
||||
return iter->second.get();
|
||||
}
|
||||
auto [iter, _] =
|
||||
table_.emplace(id, std::make_unique<BufferContext>(id, this));
|
||||
return iter->second.get();
|
||||
}
|
||||
|
||||
|
||||
void try_release_context_locked(BufferContext *context) {
|
||||
if (context->refs_context.load() != 0) {
|
||||
return;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(mutex_table_);
|
||||
if (context->refs_context.load() != 0) {
|
||||
return;
|
||||
}
|
||||
if (context->state == BufferContext::State::IDLE) {
|
||||
table_.erase(context->id);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void pin_locked(BufferContext *ctx) {
|
||||
std::lock_guard<std::mutex> lock(mutex_context_);
|
||||
if (ctx->state == BufferContext::State::IDLE) {
|
||||
return pin_at_IDLE(ctx);
|
||||
}
|
||||
if (ctx->state == BufferContext::State::IN_USE) {
|
||||
return pin_at_IN_USE(ctx);
|
||||
}
|
||||
if (ctx->state == BufferContext::State::CACHED) {
|
||||
return pin_at_CACHED(ctx);
|
||||
}
|
||||
if (ctx->state == BufferContext::State::ERROR) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
bool unpin_locked(BufferContext *ctx) {
|
||||
uint32_t prev_refs = ctx->refs_buf.fetch_sub(1);
|
||||
if (prev_refs > 1) {
|
||||
return false;
|
||||
}
|
||||
std::lock_guard<std::mutex> lock(mutex_context_);
|
||||
if (ctx->refs_buf.load() == 0 &&
|
||||
ctx->state != BufferContext::State::CACHED) {
|
||||
ctx->state = BufferContext::State::CACHED;
|
||||
LRU_insert(ctx);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void LRU_insert_locked(BufferContext *context) {
|
||||
std::lock_guard<std::mutex> lock(mutex_context_);
|
||||
LRU_insert(context);
|
||||
}
|
||||
|
||||
|
||||
void LRU_remove_locked(BufferContext *context) {
|
||||
std::lock_guard<std::mutex> lock(mutex_context_);
|
||||
LRU_remove(context);
|
||||
}
|
||||
|
||||
|
||||
uint64_t usage() const {
|
||||
return usage_;
|
||||
}
|
||||
|
||||
|
||||
private:
|
||||
void pin_at_IDLE(BufferContext *ctx) {
|
||||
ctx->state = BufferContext::State::RESERVED;
|
||||
|
||||
while (usage_ >= limit_) {
|
||||
// The tail of LRU list is the least recently used context
|
||||
BufferContext *victim = sentinel_.prev;
|
||||
if (victim == &sentinel_) { // No victim could be found
|
||||
ctx->state = BufferContext::State::ERROR;
|
||||
return;
|
||||
}
|
||||
if (victim->state == BufferContext::State::ERROR) {
|
||||
LRU_remove(victim);
|
||||
try_release_context_locked(ctx);
|
||||
continue;
|
||||
}
|
||||
if (victim->id.type == BufferID::TYPE::PARQUET) {
|
||||
victim->arrow_refs.clear();
|
||||
} else {
|
||||
free(victim->vector);
|
||||
victim->vector = nullptr;
|
||||
}
|
||||
victim->state = BufferContext::State::IDLE;
|
||||
LRU_remove(victim);
|
||||
try_release_context_locked(ctx);
|
||||
usage_ -= victim->size;
|
||||
}
|
||||
|
||||
if (ctx->id.type == BufferID::TYPE::PARQUET) {
|
||||
if (ctx->read_arrow_parquet().ok()) {
|
||||
ctx->state = BufferContext::State::IN_USE;
|
||||
ctx->refs_buf.fetch_add(ctx->arrow_refs.size());
|
||||
usage_ += ctx->size;
|
||||
} else {
|
||||
LOG_ERROR("Failed to read to %s", ctx->id.to_string().c_str());
|
||||
ctx->state = BufferContext::State::ERROR;
|
||||
}
|
||||
} else {
|
||||
if (ctx->read_vector()) {
|
||||
ctx->state = BufferContext::State::IN_USE;
|
||||
ctx->refs_buf.fetch_add(1);
|
||||
usage_ += ctx->size;
|
||||
} else {
|
||||
LOG_ERROR("Failed to read to %s", ctx->id.to_string().c_str());
|
||||
ctx->state = BufferContext::State::ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void pin_at_IN_USE(BufferContext *ctx) {
|
||||
if (ctx->id.type == BufferID::TYPE::PARQUET) {
|
||||
ctx->refs_buf.fetch_add(ctx->arrow_refs.size());
|
||||
} else {
|
||||
ctx->refs_buf.fetch_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
void pin_at_CACHED(BufferContext *ctx) {
|
||||
if (ctx->id.type == BufferID::TYPE::PARQUET) {
|
||||
ctx->refs_buf.fetch_add(ctx->arrow_refs.size());
|
||||
} else {
|
||||
ctx->refs_buf.fetch_add(1);
|
||||
}
|
||||
LRU_remove(ctx);
|
||||
ctx->state = BufferContext::State::IN_USE;
|
||||
}
|
||||
|
||||
|
||||
void LRU_insert(BufferContext *context) {
|
||||
if (context->refs_buf > 0) {
|
||||
return; // Already pinned, should not be evicted
|
||||
}
|
||||
if (context->next != nullptr || context->prev != nullptr) {
|
||||
return;
|
||||
}
|
||||
// Insert the context to the head of LRU list
|
||||
context->next = sentinel_.next;
|
||||
context->prev = &sentinel_;
|
||||
sentinel_.next = context;
|
||||
context->next->prev = context;
|
||||
inactive_ += context->size;
|
||||
}
|
||||
|
||||
|
||||
void LRU_remove(BufferContext *context) {
|
||||
if (context->next == nullptr) {
|
||||
return; // Not in LRU list
|
||||
}
|
||||
context->next->prev = context->prev;
|
||||
context->prev->next = context->next;
|
||||
context->next = nullptr;
|
||||
context->prev = nullptr;
|
||||
inactive_ -= context->size;
|
||||
}
|
||||
|
||||
private:
|
||||
using Table =
|
||||
std::unordered_map<BufferID, BufferContext::Pointer, IDHash, IDEqual>;
|
||||
|
||||
uint64_t limit_;
|
||||
std::atomic<uint64_t> usage_{0};
|
||||
std::atomic<uint64_t> inactive_{0};
|
||||
|
||||
Table table_{};
|
||||
std::mutex mutex_table_{};
|
||||
BufferContext sentinel_{BufferID{}, this}; // LRU list sentinel
|
||||
std::mutex mutex_context_{};
|
||||
};
|
||||
|
||||
|
||||
BufferManager::ArrowBufferDeleter::ArrowBufferDeleter(BufferContext *c)
|
||||
: context(c) {}
|
||||
|
||||
|
||||
void BufferManager::ArrowBufferDeleter::operator()(arrow::Buffer *) {
|
||||
context->pool->unpin_locked(context);
|
||||
}
|
||||
|
||||
|
||||
BufferHandle::BufferHandle(BufferContext *context) : context_(context) {
|
||||
if (context_ != nullptr) {
|
||||
pool_ = context_->pool;
|
||||
context_->refs_context.fetch_add(1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
BufferHandle::~BufferHandle() {
|
||||
if (context_ != nullptr) {
|
||||
uint32_t prev_refs = context_->refs_context.fetch_sub(1);
|
||||
if (prev_refs > 1) {
|
||||
return;
|
||||
}
|
||||
if (context_->state == BufferContext::State::IDLE) {
|
||||
pool_->try_release_context_locked(context_);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
std::shared_ptr<arrow::ChunkedArray> BufferHandle::pin_parquet_data() {
|
||||
pool_->pin_locked(context_);
|
||||
return context_->arrow;
|
||||
}
|
||||
|
||||
|
||||
void *BufferHandle::pin_vector_data() {
|
||||
if (!context_) {
|
||||
return nullptr;
|
||||
}
|
||||
pool_->pin_locked(context_);
|
||||
return context_->vector;
|
||||
}
|
||||
|
||||
|
||||
bool BufferHandle::unpin_vector_data() {
|
||||
if (!context_) {
|
||||
return true;
|
||||
}
|
||||
return pool_->unpin_locked(context_);
|
||||
}
|
||||
|
||||
|
||||
uint32_t BufferHandle::references() const {
|
||||
return context_->refs_buf.load();
|
||||
}
|
||||
|
||||
|
||||
uint32_t BufferHandle::size() const {
|
||||
return context_->size;
|
||||
}
|
||||
|
||||
|
||||
void BufferManager::init(uint64_t limit, uint32_t num_shards) {
|
||||
pools_.clear();
|
||||
uint64_t limit_per_shard = ailego_align(limit / num_shards, 4096);
|
||||
for (uint32_t i = 0; i < num_shards; ++i) {
|
||||
auto pool = new BufferPool(limit_per_shard);
|
||||
pools_.push_back(pool);
|
||||
}
|
||||
LOG_INFO(
|
||||
"BufferManager initialized with [%u] buffer pools, [%zu] bytes memory "
|
||||
"limit per pool, total memory limit [%zu] bytes",
|
||||
num_shards, (size_t)limit_per_shard, (size_t)limit);
|
||||
}
|
||||
|
||||
|
||||
BufferHandle BufferManager::acquire(BufferID &buffer_id) {
|
||||
static IDHash id_hash{};
|
||||
auto hash_val = id_hash(buffer_id);
|
||||
auto ctx = pools_[hash_val % pools_.size()]->acquire_locked(buffer_id);
|
||||
return BufferHandle(ctx);
|
||||
}
|
||||
|
||||
|
||||
std::unique_ptr<BufferHandle> BufferManager::acquire_ptr(BufferID &buffer_id) {
|
||||
static IDHash id_hash{};
|
||||
auto hash_val = id_hash(buffer_id);
|
||||
auto ctx = pools_[hash_val % pools_.size()]->acquire_locked(buffer_id);
|
||||
return std::make_unique<BufferHandle>(ctx);
|
||||
}
|
||||
|
||||
|
||||
uint64_t BufferManager::total_size_in_bytes() const {
|
||||
uint64_t total_usage = 0;
|
||||
for (auto pool : pools_) {
|
||||
total_usage += pool->usage();
|
||||
}
|
||||
return total_usage;
|
||||
}
|
||||
|
||||
|
||||
BufferManager::~BufferManager() {
|
||||
for (auto pool : pools_) {
|
||||
delete pool;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
} // namespace ailego
|
||||
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,269 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
|
||||
#pragma once
|
||||
|
||||
|
||||
#include <sys/stat.h>
|
||||
#include <chrono>
|
||||
#include <cstdint>
|
||||
#include <filesystem>
|
||||
#include <memory>
|
||||
#include <ailego/io/file.h>
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic push
|
||||
#pragma clang diagnostic ignored "-Wunused-parameter"
|
||||
#pragma clang diagnostic ignored "-Wshadow"
|
||||
#elif defined(__GNUC__) || defined(__GNUG__)
|
||||
#pragma GCC diagnostic push
|
||||
#pragma GCC diagnostic ignored "-Wunused-parameter"
|
||||
#pragma GCC diagnostic ignored "-Wshadow"
|
||||
#endif
|
||||
|
||||
#include <arrow/api.h>
|
||||
|
||||
#ifdef __clang__
|
||||
#pragma clang diagnostic pop
|
||||
#elif defined(__GNUC__) || defined(__GNUG__)
|
||||
#pragma GCC diagnostic pop
|
||||
#endif
|
||||
|
||||
#include "ailego/pattern/singleton.h"
|
||||
|
||||
|
||||
namespace zvec {
|
||||
|
||||
|
||||
namespace ailego {
|
||||
|
||||
|
||||
struct BufferID;
|
||||
class BufferManager;
|
||||
class BufferHandle;
|
||||
|
||||
|
||||
struct BufferID {
|
||||
struct ParquetPos {
|
||||
int column;
|
||||
int row_group;
|
||||
};
|
||||
struct VectorPos {
|
||||
uint32_t offset;
|
||||
uint32_t length;
|
||||
};
|
||||
union Position {
|
||||
explicit Position() = default;
|
||||
ParquetPos forward;
|
||||
VectorPos vector;
|
||||
};
|
||||
enum TYPE {
|
||||
PARQUET = 1,
|
||||
VECTOR = 2,
|
||||
UNKNOWN = 0,
|
||||
};
|
||||
|
||||
|
||||
static std::uint64_t getLastModifiedNs(const std::filesystem::path &p) {
|
||||
auto ftime = std::filesystem::last_write_time(p);
|
||||
return static_cast<std::uint64_t>(ftime.time_since_epoch().count());
|
||||
}
|
||||
|
||||
// Cross-platform helper to get nanosecond modification time
|
||||
// static long get_st_mtime_nsec(const struct stat &file_stat) {
|
||||
// #ifdef __APPLE__
|
||||
// return file_stat.st_mtim.tv_nsec;
|
||||
// #else
|
||||
// return file_stat.st_mtim.tv_nsec;
|
||||
// #endif
|
||||
// }
|
||||
|
||||
static BufferID ParquetID(const std::string &file_name, int column,
|
||||
int row_group) {
|
||||
BufferID buffer_id{};
|
||||
buffer_id.type = TYPE::PARQUET;
|
||||
buffer_id.file_name = file_name;
|
||||
buffer_id.pos.forward.column = column;
|
||||
buffer_id.pos.forward.row_group = row_group;
|
||||
struct stat file_stat;
|
||||
if (stat(file_name.c_str(), &file_stat) == 0) {
|
||||
// file_stat.st_ino contains the inode number
|
||||
// file_stat.st_dev contains the device ID
|
||||
// Together they uniquely identify a file
|
||||
buffer_id.file_id = file_stat.st_ino;
|
||||
std::filesystem::path p(file_name);
|
||||
buffer_id.mtime = getLastModifiedNs(p);
|
||||
}
|
||||
return buffer_id;
|
||||
}
|
||||
|
||||
static BufferID VectorID(const std::string &file_name, uint32_t offset,
|
||||
uint32_t length) {
|
||||
BufferID buffer_id{};
|
||||
buffer_id.type = TYPE::VECTOR;
|
||||
buffer_id.file_name = file_name;
|
||||
struct stat file_stat;
|
||||
if (stat(file_name.c_str(), &file_stat) == 0) {
|
||||
buffer_id.file_id = file_stat.st_ino;
|
||||
std::filesystem::path p(file_name);
|
||||
buffer_id.mtime = getLastModifiedNs(p);
|
||||
}
|
||||
buffer_id.pos.vector.offset = offset;
|
||||
buffer_id.pos.vector.length = length;
|
||||
return buffer_id;
|
||||
}
|
||||
|
||||
explicit BufferID() = default;
|
||||
|
||||
// Type of the file backing this buffer
|
||||
TYPE type{UNKNOWN};
|
||||
|
||||
// Name of the file backing this buffer
|
||||
std::string file_name{};
|
||||
|
||||
// Unique file id
|
||||
uint64_t file_id{};
|
||||
|
||||
long mtime{};
|
||||
|
||||
// To identify which part of the backing file should be loaded into the buffer
|
||||
Position pos{};
|
||||
|
||||
|
||||
// Get the forward ID
|
||||
const inline struct ParquetPos &parquet() const {
|
||||
return pos.forward;
|
||||
}
|
||||
|
||||
|
||||
// Get the vector ID
|
||||
const inline struct VectorPos &vector() const {
|
||||
return pos.vector;
|
||||
}
|
||||
|
||||
|
||||
// Get debug string
|
||||
const std::string to_string() const {
|
||||
std::string msg{"Buffer["};
|
||||
if (type == TYPE::PARQUET) {
|
||||
msg += "parquet: " + file_name + "[" + std::to_string(file_id) + "]" +
|
||||
", column: " + std::to_string(parquet().column) +
|
||||
", row_group: " + std::to_string(parquet().row_group);
|
||||
} else if (type == TYPE::VECTOR) {
|
||||
msg += "vector: " + file_name + "[" + std::to_string(file_id) + "]" +
|
||||
", offset: " + std::to_string(vector().offset);
|
||||
} else {
|
||||
msg += "unknown";
|
||||
}
|
||||
msg += ", mtime: " + std::to_string(mtime);
|
||||
msg += "]";
|
||||
return msg;
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Thread-safe LRU buffer implementation.
|
||||
class BufferManager : public Singleton<BufferManager> {
|
||||
friend BufferHandle;
|
||||
|
||||
public:
|
||||
void init(uint64_t limit, uint32_t num_shards = 1);
|
||||
|
||||
BufferHandle acquire(BufferID &buffer_id);
|
||||
|
||||
std::unique_ptr<BufferHandle> acquire_ptr(BufferID &buffer_id);
|
||||
|
||||
uint64_t total_size_in_bytes() const;
|
||||
|
||||
~BufferManager();
|
||||
|
||||
private:
|
||||
struct BufferContext;
|
||||
|
||||
class BufferPool;
|
||||
|
||||
// Custom deleter for Arrow buffer that automatically notifies us when the
|
||||
// buffer is no longer referenced by Arrow
|
||||
struct ArrowBufferDeleter {
|
||||
explicit ArrowBufferDeleter(BufferContext *c);
|
||||
BufferContext *context;
|
||||
// Only reduces the reference count but does not actually release the
|
||||
// buffer, since the buffer memory is managed by the BufferManager.
|
||||
void operator()(arrow::Buffer *);
|
||||
};
|
||||
|
||||
std::vector<BufferPool *> pools_;
|
||||
};
|
||||
|
||||
|
||||
class BufferHandle {
|
||||
public:
|
||||
typedef std::unique_ptr<BufferHandle> Pointer;
|
||||
|
||||
explicit BufferHandle(BufferManager::BufferContext *context = nullptr);
|
||||
BufferHandle(const BufferHandle &) = delete;
|
||||
BufferHandle(BufferHandle &&) = default;
|
||||
BufferHandle &operator=(const BufferHandle &) = delete;
|
||||
BufferHandle &operator=(BufferHandle &&) = default;
|
||||
|
||||
|
||||
~BufferHandle();
|
||||
|
||||
|
||||
// Pin parquet data in memory by allocating arrow buffers of appropriate size
|
||||
// and reading data from the backing file.
|
||||
// The lifecycle of the allocated memory is automatically managed through
|
||||
// shared pointers. The buffers are guaranteed to be held until they are not
|
||||
// referenced.
|
||||
// Returns a pointer to the loaded ChunkedArray in Arrow format.
|
||||
std::shared_ptr<arrow::ChunkedArray> pin_parquet_data();
|
||||
|
||||
|
||||
// Pin vector data in memory by allocating a buffer of appropriate size and
|
||||
// loading data from the backing file.
|
||||
// The memory is guaranteed to be held until unpin() is called. The caller
|
||||
// must call unpin() to release the memory when it is no longer needed.
|
||||
// Returns a raw memory address.
|
||||
void *pin_vector_data();
|
||||
|
||||
|
||||
// Reduce the reference count for this vector buffer.
|
||||
// Returns true if this was the last reference.
|
||||
// When reference count is zero, the buffer is moved to the eviction list and
|
||||
// becomes eligible for removal under memory pressure.
|
||||
bool unpin_vector_data();
|
||||
|
||||
|
||||
// Get the current reference count.
|
||||
uint32_t references() const;
|
||||
|
||||
|
||||
// Get the buffer size.
|
||||
uint32_t size() const;
|
||||
|
||||
|
||||
private:
|
||||
using BufferContext = BufferManager::BufferContext;
|
||||
using BufferPool = BufferManager::BufferPool;
|
||||
|
||||
BufferContext *context_;
|
||||
BufferPool *pool_;
|
||||
};
|
||||
|
||||
|
||||
} // namespace ailego
|
||||
|
||||
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,443 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "bitmap.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
size_t Bitset::BitwiseAndCardinality(const Bitset &lhs, const Bitset &rhs) {
|
||||
return BitsetHelper::BitwiseAndCardinality(
|
||||
lhs.array_.data(), rhs.array_.data(),
|
||||
std::min(lhs.array_.size(), rhs.array_.size()));
|
||||
}
|
||||
|
||||
size_t Bitset::BitwiseAndnotCardinality(const Bitset &lhs, const Bitset &rhs) {
|
||||
size_t lsize = lhs.array_.size();
|
||||
size_t rsize = rhs.array_.size();
|
||||
|
||||
if (lsize > rsize) {
|
||||
return (
|
||||
BitsetHelper::BitwiseAndnotCardinality(lhs.array_.data(),
|
||||
rhs.array_.data(), rsize) +
|
||||
BitsetHelper::Cardinality(lhs.array_.data() + rsize, lsize - rsize));
|
||||
}
|
||||
return BitsetHelper::BitwiseAndnotCardinality(lhs.array_.data(),
|
||||
rhs.array_.data(), lsize);
|
||||
}
|
||||
|
||||
size_t Bitset::BitwiseXorCardinality(const Bitset &lhs, const Bitset &rhs) {
|
||||
size_t lsize = lhs.array_.size();
|
||||
size_t rsize = rhs.array_.size();
|
||||
|
||||
if (lsize < rsize) {
|
||||
return (
|
||||
BitsetHelper::BitwiseXorCardinality(lhs.array_.data(),
|
||||
rhs.array_.data(), lsize) +
|
||||
BitsetHelper::Cardinality(rhs.array_.data() + lsize, rsize - lsize));
|
||||
} else if (lsize > rsize) {
|
||||
return (
|
||||
BitsetHelper::BitwiseXorCardinality(lhs.array_.data(),
|
||||
rhs.array_.data(), rsize) +
|
||||
BitsetHelper::Cardinality(lhs.array_.data() + rsize, lsize - rsize));
|
||||
}
|
||||
return BitsetHelper::BitwiseXorCardinality(lhs.array_.data(),
|
||||
rhs.array_.data(), lsize);
|
||||
}
|
||||
|
||||
size_t Bitset::BitwiseOrCardinality(const Bitset &lhs, const Bitset &rhs) {
|
||||
size_t lsize = lhs.array_.size();
|
||||
size_t rsize = rhs.array_.size();
|
||||
|
||||
if (lsize < rsize) {
|
||||
return (
|
||||
BitsetHelper::BitwiseOrCardinality(lhs.array_.data(), rhs.array_.data(),
|
||||
lsize) +
|
||||
BitsetHelper::Cardinality(rhs.array_.data() + lsize, rsize - lsize));
|
||||
} else if (lsize > rsize) {
|
||||
return (
|
||||
BitsetHelper::BitwiseOrCardinality(lhs.array_.data(), rhs.array_.data(),
|
||||
rsize) +
|
||||
BitsetHelper::Cardinality(lhs.array_.data() + rsize, lsize - rsize));
|
||||
}
|
||||
return BitsetHelper::BitwiseOrCardinality(lhs.array_.data(),
|
||||
rhs.array_.data(), lsize);
|
||||
}
|
||||
|
||||
void Bitmap::clear(void) {
|
||||
for (std::vector<Bucket *>::iterator iter = array_.begin();
|
||||
iter != array_.end(); ++iter) {
|
||||
delete (*iter);
|
||||
}
|
||||
array_.clear();
|
||||
}
|
||||
|
||||
void Bitmap::copy(const Bitmap &rhs) {
|
||||
this->clear();
|
||||
|
||||
for (std::vector<Bucket *>::const_iterator iter = rhs.array_.begin();
|
||||
iter != rhs.array_.end(); ++iter) {
|
||||
Bucket *bucket = NULL;
|
||||
if (*iter) {
|
||||
bucket = new Bucket(*(*iter));
|
||||
}
|
||||
array_.push_back(bucket);
|
||||
}
|
||||
}
|
||||
|
||||
void Bitmap::shrink_to_fit(void) {
|
||||
size_t shrink_count = 0;
|
||||
std::vector<Bucket *>::reverse_iterator iter;
|
||||
|
||||
for (iter = array_.rbegin(); iter != array_.rend(); ++iter) {
|
||||
if (*iter) {
|
||||
if (!(*iter)->test_none()) {
|
||||
break;
|
||||
}
|
||||
delete (*iter);
|
||||
*iter = NULL;
|
||||
}
|
||||
++shrink_count;
|
||||
}
|
||||
for (; iter != array_.rend(); ++iter) {
|
||||
if ((*iter) && (*iter)->test_none()) {
|
||||
delete (*iter);
|
||||
*iter = NULL;
|
||||
}
|
||||
}
|
||||
if (shrink_count != 0) {
|
||||
array_.resize(array_.size() - shrink_count);
|
||||
}
|
||||
}
|
||||
|
||||
bool Bitmap::test(size_t num) const {
|
||||
// High 16 bits
|
||||
size_t offset = num >> 16;
|
||||
|
||||
if (offset < array_.size()) {
|
||||
const Bucket *bucket = array_[offset];
|
||||
if (bucket) {
|
||||
// Low 16 bits
|
||||
return bucket->test(static_cast<uint16_t>(num));
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
void Bitmap::set(size_t num) {
|
||||
// High 16 bits
|
||||
size_t offset = num >> 16;
|
||||
if (offset >= array_.size()) {
|
||||
array_.resize(offset + 1, NULL);
|
||||
}
|
||||
|
||||
Bucket *&bucket = array_[offset];
|
||||
if (!bucket) {
|
||||
bucket = new Bucket;
|
||||
}
|
||||
// Low 16 bits
|
||||
bucket->set(static_cast<uint16_t>(num));
|
||||
}
|
||||
|
||||
void Bitmap::reset(size_t num) {
|
||||
// High 16 bits
|
||||
size_t offset = num >> 16;
|
||||
if (offset >= array_.size()) {
|
||||
array_.resize(offset + 1, NULL);
|
||||
}
|
||||
|
||||
if (offset < array_.size()) {
|
||||
Bucket *bucket = array_[offset];
|
||||
if (bucket) {
|
||||
// Low 16 bits
|
||||
bucket->reset(static_cast<uint16_t>(num));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Bitmap::flip(size_t num) {
|
||||
// High 16 bits
|
||||
uint16_t offset = (uint16_t)(num >> 16);
|
||||
if (offset >= array_.size()) {
|
||||
array_.resize(offset + 1, NULL);
|
||||
}
|
||||
|
||||
Bucket *&bucket = array_[offset];
|
||||
if (!bucket) {
|
||||
bucket = new Bucket;
|
||||
}
|
||||
// Low 16 bits
|
||||
bucket->flip(static_cast<uint16_t>(num));
|
||||
}
|
||||
|
||||
void Bitmap::bitwise_and(const Bitmap &rhs) {
|
||||
size_t overlap = std::min(array_.size(), rhs.array_.size());
|
||||
|
||||
for (size_t i = 0; i < overlap; ++i) {
|
||||
Bucket *&dst = array_[i];
|
||||
|
||||
if (dst) {
|
||||
const Bucket *src = rhs.array_[i];
|
||||
if (src) {
|
||||
dst->bitwise_and(*src);
|
||||
} else {
|
||||
delete dst;
|
||||
dst = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (size_t i = overlap; i < array_.size(); ++i) {
|
||||
Bucket *&dst = array_[i];
|
||||
delete dst;
|
||||
dst = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
void Bitmap::bitwise_andnot(const Bitmap &rhs) {
|
||||
size_t overlap = std::min(array_.size(), rhs.array_.size());
|
||||
|
||||
for (size_t i = 0; i < overlap; ++i) {
|
||||
Bucket *&dst = array_[i];
|
||||
|
||||
if (dst) {
|
||||
const Bucket *src = rhs.array_[i];
|
||||
if (src) {
|
||||
dst->bitwise_andnot(*src);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Bitmap::bitwise_or(const Bitmap &rhs) {
|
||||
size_t overlap = std::min(array_.size(), rhs.array_.size());
|
||||
|
||||
for (size_t i = 0; i < overlap; ++i) {
|
||||
const Bucket *src = rhs.array_[i];
|
||||
|
||||
if (src) {
|
||||
Bucket *&dst = array_[i];
|
||||
|
||||
if (dst) {
|
||||
dst->bitwise_or(*src);
|
||||
} else {
|
||||
dst = new Bucket(*src);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (size_t i = overlap; i < rhs.array_.size(); ++i) {
|
||||
const Bucket *src = rhs.array_[i];
|
||||
Bucket *bucket = NULL;
|
||||
|
||||
if (src) {
|
||||
bucket = new Bucket(*src);
|
||||
}
|
||||
array_.push_back(bucket);
|
||||
}
|
||||
}
|
||||
|
||||
void Bitmap::bitwise_xor(const Bitmap &rhs) {
|
||||
size_t overlap = std::min(array_.size(), rhs.array_.size());
|
||||
|
||||
for (size_t i = 0; i < overlap; ++i) {
|
||||
const Bucket *src = rhs.array_[i];
|
||||
|
||||
if (src) {
|
||||
Bucket *&dst = array_[i];
|
||||
|
||||
if (dst) {
|
||||
dst->bitwise_xor(*src);
|
||||
} else {
|
||||
dst = new Bucket(*src);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (size_t i = overlap; i < rhs.array_.size(); ++i) {
|
||||
const Bucket *src = rhs.array_[i];
|
||||
Bucket *bucket = NULL;
|
||||
|
||||
if (src) {
|
||||
bucket = new Bucket(*src);
|
||||
}
|
||||
array_.push_back(bucket);
|
||||
}
|
||||
}
|
||||
|
||||
void Bitmap::bitwise_not(void) {
|
||||
for (std::vector<Bucket *>::iterator iter = array_.begin();
|
||||
iter != array_.end(); ++iter) {
|
||||
Bucket *&bucket = *iter;
|
||||
if (!bucket) {
|
||||
bucket = new Bucket;
|
||||
}
|
||||
bucket->bitwise_not();
|
||||
}
|
||||
}
|
||||
|
||||
bool Bitmap::test_all(void) const {
|
||||
if (array_.empty()) {
|
||||
return false;
|
||||
}
|
||||
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
|
||||
iter != array_.end(); ++iter) {
|
||||
if (!(*iter) || !(*iter)->test_all()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Bitmap::test_any(void) const {
|
||||
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
|
||||
iter != array_.end(); ++iter) {
|
||||
if (*iter && (*iter)->test_any()) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool Bitmap::test_none(void) const {
|
||||
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
|
||||
iter != array_.end(); ++iter) {
|
||||
if (*iter && !(*iter)->test_none()) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
size_t Bitmap::cardinality(void) const {
|
||||
size_t result = 0;
|
||||
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
|
||||
iter != array_.end(); ++iter) {
|
||||
if (*iter) {
|
||||
result += (*iter)->cardinality();
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
void Bitmap::extract(size_t base, std::vector<size_t> *out) const {
|
||||
for (std::vector<Bucket *>::const_iterator iter = array_.begin();
|
||||
iter != array_.end(); ++iter) {
|
||||
if (*iter) {
|
||||
(*iter)->extract(base, out);
|
||||
}
|
||||
base += Bucket::MAX_SIZE;
|
||||
}
|
||||
}
|
||||
|
||||
size_t Bitmap::BitwiseAndCardinality(const Bitmap &lhs, const Bitmap &rhs) {
|
||||
size_t overlap = std::min(lhs.array_.size(), rhs.array_.size());
|
||||
size_t dist = 0;
|
||||
|
||||
for (size_t i = 0; i < overlap; ++i) {
|
||||
const Bucket *l = lhs.array_[i];
|
||||
const Bucket *r = rhs.array_[i];
|
||||
|
||||
if (l && r) {
|
||||
dist += Bucket::BitwiseAndCardinality(*l, *r);
|
||||
}
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
|
||||
size_t Bitmap::BitwiseAndnotCardinality(const Bitmap &lhs, const Bitmap &rhs) {
|
||||
size_t overlap = std::min(lhs.array_.size(), rhs.array_.size());
|
||||
size_t dist = 0;
|
||||
|
||||
for (size_t i = 0; i < overlap; ++i) {
|
||||
const Bucket *l = lhs.array_[i];
|
||||
if (l) {
|
||||
const Bucket *r = rhs.array_[i];
|
||||
if (r) {
|
||||
dist += Bucket::BitwiseAndnotCardinality(*l, *r);
|
||||
} else {
|
||||
dist += l->cardinality();
|
||||
}
|
||||
}
|
||||
}
|
||||
for (size_t i = overlap; i < lhs.array_.size(); ++i) {
|
||||
const Bucket *l = lhs.array_[i];
|
||||
if (l) {
|
||||
dist += l->cardinality();
|
||||
}
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
|
||||
size_t Bitmap::BitwiseXorCardinality(const Bitmap &lhs, const Bitmap &rhs) {
|
||||
size_t overlap = std::min(lhs.array_.size(), rhs.array_.size());
|
||||
size_t dist = 0;
|
||||
|
||||
for (size_t i = 0; i < overlap; ++i) {
|
||||
const Bucket *l = lhs.array_[i];
|
||||
const Bucket *r = rhs.array_[i];
|
||||
|
||||
if (l && r) {
|
||||
dist += Bucket::BitwiseXorCardinality(*l, *r);
|
||||
} else if (l) {
|
||||
dist += l->cardinality();
|
||||
} else if (r) {
|
||||
dist += r->cardinality();
|
||||
}
|
||||
}
|
||||
for (size_t i = overlap; i < lhs.array_.size(); ++i) {
|
||||
const Bucket *l = lhs.array_[i];
|
||||
if (l) {
|
||||
dist += l->cardinality();
|
||||
}
|
||||
}
|
||||
for (size_t i = overlap; i < rhs.array_.size(); ++i) {
|
||||
const Bucket *r = rhs.array_[i];
|
||||
if (r) {
|
||||
dist += r->cardinality();
|
||||
}
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
|
||||
size_t Bitmap::BitwiseOrCardinality(const Bitmap &lhs, const Bitmap &rhs) {
|
||||
size_t overlap = std::min(lhs.array_.size(), rhs.array_.size());
|
||||
size_t dist = 0;
|
||||
|
||||
for (size_t i = 0; i < overlap; ++i) {
|
||||
const Bucket *l = lhs.array_[i];
|
||||
const Bucket *r = rhs.array_[i];
|
||||
|
||||
if (l && r) {
|
||||
dist += Bucket::BitwiseOrCardinality(*l, *r);
|
||||
} else if (l) {
|
||||
dist += l->cardinality();
|
||||
} else if (r) {
|
||||
dist += r->cardinality();
|
||||
}
|
||||
}
|
||||
for (size_t i = overlap; i < lhs.array_.size(); ++i) {
|
||||
const Bucket *l = lhs.array_[i];
|
||||
if (l) {
|
||||
dist += l->cardinality();
|
||||
}
|
||||
}
|
||||
for (size_t i = overlap; i < rhs.array_.size(); ++i) {
|
||||
const Bucket *r = rhs.array_[i];
|
||||
if (r) {
|
||||
dist += r->cardinality();
|
||||
}
|
||||
}
|
||||
return dist;
|
||||
}
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,503 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <vector>
|
||||
#include <ailego/utility/bitset_helper.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! Fixed Bitset
|
||||
*/
|
||||
template <size_t N, typename = typename std::enable_if<N % 32 == 0>::type>
|
||||
class FixedBitset {
|
||||
public:
|
||||
enum { MAX_SIZE = N };
|
||||
|
||||
//! Constructor
|
||||
FixedBitset(void) {
|
||||
memset(array_, 0, sizeof(array_));
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
FixedBitset(const FixedBitset &rhs) {
|
||||
memcpy(array_, rhs.array_, sizeof(array_));
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
~FixedBitset(void) {}
|
||||
|
||||
//! Assignment
|
||||
FixedBitset &operator=(const FixedBitset &rhs) {
|
||||
memcpy(array_, rhs.array_, sizeof(array_));
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Retrieve data pointer
|
||||
uint32_t *data(void) {
|
||||
return reinterpret_cast<uint32_t *>(array_);
|
||||
}
|
||||
|
||||
//! Retrieve data pointer
|
||||
const uint32_t *data(void) const {
|
||||
return reinterpret_cast<const uint32_t *>(array_);
|
||||
}
|
||||
|
||||
//! Retrieve count of bits in set
|
||||
constexpr size_t size(void) const {
|
||||
return MAX_SIZE;
|
||||
}
|
||||
|
||||
// !Clear the bitset
|
||||
void clear(void) {
|
||||
memset(array_, 0, sizeof(array_));
|
||||
}
|
||||
|
||||
//! Test a bit in bitset
|
||||
bool test(size_t num) const {
|
||||
ailego_assert_with(N > num, "overflow argument");
|
||||
return ((array_[num >> 5] & (1u << (num & 0x1f))) != 0);
|
||||
}
|
||||
|
||||
//! Set a bit in bitset
|
||||
void set(size_t num) {
|
||||
ailego_assert_with(N > num, "overflow argument");
|
||||
uint32_t mask = (1u << (num & 0x1f));
|
||||
array_[num >> 5] |= mask;
|
||||
}
|
||||
|
||||
//! Clear a bit in bitset
|
||||
void reset(size_t num) {
|
||||
ailego_assert_with(N > num, "overflow argument");
|
||||
uint32_t mask = (1u << (num & 0x1f));
|
||||
array_[num >> 5] &= ~mask;
|
||||
}
|
||||
|
||||
//! Toggle a bit in bitset
|
||||
void flip(size_t num) {
|
||||
ailego_assert_with(N > num, "overflow argument");
|
||||
uint32_t mask = (1u << (num & 0x1f));
|
||||
array_[num >> 5] ^= mask;
|
||||
}
|
||||
|
||||
//! Perform binary AND
|
||||
void bitwise_and(const FixedBitset &rhs) {
|
||||
BitsetHelper::BitwiseAnd(array_, rhs.array_, ((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Perform binary AND NOT
|
||||
void bitwise_andnot(const FixedBitset &rhs) {
|
||||
BitsetHelper::BitwiseAndnot(array_, rhs.array_, ((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Perform binary OR
|
||||
void bitwise_or(const FixedBitset &rhs) {
|
||||
BitsetHelper::BitwiseOr(array_, rhs.array_, ((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Perform binary XOR
|
||||
void bitwise_xor(const FixedBitset &rhs) {
|
||||
BitsetHelper::BitwiseXor(array_, rhs.array_, ((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Perform binary NOT
|
||||
void bitwise_not(void) {
|
||||
BitsetHelper::BitwiseNot(array_, ((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Check if all bits are set to true
|
||||
bool test_all(void) const {
|
||||
return BitsetHelper::TestAll(array_, ((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Check if any bits are set to true
|
||||
bool test_any(void) const {
|
||||
return BitsetHelper::TestAny(array_, ((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Check if none of the bits are set to true
|
||||
bool test_none(void) const {
|
||||
return BitsetHelper::TestNone(array_, ((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Compute the cardinality of a bitset
|
||||
size_t cardinality(void) const {
|
||||
return BitsetHelper::Cardinality(array_, ((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Extract the bitset to an array
|
||||
void extract(size_t base, std::vector<size_t> *out) const {
|
||||
const uint32_t *iter = array_;
|
||||
const uint32_t *last = array_ + ((N + 0x1f) >> 5);
|
||||
|
||||
for (; iter != last; ++iter) {
|
||||
uint32_t w = *iter;
|
||||
|
||||
while (w != 0) {
|
||||
uint32_t c = ailego_ctz32(w);
|
||||
w &= ~(1u << c);
|
||||
out->push_back(base + c);
|
||||
}
|
||||
base += 32u;
|
||||
}
|
||||
}
|
||||
|
||||
//! Extract the bitset to an array
|
||||
void extract(std::vector<size_t> *out) const {
|
||||
this->extract(0, out);
|
||||
}
|
||||
|
||||
//! Compute the AND cardinality between two bitsets
|
||||
static size_t BitwiseAndCardinality(const FixedBitset &lhs,
|
||||
const FixedBitset &rhs) {
|
||||
return BitsetHelper::BitwiseAndCardinality(lhs.array_, rhs.array_,
|
||||
((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Compute the ANDNOT cardinality between two bitsets
|
||||
static size_t BitwiseAndnotCardinality(const FixedBitset &lhs,
|
||||
const FixedBitset &rhs) {
|
||||
return BitsetHelper::BitwiseAndnotCardinality(lhs.array_, rhs.array_,
|
||||
((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Compute the XOR cardinality between two bitsets
|
||||
static size_t BitwiseXorCardinality(const FixedBitset &lhs,
|
||||
const FixedBitset &rhs) {
|
||||
return BitsetHelper::BitwiseXorCardinality(lhs.array_, rhs.array_,
|
||||
((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Compute the OR cardinality between two bitsets
|
||||
static size_t BitwiseOrCardinality(const FixedBitset &lhs,
|
||||
const FixedBitset &rhs) {
|
||||
return BitsetHelper::BitwiseOrCardinality(lhs.array_, rhs.array_,
|
||||
((N + 0x1f) >> 5));
|
||||
}
|
||||
|
||||
//! Convert a array pointer to bitset pointer
|
||||
static FixedBitset *Cast(uint32_t *arr) {
|
||||
return reinterpret_cast<FixedBitset<N> *>(arr);
|
||||
}
|
||||
|
||||
//! Convert a array pointer to bitset pointer
|
||||
static const FixedBitset *Cast(const uint32_t *arr) {
|
||||
return reinterpret_cast<const FixedBitset<N> *>(arr);
|
||||
}
|
||||
|
||||
//! Convert a array pointer to bitset pointer
|
||||
static FixedBitset *Cast(uint64_t *arr) {
|
||||
return reinterpret_cast<FixedBitset<N> *>(arr);
|
||||
}
|
||||
|
||||
//! Convert a array pointer to bitset pointer
|
||||
static const FixedBitset *Cast(const uint64_t *arr) {
|
||||
return reinterpret_cast<const FixedBitset<N> *>(arr);
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t array_[(N + 0x1f) >> 5];
|
||||
};
|
||||
|
||||
/*! Fixed Bitset (Special)
|
||||
*/
|
||||
template <>
|
||||
class FixedBitset<0> {
|
||||
public:
|
||||
enum { MAX_SIZE = 0 };
|
||||
|
||||
//! Retrieve max size of bitset
|
||||
constexpr size_t size(void) const {
|
||||
return MAX_SIZE;
|
||||
}
|
||||
};
|
||||
|
||||
/*! Bitset
|
||||
*/
|
||||
class Bitset {
|
||||
public:
|
||||
//! Constructor
|
||||
Bitset(void) : array_() {}
|
||||
|
||||
//! Constructor
|
||||
Bitset(size_t bits) : array_((bits + 0x1f) >> 5) {}
|
||||
|
||||
//! Constructor
|
||||
Bitset(const Bitset &rhs) : array_(rhs.array_) {}
|
||||
|
||||
//! Constructor
|
||||
Bitset(Bitset &&rhs) : array_(std::move(rhs.array_)) {}
|
||||
|
||||
//! Destructor
|
||||
~Bitset(void) {}
|
||||
|
||||
//! Assignment
|
||||
Bitset &operator=(const Bitset &rhs) {
|
||||
array_ = rhs.array_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Bitset &operator=(Bitset &&rhs) {
|
||||
array_ = std::move(rhs.array_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Retrieve data pointer
|
||||
uint32_t *data(void) {
|
||||
return array_.data();
|
||||
}
|
||||
|
||||
//! Retrieve data pointer
|
||||
const uint32_t *data(void) const {
|
||||
return array_.data();
|
||||
}
|
||||
|
||||
//! Retrieve count of bits in set
|
||||
size_t size(void) const {
|
||||
return (array_.size() << 5);
|
||||
}
|
||||
|
||||
//! Resize the bitset
|
||||
void resize(size_t bits) {
|
||||
array_.resize((bits + 0x1f) >> 5);
|
||||
}
|
||||
|
||||
// !Clear the bitset
|
||||
void clear(void) {
|
||||
array_.clear();
|
||||
}
|
||||
|
||||
//! Test a bit in bitset
|
||||
bool test(size_t num) const {
|
||||
ailego_assert_with(this->size() > num, "overflow argument");
|
||||
return ((array_[num >> 5] & (1u << (num & 0x1f))) != 0);
|
||||
}
|
||||
|
||||
//! Set a bit in bitset
|
||||
void set(size_t num) {
|
||||
ailego_assert_with(this->size() > num, "overflow argument");
|
||||
uint32_t mask = (1u << (num & 0x1f));
|
||||
array_[num >> 5] |= mask;
|
||||
}
|
||||
|
||||
//! Clear a bit in bitset
|
||||
void reset(size_t num) {
|
||||
ailego_assert_with(this->size() > num, "overflow argument");
|
||||
uint32_t mask = (1u << (num & 0x1f));
|
||||
array_[num >> 5] &= ~mask;
|
||||
}
|
||||
|
||||
//! Toggle a bit in bitset
|
||||
void flip(size_t num) {
|
||||
ailego_assert_with(this->size() > num, "overflow argument");
|
||||
uint32_t mask = (1u << (num & 0x1f));
|
||||
array_[num >> 5] ^= mask;
|
||||
}
|
||||
|
||||
//! Perform binary AND
|
||||
void bitwise_and(const Bitset &rhs) {
|
||||
BitsetHelper::BitwiseAnd(array_.data(), rhs.array_.data(),
|
||||
std::min(array_.size(), rhs.array_.size()));
|
||||
}
|
||||
|
||||
//! Perform binary AND NOT
|
||||
void bitwise_andnot(const Bitset &rhs) {
|
||||
BitsetHelper::BitwiseAndnot(array_.data(), rhs.array_.data(),
|
||||
std::min(array_.size(), rhs.array_.size()));
|
||||
}
|
||||
|
||||
//! Perform binary OR
|
||||
void bitwise_or(const Bitset &rhs) {
|
||||
BitsetHelper::BitwiseOr(array_.data(), rhs.array_.data(),
|
||||
std::min(array_.size(), rhs.array_.size()));
|
||||
}
|
||||
|
||||
//! Perform binary XOR
|
||||
void bitwise_xor(const Bitset &rhs) {
|
||||
BitsetHelper::BitwiseXor(array_.data(), rhs.array_.data(),
|
||||
std::min(array_.size(), rhs.array_.size()));
|
||||
}
|
||||
|
||||
//! Perform binary NOT
|
||||
void bitwise_not(void) {
|
||||
BitsetHelper::BitwiseNot(array_.data(), array_.size());
|
||||
}
|
||||
|
||||
//! Check if all bits are set to true
|
||||
bool test_all(void) const {
|
||||
return BitsetHelper::TestAll(array_.data(), array_.size());
|
||||
}
|
||||
|
||||
//! Check if any bits are set to true
|
||||
bool test_any(void) const {
|
||||
return BitsetHelper::TestAny(array_.data(), array_.size());
|
||||
}
|
||||
|
||||
//! Check if none of the bits are set to true
|
||||
bool test_none(void) const {
|
||||
return BitsetHelper::TestNone(array_.data(), array_.size());
|
||||
}
|
||||
|
||||
//! Compute the cardinality of a bitset
|
||||
size_t cardinality(void) const {
|
||||
return BitsetHelper::Cardinality(array_.data(), array_.size());
|
||||
}
|
||||
|
||||
//! Extract the bitset to an array
|
||||
void extract(size_t base, std::vector<size_t> *out) const {
|
||||
const uint32_t *iter = array_.data();
|
||||
const uint32_t *last = array_.data() + array_.size();
|
||||
|
||||
for (; iter != last; ++iter) {
|
||||
uint32_t w = *iter;
|
||||
|
||||
while (w != 0) {
|
||||
uint32_t c = ailego_ctz32(w);
|
||||
w &= ~(1u << c);
|
||||
out->push_back(base + c);
|
||||
}
|
||||
base += 32u;
|
||||
}
|
||||
}
|
||||
|
||||
//! Extract the bitset to an array
|
||||
void extract(std::vector<size_t> *out) const {
|
||||
this->extract(0, out);
|
||||
}
|
||||
|
||||
//! Compute the AND cardinality between two bitsets
|
||||
static size_t BitwiseAndCardinality(const Bitset &lhs, const Bitset &rhs);
|
||||
|
||||
//! Compute the ANDNOT cardinality between two bitsets
|
||||
static size_t BitwiseAndnotCardinality(const Bitset &lhs, const Bitset &rhs);
|
||||
|
||||
//! Compute the XOR cardinality between two bitsets
|
||||
static size_t BitwiseXorCardinality(const Bitset &lhs, const Bitset &rhs);
|
||||
|
||||
//! Compute the OR cardinality between two bitsets
|
||||
static size_t BitwiseOrCardinality(const Bitset &lhs, const Bitset &rhs);
|
||||
|
||||
private:
|
||||
std::vector<uint32_t> array_;
|
||||
};
|
||||
|
||||
/*! Bitmap
|
||||
*/
|
||||
class Bitmap {
|
||||
public:
|
||||
typedef FixedBitset<65536u> Bucket;
|
||||
|
||||
//! Constructor
|
||||
Bitmap(void) : array_() {}
|
||||
|
||||
//! Constructor
|
||||
Bitmap(const Bitmap &rhs) {
|
||||
this->copy(rhs);
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
~Bitmap(void) {
|
||||
this->clear();
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Bitmap &operator=(const Bitmap &rhs) {
|
||||
this->copy(rhs);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Retrieve bucket size of bitmap
|
||||
size_t bucket_size(void) const {
|
||||
return array_.size();
|
||||
}
|
||||
|
||||
// !Clear the bitmap
|
||||
void clear(void);
|
||||
|
||||
//! Remove the none buckets
|
||||
void shrink_to_fit(void);
|
||||
|
||||
//! Test a bit in bitmap
|
||||
bool test(size_t num) const;
|
||||
|
||||
//! Set a bit in bitmap
|
||||
void set(size_t num);
|
||||
|
||||
//! Reset a bit in bitmap
|
||||
void reset(size_t num);
|
||||
|
||||
//! Toggle a bit in bitmap
|
||||
void flip(size_t num);
|
||||
|
||||
//! Perform binary AND
|
||||
void bitwise_and(const Bitmap &rhs);
|
||||
|
||||
//! Perform binary AND NOT
|
||||
void bitwise_andnot(const Bitmap &rhs);
|
||||
|
||||
//! Perform binary OR
|
||||
void bitwise_or(const Bitmap &rhs);
|
||||
|
||||
//! Perform binary XOR
|
||||
void bitwise_xor(const Bitmap &rhs);
|
||||
|
||||
//! Perform binary NOT (It will expand the whole map)
|
||||
void bitwise_not(void);
|
||||
|
||||
//! Check if all bits are set to true
|
||||
bool test_all(void) const;
|
||||
|
||||
//! Check if any bits are set to true
|
||||
bool test_any(void) const;
|
||||
|
||||
//! Check if none of the bits are set to true
|
||||
bool test_none(void) const;
|
||||
|
||||
//! Compute the cardinality of a bitmap
|
||||
size_t cardinality(void) const;
|
||||
|
||||
//! Extract the bitmap to an array
|
||||
void extract(size_t base, std::vector<size_t> *out) const;
|
||||
|
||||
//! Extract the bitmap to an array
|
||||
void extract(std::vector<size_t> *out) const {
|
||||
this->extract(0, out);
|
||||
}
|
||||
|
||||
//! Compute the AND cardinality between two bitmaps
|
||||
static size_t BitwiseAndCardinality(const Bitmap &lhs, const Bitmap &rhs);
|
||||
|
||||
//! Compute the ANDNOT cardinality between two bitmaps
|
||||
static size_t BitwiseAndnotCardinality(const Bitmap &lhs, const Bitmap &rhs);
|
||||
|
||||
//! Compute the XOR cardinality between two bitmaps
|
||||
static size_t BitwiseXorCardinality(const Bitmap &lhs, const Bitmap &rhs);
|
||||
|
||||
//! Compute the OR cardinality between two bitmaps
|
||||
static size_t BitwiseOrCardinality(const Bitmap &lhs, const Bitmap &rhs);
|
||||
|
||||
protected:
|
||||
//! Copy the content from another bitmap
|
||||
void copy(const Bitmap &rhs);
|
||||
|
||||
private:
|
||||
std::vector<Bucket *> array_;
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,131 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <ailego/internal/platform.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! AiLego Blob Wrap
|
||||
*/
|
||||
class BlobWrap {
|
||||
public:
|
||||
//! Constructor
|
||||
BlobWrap(void) : buffer_(nullptr), size_(0u) {}
|
||||
|
||||
//! Constructor
|
||||
BlobWrap(const BlobWrap &rhs) : buffer_(rhs.buffer_), size_(rhs.size_) {}
|
||||
|
||||
//! Constructor
|
||||
BlobWrap(BlobWrap &&rhs) : buffer_(rhs.buffer_), size_(rhs.size_) {
|
||||
rhs.buffer_ = nullptr;
|
||||
rhs.size_ = 0u;
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
BlobWrap(const void *buf, size_t len)
|
||||
: buffer_(const_cast<void *>(buf)), size_(len) {}
|
||||
|
||||
//! Constructor
|
||||
BlobWrap(const std::string &buf)
|
||||
: buffer_(const_cast<char *>(buf.data())), size_(buf.size()) {}
|
||||
|
||||
//! Destructor
|
||||
~BlobWrap(void) {}
|
||||
|
||||
//! Assignment
|
||||
BlobWrap &operator=(const BlobWrap &rhs) {
|
||||
buffer_ = rhs.buffer_;
|
||||
size_ = rhs.size_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
BlobWrap &operator=(BlobWrap &&rhs) {
|
||||
buffer_ = rhs.buffer_;
|
||||
size_ = rhs.size_;
|
||||
rhs.buffer_ = nullptr;
|
||||
rhs.size_ = 0u;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Test if the blob is valid
|
||||
bool is_valid(void) const {
|
||||
return (buffer_ && size_);
|
||||
}
|
||||
|
||||
//! Mount a buffer as blob
|
||||
void mount(void *buf, size_t len) {
|
||||
buffer_ = buf;
|
||||
size_ = len;
|
||||
}
|
||||
|
||||
//! Mount a string as blob
|
||||
void mount(std::string &buf) {
|
||||
buffer_ = const_cast<char *>(buf.data());
|
||||
size_ = buf.size();
|
||||
}
|
||||
|
||||
//! Umount the buffer of blob
|
||||
void umount(void) {
|
||||
buffer_ = nullptr;
|
||||
size_ = 0u;
|
||||
}
|
||||
|
||||
//! Retrieve buffer of blob
|
||||
void *buffer(void) {
|
||||
return buffer_;
|
||||
}
|
||||
|
||||
//! Retrieve buffer of blob
|
||||
const void *buffer(void) const {
|
||||
return buffer_;
|
||||
}
|
||||
|
||||
//! Retrieve size of blob
|
||||
size_t size(void) const {
|
||||
return size_;
|
||||
}
|
||||
|
||||
//! Copy a buffer into blob
|
||||
void copy(const void *buf, size_t len) {
|
||||
memcpy(buffer_, buf, std::min(size_, len));
|
||||
}
|
||||
|
||||
//! Copy a blob to blob
|
||||
void copy(const BlobWrap &rhs) {
|
||||
memcpy(buffer_, rhs.buffer_, std::min(size_, rhs.size_));
|
||||
}
|
||||
|
||||
//! Copy a string to blob
|
||||
void copy(const std::string &str) {
|
||||
memcpy(buffer_, str.data(), std::min(size_, str.size()));
|
||||
}
|
||||
|
||||
//! Zero the buffer of blob
|
||||
void zero(void) {
|
||||
memset(buffer_, 0, size_);
|
||||
}
|
||||
|
||||
private:
|
||||
void *buffer_;
|
||||
size_t size_;
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,279 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cmath>
|
||||
#include <ailego/internal/platform.h>
|
||||
#include <ailego/utility/type_helper.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! Bloom Filter Calculator
|
||||
*/
|
||||
struct BloomFilterCalculator {
|
||||
/**
|
||||
* \brief Calculate probability of false positives
|
||||
* \param n Number of items in the filter
|
||||
* \param m Number of bits in the filter
|
||||
* \param k Number of hash functions
|
||||
* \return Probability of false positives
|
||||
*/
|
||||
static double Probability(size_t n, size_t m, size_t k) {
|
||||
return std::pow(1.0 - std::exp(-((double)k / (double)m * (double)n)), k);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Calculate number of items in the filter
|
||||
* \param m Number of bits in the filter
|
||||
* \param k Number of hash functions
|
||||
* \param p Probability of false positives
|
||||
* \return Number of items in the filter
|
||||
*/
|
||||
static size_t NumberOfItems(size_t m, size_t k, double p) {
|
||||
return (size_t)std::ceil(
|
||||
-((double)m / (double)k *
|
||||
std::log(1.0 - std::exp(std::log(p) / (double)k))));
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Calculate number of bits in the filter
|
||||
* \param n Number of items in the filter
|
||||
* \param p Probability of false positives
|
||||
* \return Number of bits in the filter
|
||||
*/
|
||||
static size_t NumberOfBits(size_t n, double p) {
|
||||
return (size_t)std::ceil((double)n * std::log(p) /
|
||||
std::log(1.0 / std::pow(2.0, std::log(2.0))));
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Calculate number of bits in the filter
|
||||
* \param n Number of items in the filter
|
||||
* \param k Number of hash functions
|
||||
* \param p Probability of false positives
|
||||
* \return Number of bits in the filter
|
||||
*/
|
||||
static size_t NumberOfBits(size_t n, size_t k, double p) {
|
||||
return (size_t)std::ceil(-((double)k * (double)n /
|
||||
std::log(1.0 - std::pow(p, 1.0 / (double)k))));
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Calculate number of bytes in the filter
|
||||
* \param n Number of items in the filter
|
||||
* \param p Probability of false positives
|
||||
* \return Number of bytes in the filter
|
||||
*/
|
||||
static size_t NumberOfBytes(size_t n, double p) {
|
||||
return ((NumberOfBits(n, p) + 7) >> 3);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Calculate number of bits in the filter
|
||||
* \param n Number of items in the filter
|
||||
* \param k Number of hash functions
|
||||
* \param p Probability of false positives
|
||||
* \return Number of bits in the filter
|
||||
*/
|
||||
static size_t NumberOfBytes(size_t n, size_t k, double p) {
|
||||
return ((NumberOfBits(n, k, p) + 7) >> 3);
|
||||
}
|
||||
|
||||
/**
|
||||
* \brief Calculate number of hash functions
|
||||
* \param n Number of items in the filter
|
||||
* \param m Number of bits in the filter
|
||||
* \return Number of hash functions
|
||||
*/
|
||||
static size_t NumberOfHash(size_t n, size_t m) {
|
||||
return (size_t)std::round((double)m / (double)n * std::log(2.0));
|
||||
}
|
||||
};
|
||||
|
||||
/*! Bloom Filter
|
||||
*/
|
||||
template <size_t K>
|
||||
class BloomFilter {
|
||||
public:
|
||||
//! Constructor
|
||||
BloomFilter(void) {}
|
||||
|
||||
//! Constructor
|
||||
BloomFilter(size_t n, double p) {
|
||||
if (n > 0 && p > 0.0 && p < 1.0) {
|
||||
capacity_ = n;
|
||||
bits_count_ = BloomFilterCalculator::NumberOfBits(n, K, p);
|
||||
bits_count_ = ((bits_count_ + 31) >> 5) << 5;
|
||||
probability_ = BloomFilterCalculator::Probability(n, bits_count_, K);
|
||||
bitset_ = new uint32_t[bits_count_ >> 5];
|
||||
memset(bitset_, 0, (bits_count_ >> 3));
|
||||
}
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
BloomFilter(BloomFilter &&rhs)
|
||||
: bitset_(rhs.bitset_),
|
||||
bits_count_(rhs.bits_count_),
|
||||
capacity_(rhs.capacity_),
|
||||
count_(rhs.count_),
|
||||
probability_(rhs.probability_) {
|
||||
rhs.bitset_ = nullptr;
|
||||
rhs.bits_count_ = 0u;
|
||||
rhs.capacity_ = 0u;
|
||||
rhs.count_ = 0u;
|
||||
rhs.probability_ = 0u;
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
~BloomFilter(void) {
|
||||
delete[] bitset_;
|
||||
}
|
||||
|
||||
//! Test if the filter is valid
|
||||
bool is_valid(void) const {
|
||||
return (bitset_ != nullptr);
|
||||
}
|
||||
|
||||
//! Reset the bloom filter
|
||||
bool reset(size_t n, double p) {
|
||||
if (n <= 0 || p <= 0.0 || p >= 1.0) {
|
||||
return false;
|
||||
}
|
||||
delete[] bitset_;
|
||||
capacity_ = n;
|
||||
count_ = 0u;
|
||||
bits_count_ = BloomFilterCalculator::NumberOfBits(n, K, p);
|
||||
bits_count_ = ((bits_count_ + 31) >> 5) << 5;
|
||||
probability_ = BloomFilterCalculator::Probability(n, bits_count_, K);
|
||||
bitset_ = new (std::nothrow) uint32_t[bits_count_ >> 5];
|
||||
if (!bitset_) {
|
||||
return false;
|
||||
}
|
||||
memset(bitset_, 0, (bits_count_ >> 3));
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Clear the bloom filter
|
||||
void clear(void) {
|
||||
if (bitset_) {
|
||||
memset(bitset_, 0, (bits_count_ >> 3));
|
||||
count_ = 0u;
|
||||
}
|
||||
}
|
||||
|
||||
//! Insert a item into bloom filter
|
||||
template <typename... TArgs,
|
||||
typename = typename std::enable_if<
|
||||
Conjunction<std::is_integral<TArgs>...>::value &&
|
||||
sizeof...(TArgs) == K>::type>
|
||||
bool insert(TArgs... vals) {
|
||||
if (count_ >= capacity_) {
|
||||
return false;
|
||||
}
|
||||
this->set_bits(vals...);
|
||||
++count_;
|
||||
return true;
|
||||
}
|
||||
|
||||
//! Force insert a item into bloom filter
|
||||
template <typename... TArgs,
|
||||
typename = typename std::enable_if<
|
||||
Conjunction<std::is_integral<TArgs>...>::value &&
|
||||
sizeof...(TArgs) == K>::type>
|
||||
void force_insert(TArgs... vals) {
|
||||
this->set_bits(vals...);
|
||||
++count_;
|
||||
}
|
||||
|
||||
//! Insert a item into bloom filter
|
||||
template <typename... TArgs,
|
||||
typename = typename std::enable_if<
|
||||
Conjunction<std::is_integral<TArgs>...>::value &&
|
||||
sizeof...(TArgs) == K>::type>
|
||||
bool has(TArgs... vals) const {
|
||||
return this->test_bits(vals...);
|
||||
}
|
||||
|
||||
//! Retrieve count of bits in bloom filter
|
||||
size_t bits_count(void) const {
|
||||
return bits_count_;
|
||||
}
|
||||
|
||||
//! Retrieve capacity of bloom filter
|
||||
size_t capacity(void) const {
|
||||
return capacity_;
|
||||
}
|
||||
|
||||
//! Retrieve count of items in bloom filter
|
||||
size_t count(void) const {
|
||||
return count_;
|
||||
}
|
||||
|
||||
//! Retrieve probability of false positives
|
||||
double probability(void) const {
|
||||
return probability_;
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Disable them
|
||||
BloomFilter(const BloomFilter &) = delete;
|
||||
BloomFilter &operator=(const BloomFilter &) = delete;
|
||||
|
||||
//! Set bits in bloom filter
|
||||
template <typename TArg>
|
||||
void set_bits(TArg val) {
|
||||
size_t num = static_cast<size_t>(val) % bits_count_;
|
||||
bitset_[num >> 5] |= (1u << (num & 0x1f));
|
||||
}
|
||||
|
||||
//! Set bits in bloom filter
|
||||
template <typename TArg, typename... TArgs>
|
||||
void set_bits(TArg val, TArgs... vals) {
|
||||
this->set_bits(val);
|
||||
this->set_bits(vals...);
|
||||
}
|
||||
|
||||
//! Test bits in bloom filter
|
||||
template <typename TArg>
|
||||
bool test_bits(TArg val) const {
|
||||
size_t num = static_cast<size_t>(val) % bits_count_;
|
||||
return ((bitset_[num >> 5] & (1u << (num & 0x1f))) != 0);
|
||||
}
|
||||
|
||||
//! Test bits in bloom filter
|
||||
template <typename TArg, typename... TArgs>
|
||||
bool test_bits(TArg val, TArgs... vals) const {
|
||||
if (!this->test_bits(val)) {
|
||||
return false;
|
||||
}
|
||||
return this->test_bits(vals...);
|
||||
}
|
||||
|
||||
private:
|
||||
uint32_t *bitset_{nullptr};
|
||||
size_t bits_count_{0u};
|
||||
size_t capacity_{0u};
|
||||
size_t count_{0u};
|
||||
double probability_{0.0};
|
||||
};
|
||||
|
||||
/*! Bloom Filter (Special)
|
||||
*/
|
||||
template <>
|
||||
struct BloomFilter<0> {};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,414 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <string>
|
||||
#include <typeinfo>
|
||||
#include <ailego/utility/type_helper.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
namespace internal {
|
||||
|
||||
/*! Cube Policy
|
||||
*/
|
||||
struct CubePolicy {
|
||||
//! Destructor
|
||||
virtual ~CubePolicy(void) {}
|
||||
|
||||
//! Assign `src` to `dst`
|
||||
virtual void assign(const void *src, void **dst) = 0;
|
||||
|
||||
//! Cleanup value
|
||||
virtual void cleanup(void **val) = 0;
|
||||
|
||||
//! Clone value of `src` to `dst`
|
||||
virtual void clone(void *const *src, void **dst) = 0;
|
||||
|
||||
//! Move `src` to `dst`
|
||||
virtual void move(void *src, void **dst) = 0;
|
||||
|
||||
//! Retrieve size
|
||||
virtual size_t size(void) const = 0;
|
||||
|
||||
//! Retrieve type information
|
||||
virtual const std::type_info &type(void) const = 0;
|
||||
|
||||
//! Retrieve value
|
||||
virtual void *value(void **src) = 0;
|
||||
|
||||
//! Retrieve value
|
||||
virtual const void *value(void *const *src) const = 0;
|
||||
};
|
||||
|
||||
/*! Small Cube Policy
|
||||
*/
|
||||
template <typename T>
|
||||
struct SmallCubePolicy : public CubePolicy {
|
||||
//! Assign `src` to `dst`
|
||||
void assign(const void *src, void **dst) {
|
||||
new (dst) T(*reinterpret_cast<const T *>(src));
|
||||
}
|
||||
|
||||
//! Cleanup value
|
||||
void cleanup(void **val) {
|
||||
reinterpret_cast<T *>(val)->~T();
|
||||
}
|
||||
|
||||
//! Clone value of `src` to `dst`
|
||||
void clone(void *const *src, void **dst) {
|
||||
new (dst) T(*reinterpret_cast<const T *>(src));
|
||||
}
|
||||
|
||||
//! Move `src` to `dst`
|
||||
void move(void *src, void **dst) {
|
||||
new (dst) T(std::move(*reinterpret_cast<T *>(src)));
|
||||
}
|
||||
|
||||
//! Retrieve size
|
||||
size_t size(void) const {
|
||||
return sizeof(T);
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
const std::type_info &type(void) const {
|
||||
return typeid(T);
|
||||
}
|
||||
|
||||
//! Retrieve value
|
||||
void *value(void **src) {
|
||||
return reinterpret_cast<void *>(src);
|
||||
}
|
||||
|
||||
//! Retrieve value
|
||||
const void *value(void *const *src) const {
|
||||
return reinterpret_cast<const void *>(src);
|
||||
}
|
||||
};
|
||||
|
||||
/*! Large Cube Policy
|
||||
*/
|
||||
template <typename T>
|
||||
struct LargeCubePolicy : public CubePolicy {
|
||||
//! Assign `src` to `dst`
|
||||
void assign(const void *src, void **dst) {
|
||||
*dst = new T(*reinterpret_cast<const T *>(src));
|
||||
}
|
||||
|
||||
//! Cleanup value
|
||||
void cleanup(void **val) {
|
||||
delete (reinterpret_cast<T *>(*val));
|
||||
}
|
||||
|
||||
//! Clone value of `src` to `dst`
|
||||
void clone(void *const *src, void **dst) {
|
||||
*dst = new T(**reinterpret_cast<T *const *>(src));
|
||||
}
|
||||
|
||||
//! Move `src` to `dst`
|
||||
void move(void *src, void **dst) {
|
||||
*dst = new T(std::move(*reinterpret_cast<T *>(src)));
|
||||
}
|
||||
|
||||
//! Retrieve size
|
||||
size_t size(void) const {
|
||||
return sizeof(T);
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
const std::type_info &type(void) const {
|
||||
return typeid(T);
|
||||
}
|
||||
|
||||
//! Retrieve value
|
||||
void *value(void **src) {
|
||||
return *src;
|
||||
}
|
||||
|
||||
//! Retrieve value
|
||||
const void *value(void *const *src) const {
|
||||
return *src;
|
||||
}
|
||||
};
|
||||
|
||||
/*! Policy Selector
|
||||
*/
|
||||
template <typename T, typename = void>
|
||||
struct PolicySelector {
|
||||
typedef LargeCubePolicy<T> Type;
|
||||
};
|
||||
|
||||
/*! Policy Selector
|
||||
*/
|
||||
template <typename T>
|
||||
struct PolicySelector<
|
||||
T, typename std::enable_if<sizeof(T) <= sizeof(void *)>::type> {
|
||||
typedef SmallCubePolicy<T> Type;
|
||||
};
|
||||
|
||||
} // namespace internal
|
||||
|
||||
/*! Cube class
|
||||
*/
|
||||
class Cube {
|
||||
public:
|
||||
//! Constructor
|
||||
Cube(void) : policy_(Cube::Policy<Cube::EmptyPolicy>()), object_(nullptr) {}
|
||||
|
||||
//! Constructor
|
||||
template <typename T>
|
||||
Cube(const T &rhs) : policy_(Cube::Policy<T>()), object_(nullptr) {
|
||||
policy_->assign(&rhs, &object_);
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
template <typename T, typename = typename std::enable_if<
|
||||
!std::is_same<Cube &, T>::value &&
|
||||
!std::is_same<T &, T>::value>::type>
|
||||
Cube(T &&rhs) : policy_(Cube::Policy<T>()), object_(nullptr) {
|
||||
policy_->move(&rhs, &object_);
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
Cube(const char *str)
|
||||
: policy_(Cube::Policy<std::string>()), object_(nullptr) {
|
||||
std::string rhs(str);
|
||||
policy_->move(&rhs, &object_);
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
Cube(char str[]) : policy_(Cube::Policy<std::string>()), object_(nullptr) {
|
||||
std::string rhs(str);
|
||||
policy_->move(&rhs, &object_);
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
Cube(const Cube &rhs) : policy_(rhs.policy_), object_(nullptr) {
|
||||
policy_->clone(&rhs.object_, &object_);
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
Cube(Cube &&rhs) : policy_(rhs.policy_), object_(rhs.object_) {
|
||||
rhs.policy_ = Cube::Policy<Cube::EmptyPolicy>();
|
||||
rhs.object_ = nullptr;
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
~Cube(void) {
|
||||
policy_->cleanup(&object_);
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
template <typename T>
|
||||
Cube &operator=(const T &rhs) {
|
||||
this->assign(rhs);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
template <typename T, typename = typename std::enable_if<
|
||||
!std::is_same<Cube &, T>::value &&
|
||||
!std::is_same<T &, T>::value>::type>
|
||||
Cube &operator=(T &&rhs) {
|
||||
this->assign(std::forward<T>(rhs));
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Cube &operator=(const Cube &rhs) {
|
||||
this->assign(rhs);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Cube &operator=(Cube &&rhs) {
|
||||
this->assign(std::forward<Cube>(rhs));
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Cube &operator=(const char *str) {
|
||||
this->assign(str);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Cube &operator=(char str[]) {
|
||||
this->assign(str);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Retrieve object in original type
|
||||
template <typename T>
|
||||
operator T &() {
|
||||
return this->cast<T>();
|
||||
}
|
||||
|
||||
//! Retrieve object in original type
|
||||
template <typename T>
|
||||
operator const T &() const {
|
||||
return this->cast<T>();
|
||||
}
|
||||
|
||||
//! Assign content
|
||||
template <typename T>
|
||||
void assign(const T &rhs) {
|
||||
policy_->cleanup(&object_);
|
||||
policy_ = Cube::Policy<T>();
|
||||
policy_->assign(&rhs, &object_);
|
||||
}
|
||||
|
||||
//! Assign content
|
||||
template <typename T, typename = typename std::enable_if<
|
||||
!std::is_same<Cube &, T>::value &&
|
||||
!std::is_same<T &, T>::value>::type>
|
||||
void assign(T &&rhs) {
|
||||
policy_->cleanup(&object_);
|
||||
policy_ = Cube::Policy<T>();
|
||||
policy_->move(&rhs, &object_);
|
||||
}
|
||||
|
||||
//! Assign content from another Cube
|
||||
void assign(const Cube &rhs) {
|
||||
policy_->cleanup(&object_);
|
||||
policy_ = rhs.policy_;
|
||||
policy_->clone(&rhs.object_, &object_);
|
||||
}
|
||||
|
||||
//! Assign content from another Cube
|
||||
void assign(Cube &&rhs) {
|
||||
if (this != &rhs) {
|
||||
policy_->cleanup(&object_);
|
||||
policy_ = rhs.policy_;
|
||||
object_ = rhs.object_;
|
||||
rhs.policy_ = Cube::Policy<Cube::EmptyPolicy>();
|
||||
rhs.object_ = nullptr;
|
||||
}
|
||||
}
|
||||
|
||||
//! Assign content
|
||||
void assign(const char *str) {
|
||||
policy_->cleanup(&object_);
|
||||
policy_ = Cube::Policy<std::string>();
|
||||
std::string rhs(str);
|
||||
policy_->move(&rhs, &object_);
|
||||
}
|
||||
|
||||
//! Assign content
|
||||
void assign(char str[]) {
|
||||
policy_->cleanup(&object_);
|
||||
policy_ = Cube::Policy<std::string>();
|
||||
std::string rhs(str);
|
||||
policy_->move(&rhs, &object_);
|
||||
}
|
||||
|
||||
//! Swap the content with another Cube
|
||||
Cube &swap(Cube &rhs) {
|
||||
std::swap(policy_, rhs.policy_);
|
||||
std::swap(object_, rhs.object_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Cast to the original type
|
||||
template <typename T>
|
||||
T &cast(void) {
|
||||
if (policy_ != Cube::Policy<T>()) {
|
||||
throw std::bad_cast();
|
||||
}
|
||||
return *reinterpret_cast<T *>(policy_->value(&object_));
|
||||
}
|
||||
|
||||
//! Cast to the original type
|
||||
template <typename T>
|
||||
const T &cast(void) const {
|
||||
if (policy_ != Cube::Policy<T>()) {
|
||||
throw std::bad_cast();
|
||||
}
|
||||
return *reinterpret_cast<const T *>(policy_->value(&object_));
|
||||
}
|
||||
|
||||
//! Cast to the original type (unsafe)
|
||||
template <typename T>
|
||||
T &unsafe_cast(void) {
|
||||
return *reinterpret_cast<T *>(policy_->value(&object_));
|
||||
}
|
||||
|
||||
//! Cast to the original type (unsafe)
|
||||
template <typename T>
|
||||
const T &unsafe_cast(void) const {
|
||||
return *reinterpret_cast<const T *>(policy_->value(&object_));
|
||||
}
|
||||
|
||||
//! Test if the Cube is empty
|
||||
bool empty(void) const {
|
||||
return (policy_ == Cube::Policy<Cube::EmptyPolicy>());
|
||||
}
|
||||
|
||||
//! Reset Cube allocated memory
|
||||
void reset(void) {
|
||||
policy_->cleanup(&object_);
|
||||
policy_ = Cube::Policy<Cube::EmptyPolicy>();
|
||||
object_ = nullptr;
|
||||
}
|
||||
|
||||
//! Test if the Cube is compatible with another one
|
||||
bool compatible(const Cube &rhs) const {
|
||||
return (policy_ == rhs.policy_ || policy_->type() == rhs.policy_->type());
|
||||
}
|
||||
|
||||
//! Test if the Cube is compatible with another one
|
||||
template <typename T>
|
||||
bool compatible(void) const {
|
||||
return (policy_ == Cube::Policy<T>() ||
|
||||
policy_->type() == Cube::Policy<T>()->type());
|
||||
}
|
||||
|
||||
//! Retrieve size
|
||||
size_t size(void) const {
|
||||
return (!this->empty() ? policy_->size() : 0u);
|
||||
}
|
||||
|
||||
//! Retrieve type information
|
||||
const std::type_info &type(void) const {
|
||||
return (!this->empty() ? policy_->type() : typeid(void));
|
||||
}
|
||||
|
||||
protected:
|
||||
/*! Empty Policy
|
||||
*/
|
||||
struct EmptyPolicy {};
|
||||
|
||||
//! Make a static policy object
|
||||
template <typename T>
|
||||
static internal::CubePolicy *MakePolicy(void) {
|
||||
static typename internal::PolicySelector<T>::Type policy;
|
||||
return (&policy);
|
||||
}
|
||||
|
||||
//! Retrieve a static policy object
|
||||
template <typename T>
|
||||
static internal::CubePolicy *Policy(void) {
|
||||
return MakePolicy<typename UnderlyingType<T>::type>();
|
||||
}
|
||||
|
||||
private:
|
||||
//! Members
|
||||
internal::CubePolicy *policy_;
|
||||
void *object_;
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,234 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <algorithm>
|
||||
#include <functional>
|
||||
#include <limits>
|
||||
#include <utility>
|
||||
#include <vector>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! Heap Adapter
|
||||
*/
|
||||
template <typename T, typename TCompare = std::less<T>,
|
||||
typename TBase = std::vector<T>>
|
||||
class Heap : public TBase {
|
||||
public:
|
||||
//! Constructor
|
||||
Heap(void)
|
||||
: TBase(), limit_(std::numeric_limits<size_t>::max()), compare_() {}
|
||||
|
||||
//! Constructor
|
||||
template <typename... Args>
|
||||
Heap(size_t max, Args &&...args)
|
||||
: TBase(),
|
||||
limit_(std::max<size_t>(max, 1u)),
|
||||
compare_(std::forward<Args>(args)...) {
|
||||
TBase::reserve(limit_);
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
Heap(const Heap &rhs)
|
||||
: TBase(rhs), limit_(rhs.limit_), compare_(rhs.compare_) {}
|
||||
|
||||
//! Constructor
|
||||
Heap(Heap &&rhs)
|
||||
: TBase(std::move(rhs)),
|
||||
limit_(rhs.limit_),
|
||||
compare_(std::move(rhs.compare_)) {}
|
||||
|
||||
//! Constructor
|
||||
Heap(const TBase &rhs)
|
||||
: TBase(rhs), limit_(std::numeric_limits<size_t>::max()), compare_() {
|
||||
std::make_heap(TBase::begin(), TBase::end(), compare_);
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
Heap(TBase &&rhs)
|
||||
: TBase(std::move(rhs)),
|
||||
limit_(std::numeric_limits<size_t>::max()),
|
||||
compare_() {
|
||||
std::make_heap(TBase::begin(), TBase::end(), compare_);
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Heap &operator=(const Heap &rhs) {
|
||||
TBase::operator=(static_cast<const TBase &>(rhs));
|
||||
limit_ = rhs.limit_;
|
||||
compare_ = rhs.compare_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Heap &operator=(Heap &&rhs) {
|
||||
TBase::operator=(std::move(static_cast<TBase &&>(rhs)));
|
||||
limit_ = rhs.limit_;
|
||||
compare_ = std::move(rhs.compare_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Exchange the content
|
||||
void swap(Heap &rhs) {
|
||||
TBase::swap(static_cast<TBase &>(rhs));
|
||||
std::swap(limit_, rhs.limit_);
|
||||
std::swap(compare_, rhs.compare_);
|
||||
}
|
||||
|
||||
//! Pop the front element
|
||||
void pop(void) {
|
||||
if (TBase::size() > 1) {
|
||||
auto last = TBase::end() - 1;
|
||||
this->replace_heap(TBase::begin(), last, std::move(*last));
|
||||
}
|
||||
TBase::pop_back();
|
||||
}
|
||||
|
||||
//! Insert a new element into the heap
|
||||
template <class... TArgs>
|
||||
void emplace(TArgs &&...args) {
|
||||
if (this->full()) {
|
||||
typename std::remove_reference<T>::type val(std::forward<TArgs>(args)...);
|
||||
|
||||
auto first = TBase::begin();
|
||||
if (compare_(val, *first)) {
|
||||
this->replace_heap(first, TBase::end(), std::move(val));
|
||||
}
|
||||
} else {
|
||||
TBase::emplace_back(std::forward<TArgs>(args)...);
|
||||
std::push_heap(TBase::begin(), TBase::end(), compare_);
|
||||
}
|
||||
}
|
||||
|
||||
//! Insert a new element into the heap
|
||||
void push(const T &val) {
|
||||
if (this->full()) {
|
||||
auto first = TBase::begin();
|
||||
if (compare_(val, *first)) {
|
||||
this->replace_heap(first, TBase::end(), val);
|
||||
}
|
||||
} else {
|
||||
TBase::push_back(val);
|
||||
std::push_heap(TBase::begin(), TBase::end(), compare_);
|
||||
}
|
||||
}
|
||||
|
||||
//! Insert a new element into the heap
|
||||
void push(T &&val) {
|
||||
if (this->full()) {
|
||||
auto first = TBase::begin();
|
||||
if (compare_(val, *first)) {
|
||||
this->replace_heap(first, TBase::end(), std::move(val));
|
||||
}
|
||||
} else {
|
||||
TBase::push_back(std::move(val));
|
||||
std::push_heap(TBase::begin(), TBase::end(), compare_);
|
||||
}
|
||||
}
|
||||
|
||||
//! Retrieve the limit of heap
|
||||
size_t limit(void) const {
|
||||
return limit_;
|
||||
}
|
||||
|
||||
//! Limit the heap with max size
|
||||
void limit(size_t max) {
|
||||
limit_ = std::max<size_t>(max, 1u);
|
||||
TBase::reserve(limit_);
|
||||
}
|
||||
|
||||
//! Unlimit the size of heap
|
||||
void unlimit(void) {
|
||||
limit_ = std::numeric_limits<size_t>::max();
|
||||
}
|
||||
|
||||
//! Check whether the heap is full
|
||||
bool full(void) const {
|
||||
return (TBase::size() == limit_);
|
||||
}
|
||||
|
||||
//! Update the heap
|
||||
void update(void) {
|
||||
std::make_heap(TBase::begin(), TBase::end(), compare_);
|
||||
while (limit_ < TBase::size()) {
|
||||
this->pop();
|
||||
}
|
||||
}
|
||||
|
||||
//! Sort the elements in the heap
|
||||
void sort(void) {
|
||||
std::sort(TBase::begin(), TBase::end(), compare_);
|
||||
}
|
||||
|
||||
protected:
|
||||
//! Replace the top element of heap
|
||||
template <typename TRandomIterator, typename TValue>
|
||||
void replace_heap(TRandomIterator first, TRandomIterator last, TValue &&val) {
|
||||
using _DistanceType =
|
||||
typename std::iterator_traits<TRandomIterator>::difference_type;
|
||||
|
||||
_DistanceType hole = 0;
|
||||
_DistanceType count = _DistanceType(last - first);
|
||||
|
||||
if (count > 1) {
|
||||
_DistanceType child = (hole << 1) + 1;
|
||||
|
||||
while (child < count) {
|
||||
_DistanceType right_child = child + 1;
|
||||
|
||||
if (right_child < count &&
|
||||
compare_(*(first + child), *(first + right_child))) {
|
||||
child = right_child;
|
||||
}
|
||||
if (!compare_(val, *(first + child))) {
|
||||
break;
|
||||
}
|
||||
*(first + hole) = std::move(*(first + child));
|
||||
hole = child;
|
||||
child = (hole << 1) + 1;
|
||||
}
|
||||
}
|
||||
*(first + hole) = std::forward<TValue>(val);
|
||||
}
|
||||
|
||||
private:
|
||||
size_t limit_;
|
||||
TCompare compare_;
|
||||
};
|
||||
|
||||
/*! Key Value Heap Comparer
|
||||
*/
|
||||
template <typename TKey, typename TValue, typename TCompare = std::less<TValue>>
|
||||
struct KeyValueHeapComparer {
|
||||
//! Function call
|
||||
bool operator()(const std::pair<TKey, TValue> &lhs,
|
||||
const std::pair<TKey, TValue> &rhs) const {
|
||||
return compare_(lhs.second, rhs.second);
|
||||
}
|
||||
|
||||
private:
|
||||
TCompare compare_;
|
||||
};
|
||||
|
||||
/*! Key Value Heap
|
||||
*/
|
||||
template <typename TKey, typename TValue, typename TCompare = std::less<TValue>>
|
||||
using KeyValueHeap =
|
||||
Heap<std::pair<TKey, TValue>, KeyValueHeapComparer<TKey, TValue, TCompare>>;
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,268 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <map>
|
||||
#include <string>
|
||||
#include "cube.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! Hypercube
|
||||
*/
|
||||
class Hypercube {
|
||||
public:
|
||||
//! Constructor
|
||||
Hypercube(void) : cubes_() {}
|
||||
|
||||
//! Constructor
|
||||
Hypercube(const Hypercube &rhs) : cubes_(rhs.cubes_) {}
|
||||
|
||||
//! Constructor
|
||||
Hypercube(Hypercube &&rhs) : cubes_() {
|
||||
cubes_.swap(rhs.cubes_);
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
~Hypercube(void) {}
|
||||
|
||||
//! Assignment
|
||||
Hypercube &operator=(const Hypercube &rhs) {
|
||||
cubes_ = rhs.cubes_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Hypercube &operator=(Hypercube &&rhs) {
|
||||
cubes_ = std::move(rhs.cubes_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
Cube &operator[](const std::string &key) {
|
||||
return cubes_[key];
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
Cube &operator[](std::string &&key) {
|
||||
return cubes_[std::forward<std::string>(key)];
|
||||
}
|
||||
|
||||
//! Test if the element is exist
|
||||
bool has(const std::string &key) const {
|
||||
return (cubes_.find(key) != cubes_.end());
|
||||
}
|
||||
|
||||
//! Test if the hyper cube is empty
|
||||
bool empty(void) const {
|
||||
return cubes_.empty();
|
||||
}
|
||||
|
||||
//! Insert a key-value pair into map
|
||||
bool insert(const std::string &key, Cube &&val) {
|
||||
return cubes_.emplace(key, std::forward<Cube>(val)).second;
|
||||
}
|
||||
|
||||
//! Insert a key-value pair into map
|
||||
bool insert(std::string &&key, Cube &&val) {
|
||||
return cubes_
|
||||
.emplace(std::forward<std::string>(key), std::forward<Cube>(val))
|
||||
.second;
|
||||
}
|
||||
|
||||
//! Insert a key-value pair into map
|
||||
template <typename T>
|
||||
bool insert(const std::string &key, T &&val) {
|
||||
return cubes_.emplace(key, Cube(std::forward<T>(val))).second;
|
||||
}
|
||||
|
||||
//! Insert a key-value pair into map
|
||||
template <typename T>
|
||||
bool insert(std::string &&key, T &&val) {
|
||||
return cubes_
|
||||
.emplace(std::forward<std::string>(key), Cube(std::forward<T>(val)))
|
||||
.second;
|
||||
}
|
||||
|
||||
//! Insert or assign a key-value pair to map
|
||||
void insert_or_assign(const std::string &key, Cube &&val) {
|
||||
auto it = cubes_.lower_bound(key);
|
||||
if (it != cubes_.end() && it->first == key) {
|
||||
it->second = std::forward<Cube>(val);
|
||||
} else {
|
||||
cubes_.emplace_hint(it, key, std::forward<Cube>(val));
|
||||
}
|
||||
}
|
||||
|
||||
//! Insert or assign a key-value pair to map
|
||||
void insert_or_assign(std::string &&key, Cube &&val) {
|
||||
auto it = cubes_.lower_bound(key);
|
||||
if (it != cubes_.end() && it->first == key) {
|
||||
it->second = std::forward<Cube>(val);
|
||||
} else {
|
||||
cubes_.emplace_hint(it, std::forward<std::string>(key),
|
||||
std::forward<Cube>(val));
|
||||
}
|
||||
}
|
||||
|
||||
//! Insert or assign a key-value pair to map
|
||||
template <typename T>
|
||||
void insert_or_assign(const std::string &key, T &&val) {
|
||||
auto it = cubes_.lower_bound(key);
|
||||
if (it != cubes_.end() && it->first == key) {
|
||||
it->second = Cube(std::forward<T>(val));
|
||||
} else {
|
||||
cubes_.emplace_hint(it, key, Cube(std::forward<T>(val)));
|
||||
}
|
||||
}
|
||||
|
||||
//! Insert or assign a key-value pair to map
|
||||
template <typename T>
|
||||
void insert_or_assign(std::string &&key, T &&val) {
|
||||
auto it = cubes_.lower_bound(key);
|
||||
if (it != cubes_.end() && it->first == key) {
|
||||
it->second = Cube(std::forward<T>(val));
|
||||
} else {
|
||||
cubes_.emplace_hint(it, std::forward<std::string>(key),
|
||||
Cube(std::forward<T>(val)));
|
||||
}
|
||||
}
|
||||
|
||||
//! Clear the map
|
||||
void clear(void) {
|
||||
cubes_.clear();
|
||||
}
|
||||
|
||||
//! Swap the map
|
||||
void swap(Hypercube &rhs) {
|
||||
cubes_.swap(rhs.cubes_);
|
||||
}
|
||||
|
||||
//! Erase the pair via a key
|
||||
bool erase(const std::string &key) {
|
||||
auto iter = cubes_.find(key);
|
||||
if (iter != cubes_.end()) {
|
||||
cubes_.erase(iter);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value via a key
|
||||
bool get(const std::string &key, Cube *out) const {
|
||||
auto iter = cubes_.find(key);
|
||||
if (iter != cubes_.end()) {
|
||||
*out = iter->second;
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value via a key
|
||||
Cube *get(const std::string &key) {
|
||||
auto iter = cubes_.find(key);
|
||||
if (iter != cubes_.end()) {
|
||||
return &iter->second;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//! Retrieve the value via a key
|
||||
const Cube *get(const std::string &key) const {
|
||||
auto iter = cubes_.find(key);
|
||||
if (iter != cubes_.end()) {
|
||||
return &iter->second;
|
||||
}
|
||||
return nullptr;
|
||||
}
|
||||
|
||||
//! Retrieve the value via a key
|
||||
template <typename T>
|
||||
bool get(const std::string &key, T *out) const {
|
||||
auto iter = cubes_.find(key);
|
||||
if (iter != cubes_.end()) {
|
||||
if (iter->second.compatible<T>()) {
|
||||
*out = iter->second.unsafe_cast<T>();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value via a key
|
||||
template <typename T>
|
||||
T &get(const std::string &key, T &def) {
|
||||
auto iter = cubes_.find(key);
|
||||
if (iter != cubes_.end()) {
|
||||
if (iter->second.compatible<T>()) {
|
||||
return iter->second.unsafe_cast<T>();
|
||||
}
|
||||
}
|
||||
return def;
|
||||
}
|
||||
|
||||
//! Retrieve the value via a key
|
||||
template <typename T>
|
||||
const T &get(const std::string &key, const T &def) const {
|
||||
auto iter = cubes_.find(key);
|
||||
if (iter != cubes_.end()) {
|
||||
if (iter->second.compatible<T>()) {
|
||||
return iter->second.unsafe_cast<T>();
|
||||
}
|
||||
}
|
||||
return def;
|
||||
}
|
||||
|
||||
//! Merge another hyper cube
|
||||
void merge(const Hypercube &rhs) {
|
||||
for (const auto &it : rhs.cubes_) {
|
||||
auto iter = cubes_.find(it.first);
|
||||
if (iter != cubes_.end()) {
|
||||
iter->second = it.second;
|
||||
} else {
|
||||
cubes_.emplace(it.first, it.second);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//! Merge another hyper cube
|
||||
void merge(Hypercube &&rhs) {
|
||||
for (auto &it : rhs.cubes_) {
|
||||
auto iter = cubes_.find(it.first);
|
||||
if (iter != cubes_.end()) {
|
||||
iter->second = std::move(it.second);
|
||||
} else {
|
||||
cubes_.emplace(std::move(it.first), std::move(it.second));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//! Retrieve the cubes
|
||||
const std::map<std::string, Cube> &cubes(void) const {
|
||||
return cubes_;
|
||||
}
|
||||
|
||||
//! Retrieve the cubes
|
||||
std::map<std::string, Cube> *mutable_cubes(void) {
|
||||
return &cubes_;
|
||||
}
|
||||
|
||||
private:
|
||||
std::map<std::string, Cube> cubes_;
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,139 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#include "params.h"
|
||||
#include <cstring>
|
||||
#include <ailego/encoding/json.h>
|
||||
#include "ailego/logger/logger.h"
|
||||
|
||||
//! Global environ variable
|
||||
extern char **environ;
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
static void ParseFromJsonObject(const ailego::JsonObject &obj, Params *params) {
|
||||
for (ailego::JsonObject::const_iterator it = obj.begin(); it != obj.end();
|
||||
++it) {
|
||||
const ailego::JsonValue &val = it->value();
|
||||
|
||||
if (val.is_boolean()) {
|
||||
params->set(it->key().as_stl_string(), val.as_bool());
|
||||
} else if (val.is_integer()) {
|
||||
params->set(it->key().as_stl_string(),
|
||||
static_cast<int64_t>(val.as_integer()));
|
||||
} else if (val.is_float()) {
|
||||
params->set(it->key().as_stl_string(), val.as_float());
|
||||
} else if (val.is_string()) {
|
||||
params->set(it->key().as_stl_string(),
|
||||
val.as_string().decode().as_stl_string());
|
||||
} else if (val.is_object()) {
|
||||
Params subparams;
|
||||
ParseFromJsonObject(val.as_object(), &subparams);
|
||||
params->set(it->key().as_stl_string(), std::move(subparams));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
bool Params::ParseFromBuffer(const std::string &buf, Params *params) {
|
||||
ailego::JsonValue val;
|
||||
ailego::JsonParser parser;
|
||||
|
||||
parser.set_comment(true);
|
||||
parser.set_simple(true);
|
||||
parser.set_squote(true);
|
||||
parser.set_unstrict(false);
|
||||
if (!parser.parse(buf.c_str(), &val)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!val.is_object()) {
|
||||
return false;
|
||||
}
|
||||
ParseFromJsonObject(val.as_object(), params);
|
||||
return true;
|
||||
}
|
||||
|
||||
void Params::ParseFromEnvironment(Params *params) {
|
||||
// Dump all environ string
|
||||
for (size_t i = 0; environ[i]; ++i) {
|
||||
const char *env = environ[i];
|
||||
const char *p = std::strchr(env, '=');
|
||||
if (p) {
|
||||
params->set(std::string(env, p - env), std::string(p + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static void SerializeToJsonObject(const Params ¶ms,
|
||||
ailego::JsonObject *obj) {
|
||||
for (const auto &it : params.hypercube().cubes()) {
|
||||
const ailego::Cube &cube = it.second;
|
||||
const char *key = it.first.c_str();
|
||||
|
||||
if (cube.compatible<std::string>()) {
|
||||
const auto &val = cube.unsafe_cast<std::string>();
|
||||
ailego::JsonString str(val.data(), val.size());
|
||||
obj->set(key, ailego::JsonValue(str.encode()));
|
||||
} else if (cube.compatible<unsigned long long int>()) {
|
||||
obj->set(key,
|
||||
ailego::JsonValue(cube.unsafe_cast<unsigned long long int>()));
|
||||
} else if (cube.compatible<long long int>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<long long int>()));
|
||||
} else if (cube.compatible<unsigned long int>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<unsigned long int>()));
|
||||
} else if (cube.compatible<long int>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<long int>()));
|
||||
} else if (cube.compatible<unsigned int>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<unsigned int>()));
|
||||
} else if (cube.compatible<int>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<int>()));
|
||||
} else if (cube.compatible<unsigned short int>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<unsigned short int>()));
|
||||
} else if (cube.compatible<short int>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<short int>()));
|
||||
} else if (cube.compatible<unsigned char>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<unsigned char>()));
|
||||
} else if (cube.compatible<char>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<char>()));
|
||||
} else if (cube.compatible<signed char>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<signed char>()));
|
||||
} else if (cube.compatible<bool>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<bool>()));
|
||||
} else if (cube.compatible<float>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<float>()));
|
||||
} else if (cube.compatible<double>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<double>()));
|
||||
} else if (cube.compatible<long double>()) {
|
||||
obj->set(key, ailego::JsonValue(cube.unsafe_cast<long double>()));
|
||||
} else if (cube.compatible<Params>()) {
|
||||
ailego::JsonObject subobj;
|
||||
SerializeToJsonObject(cube.unsafe_cast<Params>(), &subobj);
|
||||
obj->set(key, ailego::JsonValue(subobj));
|
||||
} else {
|
||||
LOG_WARN("Unsupported serializing \'%s\' <%s>.", key, cube.type().name());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void Params::SerializeToBuffer(const Params ¶ms, std::string *buf) {
|
||||
if (buf != nullptr) {
|
||||
ailego::JsonObject obj;
|
||||
SerializeToJsonObject(params, &obj);
|
||||
buf->assign(ailego::JsonValue(obj).as_json_string().as_stl_string());
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,776 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <ailego/container/hypercube.h>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
//! Trying compatible with T
|
||||
#define _TRYING_COMPATIBLE(cube, T, out) \
|
||||
if (cube->compatible<T>()) \
|
||||
return ( \
|
||||
*out = static_cast<typename std::remove_pointer<decltype(out)>::type>( \
|
||||
cube->unsafe_cast<T>()), \
|
||||
true)
|
||||
|
||||
//! Trying compatible with T (Boolean)
|
||||
#define _TRYING_COMPATIBLE_BOOL(cube, T, out) \
|
||||
if (cube->compatible<T>()) return (*out = !!cube->unsafe_cast<T>(), true)
|
||||
|
||||
//! Trying compatible with T (String)
|
||||
#define _TRYING_COMPATIBLE_STRING(cube, T, out) \
|
||||
if (cube->compatible<T>()) \
|
||||
return (out->assign(std::to_string(cube->unsafe_cast<T>())), true)
|
||||
|
||||
//! Trying convert string
|
||||
#define _TRYING_CONVERT_STRING(cube, out) \
|
||||
if (cube->compatible<std::string>()) \
|
||||
return (*out = Params::StringCast<std::remove_pointer<decltype(out)>::type>( \
|
||||
cube->unsafe_cast<std::string>()), \
|
||||
true)
|
||||
|
||||
/*! Index Params
|
||||
*/
|
||||
class Params {
|
||||
public:
|
||||
//! Constructor
|
||||
Params(void) : hypercube_() {}
|
||||
|
||||
//! Constructor
|
||||
Params(const Params &rhs) : hypercube_(rhs.hypercube_) {}
|
||||
|
||||
//! Constructor
|
||||
Params(Params &&rhs) : hypercube_() {
|
||||
hypercube_.swap(rhs.hypercube_);
|
||||
}
|
||||
|
||||
//! Destructor
|
||||
~Params(void) {}
|
||||
|
||||
//! Assignment
|
||||
Params &operator=(const Params &rhs) {
|
||||
hypercube_ = rhs.hypercube_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Params &operator=(Params &&rhs) {
|
||||
hypercube_.swap(rhs.hypercube_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
ailego::Cube &operator[](const std::string &key) {
|
||||
return hypercube_[key];
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
ailego::Cube &operator[](std::string &&key) {
|
||||
return hypercube_[std::move(key)];
|
||||
}
|
||||
|
||||
//! Test if the element is exist
|
||||
bool has(const std::string &key) const {
|
||||
return hypercube_.has(key);
|
||||
}
|
||||
|
||||
//! Test if the map is empty
|
||||
bool empty(void) const {
|
||||
return hypercube_.empty();
|
||||
}
|
||||
|
||||
//! Clear the map
|
||||
void clear(void) {
|
||||
hypercube_.clear();
|
||||
}
|
||||
|
||||
//! Erase the pair via a key
|
||||
bool erase(const std::string &key) {
|
||||
return hypercube_.erase(key);
|
||||
}
|
||||
|
||||
//! Merge another index params
|
||||
void merge(const Params &rhs) {
|
||||
hypercube_.merge(rhs.hypercube_);
|
||||
}
|
||||
|
||||
//! Merge another index params
|
||||
void merge(Params &&rhs) {
|
||||
hypercube_.merge(std::move(rhs.hypercube_));
|
||||
}
|
||||
|
||||
//! Set the value of key in T
|
||||
template <typename T>
|
||||
bool insert(const std::string &key, T &&val) {
|
||||
return hypercube_.insert<T>(key, std::forward<T>(val));
|
||||
}
|
||||
|
||||
//! Set the value of key in T
|
||||
template <typename T>
|
||||
bool insert(std::string &&key, T &&val) {
|
||||
return hypercube_.insert<T>(std::forward<std::string>(key),
|
||||
std::forward<T>(val));
|
||||
}
|
||||
|
||||
//! Set the value of key in T
|
||||
template <typename T>
|
||||
void set(const std::string &key, T &&val) {
|
||||
hypercube_.insert_or_assign<T>(key, std::forward<T>(val));
|
||||
}
|
||||
|
||||
//! Set the value of key in T
|
||||
template <typename T>
|
||||
void set(std::string &&key, T &&val) {
|
||||
hypercube_.insert_or_assign<T>(std::forward<std::string>(key),
|
||||
std::forward<T>(val));
|
||||
}
|
||||
|
||||
//! Retrieve the value in boolean
|
||||
bool get(const std::string &key, bool *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, char, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, short int, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, int, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, long int, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, float, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, double, out);
|
||||
_TRYING_COMPATIBLE_BOOL(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'char'
|
||||
bool get(const std::string &key, char *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'unsigned char'
|
||||
bool get(const std::string &key, unsigned char *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'signed char'
|
||||
bool get(const std::string &key, signed char *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'short int'
|
||||
bool get(const std::string &key, short int *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'unsigned short int'
|
||||
bool get(const std::string &key, unsigned short int *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'int'
|
||||
bool get(const std::string &key, int *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'unsigned int'
|
||||
bool get(const std::string &key, unsigned int *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'long int'
|
||||
bool get(const std::string &key, long int *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'unsigned long int'
|
||||
bool get(const std::string &key, unsigned long int *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'long long int'
|
||||
bool get(const std::string &key, long long int *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'unsigned long long int'
|
||||
bool get(const std::string &key, unsigned long long int *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'float'
|
||||
bool get(const std::string &key, float *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'double'
|
||||
bool get(const std::string &key, double *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in 'long double'
|
||||
bool get(const std::string &key, long double *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, long double, out);
|
||||
_TRYING_COMPATIBLE(cube, double, out);
|
||||
_TRYING_COMPATIBLE(cube, float, out);
|
||||
_TRYING_COMPATIBLE(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE(cube, long int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE(cube, int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE(cube, short int, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE(cube, char, out);
|
||||
_TRYING_COMPATIBLE(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE(cube, bool, out);
|
||||
_TRYING_CONVERT_STRING(cube, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in string
|
||||
bool get(const std::string &key, std::string *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, std::string, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, bool, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, char, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, unsigned char, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, signed char, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, short int, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, unsigned short int, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, int, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, unsigned int, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, long int, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, unsigned long int, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, long long int, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, unsigned long long int, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, float, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, double, out);
|
||||
_TRYING_COMPATIBLE_STRING(cube, long double, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in T
|
||||
template <typename T>
|
||||
bool get(const std::string &key, T *out) const {
|
||||
const ailego::Cube *cube = hypercube_.get(key);
|
||||
if (cube) {
|
||||
_TRYING_COMPATIBLE(cube, T, out);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
//! Retrieve the value in boolean
|
||||
bool get_as_bool(const std::string &key) const {
|
||||
bool result = false;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in int8
|
||||
int8_t get_as_int8(const std::string &key) const {
|
||||
int8_t result = 0;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in int16
|
||||
int16_t get_as_int16(const std::string &key) const {
|
||||
int16_t result = 0;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in int32
|
||||
int32_t get_as_int32(const std::string &key) const {
|
||||
int32_t result = 0;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in int64
|
||||
int64_t get_as_int64(const std::string &key) const {
|
||||
int64_t result = 0;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in uint8
|
||||
uint8_t get_as_uint8(const std::string &key) const {
|
||||
uint8_t result = 0;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in uint16
|
||||
uint16_t get_as_uint16(const std::string &key) const {
|
||||
uint16_t result = 0;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in uint32
|
||||
uint32_t get_as_uint32(const std::string &key) const {
|
||||
uint32_t result = 0;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in uint64
|
||||
uint64_t get_as_uint64(const std::string &key) const {
|
||||
uint64_t result = 0;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in float
|
||||
float get_as_float(const std::string &key) const {
|
||||
float result = 0.0f;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in double
|
||||
double get_as_double(const std::string &key) const {
|
||||
double result = 0.0f;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the value in string
|
||||
std::string get_as_string(const std::string &key) const {
|
||||
std::string result;
|
||||
this->get(key, &result);
|
||||
return result;
|
||||
}
|
||||
|
||||
//! Retrieve the debug string
|
||||
std::string debug_string(void) const {
|
||||
std::string str;
|
||||
SerializeToBuffer(*this, &str);
|
||||
return str;
|
||||
}
|
||||
|
||||
//! Retrieve the map of parameters
|
||||
const ailego::Hypercube &hypercube(void) const {
|
||||
return hypercube_;
|
||||
}
|
||||
|
||||
//! Retrieve the map of parameters
|
||||
ailego::Hypercube *mutable_hypercube(void) {
|
||||
return &hypercube_;
|
||||
}
|
||||
|
||||
//! Parse parameters from buffer (Json format)
|
||||
static bool ParseFromBuffer(const std::string &buf, Params *params);
|
||||
|
||||
//! Parse parameters from OS environment
|
||||
static void ParseFromEnvironment(Params *params);
|
||||
|
||||
//! Serialize parameters into buffer
|
||||
static void SerializeToBuffer(const Params ¶ms, std::string *buf);
|
||||
|
||||
protected:
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, float>::value, T>::type {
|
||||
return std::strtof(str.c_str(), nullptr);
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, double>::value, T>::type {
|
||||
return std::strtod(str.c_str(), nullptr);
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, long double>::value, T>::type {
|
||||
return std::strtold(str.c_str(), nullptr);
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, char>::value, T>::type {
|
||||
return static_cast<char>(std::strtol(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, signed char>::value, T>::type {
|
||||
return static_cast<signed char>(std::strtol(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, unsigned char>::value, T>::type {
|
||||
return static_cast<unsigned char>(std::strtoul(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, short int>::value, T>::type {
|
||||
return static_cast<short int>(std::strtol(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, int>::value, T>::type {
|
||||
return static_cast<int>(std::strtol(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, long int>::value, T>::type {
|
||||
return static_cast<long int>(std::strtol(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, long long int>::value, T>::type {
|
||||
return static_cast<long long int>(std::strtoll(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, unsigned short int>::value,
|
||||
T>::type {
|
||||
return static_cast<unsigned short int>(
|
||||
std::strtoul(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, unsigned int>::value, T>::type {
|
||||
return static_cast<unsigned int>(std::strtoul(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, unsigned long int>::value,
|
||||
T>::type {
|
||||
return static_cast<unsigned long int>(
|
||||
std::strtoul(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, unsigned long long int>::value,
|
||||
T>::type {
|
||||
return static_cast<unsigned long long int>(
|
||||
std::strtoull(str.c_str(), nullptr, 0));
|
||||
}
|
||||
|
||||
//! Convert string type to another type
|
||||
template <typename T>
|
||||
static auto StringCast(const std::string &str) ->
|
||||
typename std::enable_if<std::is_same<T, bool>::value, T>::type {
|
||||
if (str.empty()) {
|
||||
return false;
|
||||
}
|
||||
char c = str[0];
|
||||
if (c == 'Y' || c == 'T' || c == 'y' || c == 't') {
|
||||
return true;
|
||||
}
|
||||
return !!std::strtof(str.c_str(), nullptr);
|
||||
}
|
||||
|
||||
private:
|
||||
ailego::Hypercube hypercube_;
|
||||
};
|
||||
|
||||
#undef _TRYING_COMPATIBLE
|
||||
#undef _TRYING_COMPATIBLE
|
||||
#undef _TRYING_COMPATIBLE_BOOL
|
||||
#undef _TRYING_COMPATIBLE_STRING
|
||||
#undef _TRYING_CONVERT_STRING
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,140 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <random>
|
||||
#include <vector>
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! Sampling Reservoir
|
||||
*/
|
||||
template <typename T, typename Allocator = std::allocator<T>>
|
||||
class Reservoir {
|
||||
public:
|
||||
//! Constructor
|
||||
Reservoir(size_t cnt)
|
||||
: samples_(cnt), total_(0), mt_(std::random_device()()), pool_() {
|
||||
pool_.reserve(samples_);
|
||||
}
|
||||
|
||||
//! Constructor
|
||||
Reservoir(const Reservoir &rhs)
|
||||
: samples_(rhs.samples_),
|
||||
total_(rhs.total_),
|
||||
mt_(std::random_device()()),
|
||||
pool_(rhs.pool_) {}
|
||||
|
||||
//! Constructor
|
||||
Reservoir(Reservoir &&rhs)
|
||||
: samples_(rhs.samples_),
|
||||
total_(rhs.total_),
|
||||
mt_(std::random_device()()),
|
||||
pool_(std::move(rhs.pool_)) {}
|
||||
|
||||
//! Destructor
|
||||
~Reservoir(void) {}
|
||||
|
||||
//! Assignment
|
||||
Reservoir &operator=(const Reservoir &rhs) {
|
||||
samples_ = rhs.samples_;
|
||||
total_ = rhs.total_;
|
||||
pool_ = rhs.pool_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
Reservoir &operator=(Reservoir &&rhs) {
|
||||
samples_ = rhs.samples_;
|
||||
total_ = rhs.total_;
|
||||
pool_ = std::move(rhs.pool_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Retrieve pool of reservoir
|
||||
std::vector<T, Allocator> *mutable_pool(void) {
|
||||
return &pool_;
|
||||
}
|
||||
|
||||
//! Retrieve pool of reservoir
|
||||
const std::vector<T, Allocator> &pool(void) const {
|
||||
return pool_;
|
||||
}
|
||||
|
||||
//! Retrieve count of samples
|
||||
size_t samples(void) const {
|
||||
return samples_;
|
||||
}
|
||||
|
||||
//! Retrieve total count of filling
|
||||
size_t total(void) const {
|
||||
return total_;
|
||||
}
|
||||
|
||||
//! Reset the reservoir
|
||||
void reset(void) {
|
||||
total_ = 0;
|
||||
pool_.clear();
|
||||
pool_.reserve(samples_);
|
||||
}
|
||||
|
||||
//! Fill the reservoir
|
||||
void fill(const T &item) {
|
||||
if (samples_ > 0) {
|
||||
if (pool_.size() >= samples_) {
|
||||
std::uniform_int_distribution<size_t> dt(0, total_);
|
||||
size_t i = dt(mt_);
|
||||
|
||||
if (i < samples_) {
|
||||
pool_[i] = item;
|
||||
}
|
||||
} else {
|
||||
pool_.push_back(item);
|
||||
}
|
||||
}
|
||||
++total_;
|
||||
}
|
||||
|
||||
//! Fill the reservoir
|
||||
void fill(T &&item) {
|
||||
if (samples_ > 0) {
|
||||
if (pool_.size() >= samples_) {
|
||||
std::uniform_int_distribution<size_t> dt(0, total_);
|
||||
size_t i = dt(mt_);
|
||||
|
||||
if (i < samples_) {
|
||||
pool_[i] = std::move(item);
|
||||
}
|
||||
} else {
|
||||
pool_.push_back(std::move(item));
|
||||
}
|
||||
}
|
||||
++total_;
|
||||
}
|
||||
|
||||
private:
|
||||
//! Disable them
|
||||
Reservoir(void) = delete;
|
||||
|
||||
//! Members
|
||||
size_t samples_;
|
||||
size_t total_;
|
||||
std::mt19937 mt_;
|
||||
std::vector<T, Allocator> pool_;
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,555 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "vector.h"
|
||||
|
||||
namespace zvec {
|
||||
namespace ailego {
|
||||
|
||||
/*! Numerical Vector Array
|
||||
*/
|
||||
template <typename T,
|
||||
typename =
|
||||
typename std::enable_if<IsTriviallyCopyable<T>::value>::type>
|
||||
class NumericalVectorArray {
|
||||
public:
|
||||
//! Type of value
|
||||
using ValueType = typename NumericalVector<T>::ValueType;
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorArray(void) {}
|
||||
|
||||
//! Constructor
|
||||
explicit NumericalVectorArray(size_t dim) : dimension_(dim) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorArray(const NumericalVectorArray &rhs)
|
||||
: dimension_(rhs.dimension_), buffer_(rhs.buffer_) {}
|
||||
|
||||
//! Constructor
|
||||
NumericalVectorArray(NumericalVectorArray &&rhs)
|
||||
: dimension_(rhs.dimension_), buffer_(std::move(rhs.buffer_)) {}
|
||||
|
||||
//! Assignment
|
||||
NumericalVectorArray &operator=(const NumericalVectorArray &rhs) {
|
||||
dimension_ = rhs.dimension_;
|
||||
buffer_ = rhs.buffer_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
NumericalVectorArray &operator=(NumericalVectorArray &&rhs) {
|
||||
dimension_ = rhs.dimension_;
|
||||
buffer_ = std::move(rhs.buffer_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
ValueType *operator[](size_t i) {
|
||||
return (reinterpret_cast<ValueType *>(&buffer_[0]) + i * dimension_);
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
const ValueType *operator[](size_t i) const {
|
||||
return (reinterpret_cast<const ValueType *>(buffer_.data()) +
|
||||
i * dimension_);
|
||||
}
|
||||
|
||||
//! Append a vector
|
||||
void append(const ValueType *vec, size_t dim) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
buffer_.append(reinterpret_cast<const char *>(vec),
|
||||
dim * sizeof(ValueType));
|
||||
}
|
||||
|
||||
//! Append vectors
|
||||
void append(const ValueType *vec, size_t dim, size_t cnt) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
buffer_.append(reinterpret_cast<const char *>(vec),
|
||||
cnt * dim * sizeof(ValueType));
|
||||
}
|
||||
|
||||
//! Append a vector
|
||||
void append(const NumericalVector<ValueType> &vec) {
|
||||
this->append(vec.data(), vec.dimension());
|
||||
}
|
||||
|
||||
//! Replace a vector
|
||||
void replace(size_t index, const ValueType *vec, size_t dim) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
size_t element_size = dim * sizeof(ValueType);
|
||||
buffer_.replace(index * element_size, element_size,
|
||||
reinterpret_cast<const char *>(vec), element_size);
|
||||
}
|
||||
|
||||
//! Replace a vector
|
||||
void replace(size_t index, const ValueType *vec, size_t dim, size_t cnt) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
size_t element_size = dim * sizeof(ValueType);
|
||||
size_t total = element_size * cnt;
|
||||
buffer_.replace(index * element_size, total,
|
||||
reinterpret_cast<const char *>(vec), total);
|
||||
}
|
||||
|
||||
//! Replace a vector
|
||||
void replace(size_t index, const NumericalVector<ValueType> &vec) {
|
||||
this->replace(index, vec.data(), vec.dimension());
|
||||
}
|
||||
|
||||
//! Request a change in capacity
|
||||
void reserve(size_t n) {
|
||||
buffer_.reserve(n * dimension_ * sizeof(ValueType));
|
||||
}
|
||||
|
||||
//! Resize the array to a length of n elements
|
||||
void resize(size_t n) {
|
||||
buffer_.resize(n * dimension_ * sizeof(ValueType));
|
||||
}
|
||||
|
||||
//! Clear the vector array
|
||||
void clear(void) {
|
||||
buffer_.clear();
|
||||
}
|
||||
|
||||
//! Reset the vector array
|
||||
void reset(size_t dim) {
|
||||
dimension_ = dim;
|
||||
buffer_.clear();
|
||||
}
|
||||
|
||||
//! Requests the removal of unused capacity.
|
||||
void shrink_to_fit(void) {
|
||||
buffer_.shrink_to_fit();
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
ValueType *data(void) {
|
||||
return reinterpret_cast<ValueType *>(&buffer_[0]);
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
const ValueType *data(void) const {
|
||||
return reinterpret_cast<const ValueType *>(buffer_.data());
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
ValueType *at(size_t i) {
|
||||
if (ailego_unlikely(i >= this->count())) {
|
||||
throw std::out_of_range("Index overflow");
|
||||
}
|
||||
return (reinterpret_cast<ValueType *>(&buffer_[0]) + i * dimension_);
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
const ValueType *at(size_t i) const {
|
||||
if (ailego_unlikely(i >= this->count())) {
|
||||
throw std::out_of_range("Index overflow");
|
||||
}
|
||||
return (reinterpret_cast<const ValueType *>(buffer_.data()) +
|
||||
i * dimension_);
|
||||
}
|
||||
|
||||
//! Test if the array is empty
|
||||
bool empty(void) const {
|
||||
return buffer_.empty();
|
||||
}
|
||||
|
||||
//! Retrieve count of vectors
|
||||
size_t count(void) const {
|
||||
return (dimension_ > 0 ? buffer_.size() / (dimension_ * sizeof(ValueType))
|
||||
: 0u);
|
||||
}
|
||||
|
||||
//! Retrieve dimension of vector
|
||||
size_t dimension(void) const {
|
||||
return dimension_;
|
||||
}
|
||||
|
||||
//! Retrieve size of array in bytes
|
||||
size_t bytes(void) const {
|
||||
return buffer_.size();
|
||||
}
|
||||
|
||||
private:
|
||||
size_t dimension_{0u};
|
||||
std::string buffer_{};
|
||||
};
|
||||
|
||||
/*! Nibble Vector Array
|
||||
*/
|
||||
template <typename T,
|
||||
typename = typename std::enable_if<std::is_integral<T>::value>::type>
|
||||
class NibbleVectorArray {
|
||||
public:
|
||||
//! Type of value
|
||||
using ValueType = typename NibbleVector<T>::ValueType;
|
||||
using StoreType = typename NibbleVector<T>::StoreType;
|
||||
|
||||
//! Constructor
|
||||
NibbleVectorArray(void) {}
|
||||
|
||||
//! Constructor
|
||||
explicit NibbleVectorArray(size_t dim)
|
||||
: dimension_((dim + (sizeof(ValueType) << 1) - 1) /
|
||||
(sizeof(ValueType) << 1) * sizeof(ValueType)
|
||||
<< 1) {}
|
||||
|
||||
//! Constructor
|
||||
NibbleVectorArray(const NibbleVectorArray &rhs)
|
||||
: dimension_(rhs.dimension_), buffer_(rhs.buffer_) {}
|
||||
|
||||
//! Constructor
|
||||
NibbleVectorArray(NibbleVectorArray &&rhs)
|
||||
: dimension_(rhs.dimension_), buffer_(std::move(rhs.buffer_)) {}
|
||||
|
||||
//! Assignment
|
||||
NibbleVectorArray &operator=(const NibbleVectorArray &rhs) {
|
||||
dimension_ = rhs.dimension_;
|
||||
buffer_ = rhs.buffer_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
NibbleVectorArray &operator=(NibbleVectorArray &&rhs) {
|
||||
dimension_ = rhs.dimension_;
|
||||
buffer_ = std::move(rhs.buffer_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
StoreType *operator[](size_t i) {
|
||||
return reinterpret_cast<StoreType *>(&buffer_[0] + i * (dimension_ >> 1));
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
const StoreType *operator[](size_t i) const {
|
||||
return reinterpret_cast<const StoreType *>(&buffer_[0] +
|
||||
i * (dimension_ >> 1));
|
||||
}
|
||||
|
||||
//! Append a vector
|
||||
void append(const StoreType *vec, size_t dim) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
buffer_.append(reinterpret_cast<const char *>(vec), dim >> 1);
|
||||
}
|
||||
|
||||
//! Append vectors
|
||||
void append(const StoreType *vec, size_t dim, size_t cnt) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
buffer_.append(reinterpret_cast<const char *>(vec), cnt * (dim >> 1));
|
||||
}
|
||||
|
||||
//! Append a vector
|
||||
void append(const NibbleVector<ValueType> &vec) {
|
||||
this->append(vec.data(), vec.dimension());
|
||||
}
|
||||
|
||||
//! Replace a vector
|
||||
void replace(size_t index, const StoreType *vec, size_t dim) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
size_t element_size = (dim >> 1);
|
||||
buffer_.replace(index * element_size, element_size,
|
||||
reinterpret_cast<const char *>(vec), element_size);
|
||||
}
|
||||
|
||||
//! Replace a vector
|
||||
void replace(size_t index, const StoreType *vec, size_t dim, size_t cnt) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
size_t element_size = (dim >> 1);
|
||||
size_t total = element_size * cnt;
|
||||
buffer_.replace(index * element_size, total,
|
||||
reinterpret_cast<const char *>(vec), total);
|
||||
}
|
||||
|
||||
//! Replace a vector
|
||||
void replace(size_t index, const NibbleVector<ValueType> &vec) {
|
||||
this->replace(index, vec.data(), vec.dimension());
|
||||
}
|
||||
|
||||
//! Request a change in capacity
|
||||
void reserve(size_t n) {
|
||||
buffer_.reserve(n * (dimension_ >> 1));
|
||||
}
|
||||
|
||||
//! Resize the array to a length of n elements
|
||||
void resize(size_t n) {
|
||||
buffer_.resize(n * (dimension_ >> 1));
|
||||
}
|
||||
|
||||
//! Clear the vector array
|
||||
void clear(void) {
|
||||
buffer_.clear();
|
||||
}
|
||||
|
||||
//! Reset the vector array
|
||||
void reset(size_t dim) {
|
||||
dimension_ = (dim + (sizeof(ValueType) << 1) - 1) /
|
||||
(sizeof(ValueType) << 1) * sizeof(ValueType)
|
||||
<< 1;
|
||||
buffer_.clear();
|
||||
}
|
||||
|
||||
//! Requests the removal of unused capacity.
|
||||
void shrink_to_fit(void) {
|
||||
buffer_.shrink_to_fit();
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
StoreType *data(void) {
|
||||
return reinterpret_cast<StoreType *>(&buffer_[0]);
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
const StoreType *data(void) const {
|
||||
return reinterpret_cast<const StoreType *>(buffer_.data());
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
StoreType *at(size_t i) {
|
||||
if (ailego_unlikely(i >= this->count())) {
|
||||
throw std::out_of_range("Index overflow");
|
||||
}
|
||||
return reinterpret_cast<StoreType *>(&buffer_[0] + i * (dimension_ >> 1));
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
const StoreType *at(size_t i) const {
|
||||
if (ailego_unlikely(i >= this->count())) {
|
||||
throw std::out_of_range("Index overflow");
|
||||
}
|
||||
return reinterpret_cast<const StoreType *>(buffer_.data() +
|
||||
i * (dimension_ >> 1));
|
||||
}
|
||||
|
||||
//! Test if the array is empty
|
||||
bool empty(void) const {
|
||||
return buffer_.empty();
|
||||
}
|
||||
|
||||
//! Retrieve count of vectors
|
||||
size_t count(void) const {
|
||||
return (dimension_ > 1 ? buffer_.size() / (dimension_ >> 1) : 0u);
|
||||
}
|
||||
|
||||
//! Retrieve dimension of vector
|
||||
size_t dimension(void) const {
|
||||
return dimension_;
|
||||
}
|
||||
|
||||
//! Retrieve size of array in bytes
|
||||
size_t bytes(void) const {
|
||||
return buffer_.size();
|
||||
}
|
||||
|
||||
private:
|
||||
size_t dimension_{0u};
|
||||
std::string buffer_{};
|
||||
};
|
||||
|
||||
/*! Binary Vector Array
|
||||
*/
|
||||
template <typename T,
|
||||
typename = typename std::enable_if<std::is_integral<T>::value>::type>
|
||||
class BinaryVectorArray {
|
||||
public:
|
||||
//! Type of value
|
||||
using ValueType = typename BinaryVector<T>::ValueType;
|
||||
|
||||
//! Constructor
|
||||
BinaryVectorArray(void) {}
|
||||
|
||||
//! Constructor
|
||||
explicit BinaryVectorArray(size_t dim)
|
||||
: dimension_((dim + (sizeof(ValueType) << 3) - 1) /
|
||||
(sizeof(ValueType) << 3) * (sizeof(ValueType) << 3)) {}
|
||||
|
||||
//! Constructor
|
||||
BinaryVectorArray(const BinaryVectorArray &rhs)
|
||||
: dimension_(rhs.dimension_), buffer_(rhs.buffer_) {}
|
||||
|
||||
//! Constructor
|
||||
BinaryVectorArray(BinaryVectorArray &&rhs)
|
||||
: dimension_(rhs.dimension_), buffer_(std::move(rhs.buffer_)) {}
|
||||
|
||||
//! Assignment
|
||||
BinaryVectorArray &operator=(const BinaryVectorArray &rhs) {
|
||||
dimension_ = rhs.dimension_;
|
||||
buffer_ = rhs.buffer_;
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Assignment
|
||||
BinaryVectorArray &operator=(BinaryVectorArray &&rhs) {
|
||||
dimension_ = rhs.dimension_;
|
||||
buffer_ = std::move(rhs.buffer_);
|
||||
return *this;
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
ValueType *operator[](size_t i) {
|
||||
return reinterpret_cast<ValueType *>(&buffer_[0] + i * (dimension_ >> 3));
|
||||
}
|
||||
|
||||
//! Overloaded operator []
|
||||
const ValueType *operator[](size_t i) const {
|
||||
return reinterpret_cast<const ValueType *>(buffer_.data() +
|
||||
i * (dimension_ >> 3));
|
||||
}
|
||||
|
||||
//! Append a vector
|
||||
void append(const ValueType *vec, size_t dim) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
buffer_.append(reinterpret_cast<const char *>(vec), (dim >> 3));
|
||||
}
|
||||
|
||||
//! Append vectors
|
||||
void append(const ValueType *vec, size_t dim, size_t cnt) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
buffer_.append(reinterpret_cast<const char *>(vec), cnt * (dim >> 3));
|
||||
}
|
||||
|
||||
//! Append a vector
|
||||
void append(const BinaryVector<ValueType> &vec) {
|
||||
this->append(vec.data(), vec.dimension());
|
||||
}
|
||||
|
||||
//! Replace a vector
|
||||
void replace(size_t index, const ValueType *vec, size_t dim) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
size_t element_size = (dim >> 3);
|
||||
buffer_.replace(index * element_size, element_size,
|
||||
reinterpret_cast<const char *>(vec), element_size);
|
||||
}
|
||||
|
||||
//! Replace a vector
|
||||
void replace(size_t index, const ValueType *vec, size_t dim, size_t cnt) {
|
||||
if (ailego_unlikely(dim != dimension_)) {
|
||||
throw std::length_error("Unmatched dimension");
|
||||
}
|
||||
size_t element_size = (dim >> 3);
|
||||
size_t total = element_size * cnt;
|
||||
buffer_.replace(index * element_size, total,
|
||||
reinterpret_cast<const char *>(vec), total);
|
||||
}
|
||||
|
||||
//! Replace a vector
|
||||
void replace(size_t index, const BinaryVector<ValueType> &vec) {
|
||||
this->replace(index, vec.data(), vec.dimension());
|
||||
}
|
||||
|
||||
//! Request a change in capacity
|
||||
void reserve(size_t n) {
|
||||
buffer_.reserve(n * (dimension_ >> 3));
|
||||
}
|
||||
|
||||
//! Resize the array to a length of n elements
|
||||
void resize(size_t n) {
|
||||
buffer_.resize(n * (dimension_ >> 3));
|
||||
}
|
||||
|
||||
//! Clear the vector array
|
||||
void clear(void) {
|
||||
buffer_.clear();
|
||||
}
|
||||
|
||||
//! Reset the vector array
|
||||
void reset(size_t dim) {
|
||||
dimension_ = (dim + (sizeof(ValueType) << 3) - 1) /
|
||||
(sizeof(ValueType) << 3) * (sizeof(ValueType) << 3);
|
||||
buffer_.clear();
|
||||
}
|
||||
|
||||
//! Requests the removal of unused capacity.
|
||||
void shrink_to_fit(void) {
|
||||
buffer_.shrink_to_fit();
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
ValueType *data(void) {
|
||||
return reinterpret_cast<ValueType *>(&buffer_[0]);
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
const ValueType *data(void) const {
|
||||
return reinterpret_cast<const ValueType *>(buffer_.data());
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
ValueType *at(size_t i) {
|
||||
if (ailego_unlikely(i >= this->count())) {
|
||||
throw std::out_of_range("Index overflow");
|
||||
}
|
||||
return reinterpret_cast<ValueType *>(&buffer_[0] + i * (dimension_ >> 3));
|
||||
}
|
||||
|
||||
//! Retrieve pointer of data
|
||||
const ValueType *at(size_t i) const {
|
||||
if (ailego_unlikely(i >= this->count())) {
|
||||
throw std::out_of_range("Index overflow");
|
||||
}
|
||||
return reinterpret_cast<const ValueType *>(buffer_.data() +
|
||||
i * (dimension_ >> 3));
|
||||
}
|
||||
|
||||
//! Test if the array is empty
|
||||
bool empty(void) const {
|
||||
return buffer_.empty();
|
||||
}
|
||||
|
||||
//! Retrieve count of vectors
|
||||
size_t count(void) const {
|
||||
return (dimension_ > 0 ? buffer_.size() / (dimension_ >> 3) : 0u);
|
||||
}
|
||||
|
||||
//! Retrieve dimension of vector
|
||||
size_t dimension(void) const {
|
||||
return dimension_;
|
||||
}
|
||||
|
||||
//! Retrieve size of array in bytes
|
||||
size_t bytes(void) const {
|
||||
return buffer_.size();
|
||||
}
|
||||
|
||||
private:
|
||||
size_t dimension_{0u};
|
||||
std::string buffer_{};
|
||||
};
|
||||
|
||||
} // namespace ailego
|
||||
} // namespace zvec
|
||||
@@ -0,0 +1,17 @@
|
||||
// Copyright 2025-present the zvec project
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include "json/mod_json_plus.h"
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user